diff options
Diffstat (limited to 'ci')
| -rwxr-xr-x | ci/check-whitespace.sh | 10 | ||||
| -rwxr-xr-x | ci/install-dependencies.sh | 60 | ||||
| -rwxr-xr-x | ci/install-sdk.ps1 | 12 | ||||
| -rwxr-xr-x | ci/lib.sh | 30 | ||||
| -rwxr-xr-x | ci/run-build-and-minimal-fuzzers.sh | 13 | ||||
| -rwxr-xr-x | ci/run-build-and-tests.sh | 2 | ||||
| -rwxr-xr-x | ci/run-docker-build.sh | 66 | ||||
| -rwxr-xr-x | ci/run-docker.sh | 47 | ||||
| -rwxr-xr-x | ci/run-style-check.sh | 25 |
9 files changed, 119 insertions, 146 deletions
diff --git a/ci/check-whitespace.sh b/ci/check-whitespace.sh index db399097a5..c40804394c 100755 --- a/ci/check-whitespace.sh +++ b/ci/check-whitespace.sh @@ -9,7 +9,7 @@ baseCommit=$1 outputFile=$2 url=$3 -if test "$#" -ne 1 && test "$#" -ne 3 +if test "$#" -ne 1 && test "$#" -ne 3 || test -z "$1" then echo "USAGE: $0 <BASE_COMMIT> [<OUTPUT_FILE> <URL>]" exit 1 @@ -21,6 +21,12 @@ commitText= commitTextmd= goodParent= +if ! git rev-parse --quiet --verify "${baseCommit}" +then + echo "Invalid <BASE_COMMIT> '${baseCommit}'" + exit 1 +fi + while read dash sha etc do case "${dash}" in @@ -67,7 +73,7 @@ then goodParent=${baseCommit: 0:7} fi - echo "A whitespace issue was found in onen of more of the commits." + echo "A whitespace issue was found in one or more of the commits." echo "Run the following command to resolve whitespace issues:" echo "git rebase --whitespace=fix ${goodParent}" diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 6ec0f85972..d020cb7aa5 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -7,7 +7,7 @@ begin_group "Install dependencies" -P4WHENCE=https://cdist2.perforce.com/perforce/r21.2 +P4WHENCE=https://cdist2.perforce.com/perforce/r23.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION JGITWHENCE=https://repo.eclipse.org/content/groups/releases//org/eclipse/jgit/org.eclipse.jgit.pgm/6.8.0.202311291450-r/org.eclipse.jgit.pgm-6.8.0.202311291450-r.sh @@ -29,41 +29,54 @@ alpine-*) apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ bash cvs gnupg perl-cgi perl-dbd-sqlite perl-io-tty >/dev/null ;; -fedora-*) +fedora-*|almalinux-*) dnf -yq update >/dev/null && dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; -ubuntu-*) +ubuntu-*|ubuntu32-*|debian-*) # Required so that apt doesn't wait for user input on certain packages. export DEBIAN_FRONTEND=noninteractive + case "$distro" in + ubuntu-*) + SVN='libsvn-perl subversion' + LANGUAGES='language-pack-is' + ;; + ubuntu32-*) + SVN= + LANGUAGES='language-pack-is' + ;; + *) + SVN='libsvn-perl subversion' + LANGUAGES='locales-all' + ;; + esac + sudo apt-get -q update sudo apt-get -q -y install \ - language-pack-is libsvn-perl apache2 cvs cvsps git gnupg subversion \ + $LANGUAGES apache2 cvs cvsps git gnupg $SVN \ make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \ tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE - mkdir --parents "$CUSTOM_PATH" - wget --quiet --directory-prefix="$CUSTOM_PATH" \ - "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" - chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" + case "$distro" in + ubuntu-*) + mkdir --parents "$CUSTOM_PATH" - wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ - -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" - rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + wget --quiet --directory-prefix="$CUSTOM_PATH" \ + "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" + chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" - wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" - chmod a+x "$CUSTOM_PATH/jgit" - ;; -ubuntu32-*) - sudo linux32 --32bit i386 sh -c ' - apt update >/dev/null && - apt install -y build-essential libcurl4-openssl-dev \ - libssl-dev libexpat-dev gettext python >/dev/null - ' + wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ + -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" + rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + + wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" + chmod a+x "$CUSTOM_PATH/jgit" + ;; + esac ;; macos-*) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 @@ -87,6 +100,10 @@ macos-*) esac case "$jobname" in +ClangFormat) + sudo apt-get -q update + sudo apt-get -q -y install clang-format + ;; StaticAnalysis) sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ @@ -103,6 +120,7 @@ Documentation) test -n "$ALREADY_HAVE_ASCIIDOCTOR" || sudo gem install --version 1.5.8 asciidoctor + sudo gem install concurrent-ruby ;; esac diff --git a/ci/install-sdk.ps1 b/ci/install-sdk.ps1 new file mode 100755 index 0000000000..66f24838a4 --- /dev/null +++ b/ci/install-sdk.ps1 @@ -0,0 +1,12 @@ +param( + [string]$directory='git-sdk', + [string]$url='https://github.com/git-for-windows/git-sdk-64/releases/download/ci-artifacts/git-sdk-x86_64-minimal.zip' +) + +Invoke-WebRequest "$url" -OutFile git-sdk.zip +Expand-Archive -LiteralPath git-sdk.zip -DestinationPath "$directory" +Remove-Item -Path git-sdk.zip + +New-Item -Path .git/info -ItemType Directory -Force +New-Item -Path .git/info/exclude -ItemType File -Force +Add-Content -Path .git/info/exclude -Value "/$directory" @@ -62,7 +62,7 @@ trap "end_group 'CI setup'" EXIT # something went wrong. # # We already enabled tracing executed commands earlier. This helps by showing -# how # environment variables are set and and dependencies are installed. +# how # environment variables are set and dependencies are installed. set -e skip_branch_tip_with_tag () { @@ -250,8 +250,13 @@ then CI_TYPE=gitlab-ci CI_BRANCH="$CI_COMMIT_REF_NAME" CI_COMMIT="$CI_COMMIT_SHA" - case "$CI_JOB_IMAGE" in - macos-*) + + case "$OS,$CI_JOB_IMAGE" in + Windows_NT,*) + CI_OS_NAME=windows + JOBS=$NUMBER_OF_PROCESSORS + ;; + *,macos-*) # GitLab CI has Python installed via multiple package managers, # most notably via asdf and Homebrew. Ensure that our builds # pick up the Homebrew one by prepending it to our PATH as the @@ -259,9 +264,12 @@ then export PATH="$(brew --prefix)/bin:$PATH" CI_OS_NAME=osx + JOBS=$(nproc) + ;; + *,alpine:*|*,fedora:*|*,ubuntu:*) + CI_OS_NAME=linux + JOBS=$(nproc) ;; - alpine:*|fedora:*|ubuntu:*) - CI_OS_NAME=linux;; *) echo "Could not identify OS image" >&2 env >&2 @@ -272,6 +280,7 @@ then CI_JOB_ID="$CI_JOB_ID" CC="${CC_PACKAGE:-${CC:-gcc}}" DONT_SKIP_TAGS=t + handle_failed_tests () { create_failed_test_artifacts return 1 @@ -280,7 +289,6 @@ then cache_dir="$HOME/none" distro=$(echo "$CI_JOB_IMAGE" | tr : -) - JOBS=$(nproc) else echo "Could not identify CI type" >&2 env >&2 @@ -336,7 +344,14 @@ ubuntu-*) fi MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE" - export GIT_TEST_HTTPD=true + case "$distro" in + ubuntu-16.04) + # Apache is too old for HTTP/2. + ;; + *) + export GIT_TEST_HTTPD=true + ;; + esac # The Linux build installs the defined dependency versions below. # The OS X build installs much more recent versions, whichever @@ -369,7 +384,6 @@ linux-musl) ;; linux-leaks|linux-reftable-leaks) export SANITIZE=leak - export GIT_TEST_PASSING_SANITIZE_LEAK=true ;; linux-asan-ubsan) export SANITIZE=address,undefined diff --git a/ci/run-build-and-minimal-fuzzers.sh b/ci/run-build-and-minimal-fuzzers.sh index af8065f349..e7b97952e7 100755 --- a/ci/run-build-and-minimal-fuzzers.sh +++ b/ci/run-build-and-minimal-fuzzers.sh @@ -13,7 +13,18 @@ group "Build fuzzers" make \ LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \ fuzz-all -for fuzzer in commit-graph config date pack-headers pack-idx ; do +fuzzers=" +commit-graph +config +credential-from-url-gently +date +pack-headers +pack-idx +parse-attr-line +url-decode-mem +" + +for fuzzer in $fuzzers; do begin_group "fuzz-$fuzzer" ./oss-fuzz/fuzz-$fuzzer -verbosity=0 -runs=1 || exit 1 end_group "fuzz-$fuzzer" diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 98dda42045..2e28d02b20 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -25,7 +25,7 @@ linux-TEST-vars) export GIT_TEST_COMMIT_GRAPH=1 export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1 export GIT_TEST_MULTI_PACK_INDEX=1 - export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1 + export GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=1 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master export GIT_TEST_NO_WRITE_REV_INDEX=1 export GIT_TEST_CHECKOUT_WORKERS=2 diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh deleted file mode 100755 index 6cd832efb9..0000000000 --- a/ci/run-docker-build.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -# -# Build and test Git inside container -# -# Usage: -# run-docker-build.sh <host-user-id> -# - -set -ex - -if test $# -ne 1 || test -z "$1" -then - echo >&2 "usage: run-docker-build.sh <host-user-id>" - exit 1 -fi - -case "$jobname" in -linux32) - switch_cmd="linux32 --32bit i386" - ;; -linux-musl) - switch_cmd= - useradd () { adduser -D "$@"; } - ;; -*) - exit 1 - ;; -esac - -"${0%/*}/install-docker-dependencies.sh" - -# If this script runs inside a docker container, then all commands are -# usually executed as root. Consequently, the host user might not be -# able to access the test output files. -# If a non 0 host user id is given, then create a user "ci" with that -# user id to make everything accessible to the host user. -HOST_UID=$1 -if test $HOST_UID -eq 0 -then - # Just in case someone does want to run the test suite as root. - CI_USER=root -else - CI_USER=ci - if test "$(id -u $CI_USER 2>/dev/null)" = $HOST_UID - then - echo "user '$CI_USER' already exists with the requested ID $HOST_UID" - else - useradd -u $HOST_UID $CI_USER - fi -fi - -# Build and test -command $switch_cmd su -m -l $CI_USER -c " - set -ex - export DEVELOPER='$DEVELOPER' - export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' - export GIT_PROVE_OPTS='$GIT_PROVE_OPTS' - export GIT_TEST_OPTS='$GIT_TEST_OPTS' - export GIT_TEST_CLONE_2GB='$GIT_TEST_CLONE_2GB' - export MAKEFLAGS='$MAKEFLAGS' - export cache_dir='$cache_dir' - cd /usr/src/git - test -n '$cache_dir' && ln -s '$cache_dir/.prove' t/.prove - make - make test -" diff --git a/ci/run-docker.sh b/ci/run-docker.sh deleted file mode 100755 index af89d1624a..0000000000 --- a/ci/run-docker.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# -# Download and run Docker image to build and test Git -# - -. ${0%/*}/lib.sh - -case "$jobname" in -linux32) - CI_CONTAINER="daald/ubuntu32:xenial" - ;; -linux-musl) - CI_CONTAINER=alpine - ;; -*) - exit 1 - ;; -esac - -docker pull "$CI_CONTAINER" - -# Use the following command to debug the docker build locally: -# <host-user-id> must be 0 if podman is used as drop-in replacement for docker -# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/sh "$CI_CONTAINER" -# root@container:/# export jobname=<jobname> -# root@container:/# /usr/src/git/ci/run-docker-build.sh <host-user-id> - -container_cache_dir=/tmp/container-cache - -docker run \ - --interactive \ - --env DEVELOPER \ - --env DEFAULT_TEST_TARGET \ - --env GIT_PROVE_OPTS \ - --env GIT_TEST_OPTS \ - --env GIT_TEST_CLONE_2GB \ - --env MAKEFLAGS \ - --env jobname \ - --env cache_dir="$container_cache_dir" \ - --volume "${PWD}:/usr/src/git" \ - --volume "$cache_dir:$container_cache_dir" \ - "$CI_CONTAINER" \ - /usr/src/git/ci/run-docker-build.sh $(id -u $USER) - -check_unignored_build_artifacts - -save_good_tree diff --git a/ci/run-style-check.sh b/ci/run-style-check.sh new file mode 100755 index 0000000000..6cd4b1d934 --- /dev/null +++ b/ci/run-style-check.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# +# Perform style check +# + +baseCommit=$1 + +# Remove optional braces of control statements (if, else, for, and while) +# according to the LLVM coding style. This avoids braces on simple +# single-statement bodies of statements but keeps braces if one side of +# if/else if/.../else cascade has multi-statement body. +# +# As this rule comes with a warning [1], we want to experiment with it +# before adding it in-tree. since the CI job for the style check is allowed +# to fail, appending the rule here allows us to validate its efficacy. +# While also ensuring that end-users are not affected directly. +# +# [1]: https://clang.llvm.org/docs/ClangFormatStyleOptions.html#removebracesllvm +{ + cat .clang-format + echo "RemoveBracesLLVM: true" +} >/tmp/clang-format-rules + +git clang-format --style=file:/tmp/clang-format-rules \ + --diff --extensions c,h "$baseCommit" |