diff options
Diffstat (limited to 'ci')
| -rw-r--r-- | ci/config/README | 14 | ||||
| -rwxr-xr-x | ci/config/allow-ref.sample | 27 | ||||
| -rwxr-xr-x | ci/install-docker-dependencies.sh | 23 | ||||
| -rwxr-xr-x | ci/lib.sh | 205 | ||||
| -rwxr-xr-x | ci/print-test-failures.sh | 6 | ||||
| -rwxr-xr-x | ci/run-build-and-tests.sh | 5 | ||||
| -rwxr-xr-x | ci/run-test-slice.sh | 5 |
7 files changed, 194 insertions, 91 deletions
diff --git a/ci/config/README b/ci/config/README new file mode 100644 index 0000000000..8de3a04e32 --- /dev/null +++ b/ci/config/README @@ -0,0 +1,14 @@ +You can configure some aspects of the GitHub Actions-based CI on a +per-repository basis by setting "variables" and "secrets" from with the +GitHub web interface. These can be found at: + + https://github.com/<user>/git/settings/secrets/actions + +The following variables can be used: + + - CI_BRANCHES + + By default, CI is run when any branch is pushed. If this variable is + non-empty, then only the branches it lists will run CI. Branch names + should be separated by spaces, and should use their shortened form + (e.g., "main", not "refs/heads/main"). diff --git a/ci/config/allow-ref.sample b/ci/config/allow-ref.sample deleted file mode 100755 index af0e076f8a..0000000000 --- a/ci/config/allow-ref.sample +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# -# Sample script for enabling/disabling GitHub Actions CI runs on -# particular refs. By default, CI is run for all branches pushed to -# GitHub. You can override this by dropping the ".sample" from the script, -# editing it, committing, and pushing the result to the "ci-config" branch of -# your repository: -# -# git checkout -b ci-config -# cp allow-ref.sample allow-ref -# $EDITOR allow-ref -# git add allow-ref -# git commit -am "implement my ci preferences" -# git push -# -# This script will then be run when any refs are pushed to that repository. It -# gets the fully qualified refname as the first argument, and should exit with -# success only for refs for which you want to run CI. - -case "$1" in -# allow one-off tests by pushing to "for-ci" or "for-ci/mybranch" -refs/heads/for-ci*) true ;; -# always build your integration branch -refs/heads/my-integration-branch) true ;; -# don't build any other branches or tags -*) false ;; -esac diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 78b7e326da..48c43f0f90 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -3,6 +3,10 @@ # Install dependencies required to build and test Git inside container # +. ${0%/*}/lib.sh + +begin_group "Install dependencies" + case "$jobname" in linux32) linux32 --32bit i386 sh -c ' @@ -12,11 +16,26 @@ linux32) ' ;; linux-musl) - apk add --update build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null + ;; +linux-*) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + + apt update -q && + apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \ + libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \ + perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ + libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ + apache2 cvs cvsps gnupg libcgi-pm-perl subversion ;; pedantic) dnf -yq update >/dev/null && dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; esac + +end_group "Install dependencies" @@ -1,16 +1,7 @@ # Library of functions shared by all CI scripts -if test true != "$GITHUB_ACTIONS" +if test true = "$GITHUB_ACTIONS" then - begin_group () { :; } - end_group () { :; } - - group () { - shift - "$@" - } - set -x -else begin_group () { need_to_end_group=t echo "::group::$1" >&2 @@ -23,27 +14,50 @@ else need_to_end_group= echo '::endgroup::' >&2 } - trap end_group EXIT +elif test true = "$GITLAB_CI" +then + begin_group () { + need_to_end_group=t + printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n" + trap "end_group '$1'" EXIT + set -x + } - group () { + end_group () { + test -n "$need_to_end_group" || return 0 set +x - begin_group "$1" - shift - # work around `dash` not supporting `set -o pipefail` - ( - "$@" 2>&1 - echo $? >exit.status - ) | - sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' - res=$(cat exit.status) - rm exit.status - end_group - return $res + need_to_end_group= + printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n" + trap - EXIT } +else + begin_group () { :; } + end_group () { :; } - begin_group "CI setup" + set -x fi +group () { + group="$1" + shift + begin_group "$group" + + # work around `dash` not supporting `set -o pipefail` + ( + "$@" 2>&1 + echo $? >exit.status + ) | + sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' + res=$(cat exit.status) + rm exit.status + + end_group "$group" + return $res +} + +begin_group "CI setup" +trap "end_group 'CI setup'" EXIT + # Set 'exit on error' for all CI scripts to let the caller know that # something went wrong. # @@ -71,10 +85,32 @@ skip_branch_tip_with_tag () { fi } +# Check whether we can use the path passed via the first argument as Git +# repository. +is_usable_git_repository () { + # We require Git in our PATH, otherwise we cannot access repositories + # at all. + if ! command -v git >/dev/null + then + return 1 + fi + + # And the target directory needs to be a proper Git repository. + if ! git -C "$1" rev-parse 2>/dev/null + then + return 1 + fi +} + # Save some info about the current commit's tree, so we can skip the build # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { + if ! is_usable_git_repository . + then + return + fi + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp @@ -90,6 +126,11 @@ skip_good_tree () { return fi + if ! is_usable_git_repository . + then + return + fi + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" then # Haven't seen this tree yet, or no cached good trees file yet. @@ -121,6 +162,11 @@ skip_good_tree () { } check_unignored_build_artifacts () { + if ! is_usable_git_repository . + then + return + fi + ! git ls-files --other --exclude-standard --error-unmatch \ -- ':/*' 2>/dev/null || { @@ -133,6 +179,26 @@ handle_failed_tests () { return 1 } +create_failed_test_artifacts () { + mkdir -p t/failed-test-artifacts + + for test_exit in t/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the 'print test failures' step below." + echo "See also the 'failed-tests-*' artifacts attached to this run." + cat "t/test-results/$test_name.markup" + + trash_dir="t/trash directory.$test_name" + cp "t/test-results/$test_name.out" t/failed-test-artifacts/ + tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + done +} + # GitHub Action doesn't set TERM, which is required by tput export TERM=${TERM:-dumb} @@ -156,11 +222,8 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows_nt != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + GIT_TEST_OPTS="--write-junit-xml" + JOBS=10 elif test true = "$GITHUB_ACTIONS" then CI_TYPE=github-actions @@ -173,40 +236,63 @@ then CC="${CC_PACKAGE:-${CC:-gcc}}" DONT_SKIP_TAGS=t handle_failed_tests () { - mkdir -p t/failed-test-artifacts echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV + create_failed_test_artifacts + return 1 + } + + cache_dir="$HOME/none" - for test_exit in t/test-results/*.exit - do - test 0 != "$(cat "$test_exit")" || continue - - test_name="${test_exit%.exit}" - test_name="${test_name##*/}" - printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" - echo "The full logs are in the 'print test failures' step below." - echo "See also the 'failed-tests-*' artifacts attached to this run." - cat "t/test-results/$test_name.markup" - - trash_dir="t/trash directory.$test_name" - cp "t/test-results/$test_name.out" t/failed-test-artifacts/ - tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" - done + GIT_TEST_OPTS="--github-workflow-markup" + JOBS=10 +elif test true = "$GITLAB_CI" +then + CI_TYPE=gitlab-ci + CI_BRANCH="$CI_COMMIT_REF_NAME" + CI_COMMIT="$CI_COMMIT_SHA" + case "$CI_JOB_IMAGE" in + macos-*) + CI_OS_NAME=osx;; + alpine:*|fedora:*|ubuntu:*) + CI_OS_NAME=linux;; + *) + echo "Could not identify OS image" >&2 + env >&2 + exit 1 + ;; + esac + CI_REPO_SLUG="$CI_PROJECT_PATH" + CI_JOB_ID="$CI_JOB_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + create_failed_test_artifacts return 1 } cache_dir="$HOME/none" - export GIT_PROVE_OPTS="--timer --jobs 10" - export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) + JOBS=$(nproc) else echo "Could not identify CI type" >&2 env >&2 exit 1 fi +MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" +GIT_PROVE_OPTS="--timer --jobs $JOBS" + +GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" +case "$CI_OS_NAME" in +windows|windows_nt) + GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" + ;; +esac + +export GIT_TEST_OPTS +export GIT_PROVE_OPTS + good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" @@ -253,11 +339,9 @@ ubuntu-*) export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" ;; macos-*) - if [ "$jobname" = osx-gcc ] + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" + if [ "$jobname" != osx-gcc ] then - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" - else - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" fi ;; @@ -278,15 +362,14 @@ linux-leaks) export GIT_TEST_PASSING_SANITIZE_LEAK=true export GIT_TEST_SANITIZE_LEAK_LOG=true ;; -linux-asan) - export SANITIZE=address - ;; -linux-ubsan) - export SANITIZE=undefined +linux-asan-ubsan) + export SANITIZE=address,undefined + export NO_SVN_TESTS=LetsSaveSomeTime + MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften" ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" -end_group +end_group "CI setup" set -x diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 57277eefcd..c33ad4e3a2 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -51,6 +51,12 @@ do tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" continue ;; + gitlab-ci) + mkdir -p failed-test-artifacts + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1 diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index b098e10f52..7a1466b868 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -27,8 +27,9 @@ linux-TEST-vars) export GIT_TEST_MULTI_PACK_INDEX=1 export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master - export GIT_TEST_WRITE_REV_INDEX=1 + export GIT_TEST_NO_WRITE_REV_INDEX=1 export GIT_TEST_CHECKOUT_WORKERS=2 + export GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=1 ;; linux-clang) export GIT_TEST_DEFAULT_HASH=sha1 @@ -49,6 +50,8 @@ if test -n "$run_tests" then group "Run tests" make test || handle_failed_tests + group "Run unit tests" \ + make DEFAULT_UNIT_TEST_TARGET=unit-tests-prove unit-tests fi check_unignored_build_artifacts diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index a3c67956a8..ae8094382f 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -15,4 +15,9 @@ group "Run tests" make --quiet -C t T="$(cd t && tr '\n' ' ')" || handle_failed_tests +# We only have one unit test at the moment, so run it in the first slice +if [ "$1" == "0" ] ; then + group "Run unit tests" make --quiet -C t unit-tests-prove +fi + check_unignored_build_artifacts |