diff options
Diffstat (limited to 'ci')
| -rwxr-xr-x | ci/check-directional-formatting.bash | 27 | ||||
| -rwxr-xr-x | ci/install-dependencies.sh | 35 | ||||
| -rwxr-xr-x | ci/install-docker-dependencies.sh | 2 | ||||
| -rwxr-xr-x | ci/lib.sh | 62 | ||||
| -rwxr-xr-x | ci/print-test-failures.sh | 10 | ||||
| -rwxr-xr-x | ci/run-build-and-tests.sh | 28 | ||||
| -rwxr-xr-x | ci/run-docker-build.sh | 11 | ||||
| -rwxr-xr-x | ci/run-docker.sh | 4 |
8 files changed, 80 insertions, 99 deletions
diff --git a/ci/check-directional-formatting.bash b/ci/check-directional-formatting.bash new file mode 100755 index 0000000000..e6211b141a --- /dev/null +++ b/ci/check-directional-formatting.bash @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script verifies that the non-binary files tracked in the Git index do +# not contain any Unicode directional formatting: such formatting could be used +# to deceive reviewers into interpreting code differently from the compiler. +# This is intended to run on an Ubuntu agent in a GitHub workflow. +# +# To allow translated messages to introduce such directional formatting in the +# future, we exclude the `.po` files from this validation. +# +# Neither GNU grep nor `git grep` (not even with `-P`) handle `\u` as a way to +# specify UTF-8. +# +# To work around that, we use `printf` to produce the pattern as a byte +# sequence, and then feed that to `git grep` as a byte sequence (setting +# `LC_CTYPE` to make sure that the arguments are interpreted as intended). +# +# Note: we need to use Bash here because its `printf` interprets `\uNNNN` as +# UTF-8 code points, as desired. Running this script through Ubuntu's `dash`, +# for example, would use a `printf` that does not understand that syntax. + +# U+202a..U+2a2e: LRE, RLE, PDF, LRO and RLO +# U+2066..U+2069: LRI, RLI, FSI and PDI +regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)' + +! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \ + -- ':(exclude,attr:binary)' ':(exclude)*.po' diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 5772081b6e..dbcebad2fb 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -11,18 +11,11 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" -case "$jobname" in -linux-clang|linux-gcc) - sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" +case "$runs_on_pool" in +ubuntu-latest) sudo apt-get -q update sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ - $UBUNTU_COMMON_PKGS - case "$jobname" in - linux-gcc) - sudo apt-get -q -y install gcc-8 - ;; - esac - + $UBUNTU_COMMON_PKGS $CC_PACKAGE mkdir --parents "$P4_PATH" pushd "$P4_PATH" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" @@ -37,7 +30,7 @@ linux-clang|linux-gcc) cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . popd ;; -osx-clang|osx-gcc) +macos-latest) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 # Uncomment this if you want to run perf tests: # brew install gnu-time @@ -51,15 +44,17 @@ osx-clang|osx-gcc) brew install --cask --no-quarantine perforce } || brew install homebrew/cask/perforce - case "$jobname" in - osx-gcc) - brew install gcc@9 - # Just in case the image is updated to contain gcc@9 - # pre-installed but not linked. - brew link gcc@9 - ;; - esac + + if test -n "$CC_PACKAGE" + then + BREW_PACKAGE=${CC_PACKAGE/-/@} + brew install "$BREW_PACKAGE" + brew link "$BREW_PACKAGE" + fi ;; +esac + +case "$jobname" in StaticAnalysis) sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ @@ -77,7 +72,7 @@ Documentation) test -n "$ALREADY_HAVE_ASCIIDOCTOR" || sudo gem install --version 1.5.8 asciidoctor ;; -linux-gcc-default|linux-gcc-4.8) +linux-gcc-default) sudo apt-get -q update sudo apt-get -q -y install $UBUNTU_COMMON_PKGS ;; diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 07a8c6b199..78b7e326da 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -4,7 +4,7 @@ # case "$jobname" in -Linux32) +linux32) linux32 --32bit i386 sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev \ @@ -34,7 +34,7 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if test "$TRAVIS_DEBUG_MODE" = true || test true = "$GITHUB_ACTIONS" + if test true = "$GITHUB_ACTIONS" then return fi @@ -60,7 +60,7 @@ skip_good_tree () { cat <<-EOF $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. - The log of that build job is available at $(url_for_job_id $prev_good_job_id) + The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id To force a re-build delete the branch's cache and then hit 'Restart job'. EOF fi @@ -91,29 +91,7 @@ export MAKEFLAGS= # and installing dependencies. set -ex -if test true = "$TRAVIS" -then - CI_TYPE=travis - # When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not - # what we want here. We want the source branch instead. - CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" - CI_COMMIT="$TRAVIS_COMMIT" - CI_JOB_ID="$TRAVIS_JOB_ID" - CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" - CI_OS_NAME="$TRAVIS_OS_NAME" - CI_REPO_SLUG="$TRAVIS_REPO_SLUG" - - cache_dir="$HOME/travis-cache" - - url_for_job_id () { - echo "https://travis-ci.org/$CI_REPO_SLUG/jobs/$1" - } - - BREW_INSTALL_PACKAGES="git-lfs gettext" - export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --immediate" - MAKEFLAGS="$MAKEFLAGS --jobs=2" -elif test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" +if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" then CI_TYPE=azure-pipelines # We are running in Azure Pipelines @@ -130,10 +108,6 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - url_for_job_id () { - echo "$SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$1" - } - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" MAKEFLAGS="$MAKEFLAGS --jobs=10" @@ -182,11 +156,15 @@ export DEFAULT_TEST_TARGET=prove export GIT_TEST_CLONE_2GB=true export SKIP_DASHED_BUILT_INS=YesPlease -case "$jobname" in -linux-clang|linux-gcc) +case "$runs_on_pool" in +ubuntu-latest) + if test "$jobname" = "linux-gcc-default" + then + break + fi + if [ "$jobname" = linux-gcc ] then - export CC=gcc-8 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3" else MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2" @@ -206,24 +184,20 @@ linux-clang|linux-gcc) GIT_LFS_PATH="$HOME/custom/git-lfs" export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" ;; -osx-clang|osx-gcc) +macos-latest) if [ "$jobname" = osx-gcc ] then - export CC=gcc-9 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" else MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" fi - - # t9810 occasionally fails on Travis CI OS X - # t9816 occasionally fails with "TAP out of sequence errors" on - # Travis CI OS X - export GIT_SKIP_TESTS="t9810 t9816" ;; -linux-gcc-default) - ;; -Linux32) +esac + +case "$jobname" in +linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc @@ -231,6 +205,10 @@ linux-musl) MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" ;; +linux-leaks) + export SANITIZE=leak + export GIT_TEST_PASSING_SANITIZE_LEAK=true + ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index c70d6cdbf2..57277eefcd 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -39,8 +39,6 @@ do test_name="${test_name##*/}" trash_dir="trash directory.$test_name" case "$CI_TYPE" in - travis) - ;; azure-pipelines) mkdir -p failed-test-artifacts mv "$trash_dir" failed-test-artifacts @@ -88,11 +86,3 @@ do fi fi done - -if [ $combined_trash_size -gt 0 ] -then - echo "------------------------------------------------------------------------" - echo "Trash directories embedded in this log can be extracted by running:" - echo - echo " curl https://api.travis-ci.org/v3/job/$TRAVIS_JOB_ID/log.txt |./ci/util/extract-trash-dirs.sh" -fi diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index f3aba5d6cb..280dda7d28 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -10,16 +10,13 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -if test "$jobname" = "pedantic" -then - export DEVOPTS=pedantic -fi +export MAKE_TARGETS="all test" -make case "$jobname" in linux-gcc) export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main - make test + ;; +linux-TEST-vars) export GIT_TEST_SPLIT_INDEX=yes export GIT_TEST_MERGE_ALGORITHM=recursive export GIT_TEST_FULL_IN_PACK_ARRAY=true @@ -28,27 +25,30 @@ linux-gcc) export GIT_TEST_COMMIT_GRAPH=1 export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1 export GIT_TEST_MULTI_PACK_INDEX=1 + export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1 export GIT_TEST_ADD_I_USE_BUILTIN=1 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master export GIT_TEST_WRITE_REV_INDEX=1 export GIT_TEST_CHECKOUT_WORKERS=2 - make test ;; linux-clang) export GIT_TEST_DEFAULT_HASH=sha1 - make test + ;; +linux-sha256) export GIT_TEST_DEFAULT_HASH=sha256 - make test ;; -linux-gcc-4.8|pedantic) +pedantic) # Don't run the tests; we only care about whether Git can be - # built with GCC 4.8 or with pedantic - ;; -*) - make test + # built. + export DEVOPTS=pedantic + export MAKE_TARGETS=all ;; esac +# Any new "test" targets should not go after this "make", but should +# adjust $MAKE_TARGETS. Otherwise compilation-only targets above will +# start running tests. +make $MAKE_TARGETS check_unignored_build_artifacts save_good_tree diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index 8d47a5fda3..6cd832efb9 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -15,7 +15,7 @@ then fi case "$jobname" in -Linux32) +linux32) switch_cmd="linux32 --32bit i386" ;; linux-musl) @@ -47,15 +47,6 @@ else else useradd -u $HOST_UID $CI_USER fi - - # Due to a bug the test suite was run as root in the past, so - # a prove state file created back then is only accessible by - # root. Now that bug is fixed, the test suite is run as a - # regular user, but the prove state file coming from Travis - # CI's cache might still be owned by root. - # Make sure that this user has rights to any cached files, - # including an existing prove state file. - test -n "$cache_dir" && chown -R $HOST_UID:$HOST_UID "$cache_dir" fi # Build and test diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 37fa372052..af89d1624a 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -6,7 +6,7 @@ . ${0%/*}/lib.sh case "$jobname" in -Linux32) +linux32) CI_CONTAINER="daald/ubuntu32:xenial" ;; linux-musl) @@ -25,7 +25,7 @@ docker pull "$CI_CONTAINER" # root@container:/# export jobname=<jobname> # root@container:/# /usr/src/git/ci/run-docker-build.sh <host-user-id> -container_cache_dir=/tmp/travis-cache +container_cache_dir=/tmp/container-cache docker run \ --interactive \ |