diff options
Diffstat (limited to 'ci')
| -rwxr-xr-x | ci/install-dependencies.sh | 18 | ||||
| -rwxr-xr-x | ci/lib-travisci.sh | 107 | ||||
| -rwxr-xr-x | ci/print-test-failures.sh | 62 | ||||
| -rwxr-xr-x | ci/run-build-and-tests.sh | 22 | ||||
| -rwxr-xr-x | ci/run-build.sh | 8 | ||||
| -rwxr-xr-x | ci/run-linux32-build.sh | 50 | ||||
| -rwxr-xr-x | ci/run-linux32-docker.sh | 10 | ||||
| -rwxr-xr-x | ci/run-static-analysis.sh | 23 | ||||
| -rwxr-xr-x | ci/run-tests.sh | 10 | ||||
| -rwxr-xr-x | ci/run-windows-build.sh | 7 | ||||
| -rwxr-xr-x | ci/test-documentation.sh | 8 | ||||
| -rwxr-xr-x | ci/util/extract-trash-dirs.sh | 50 |
12 files changed, 330 insertions, 45 deletions
diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index a29246af35..75a9fd2475 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -8,27 +8,23 @@ P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION -case "${TRAVIS_OS_NAME:-linux}" in -linux) - export GIT_TEST_HTTPD=YesPlease - - mkdir --parents custom/p4 - pushd custom/p4 +case "$jobname" in +linux-clang|linux-gcc) + mkdir --parents "$P4_PATH" + pushd "$P4_PATH" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4" chmod u+x p4d chmod u+x p4 - export PATH="$(pwd):$PATH" popd - mkdir --parents custom/git-lfs - pushd custom/git-lfs + mkdir --parents "$GIT_LFS_PATH" + pushd "$GIT_LFS_PATH" wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . - export PATH="$(pwd):$PATH" popd ;; -osx) +osx-clang|osx-gcc) brew update --quiet # Uncomment this if you want to run perf tests: # brew install gnu-time diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index b3ed0a0dda..06970f7213 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -16,13 +16,114 @@ skip_branch_tip_with_tag () { if TAG=$(git describe --exact-match "$TRAVIS_BRANCH" 2>/dev/null) && test "$TAG" != "$TRAVIS_BRANCH" then - echo "Tip of $TRAVIS_BRANCH is exactly at $TAG" + echo "$(tput setaf 2)Tip of $TRAVIS_BRANCH is exactly at $TAG$(tput sgr0)" exit 0 fi } +# Save some info about the current commit's tree, so we can skip the build +# job if we encounter the same tree again and can provide a useful info +# message. +save_good_tree () { + echo "$(git rev-parse $TRAVIS_COMMIT^{tree}) $TRAVIS_COMMIT $TRAVIS_JOB_NUMBER $TRAVIS_JOB_ID" >>"$good_trees_file" + # limit the file size + tail -1000 "$good_trees_file" >"$good_trees_file".tmp + mv "$good_trees_file".tmp "$good_trees_file" +} + +# Skip the build job if the same tree has already been built and tested +# successfully before (e.g. because the branch got rebased, changing only +# the commit messages). +skip_good_tree () { + if ! good_tree_info="$(grep "^$(git rev-parse $TRAVIS_COMMIT^{tree}) " "$good_trees_file")" + then + # Haven't seen this tree yet, or no cached good trees file yet. + # Continue the build job. + return + fi + + echo "$good_tree_info" | { + read tree prev_good_commit prev_good_job_number prev_good_job_id + + if test "$TRAVIS_JOB_ID" = "$prev_good_job_id" + then + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit has already been built and tested successfully by this build job. + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + else + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. + The log of that build job is available at https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$prev_good_job_id + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + fi + } + + exit 0 +} + +check_unignored_build_artifacts () +{ + ! git ls-files --other --exclude-standard --error-unmatch \ + -- ':/*' 2>/dev/null || + { + echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)" + false + } +} + # Set 'exit on error' for all CI scripts to let the caller know that -# something went wrong -set -e +# something went wrong. +# Set tracing executed commands, primarily setting environment variables +# and installing dependencies. +set -ex + +cache_dir="$HOME/travis-cache" +good_trees_file="$cache_dir/good-trees" + +mkdir -p "$cache_dir" skip_branch_tip_with_tag +skip_good_tree + +if test -z "$jobname" +then + jobname="$TRAVIS_OS_NAME-$CC" +fi + +export DEVELOPER=1 +export DEFAULT_TEST_TARGET=prove +export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" +export GIT_TEST_OPTS="--verbose-log -x --immediate" +export GIT_TEST_CLONE_2GB=YesPlease +if [ "$jobname" = linux-gcc ]; then + export CC=gcc-8 +fi + +case "$jobname" in +linux-clang|linux-gcc) + export GIT_TEST_HTTPD=YesPlease + + # The Linux build installs the defined dependency versions below. + # The OS X build installs the latest available versions. Keep that + # in mind when you encounter a broken OS X build! + export LINUX_P4_VERSION="16.2" + export LINUX_GIT_LFS_VERSION="1.5.2" + + P4_PATH="$HOME/custom/p4" + GIT_LFS_PATH="$HOME/custom/git-lfs" + export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" + ;; +osx-clang|osx-gcc) + # t9810 occasionally fails on Travis CI OS X + # t9816 occasionally fails with "TAP out of sequence errors" on + # Travis CI OS X + export GIT_SKIP_TESTS="t9810 t9816" + ;; +GETTEXT_POISON) + export GETTEXT_POISON=YesPlease + ;; +esac diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 8c8973cbf3..d55460a212 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -5,7 +5,27 @@ . ${0%/*}/lib-travisci.sh -for TEST_EXIT in t/test-results/*.exit +# Tracing executed commands would produce too much noise in the loop below. +set +x + +cd t/ + +if ! ls test-results/*.exit >/dev/null 2>/dev/null +then + echo "Build job failed before the tests could have been run" + exit +fi + +case "$jobname" in +osx-clang|osx-gcc) + # base64 in OSX doesn't wrap its output at 76 columns by + # default, but prints a single, very long line. + base64_opts="-b 76" + ;; +esac + +combined_trash_size=0 +for TEST_EXIT in test-results/*.exit do if [ "$(cat "$TEST_EXIT")" != "0" ] then @@ -14,5 +34,45 @@ do echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)" echo "------------------------------------------------------------------------" cat "${TEST_OUT}" + + test_name="${TEST_EXIT%.exit}" + test_name="${test_name##*/}" + trash_dir="trash directory.$test_name" + trash_tgz_b64="trash.$test_name.base64" + if [ -d "$trash_dir" ] + then + tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64" + + trash_size=$(wc -c <"$trash_tgz_b64") + if [ $trash_size -gt 1048576 ] + then + # larger than 1MB + echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)" + continue + fi + + new_combined_trash_size=$(($combined_trash_size + $trash_size)) + if [ $new_combined_trash_size -gt 1048576 ] + then + echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)" + continue + fi + combined_trash_size=$new_combined_trash_size + + # DO NOT modify these two 'echo'-ed strings below + # without updating 'ci/util/extract-trash-dirs.sh' + # as well. + echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)" + cat "$trash_tgz_b64" + echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)" + fi fi done + +if [ $combined_trash_size -gt 0 ] +then + echo "------------------------------------------------------------------------" + echo "Trash directories embedded in this log can be extracted by running:" + echo + echo " curl https://api.travis-ci.org/v3/job/$TRAVIS_JOB_ID/log.txt |./ci/util/extract-trash-dirs.sh" +fi diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh new file mode 100755 index 0000000000..4b04c75b7f --- /dev/null +++ b/ci/run-build-and-tests.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Build and test Git +# + +. ${0%/*}/lib-travisci.sh + +ln -s "$cache_dir/.prove" t/.prove + +make --jobs=2 +make --quiet test +if test "$jobname" = "linux-gcc" +then + export GIT_TEST_SPLIT_INDEX=yes + export GIT_TEST_FULL_IN_PACK_ARRAY=true + export GIT_TEST_OE_SIZE=10 + make --quiet test +fi + +check_unignored_build_artifacts + +save_good_tree diff --git a/ci/run-build.sh b/ci/run-build.sh deleted file mode 100755 index 4f940d1032..0000000000 --- a/ci/run-build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# Build Git -# - -. ${0%/*}/lib-travisci.sh - -make --jobs=2 diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index e30fb2cddc..2c60d2e70a 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -3,28 +3,58 @@ # Build and test Git in a 32-bit environment # # Usage: -# run-linux32-build.sh [host-user-id] +# run-linux32-build.sh <host-user-id> # +set -ex + +if test $# -ne 1 || test -z "$1" +then + echo >&2 "usage: run-linux32-build.sh <host-user-id>" + exit 1 +fi + # Update packages to the latest available versions linux32 --32bit i386 sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext python >/dev/null -' && +' # If this script runs inside a docker container, then all commands are # usually executed as root. Consequently, the host user might not be # able to access the test output files. -# If a host user id is given, then create a user "ci" with the host user -# id to make everything accessible to the host user. -HOST_UID=$1 && -CI_USER=$USER && -test -z $HOST_UID || (CI_USER="ci" && useradd -u $HOST_UID $CI_USER) && +# If a non 0 host user id is given, then create a user "ci" with that +# user id to make everything accessible to the host user. +HOST_UID=$1 +if test $HOST_UID -eq 0 +then + # Just in case someone does want to run the test suite as root. + CI_USER=root +else + CI_USER=ci + if test "$(id -u $CI_USER 2>/dev/null)" = $HOST_UID + then + echo "user '$CI_USER' already exists with the requested ID $HOST_UID" + else + useradd -u $HOST_UID $CI_USER + fi + + # Due to a bug the test suite was run as root in the past, so + # a prove state file created back then is only accessible by + # root. Now that bug is fixed, the test suite is run as a + # regular user, but the prove state file coming from Travis + # CI's cache might still be owned by root. + # Make sure that this user has rights to any cached files, + # including an existing prove state file. + test -n "$cache_dir" && chown -R $HOST_UID:$HOST_UID "$cache_dir" +fi # Build and test linux32 --32bit i386 su -m -l $CI_USER -c ' - cd /usr/src/git && - make --jobs=2 && - make --quiet test + set -ex + cd /usr/src/git + test -n "$cache_dir" && ln -s "$cache_dir/.prove" t/.prove + make --jobs=2 + make --quiet test ' diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index 0edf63acfa..21637903ce 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -9,7 +9,9 @@ docker pull daald/ubuntu32:xenial # Use the following command to debug the docker build locally: # $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial -# root@container:/# /usr/src/git/ci/run-linux32-build.sh +# root@container:/# /usr/src/git/ci/run-linux32-build.sh <host-user-id> + +container_cache_dir=/tmp/travis-cache docker run \ --interactive \ @@ -18,6 +20,12 @@ docker run \ --env GIT_PROVE_OPTS \ --env GIT_TEST_OPTS \ --env GIT_TEST_CLONE_2GB \ + --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ + --volume "$cache_dir:$container_cache_dir" \ daald/ubuntu32:xenial \ /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) + +check_unignored_build_artifacts + +save_good_tree diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index 68dd0f080e..5688f261d0 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -5,4 +5,25 @@ . ${0%/*}/lib-travisci.sh -make coccicheck +make --jobs=2 coccicheck + +set +x + +fail= +for cocci_patch in contrib/coccinelle/*.patch +do + if test -s "$cocci_patch" + then + echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)" + cat "$cocci_patch" + fail=UnfortunatelyYes + fi +done + +if test -n "$fail" +then + echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)" + exit 1 +fi + +save_good_tree diff --git a/ci/run-tests.sh b/ci/run-tests.sh deleted file mode 100755 index f0c743de94..0000000000 --- a/ci/run-tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# -# Test Git -# - -. ${0%/*}/lib-travisci.sh - -mkdir -p $HOME/travis-cache -ln -s $HOME/travis-cache/.prove t/.prove -make --quiet test diff --git a/ci/run-windows-build.sh b/ci/run-windows-build.sh index 8757b3a97c..d99a180e52 100755 --- a/ci/run-windows-build.sh +++ b/ci/run-windows-build.sh @@ -69,6 +69,10 @@ esac echo "Visual Studio Team Services Build #${BUILD_ID}" +# Tracing execued commands would produce too much noise in the waiting +# loop below. +set +x + # Wait until build job finished STATUS= RESULT= @@ -90,7 +94,10 @@ done # Print log echo "" echo "" +set -x gfwci "action=log&buildId=$BUILD_ID" | cut -c 30- # Set exit code for TravisCI test "$RESULT" = "success" + +save_good_tree diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index 7a0a848e83..a20de9ca12 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -18,6 +18,9 @@ test -s Documentation/git.xml test -s Documentation/git.1 grep '<meta name="generator" content="AsciiDoc ' Documentation/git.html +rm -f stdout.log stderr.log +check_unignored_build_artifacts + # Build docs with AsciiDoctor make clean make --jobs=2 USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.log >&2) @@ -25,3 +28,8 @@ sed '/^GIT_VERSION = / d' stderr.log ! test -s stderr.log test -s Documentation/git.html grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html + +rm -f stdout.log stderr.log +check_unignored_build_artifacts + +save_good_tree diff --git a/ci/util/extract-trash-dirs.sh b/ci/util/extract-trash-dirs.sh new file mode 100755 index 0000000000..8e67bec21a --- /dev/null +++ b/ci/util/extract-trash-dirs.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +error () { + echo >&2 "error: $@" + exit 1 +} + +find_embedded_trash () { + while read -r line + do + case "$line" in + *Start\ of\ trash\ directory\ of\ \'t[0-9][0-9][0-9][0-9]-*\':*) + test_name="${line#*\'}" + test_name="${test_name%\'*}" + + return 0 + esac + done + + return 1 +} + +extract_embedded_trash () { + while read -r line + do + case "$line" in + *End\ of\ trash\ directory\ of\ \'$test_name\'*) + return + ;; + *) + printf '%s\n' "$line" + ;; + esac + done + + error "unexpected end of input" +} + +# Raw logs from Linux build jobs have CRLF line endings, while OSX +# build jobs mostly have CRCRLF, except an odd line every now and +# then that has CRCRCRLF. 'base64 -d' from 'coreutils' doesn't like +# CRs and complains about "invalid input", so remove all CRs at the +# end of lines. +sed -e 's/\r*$//' | \ +while find_embedded_trash +do + echo "Extracting trash directory of '$test_name'" + + extract_embedded_trash |base64 -d |tar xzp +done |