diff options
Diffstat (limited to 't/test-lib.sh')
| -rw-r--r-- | t/test-lib.sh | 505 |
1 files changed, 377 insertions, 128 deletions
diff --git a/t/test-lib.sh b/t/test-lib.sh index 0f1faa24b2..42b1a0aa7f 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -71,19 +71,227 @@ then exit 1 fi +# Parse options while taking care to leave $@ intact, so we will still +# have all the original command line options when executing the test +# script again for '--tee' and '--verbose-log' below. +store_arg_to= +prev_opt= +for opt +do + if test -n "$store_arg_to" + then + eval $store_arg_to=\$opt + store_arg_to= + prev_opt= + continue + fi + + case "$opt" in + -d|--d|--de|--deb|--debu|--debug) + debug=t ;; + -i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate) + immediate=t ;; + -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests) + GIT_TEST_LONG=t; export GIT_TEST_LONG ;; + -r) + store_arg_to=run_list + ;; + --run=*) + run_list=${opt#--*=} ;; + -h|--h|--he|--hel|--help) + help=t ;; + -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) + verbose=t ;; + --verbose-only=*) + verbose_only=${opt#--*=} + ;; + -q|--q|--qu|--qui|--quie|--quiet) + # Ignore --quiet under a TAP::Harness. Saying how many tests + # passed without the ok/not ok details is always an error. + test -z "$HARNESS_ACTIVE" && quiet=t ;; + --with-dashes) + with_dashes=t ;; + --no-bin-wrappers) + no_bin_wrappers=t ;; + --no-color) + color= ;; + --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind) + valgrind=memcheck + tee=t + ;; + --valgrind=*) + valgrind=${opt#--*=} + tee=t + ;; + --valgrind-only=*) + valgrind_only=${opt#--*=} + tee=t + ;; + --tee) + tee=t ;; + --root=*) + root=${opt#--*=} ;; + --chain-lint) + GIT_TEST_CHAIN_LINT=1 ;; + --no-chain-lint) + GIT_TEST_CHAIN_LINT=0 ;; + -x) + trace=t ;; + -V|--verbose-log) + verbose_log=t + tee=t + ;; + --write-junit-xml) + write_junit_xml=t + ;; + --stress) + stress=t ;; + --stress=*) + stress=${opt#--*=} + case "$stress" in + *[^0-9]*|0*|"") + echo "error: --stress=<N> requires the number of jobs to run" >&2 + exit 1 + ;; + *) # Good. + ;; + esac + ;; + *) + echo "error: unknown test option '$opt'" >&2; exit 1 ;; + esac + + prev_opt=$opt +done +if test -n "$store_arg_to" +then + echo "error: $prev_opt requires an argument" >&2 + exit 1 +fi + +if test -n "$valgrind_only" +then + test -z "$valgrind" && valgrind=memcheck + test -z "$verbose" && verbose_only="$valgrind_only" +elif test -n "$valgrind" +then + test -z "$verbose_log" && verbose=t +fi + +if test -n "$stress" +then + verbose=t + trace=t + immediate=t +fi + +TEST_STRESS_JOB_SFX="${GIT_TEST_STRESS_JOB_NR:+.stress-$GIT_TEST_STRESS_JOB_NR}" +TEST_NAME="$(basename "$0" .sh)" +TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results" +TEST_RESULTS_BASE="$TEST_RESULTS_DIR/$TEST_NAME$TEST_STRESS_JOB_SFX" +TRASH_DIRECTORY="trash directory.$TEST_NAME$TEST_STRESS_JOB_SFX" +test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY" +case "$TRASH_DIRECTORY" in +/*) ;; # absolute path is good + *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;; +esac + +# If --stress was passed, run this test repeatedly in several parallel loops. +if test "$GIT_TEST_STRESS_STARTED" = "done" +then + : # Don't stress test again. +elif test -n "$stress" +then + if test "$stress" != t + then + job_count=$stress + elif test -n "$GIT_TEST_STRESS_LOAD" + then + job_count="$GIT_TEST_STRESS_LOAD" + elif job_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null) && + test -n "$job_count" + then + job_count=$((2 * $job_count)) + else + job_count=8 + fi + + mkdir -p "$TEST_RESULTS_DIR" + stressfail="$TEST_RESULTS_BASE.stress-failed" + rm -f "$stressfail" + + stress_exit=0 + trap ' + kill $job_pids 2>/dev/null + wait + stress_exit=1 + ' TERM INT HUP + + job_pids= + job_nr=0 + while test $job_nr -lt "$job_count" + do + ( + GIT_TEST_STRESS_STARTED=done + GIT_TEST_STRESS_JOB_NR=$job_nr + export GIT_TEST_STRESS_STARTED GIT_TEST_STRESS_JOB_NR + + trap ' + kill $test_pid 2>/dev/null + wait + exit 1 + ' TERM INT + + cnt=0 + while ! test -e "$stressfail" + do + $TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 & + test_pid=$! + + if wait $test_pid + then + printf "OK %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt + else + echo $GIT_TEST_STRESS_JOB_NR >>"$stressfail" + printf "FAIL %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt + fi + cnt=$(($cnt + 1)) + done + ) & + job_pids="$job_pids $!" + job_nr=$(($job_nr + 1)) + done + + wait + + if test -f "$stressfail" + then + echo "Log(s) of failed test run(s):" + for failed_job_nr in $(sort -n "$stressfail") + do + echo "Contents of '$TEST_RESULTS_BASE.stress-$failed_job_nr.out':" + cat "$TEST_RESULTS_BASE.stress-$failed_job_nr.out" + done + rm -rf "$TRASH_DIRECTORY.stress-failed" + # Move the last one. + mv "$TRASH_DIRECTORY.stress-$failed_job_nr" "$TRASH_DIRECTORY.stress-failed" + fi + + exit $stress_exit +fi + # if --tee was passed, write the output not only to the terminal, but # additionally to the file test-results/$BASENAME.out, too. -case "$GIT_TEST_TEE_STARTED, $* " in -done,*) - # do not redirect again - ;; -*' --tee '*|*' --va'*|*' -V '*|*' --verbose-log '*) - mkdir -p "$TEST_OUTPUT_DIRECTORY/test-results" - BASE="$TEST_OUTPUT_DIRECTORY/test-results/$(basename "$0" .sh)" +if test "$GIT_TEST_TEE_STARTED" = "done" +then + : # do not redirect again +elif test -n "$tee" +then + mkdir -p "$TEST_RESULTS_DIR" # Make this filename available to the sub-process in case it is using # --verbose-log. - GIT_TEST_TEE_OUTPUT_FILE=$BASE.out + GIT_TEST_TEE_OUTPUT_FILE=$TEST_RESULTS_BASE.out export GIT_TEST_TEE_OUTPUT_FILE # Truncate before calling "tee -a" to get rid of the results @@ -91,11 +299,38 @@ done,*) >"$GIT_TEST_TEE_OUTPUT_FILE" (GIT_TEST_TEE_STARTED=done ${TEST_SHELL_PATH} "$0" "$@" 2>&1; - echo $? >"$BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE" - test "$(cat "$BASE.exit")" = 0 + echo $? >"$TEST_RESULTS_BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE" + test "$(cat "$TEST_RESULTS_BASE.exit")" = 0 exit - ;; -esac +fi + +if test -n "$trace" && test -n "$test_untraceable" +then + # '-x' tracing requested, but this test script can't be reliably + # traced, unless it is run with a Bash version supporting + # BASH_XTRACEFD (introduced in Bash v4.1). + # + # Perform this version check _after_ the test script was + # potentially re-executed with $TEST_SHELL_PATH for '--tee' or + # '--verbose-log', so the right shell is checked and the + # warning is issued only once. + if test -n "$BASH_VERSION" && eval ' + test ${BASH_VERSINFO[0]} -gt 4 || { + test ${BASH_VERSINFO[0]} -eq 4 && + test ${BASH_VERSINFO[1]} -ge 1 + } + ' + then + : Executed by a Bash version supporting BASH_XTRACEFD. Good. + else + echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD" + trace= + fi +fi +if test -n "$trace" && test -z "$verbose_log" +then + verbose=t +fi # For repeatability, reset the environment to known value. # TERM is sanitized below, after saving color control sequences. @@ -193,7 +428,7 @@ fi # Add libc MALLOC and MALLOC_PERTURB test # only if we are not executing the test with valgrind -if expr " $GIT_TEST_OPTS " : ".* --valgrind " >/dev/null || +if test -n "$valgrind" || test -n "$TEST_NO_MALLOC_CHECK" then setup_malloc_check () { @@ -264,100 +499,6 @@ test "x$TERM" != "xdumb" && ( ) && color=t -while test "$#" -ne 0 -do - case "$1" in - -d|--d|--de|--deb|--debu|--debug) - debug=t; shift ;; - -i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate) - immediate=t; shift ;; - -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests) - GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;; - -r) - shift; test "$#" -ne 0 || { - echo 'error: -r requires an argument' >&2; - exit 1; - } - run_list=$1; shift ;; - --run=*) - run_list=${1#--*=}; shift ;; - -h|--h|--he|--hel|--help) - help=t; shift ;; - -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) - verbose=t; shift ;; - --verbose-only=*) - verbose_only=${1#--*=} - shift ;; - -q|--q|--qu|--qui|--quie|--quiet) - # Ignore --quiet under a TAP::Harness. Saying how many tests - # passed without the ok/not ok details is always an error. - test -z "$HARNESS_ACTIVE" && quiet=t; shift ;; - --with-dashes) - with_dashes=t; shift ;; - --no-color) - color=; shift ;; - --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind) - valgrind=memcheck - shift ;; - --valgrind=*) - valgrind=${1#--*=} - shift ;; - --valgrind-only=*) - valgrind_only=${1#--*=} - shift ;; - --tee) - shift ;; # was handled already - --root=*) - root=${1#--*=} - shift ;; - --chain-lint) - GIT_TEST_CHAIN_LINT=1 - shift ;; - --no-chain-lint) - GIT_TEST_CHAIN_LINT=0 - shift ;; - -x) - # Some test scripts can't be reliably traced with '-x', - # unless the test is run with a Bash version supporting - # BASH_XTRACEFD (introduced in Bash v4.1). Check whether - # this test is marked as such, and ignore '-x' if it - # isn't executed with a suitable Bash version. - if test -z "$test_untraceable" || { - test -n "$BASH_VERSION" && { - test ${BASH_VERSINFO[0]} -gt 4 || { - test ${BASH_VERSINFO[0]} -eq 4 && - test ${BASH_VERSINFO[1]} -ge 1 - } - } - } - then - trace=t - else - echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD" - fi - shift ;; - -V|--verbose-log) - verbose_log=t - shift ;; - *) - echo "error: unknown test option '$1'" >&2; exit 1 ;; - esac -done - -if test -n "$valgrind_only" -then - test -z "$valgrind" && valgrind=memcheck - test -z "$verbose" && verbose_only="$valgrind_only" -elif test -n "$valgrind" -then - test -z "$verbose_log" && verbose=t -fi - -if test -n "$trace" && test -z "$verbose_log" -then - verbose=t -fi - if test -n "$color" then # Save the color control sequences now rather than run tput @@ -476,7 +617,7 @@ die () { GIT_EXIT_OK= trap 'die' EXIT -trap 'exit $?' INT +trap 'exit $?' INT TERM HUP # The user-facing functions are loaded from a separate file so that # test_perf subshells can have them too @@ -486,11 +627,35 @@ trap 'exit $?' INT # the test_expect_* functions instead. test_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$*" + fi test_success=$(($test_success + 1)) say_color "" "ok $test_count - $@" } test_failure_ () { + if test -n "$write_junit_xml" + then + junit_insert="<failure message=\"not ok $test_count -" + junit_insert="$junit_insert $(xml_attr_encode "$1")\">" + junit_insert="$junit_insert $(xml_attr_encode \ + "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + test-tool path-utils skip-n-bytes \ + "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET + else + printf '%s\n' "$@" | sed 1d + fi)")" + junit_insert="$junit_insert</failure>" + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + junit_insert="$junit_insert<system-err>$(xml_attr_encode \ + "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>" + fi + write_junit_xml_testcase "$1" " $junit_insert" + fi test_failure=$(($test_failure + 1)) say_color error "not ok $test_count - $1" shift @@ -499,11 +664,19 @@ test_failure_ () { } test_known_broken_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (breakage fixed)" + fi test_fixed=$(($test_fixed+1)) say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (known breakage)" + fi test_broken=$(($test_broken+1)) say_color warn "not ok $test_count - $@ # TODO known breakage" } @@ -761,12 +934,21 @@ test_start_ () { test_count=$(($test_count+1)) maybe_setup_verbose maybe_setup_valgrind + if test -n "$write_junit_xml" + then + junit_start=$(test-tool date getnanos) + fi } test_finish_ () { echo >&3 "" maybe_teardown_valgrind maybe_teardown_verbose + if test -n "$GIT_TEST_TEE_OFFSET" + then + GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \ + "$GIT_TEST_TEE_OUTPUT_FILE") + fi } test_skip () { @@ -798,6 +980,13 @@ test_skip () { case "$to_skip" in t) + if test -n "$write_junit_xml" + then + message="$(xml_attr_encode "$skipped_reason")" + write_junit_xml_testcase "$1" \ + " <skipped message=\"$message\" />" + fi + say_color skip >&3 "skipping test: $@" say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true @@ -813,17 +1002,56 @@ test_at_end_hook_ () { : } +write_junit_xml () { + case "$1" in + --truncate) + >"$junit_xml_path" + junit_have_testcase= + shift + ;; + esac + printf '%s\n' "$@" >>"$junit_xml_path" +} + +xml_attr_encode () { + printf '%s\n' "$@" | test-tool xml-encode +} + +write_junit_xml_testcase () { + junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" + shift + junit_attrs="$junit_attrs classname=\"$this_test\"" + junit_attrs="$junit_attrs time=\"$(test-tool \ + date getnanos $junit_start)\"" + write_junit_xml "$(printf '%s\n' \ + " <testcase $junit_attrs>" "$@" " </testcase>")" + junit_have_testcase=t +} + test_done () { GIT_EXIT_OK=t + if test -n "$write_junit_xml" && test -n "$junit_xml_path" + then + test -n "$junit_have_testcase" || { + junit_start=$(test-tool date getnanos) + write_junit_xml_testcase "all tests skipped" + } + + # adjust the overall time + junit_time=$(test-tool date getnanos $junit_suite_start) + sed "s/<testsuite [^>]*/& time=\"$junit_time\"/" \ + <"$junit_xml_path" >"$junit_xml_path.new" + mv "$junit_xml_path.new" "$junit_xml_path" + + write_junit_xml " </testsuite>" "</testsuites>" + fi + if test -z "$HARNESS_ACTIVE" then - test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results" - mkdir -p "$test_results_dir" - base=${0##*/} - test_results_path="$test_results_dir/${base%.sh}.counts" + mkdir -p "$TEST_RESULTS_DIR" - cat >"$test_results_path" <<-EOF + cat >"$TEST_RESULTS_BASE.counts" <<-EOF total $test_count success $test_success fixed $test_fixed @@ -878,7 +1106,11 @@ test_done () { error "Tests passed but trash directory already removed before test cleanup; aborting" cd "$TRASH_DIRECTORY/.." && - rm -fr "$TRASH_DIRECTORY" || + rm -fr "$TRASH_DIRECTORY" || { + # try again in a bit + sleep 5; + rm -fr "$TRASH_DIRECTORY" + } || error "Tests passed but test cleanup failed; aborting" fi test_at_end_hook_ @@ -984,20 +1216,25 @@ then PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH} else # normal case, use ../bin-wrappers only unless $with_dashes: - git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" - if ! test -x "$git_bin_dir/git" + if test -n "$no_bin_wrappers" then - if test -z "$with_dashes" + with_dashes=t + else + git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" + if ! test -x "$git_bin_dir/git" then - say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" + if test -z "$with_dashes" + then + say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" + fi + with_dashes=t fi - with_dashes=t + PATH="$git_bin_dir:$PATH" fi - PATH="$git_bin_dir:$PATH" GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then - PATH="$GIT_BUILD_DIR:$PATH" + PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH" fi fi GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt @@ -1021,7 +1258,7 @@ test -d "$GIT_BUILD_DIR"/templates/blt || { error "You haven't built things yet, have you?" } -if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool +if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X then echo >&2 'You need to build test-tool:' echo >&2 'Run "make t/helper/test-tool" in the source (toplevel) directory' @@ -1029,12 +1266,6 @@ then fi # Test repository -TRASH_DIRECTORY="trash directory.$(basename "$0" .sh)" -test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY" -case "$TRASH_DIRECTORY" in -/*) ;; # absolute path is good - *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;; -esac rm -fr "$TRASH_DIRECTORY" || { GIT_EXIT_OK=t echo >&5 "FATAL: Cannot prepare test area" @@ -1051,6 +1282,7 @@ then else mkdir -p "$TRASH_DIRECTORY" fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 @@ -1064,6 +1296,23 @@ then test_done fi +if test -n "$write_junit_xml" +then + junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" + mkdir -p "$junit_xml_dir" + junit_xml_base=${0##*/} + junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" + junit_attrs="name=\"${junit_xml_base%.sh}\"" + junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ + date +%Y-%m-%dT%H:%M:%S)\"" + write_junit_xml --truncate "<testsuites>" " <testsuite $junit_attrs>" + junit_suite_start=$(test-tool date getnanos) + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + GIT_TEST_TEE_OFFSET=0 + fi +fi + # Provide an implementation of the 'yes' utility yes () { if test $# = 0 |