diff options
Diffstat (limited to 't')
| -rw-r--r-- | t/README | 10 | ||||
| -rw-r--r-- | t/perf/perf-lib.sh | 3 | ||||
| -rwxr-xr-x | t/t0000-basic.sh | 61 | ||||
| -rw-r--r-- | t/test-lib-functions.sh | 6 | ||||
| -rw-r--r-- | t/test-lib.sh | 246 | ||||
| -rwxr-xr-x | t/valgrind/valgrind.sh | 3 |
6 files changed, 276 insertions, 53 deletions
@@ -76,6 +76,11 @@ appropriately before running "make". command being run and their output if any are also output. +--verbose-only=<pattern>:: + Like --verbose, but the effect is limited to tests with + numbers matching <pattern>. The number matched against is + simply the running count of the test within the file. + --debug:: This may help the person who is developing a new test. It causes the command defined with test_debug to run. @@ -121,6 +126,11 @@ appropriately before running "make". the 't/valgrind/' directory and use the commands under 't/valgrind/bin/'. +--valgrind-only=<pattern>:: + Like --valgrind, but the effect is limited to tests with + numbers matching <pattern>. The number matched against is + simply the running count of the test within the file. + --tee:: In addition to printing the test output to the terminal, write it to files named 't/test-results/$TEST_NAME.out'. diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index a816fbcb76..c61d5351e1 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -150,6 +150,7 @@ exit $ret' >&3 2>&4 test_perf () { + test_start_ test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test-expect-success" @@ -187,7 +188,7 @@ test_perf () { base="$perf_results_dir"/"$perf_results_prefix$(basename "$0" .sh)"."$test_count" "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".times fi - echo >&3 "" + test_finish_ } # We extend test_done to print timings at the end (./run disables this diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 0f1318056c..5c32288380 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -47,8 +47,13 @@ test_expect_failure 'pretend we have a known breakage' ' run_sub_test_lib_test () { name="$1" descr="$2" # stdin is the body of the test code + shift 2 mkdir "$name" && ( + # Pretend we're a test harness. This prevents + # test-lib from writing the counts to a file that will + # later be summarized, showing spurious "failed" tests + export HARNESS_ACTIVE=t && cd "$name" && cat >"$name.sh" <<-EOF && #!$SHELL_PATH @@ -65,7 +70,7 @@ run_sub_test_lib_test () { cat >>"$name.sh" && chmod +x "$name.sh" && export TEST_DIRECTORY && - ./"$name.sh" >out 2>err + ./"$name.sh" "$@" >out 2>err ) } @@ -215,6 +220,60 @@ test_expect_success 'pretend we have a mix of all possible results' " EOF " +test_expect_success 'test --verbose' ' + test_must_fail run_sub_test_lib_test \ + test-verbose "test verbose" --verbose <<-\EOF && + test_expect_success "passing test" true + test_expect_success "test with output" "echo foo" + test_expect_success "failing test" false + test_done + EOF + mv test-verbose/out test-verbose/out+ + grep -v "^Initialized empty" test-verbose/out+ >test-verbose/out && + check_sub_test_lib_test test-verbose <<-\EOF + > expecting success: true + > Z + > ok 1 - passing test + > Z + > expecting success: echo foo + > foo + > Z + > ok 2 - test with output + > Z + > expecting success: false + > Z + > not ok 3 - failing test + > # false + > Z + > # failed 1 among 3 test(s) + > 1..3 + EOF +' + +test_expect_success 'test --verbose-only' ' + test_must_fail run_sub_test_lib_test \ + test-verbose-only-2 "test verbose-only=2" \ + --verbose-only=2 <<-\EOF && + test_expect_success "passing test" true + test_expect_success "test with output" "echo foo" + test_expect_success "failing test" false + test_done + EOF + check_sub_test_lib_test test-verbose-only-2 <<-\EOF + > ok 1 - passing test + > Z + > expecting success: echo foo + > foo + > Z + > ok 2 - test with output + > Z + > not ok 3 - failing test + > # false + > # failed 1 among 3 test(s) + > 1..3 + EOF +' + test_set_prereq HAVEIT haveit=no test_expect_success HAVEIT 'test runs if prerequisite is satisfied' ' diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 8828ff78f1..a7e9aacbb2 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -343,6 +343,7 @@ test_declared_prereq () { } test_expect_failure () { + test_start_ test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test-expect-failure" @@ -357,10 +358,11 @@ test_expect_failure () { test_known_broken_failure_ "$1" fi fi - echo >&3 "" + test_finish_ } test_expect_success () { + test_start_ test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test-expect-success" @@ -375,7 +377,7 @@ test_expect_success () { test_failure_ "$@" fi fi - echo >&3 "" + test_finish_ } # test_external runs external test scripts that provide continuous diff --git a/t/test-lib.sh b/t/test-lib.sh index eff3a653d1..97536419e6 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -184,6 +184,9 @@ do help=t; shift ;; -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) verbose=t; shift ;; + --verbose-only=*) + verbose_only=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; -q|--q|--qu|--qui|--quie|--quiet) # Ignore --quiet under a TAP::Harness. Saying how many tests # passed without the ok/not ok details is always an error. @@ -198,17 +201,39 @@ do --valgrind=*) valgrind=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --valgrind-only=*) + valgrind_only=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; + --valgrind-parallel=*) + valgrind_parallel=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; + --valgrind-only-stride=*) + valgrind_only_stride=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; + --valgrind-only-offset=*) + valgrind_only_offset=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; --tee) shift ;; # was handled already --root=*) root=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --statusprefix=*) + statusprefix=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; *) echo "error: unknown test option '$1'" >&2; exit 1 ;; esac done -test -n "$valgrind" && verbose=t +if test -n "$valgrind_only" || test -n "$valgrind_only_stride" +then + test -z "$valgrind" && valgrind=memcheck + test -z "$verbose" && verbose_only="$valgrind_only" +elif test -n "$valgrind" +then + verbose=t +fi if test -n "$color" then @@ -303,12 +328,12 @@ trap 'die' EXIT test_ok_ () { test_success=$(($test_success + 1)) - say_color "" "ok $test_count - $@" + say_color "" "${statusprefix}ok $test_count - $@" } test_failure_ () { test_failure=$(($test_failure + 1)) - say_color error "not ok $test_count - $1" + say_color error "${statusprefix}not ok $test_count - $1" shift echo "$@" | sed -e 's/^/# /' test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } @@ -316,18 +341,83 @@ test_failure_ () { test_known_broken_ok_ () { test_fixed=$(($test_fixed+1)) - say_color error "ok $test_count - $@ # TODO known breakage vanished" + say_color error "${statusprefix}ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { test_broken=$(($test_broken+1)) - say_color warn "not ok $test_count - $@ # TODO known breakage" + say_color warn "${statusprefix}not ok $test_count - $@ # TODO known breakage" } test_debug () { test "$debug" = "" || eval "$1" } +match_pattern_list () { + arg="$1" + shift + test -z "$*" && return 1 + for pattern_ + do + case "$arg" in + $pattern_) + return 0 + esac + done + return 1 +} + +maybe_teardown_verbose () { + test -z "$verbose_only" && return + exec 4>/dev/null 3>/dev/null + verbose= +} + +last_verbose=t +maybe_setup_verbose () { + test -z "$verbose_only" && return + if match_pattern_list $test_count $verbose_only || + { test -n "$valgrind_only_stride" && + expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null; } + then + exec 4>&2 3>&1 + # Emit a delimiting blank line when going from + # non-verbose to verbose. Within verbose mode the + # delimiter is printed by test_expect_*. The choice + # of the initial $last_verbose is such that before + # test 1, we do not print it. + test -z "$last_verbose" && echo >&3 "" + verbose=t + else + exec 4>/dev/null 3>/dev/null + verbose= + fi + last_verbose=$verbose +} + +maybe_teardown_valgrind () { + test -z "$GIT_VALGRIND" && return + GIT_VALGRIND_ENABLED= +} + +maybe_setup_valgrind () { + test -z "$GIT_VALGRIND" && return + if test -z "$valgrind_only" && test -z "$valgrind_only_stride" + then + GIT_VALGRIND_ENABLED=t + return + fi + GIT_VALGRIND_ENABLED= + if match_pattern_list $test_count $valgrind_only + then + GIT_VALGRIND_ENABLED=t + elif test -n "$valgrind_only_stride" && + expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null + then + GIT_VALGRIND_ENABLED=t + fi +} + test_eval_ () { # This is a separate function because some tests use # "return" to end a test_expect_success block early. @@ -337,8 +427,10 @@ test_eval_ () { test_run_ () { test_cleanup=: expecting_failure=$2 + setup_malloc_check test_eval_ "$1" eval_ret=$? + teardown_malloc_check if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure" then @@ -353,17 +445,24 @@ test_run_ () { return "$eval_ret" } -test_skip () { +test_start_ () { test_count=$(($test_count+1)) + maybe_setup_verbose + maybe_setup_valgrind +} + +test_finish_ () { + echo >&3 "" + maybe_teardown_valgrind + maybe_teardown_verbose +} + +test_skip () { to_skip= - for skp in $GIT_SKIP_TESTS - do - case $this_test.$test_count in - $skp) - to_skip=t - break - esac - done + if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS + then + to_skip=t + fi if test -z "$to_skip" && test -n "$test_prereq" && ! test_have_prereq "$test_prereq" then @@ -377,8 +476,8 @@ test_skip () { of_prereq=" of $test_prereq" fi - say_color skip >&3 "skipping test: $@" - say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" + say_color skip >&3 "${statusprefix}skipping test: $@" + say_color skip "${statusprefix}ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" : true ;; *) @@ -395,6 +494,8 @@ test_at_end_hook_ () { test_done () { GIT_EXIT_OK=t + # Note: t0000 relies on $HARNESS_ACTIVE disabling the .counts + # output file if test -z "$HARNESS_ACTIVE" then test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results" @@ -414,11 +515,11 @@ test_done () { if test "$test_fixed" != 0 then - say_color error "# $test_fixed known breakage(s) vanished; please update test(s)" + say_color error "${statusprefix}# $test_fixed known breakage(s) vanished; please update test(s)" fi if test "$test_broken" != 0 then - say_color warn "# still have $test_broken known breakage(s)" + say_color warn "${statusprefix}# still have $test_broken known breakage(s)" fi if test "$test_broken" != 0 || test "$test_fixed" != 0 then @@ -441,9 +542,9 @@ test_done () { then if test $test_remaining -gt 0 then - say_color pass "# passed all $msg" + say_color pass "${statusprefix}# passed all $msg" fi - say "1..$test_count$skip_all" + say "${statusprefix}1..$test_count$skip_all" fi test -d "$remove_trash" && @@ -457,8 +558,8 @@ test_done () { *) if test $test_external_has_tap -eq 0 then - say_color error "# failed $test_failure among $msg" - say "1..$test_count" + say_color error "${statusprefix}# failed $test_failure among $msg" + say "${statusprefix}1..$test_count" fi exit 1 ;; @@ -466,6 +567,9 @@ test_done () { esac } + +# Set up a directory that we can put in PATH which redirects all git +# calls to 'valgrind git ...'. if test -n "$valgrind" then make_symlink () { @@ -513,31 +617,43 @@ then make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit } - # override all git executables in TEST_DIRECTORY/.. - GIT_VALGRIND=$TEST_DIRECTORY/valgrind - mkdir -p "$GIT_VALGRIND"/bin - for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-* - do - make_valgrind_symlink $file - done - # special-case the mergetools loadables - make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools" - OLDIFS=$IFS - IFS=: - for path in $PATH - do - ls "$path"/git-* 2> /dev/null | - while read file + # In the case of --valgrind-parallel, we only need to do the + # wrapping once, in the main script. The worker children all + # have $valgrind_only_stride set, so we can skip based on that. + if test -z "$valgrind_only_stride" + then + # override all git executables in TEST_DIRECTORY/.. + GIT_VALGRIND=$TEST_DIRECTORY/valgrind + mkdir -p "$GIT_VALGRIND"/bin + for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-* do - make_valgrind_symlink "$file" + make_valgrind_symlink $file done - done - IFS=$OLDIFS + # special-case the mergetools loadables + make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools" + OLDIFS=$IFS + IFS=: + for path in $PATH + do + ls "$path"/git-* 2> /dev/null | + while read file + do + make_valgrind_symlink "$file" + done + done + IFS=$OLDIFS + fi PATH=$GIT_VALGRIND/bin:$PATH GIT_EXEC_PATH=$GIT_VALGRIND/bin export GIT_VALGRIND GIT_VALGRIND_MODE="$valgrind" export GIT_VALGRIND_MODE + GIT_VALGRIND_ENABLED=t + if test -n "$valgrind_only" || test -n "$valgrind_only_stride" + then + GIT_VALGRIND_ENABLED= + fi + export GIT_VALGRIND_ENABLED elif test -n "$GIT_TEST_INSTALLED" then GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) || @@ -622,21 +738,53 @@ then else mkdir -p "$TRASH_DIRECTORY" fi + +# Gross hack to spawn N sub-instances of the tests in parallel, and +# summarize the results. Note that if this is enabled, the script +# terminates at the end of this 'if' block. +if test -n "$valgrind_parallel" +then + for i in $(test_seq 1 $valgrind_parallel) + do + root="$TRASH_DIRECTORY/vgparallel-$i" + mkdir "$root" + TEST_OUTPUT_DIRECTORY="$root" \ + ${SHELL_PATH} "$0" \ + --root="$root" --statusprefix="[$i] " \ + --valgrind="$valgrind" \ + --valgrind-only-stride="$valgrind_parallel" \ + --valgrind-only-offset="$i" & + pids="$pids $!" + done + trap "kill $pids" INT TERM HUP + wait $pids + trap - INT TERM HUP + for i in $(test_seq 1 $valgrind_parallel) + do + root="$TRASH_DIRECTORY/vgparallel-$i" + eval "$(cat "$root/test-results/$(basename "$0" .sh)"-*.counts | + sed 's/^\([a-z][a-z]*\) \([0-9][0-9]*\)/inner_\1=\2/')" + test_count=$(expr $test_count + $inner_total) + test_success=$(expr $test_success + $inner_success) + test_fixed=$(expr $test_fixed + $inner_fixed) + test_broken=$(expr $test_broken + $inner_broken) + test_failure=$(expr $test_failure + $inner_failed) + done + test_done +fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 this_test=${0##*/} this_test=${this_test%%-*} -for skp in $GIT_SKIP_TESTS -do - case "$this_test" in - $skp) - say_color info >&3 "skipping test $this_test altogether" - skip_all="skip all tests in $this_test" - test_done - esac -done +if match_pattern_list "$this_test" $GIT_SKIP_TESTS +then + say_color info >&3 "skipping test $this_test altogether" + skip_all="skip all tests in $this_test" + test_done +fi # Provide an implementation of the 'yes' utility yes () { diff --git a/t/valgrind/valgrind.sh b/t/valgrind/valgrind.sh index 6b87c91b60..42153036dc 100755 --- a/t/valgrind/valgrind.sh +++ b/t/valgrind/valgrind.sh @@ -4,6 +4,9 @@ base=$(basename "$0") TOOL_OPTIONS='--leak-check=no' +test -z "$GIT_VALGRIND_ENABLED" && +exec "$GIT_VALGRIND"/../../"$base" "$@" + case "$GIT_VALGRIND_MODE" in memcheck-fast) ;; |