summaryrefslogtreecommitdiff
path: root/t/perf
diff options
context:
space:
mode:
Diffstat (limited to 't/perf')
-rwxr-xr-xt/perf/aggregate.perl106
-rwxr-xr-xt/perf/bisect_regression73
-rwxr-xr-xt/perf/bisect_run_script53
-rwxr-xr-xt/perf/p0002-read-cache.sh2
-rwxr-xr-xt/perf/p0004-lazy-init-name-hash.sh8
-rwxr-xr-xt/perf/p0007-write-cache.sh2
-rwxr-xr-xt/perf/p0071-sort.sh2
-rwxr-xr-xt/perf/p7519-fsmonitor.sh12
-rwxr-xr-xt/perf/run64
9 files changed, 266 insertions, 56 deletions
diff --git a/t/perf/aggregate.perl b/t/perf/aggregate.perl
index 821cf1498b..bc865160e7 100755
--- a/t/perf/aggregate.perl
+++ b/t/perf/aggregate.perl
@@ -4,6 +4,7 @@ use lib '../../perl/build/lib';
use strict;
use warnings;
use JSON;
+use Getopt::Long;
use Git;
sub get_times {
@@ -36,34 +37,34 @@ sub format_times {
return $out;
}
+sub usage {
+ print <<EOT;
+./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
+
+ Options:
+ --codespeed * Format output for Codespeed
+ --reponame <str> * Send given reponame to codespeed
+ --sort-by <str> * Sort output (only "regression" criteria is supported)
+ --subsection <str> * Use results from given subsection
+
+EOT
+ exit(1);
+}
+
my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
- $codespeed, $subsection, $reponame);
+ $codespeed, $sortby, $subsection, $reponame);
+
+Getopt::Long::Configure qw/ require_order /;
+
+my $rc = GetOptions("codespeed" => \$codespeed,
+ "reponame=s" => \$reponame,
+ "sort-by=s" => \$sortby,
+ "subsection=s" => \$subsection);
+usage() unless $rc;
+
while (scalar @ARGV) {
my $arg = $ARGV[0];
my $dir;
- if ($arg eq "--codespeed") {
- $codespeed = 1;
- shift @ARGV;
- next;
- }
- if ($arg eq "--subsection") {
- shift @ARGV;
- $subsection = $ARGV[0];
- shift @ARGV;
- if (! $subsection) {
- die "empty subsection";
- }
- next;
- }
- if ($arg eq "--reponame") {
- shift @ARGV;
- $reponame = $ARGV[0];
- shift @ARGV;
- if (! $reponame) {
- die "empty reponame";
- }
- next;
- }
last if -f $arg or $arg eq "--";
if (! -d $arg) {
my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
@@ -147,6 +148,11 @@ sub have_slash {
return 0;
}
+sub display_dir {
+ my ($d) = @_;
+ return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
+}
+
sub print_default_results {
my %descrs;
my $descrlen = 4; # "Test"
@@ -168,8 +174,7 @@ sub print_default_results {
my %times;
my @colwidth = ((0)x@dirs);
for my $i (0..$#dirs) {
- my $d = $dirs[$i];
- my $w = length (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
+ my $w = length display_dir($dirs[$i]);
$colwidth[$i] = $w if $w > $colwidth[$i];
}
for my $t (@subtests) {
@@ -188,8 +193,7 @@ sub print_default_results {
printf "%-${descrlen}s", "Test";
for my $i (0..$#dirs) {
- my $d = $dirs[$i];
- printf " %-$colwidth[$i]s", (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
+ printf " %-$colwidth[$i]s", display_dir($dirs[$i]);
}
print "\n";
print "-"x$totalwidth, "\n";
@@ -206,6 +210,50 @@ sub print_default_results {
}
}
+sub print_sorted_results {
+ my ($sortby) = @_;
+
+ if ($sortby ne "regression") {
+ print "Only 'regression' is supported as '--sort-by' argument\n";
+ usage();
+ }
+
+ my @evolutions;
+ for my $t (@subtests) {
+ my ($prevr, $prevu, $prevs, $prevrev);
+ for my $i (0..$#dirs) {
+ my $d = $dirs[$i];
+ my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
+ if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
+ my $percent = 100.0 * ($r - $prevr) / $prevr;
+ push @evolutions, { "percent" => $percent,
+ "test" => $t,
+ "prevrev" => $prevrev,
+ "rev" => $d,
+ "prevr" => $prevr,
+ "r" => $r,
+ "prevu" => $prevu,
+ "u" => $u,
+ "prevs" => $prevs,
+ "s" => $s};
+ }
+ ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
+ }
+ }
+
+ my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
+
+ for my $e (@sorted_evolutions) {
+ printf "%+.1f%%", $e->{percent};
+ print " " . $e->{test};
+ print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
+ print " " . format_times($e->{r}, $e->{u}, $e->{s});
+ print " " . display_dir($e->{prevrev});
+ print " " . display_dir($e->{rev});
+ print "\n";
+ }
+}
+
sub print_codespeed_results {
my ($subsection) = @_;
@@ -260,6 +308,8 @@ binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
if ($codespeed) {
print_codespeed_results($subsection);
+} elsif (defined $sortby) {
+ print_sorted_results($sortby);
} else {
print_default_results();
}
diff --git a/t/perf/bisect_regression b/t/perf/bisect_regression
new file mode 100755
index 0000000000..a94d9955d0
--- /dev/null
+++ b/t/perf/bisect_regression
@@ -0,0 +1,73 @@
+#!/bin/sh
+
+# Read a line coming from `./aggregate.perl --sort-by regression ...`
+# and automatically bisect to find the commit responsible for the
+# performance regression.
+#
+# Lines from `./aggregate.perl --sort-by regression ...` look like:
+#
+# +100.0% p7821-grep-engines-fixed.1 0.04(0.10+0.03) 0.08(0.11+0.08) v2.14.3 v2.15.1
+# +33.3% p7820-grep-engines.1 0.03(0.08+0.02) 0.04(0.08+0.02) v2.14.3 v2.15.1
+#
+
+die () {
+ echo >&2 "error: $*"
+ exit 1
+}
+
+while [ $# -gt 0 ]; do
+ arg="$1"
+ case "$arg" in
+ --help)
+ echo "usage: $0 [--config file] [--subsection subsection]"
+ exit 0
+ ;;
+ --config)
+ shift
+ GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
+ export GIT_PERF_CONFIG_FILE
+ shift ;;
+ --subsection)
+ shift
+ GIT_PERF_SUBSECTION="$1"
+ export GIT_PERF_SUBSECTION
+ shift ;;
+ --*)
+ die "unrecognised option: '$arg'" ;;
+ *)
+ die "unknown argument '$arg'"
+ ;;
+ esac
+done
+
+read -r regression subtest oldtime newtime oldrev newrev
+
+test_script=$(echo "$subtest" | sed -e 's/\(.*\)\.[0-9]*$/\1.sh/')
+test_number=$(echo "$subtest" | sed -e 's/.*\.\([0-9]*\)$/\1/')
+
+# oldtime and newtime are decimal number, not integers
+
+oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
+newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
+
+test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 ||
+ die "New time '$newtime' shoud be greater than old time '$oldtime'"
+
+tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory"
+echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'"
+echo "$newtime" >"$tmpdir/newtime" || die "Failed to write to '$tmpdir/newtime'"
+
+# Bisecting must be performed from the top level directory (even with --no-checkout)
+(
+ toplevel_dir=$(git rev-parse --show-toplevel) || die "Failed to find top level directory"
+ cd "$toplevel_dir" || die "Failed to cd into top level directory '$toplevel_dir'"
+
+ git bisect start --no-checkout "$newrev" "$oldrev" || die "Failed to start bisecting"
+
+ git bisect run t/perf/bisect_run_script "$test_script" "$test_number" "$tmpdir"
+ res="$?"
+
+ git bisect reset
+
+ exit "$res"
+)
diff --git a/t/perf/bisect_run_script b/t/perf/bisect_run_script
new file mode 100755
index 0000000000..3ebaf15521
--- /dev/null
+++ b/t/perf/bisect_run_script
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+script="$1"
+test_number="$2"
+info_dir="$3"
+
+# This aborts the bisection immediately
+die () {
+ echo >&2 "error: $*"
+ exit 255
+}
+
+bisect_head=$(git rev-parse --verify BISECT_HEAD) || die "Failed to find BISECT_HEAD ref"
+
+script_number=$(echo "$script" | sed -e "s/^p\([0-9]*\).*\$/\1/") || die "Failed to get script number for '$script'"
+
+oldtime=$(cat "$info_dir/oldtime") || die "Failed to access '$info_dir/oldtime'"
+newtime=$(cat "$info_dir/newtime") || die "Failed to access '$info_dir/newtime'"
+
+cd t/perf || die "Failed to cd into 't/perf'"
+
+result_file="$info_dir/perf_${script_number}_${bisect_head}_results.txt"
+
+GIT_PERF_DIRS_OR_REVS="$bisect_head"
+export GIT_PERF_DIRS_OR_REVS
+
+# Don't use codespeed
+GIT_PERF_CODESPEED_OUTPUT=
+GIT_PERF_SEND_TO_CODESPEED=
+export GIT_PERF_CODESPEED_OUTPUT
+export GIT_PERF_SEND_TO_CODESPEED
+
+./run "$script" >"$result_file" 2>&1 || die "Failed to run perf test '$script'"
+
+rtime=$(sed -n "s/^$script_number\.$test_number:.*\([0-9]\+\.[0-9]\+\)(.*).*\$/\1/p" "$result_file")
+
+echo "newtime: $newtime"
+echo "rtime: $rtime"
+echo "oldtime: $oldtime"
+
+# Compare ($newtime - $rtime) with ($rtime - $oldtime)
+# Times are decimal number, not integers
+
+if test $(echo "$newtime" "$rtime" "$oldtime" | awk '{ print ($1 - $2 > $2 - $3) }') = 1
+then
+ # Current commit is considered "good/old"
+ echo "$rtime" >"$info_dir/oldtime"
+ exit 0
+else
+ # Current commit is considered "bad/new"
+ echo "$rtime" >"$info_dir/newtime"
+ exit 1
+fi
diff --git a/t/perf/p0002-read-cache.sh b/t/perf/p0002-read-cache.sh
index 9180ae9343..cdd105a594 100755
--- a/t/perf/p0002-read-cache.sh
+++ b/t/perf/p0002-read-cache.sh
@@ -8,7 +8,7 @@ test_perf_default_repo
count=1000
test_perf "read_cache/discard_cache $count times" "
- test-read-cache $count
+ test-tool read-cache $count
"
test_done
diff --git a/t/perf/p0004-lazy-init-name-hash.sh b/t/perf/p0004-lazy-init-name-hash.sh
index 8de5a98cfc..1afc08fe7f 100755
--- a/t/perf/p0004-lazy-init-name-hash.sh
+++ b/t/perf/p0004-lazy-init-name-hash.sh
@@ -7,8 +7,8 @@ test_perf_large_repo
test_checkout_worktree
test_expect_success 'verify both methods build the same hashmaps' '
- test-lazy-init-name-hash --dump --single >out.single &&
- if test-lazy-init-name-hash --dump --multi >out.multi
+ test-tool lazy-init-name-hash --dump --single >out.single &&
+ if test-tool lazy-init-name-hash --dump --multi >out.multi
then
test_set_prereq REPO_BIG_ENOUGH_FOR_MULTI &&
sort <out.single >sorted.single &&
@@ -46,11 +46,11 @@ test_expect_success 'calibrate' '
'
test_perf "single-threaded, $desc" "
- test-lazy-init-name-hash --single --count=$count
+ test-tool lazy-init-name-hash --single --count=$count
"
test_perf REPO_BIG_ENOUGH_FOR_MULTI "multi-threaded, $desc" "
- test-lazy-init-name-hash --multi --count=$count
+ test-tool lazy-init-name-hash --multi --count=$count
"
test_done
diff --git a/t/perf/p0007-write-cache.sh b/t/perf/p0007-write-cache.sh
index 261fe92fd9..09595264f0 100755
--- a/t/perf/p0007-write-cache.sh
+++ b/t/perf/p0007-write-cache.sh
@@ -23,7 +23,7 @@ test_expect_success "setup repo" '
count=3
test_perf "write_locked_index $count times ($nr_files files)" "
- test-write-cache $count
+ test-tool write-cache $count
"
test_done
diff --git a/t/perf/p0071-sort.sh b/t/perf/p0071-sort.sh
index 7c9a35a646..6e924f5fa3 100755
--- a/t/perf/p0071-sort.sh
+++ b/t/perf/p0071-sort.sh
@@ -16,7 +16,7 @@ test_perf 'sort(1)' '
'
test_perf 'string_list_sort()' '
- test-string-list sort <unsorted >actual
+ test-tool string-list sort <unsorted >actual
'
test_expect_success 'string_list_sort() sorts like sort(1)' '
diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh
index 65e145c02d..def7ecdbc7 100755
--- a/t/perf/p7519-fsmonitor.sh
+++ b/t/perf/p7519-fsmonitor.sh
@@ -118,7 +118,7 @@ test_expect_success "setup for fsmonitor" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
@@ -126,7 +126,7 @@ test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
@@ -134,7 +134,7 @@ test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
@@ -148,7 +148,7 @@ test_expect_success "setup without fsmonitor" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
@@ -156,7 +156,7 @@ test_perf "status (fsmonitor=$INTEGRATION_SCRIPT)" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
@@ -164,7 +164,7 @@ test_perf "status -uno (fsmonitor=$INTEGRATION_SCRIPT)" '
'
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
- test-drop-caches
+ test-tool drop-caches
fi
test_perf "status -uall (fsmonitor=$INTEGRATION_SCRIPT)" '
diff --git a/t/perf/run b/t/perf/run
index 1a100d6134..9aaa733c77 100755
--- a/t/perf/run
+++ b/t/perf/run
@@ -1,21 +1,34 @@
#!/bin/sh
-case "$1" in
+die () {
+ echo >&2 "error: $*"
+ exit 1
+}
+
+while [ $# -gt 0 ]; do
+ arg="$1"
+ case "$arg" in
+ --)
+ break ;;
--help)
- echo "usage: $0 [--config file] [other_git_tree...] [--] [test_scripts]"
- exit 0
- ;;
+ echo "usage: $0 [--config file] [--subsection subsec] [other_git_tree...] [--] [test_scripts]"
+ exit 0 ;;
--config)
shift
GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
export GIT_PERF_CONFIG_FILE
shift ;;
-esac
-
-die () {
- echo >&2 "error: $*"
- exit 1
-}
+ --subsection)
+ shift
+ GIT_PERF_SUBSECTION="$1"
+ export GIT_PERF_SUBSECTION
+ shift ;;
+ --*)
+ die "unrecognised option: '$arg'" ;;
+ *)
+ break ;;
+ esac
+done
run_one_dir () {
if test $# -eq 0; then
@@ -106,7 +119,6 @@ get_var_from_env_or_config () {
conf_sec="$2"
conf_var="$3"
conf_opts="$4" # optional
- # $5 can be set to a default value
# Do nothing if the env variable is already set
eval "test -z \"\${$env_var+x}\"" || return
@@ -122,13 +134,12 @@ get_var_from_env_or_config () {
fi
var="$conf_sec.$conf_var"
conf_value=$(git config $conf_opts -f "$GIT_PERF_CONFIG_FILE" "$var") &&
- eval "$env_var=\"$conf_value\"" && return
-
- test -n "${5+x}" && eval "$env_var=\"$5\""
+ eval "$env_var=\"$conf_value\""
}
run_subsection () {
- get_var_from_env_or_config "GIT_PERF_REPEAT_COUNT" "perf" "repeatCount" "--int" 3
+ get_var_from_env_or_config "GIT_PERF_REPEAT_COUNT" "perf" "repeatCount" "--int"
+ : ${GIT_PERF_REPEAT_COUNT:=3}
export GIT_PERF_REPEAT_COUNT
get_var_from_env_or_config "GIT_PERF_DIRS_OR_REVS" "perf" "dirsOrRevs"
@@ -174,9 +185,32 @@ get_subsections "perf" >test-results/run_subsections.names
if test $(wc -l <test-results/run_subsections.names) -eq 0
then
+ if test -n "$GIT_PERF_SUBSECTION"
+ then
+ if test -n "$GIT_PERF_CONFIG_FILE"
+ then
+ die "no subsections are defined in config file '$GIT_PERF_CONFIG_FILE'"
+ else
+ die "subsection '$GIT_PERF_SUBSECTION' defined without a config file"
+ fi
+ fi
(
run_subsection "$@"
)
+elif test -n "$GIT_PERF_SUBSECTION"
+then
+ egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names >/dev/null ||
+ die "subsection '$GIT_PERF_SUBSECTION' not found in '$GIT_PERF_CONFIG_FILE'"
+
+ egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names | while read -r subsec
+ do
+ (
+ GIT_PERF_SUBSECTION="$subsec"
+ export GIT_PERF_SUBSECTION
+ echo "======== Run for subsection '$GIT_PERF_SUBSECTION' ========"
+ run_subsection "$@"
+ )
+ done
else
while read -r subsec
do
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-21 08:55:18 +0000