From 1942d48380fec53f76361e9adebef15b5db9628a Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 28 Aug 2020 15:45:12 +0000 Subject: maintenance: optionally skip --auto process Some commands run 'git maintenance run --auto --[no-]quiet' after doing their normal work, as a way to keep repositories clean as they are used. Currently, users who do not want this maintenance to occur would set the 'gc.auto' config option to 0 to avoid the 'gc' task from running. However, this does not stop the extra process invocation. On Windows, this extra process invocation can be more expensive than necessary. Allow users to drop this extra process by setting 'maintenance.auto' to 'false'. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 't') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 55116c2f04..c7caaa7a55 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -28,6 +28,19 @@ test_expect_success 'run [--auto|--quiet]' ' test_subcommand git gc --no-quiet .enabled' ' git config maintenance.gc.enabled false && git config maintenance.commit-graph.enabled true && -- cgit v1.2.3 From b08ff1fee000230ad22fee823aabff2613bd4e62 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 11 Sep 2020 17:49:15 +0000 Subject: maintenance: add --schedule option and config Maintenance currently triggers when certain data-size thresholds are met, such as number of pack-files or loose objects. Users may want to run certain maintenance tasks based on frequency instead. For example, a user may want to perform a 'prefetch' task every hour, or 'gc' task every day. To help these users, update the 'git maintenance run' command to include a '--schedule=' option. The allowed frequencies are 'hourly', 'daily', and 'weekly'. These values are also allowed in a new config value 'maintenance..schedule'. The 'git maintenance run --schedule=' checks the '*.schedule' config value for each enabled task to see if the configured frequency is at least as frequent as the frequency from the '--schedule' argument. We use the following order, for full clarity: 'hourly' > 'daily' > 'weekly' Use new 'enum schedule_priority' to track these values numerically. The following cron table would run the scheduled tasks with the correct frequencies: 0 1-23 * * * git -C maintenance run --schedule=hourly 0 0 * * 1-6 git -C maintenance run --schedule=daily 0 0 * * 0 git -C maintenance run --schedule=weekly This cron schedule will run --schedule=hourly every hour except at midnight. This avoids a concurrent run with the --schedule=daily that runs at midnight every day except the first day of the week. This avoids a concurrent run with the --schedule=weekly that runs at midnight on the first day of the week. Since --schedule=daily also runs the 'hourly' tasks and --schedule=weekly runs the 'hourly' and 'daily' tasks, we will still see all tasks run with the proper frequencies. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 't') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index c7caaa7a55..33d73cd01c 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -260,4 +260,44 @@ test_expect_success 'maintenance.incremental-repack.auto' ' test_subcommand git multi-pack-index write --no-progress err && + test_i18ngrep "at most one" err +' + +test_expect_success 'invalid --schedule value' ' + test_must_fail git maintenance run --schedule=annually 2>err && + test_i18ngrep "unrecognized --schedule" err +' + +test_expect_success '--schedule inheritance weekly -> daily -> hourly' ' + git config maintenance.loose-objects.enabled true && + git config maintenance.loose-objects.schedule hourly && + git config maintenance.commit-graph.enabled true && + git config maintenance.commit-graph.schedule daily && + git config maintenance.incremental-repack.enabled true && + git config maintenance.incremental-repack.schedule weekly && + + GIT_TRACE2_EVENT="$(pwd)/hourly.txt" \ + git maintenance run --schedule=hourly 2>/dev/null && + test_subcommand git prune-packed --quiet /dev/null && + test_subcommand git prune-packed --quiet /dev/null && + test_subcommand git prune-packed --quiet Date: Fri, 11 Sep 2020 17:49:16 +0000 Subject: for-each-repo: run subcommands on configured repos It can be helpful to store a list of repositories in global or system config and then iterate Git commands on that list. Create a new builtin that makes this process simple for experts. We will use this builtin to run scheduled maintenance on all configured repositories in a future change. The test is very simple, but does highlight that the "--" argument is optional. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t0068-for-each-repo.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 t/t0068-for-each-repo.sh (limited to 't') diff --git a/t/t0068-for-each-repo.sh b/t/t0068-for-each-repo.sh new file mode 100755 index 0000000000..136b4ec839 --- /dev/null +++ b/t/t0068-for-each-repo.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='git for-each-repo builtin' + +. ./test-lib.sh + +test_expect_success 'run based on configured value' ' + git init one && + git init two && + git init three && + git -C two commit --allow-empty -m "DID NOT RUN" && + git config run.key "$TRASH_DIRECTORY/one" && + git config --add run.key "$TRASH_DIRECTORY/three" && + git for-each-repo --config=run.key commit --allow-empty -m "ran" && + git -C one log -1 --pretty=format:%s >message && + grep ran message && + git -C two log -1 --pretty=format:%s >message && + ! grep ran message && + git -C three log -1 --pretty=format:%s >message && + grep ran message && + git for-each-repo --config=run.key -- commit --allow-empty -m "ran again" && + git -C one log -1 --pretty=format:%s >message && + grep again message && + git -C two log -1 --pretty=format:%s >message && + ! grep again message && + git -C three log -1 --pretty=format:%s >message && + grep again message +' + +test_done -- cgit v1.2.3 From 0c18b700810ab2e2a4fac0d5b54a817141198a27 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 11 Sep 2020 17:49:17 +0000 Subject: maintenance: add [un]register subcommands In preparation for launching background maintenance from the 'git maintenance' builtin, create register/unregister subcommands. These commands update the new 'maintenance.repos' config option in the global config so the background maintenance job knows which repositories to maintain. These commands allow users to add a repository to the background maintenance list without disrupting the actual maintenance mechanism. For example, a user can run 'git maintenance register' when no background maintenance is running and it will not start the background maintenance. A later update to start running background maintenance will then pick up this repository automatically. The opposite example is that a user can run 'git maintenance unregister' to remove the current repository from background maintenance without halting maintenance for other repositories. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 't') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 33d73cd01c..8f383d01d9 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -9,7 +9,7 @@ GIT_TEST_MULTI_PACK_INDEX=0 test_expect_success 'help text' ' test_expect_code 129 git maintenance -h 2>err && - test_i18ngrep "usage: git maintenance run" err && + test_i18ngrep "usage: git maintenance " err && test_expect_code 128 git maintenance barf 2>err && test_i18ngrep "invalid subcommand: barf" err && test_expect_code 129 git maintenance 2>err && @@ -300,4 +300,19 @@ test_expect_success '--schedule inheritance weekly -> daily -> hourly' ' test_subcommand git multi-pack-index write --no-progress before && + git maintenance register && + git config --global --get-all maintenance.repo >actual && + cp before after && + pwd >>after && + test_cmp after actual && + git maintenance unregister && + git config --global --get-all maintenance.repo >actual && + test_cmp before actual +' + test_done -- cgit v1.2.3 From 2fec604f8df51fcf47551723e68a3447915b9eb9 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 11 Sep 2020 17:49:18 +0000 Subject: maintenance: add start/stop subcommands Add new subcommands to 'git maintenance' that start or stop background maintenance using 'cron', when available. This integration is as simple as I could make it, barring some implementation complications. The schedule is laid out as follows: 0 1-23 * * * $cmd maintenance run --schedule=hourly 0 0 * * 1-6 $cmd maintenance run --schedule=daily 0 0 * * 0 $cmd maintenance run --schedule=weekly where $cmd is a properly-qualified 'git for-each-repo' execution: $cmd=$path/git --exec-path=$path for-each-repo --config=maintenance.repo where $path points to the location of the Git executable running 'git maintenance start'. This is critical for systems with multiple versions of Git. Specifically, macOS has a system version at '/usr/bin/git' while the version that users can install resides at '/usr/local/bin/git' (symlinked to '/usr/local/libexec/git-core/git'). This will also use your locally-built version if you build and run this in your development environment without installing first. This conditional schedule avoids having cron launch multiple 'git for-each-repo' commands in parallel. Such parallel commands would likely lead to the 'hourly' and 'daily' tasks competing over the object database lock. This could lead to to some tasks never being run! Since the --schedule= argument will run all tasks with _at least_ the given frequency, the daily runs will also run the hourly tasks. Similarly, the weekly runs will also run the daily and hourly tasks. The GIT_TEST_CRONTAB environment variable is not intended for users to edit, but instead as a way to mock the 'crontab [-l]' command. This variable is set in test-lib.sh to avoid a future test from accidentally running anything with the cron integration from modifying the user's schedule. We use GIT_TEST_CRONTAB='test-tool crontab ' in our tests to check how the schedule is modified in 'git maintenance (start|stop)' commands. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/helper/test-crontab.c | 35 +++++++++++++++++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/t7900-maintenance.sh | 28 ++++++++++++++++++++++++++++ t/test-lib.sh | 6 ++++++ 5 files changed, 71 insertions(+) create mode 100644 t/helper/test-crontab.c (limited to 't') diff --git a/t/helper/test-crontab.c b/t/helper/test-crontab.c new file mode 100644 index 0000000000..e7c0137a47 --- /dev/null +++ b/t/helper/test-crontab.c @@ -0,0 +1,35 @@ +#include "test-tool.h" +#include "cache.h" + +/* + * Usage: test-tool cron [-l] + * + * If -l is specified, then write the contents of to stdout. + * Otherwise, write from stdin into . + */ +int cmd__crontab(int argc, const char **argv) +{ + int a; + FILE *from, *to; + + if (argc == 3 && !strcmp(argv[2], "-l")) { + from = fopen(argv[1], "r"); + if (!from) + return 0; + to = stdout; + } else if (argc == 2) { + from = stdin; + to = fopen(argv[1], "w"); + } else + return error("unknown arguments"); + + while ((a = fgetc(from)) != EOF) + fputc(a, to); + + if (argc == 3) + fclose(from); + else + fclose(to); + + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 590b2efca7..432b49d948 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -18,6 +18,7 @@ static struct test_cmd cmds[] = { { "bloom", cmd__bloom }, { "chmtime", cmd__chmtime }, { "config", cmd__config }, + { "crontab", cmd__crontab }, { "ctype", cmd__ctype }, { "date", cmd__date }, { "delta", cmd__delta }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index ddc8e990e9..7c3281e071 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -8,6 +8,7 @@ int cmd__advise_if_enabled(int argc, const char **argv); int cmd__bloom(int argc, const char **argv); int cmd__chmtime(int argc, const char **argv); int cmd__config(int argc, const char **argv); +int cmd__crontab(int argc, const char **argv); int cmd__ctype(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 8f383d01d9..7715e40391 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -315,4 +315,32 @@ test_expect_success 'register and unregister' ' test_cmp before actual ' +test_expect_success 'start from empty cron table' ' + GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance start && + + # start registers the repo + git config --get --global maintenance.repo "$(pwd)" && + + grep "for-each-repo --config=maintenance.repo maintenance run --schedule=daily" cron.txt && + grep "for-each-repo --config=maintenance.repo maintenance run --schedule=hourly" cron.txt && + grep "for-each-repo --config=maintenance.repo maintenance run --schedule=weekly" cron.txt +' + +test_expect_success 'stop from existing schedule' ' + GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance stop && + + # stop does not unregister the repo + git config --get --global maintenance.repo "$(pwd)" && + + # Operation is idempotent + GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance stop && + test_must_be_empty cron.txt +' + +test_expect_success 'start preserves existing schedule' ' + echo "Important information!" >cron.txt && + GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance start && + grep "Important information!" cron.txt +' + test_done diff --git a/t/test-lib.sh b/t/test-lib.sh index ef31f40037..4a60d1ed76 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1702,3 +1702,9 @@ test_lazy_prereq SHA1 ' test_lazy_prereq REBASE_P ' test -z "$GIT_TEST_SKIP_REBASE_P" ' + +# Ensure that no test accidentally triggers a Git command +# that runs 'crontab', affecting a user's cron schedule. +# Tests that verify the cron integration must set this locally +# to avoid errors. +GIT_TEST_CRONTAB="exit 1" -- cgit v1.2.3 From a4cb1a2339c1aa4ba71ebc6d2b07d537e1071124 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 15 Oct 2020 17:22:02 +0000 Subject: maintenance: create maintenance.strategy config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To provide an on-ramp for users to use background maintenance without several 'git config' commands, create a 'maintenance.strategy' config option. Currently, the only important value is 'incremental' which assigns the following schedule: * gc: never * prefetch: hourly * commit-graph: hourly * loose-objects: daily * incremental-repack: daily These tasks are chosen to minimize disruptions to foreground Git commands and use few compute resources. The 'maintenance.strategy' is intended as a baseline that can be customzied further by manually assigning 'maintenance..enabled' and 'maintenance..schedule' config options, which will override any recommendation from 'maintenance.strategy'. This operates similarly to config options like 'feature.experimental' which operate as "meta" config options that change default config values. This presents a way forward for updating the 'incremental' strategy in the future or adding new strategies. For example, a potential strategy could be to include a 'full' strategy that runs the 'gc' task weekly and no other tasks by default. Helped-by: Martin Ågren Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 't') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 7715e40391..7440a0ea19 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -300,6 +300,55 @@ test_expect_success '--schedule inheritance weekly -> daily -> hourly' ' test_subcommand git multi-pack-index write --no-progress Date: Thu, 15 Oct 2020 17:22:03 +0000 Subject: maintenance: use 'incremental' strategy by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'git maintenance (register|start)' subcommands add the current repository to the global Git config so maintenance will operate on that repository. It does not specify what maintenance should occur or how often. To make it simple for users to start background maintenance with a recommended schedlue, update the 'maintenance.strategy' config option in both the 'register' and 'start' subcommands. This allows users to customize beyond the defaults using individual 'maintenance..schedule' options, but also the user can opt-out of this strategy using 'maintenance.strategy=none'. Helped-by: Martin Ågren Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 't') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 7440a0ea19..20184e96e1 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -354,11 +354,14 @@ test_expect_success 'register and unregister' ' git config --global --add maintenance.repo /existing1 && git config --global --add maintenance.repo /existing2 && git config --global --get-all maintenance.repo >before && + git maintenance register && - git config --global --get-all maintenance.repo >actual && - cp before after && - pwd >>after && - test_cmp after actual && + test_cmp_config false maintenance.auto && + git config --global --get-all maintenance.repo >between && + cp before expect && + pwd >>expect && + test_cmp expect between && + git maintenance unregister && git config --global --get-all maintenance.repo >actual && test_cmp before actual @@ -392,4 +395,13 @@ test_expect_success 'start preserves existing schedule' ' grep "Important information!" cron.txt ' +test_expect_success 'register preserves existing strategy' ' + git config maintenance.strategy none && + git maintenance register && + test_config maintenance.strategy none && + git config --unset maintenance.strategy && + git maintenance register && + test_config maintenance.strategy incremental +' + test_done -- cgit v1.2.3