From a16eb6b1ff333f3e26b99f17ef3bb7dbf8135f39 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 24 Aug 2021 15:44:00 +0000 Subject: maintenance: skip bootout/bootstrap when plist is registered On macOS, we use launchctl to manage the background maintenance schedule. This uses a set of .plist files to describe the schedule, but these files are also registered with 'launchctl bootstrap'. If multiple 'git maintenance start' commands run concurrently, then they can collide replacing these schedule files and registering them with launchctl. To avoid extra launchctl commands, do a check for the .plist files on disk and check if they are registered using 'launchctl list '. This command will return with exit code 0 if it exists, or exit code 113 if it does not. We can test this behavior using the GIT_TEST_MAINT_SCHEDULER environment variable. Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 't/t7900-maintenance.sh') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 58f46c77e6..fc16ac2258 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -582,6 +582,23 @@ test_expect_success 'start and stop macOS maintenance' ' test_line_count = 0 actual ' +test_expect_success 'use launchctl list to prevent extra work' ' + # ensure we are registered + GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance start && + + # do it again on a fresh args file + rm -f args && + GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance start && + + ls "$HOME/Library/LaunchAgents" >actual && + cat >expect <<-\EOF && + list org.git-scm.git.hourly + list org.git-scm.git.daily + list org.git-scm.git.weekly + EOF + test_cmp expect args +' + test_expect_success 'start and stop Windows maintenance' ' write_script print-args <<-\EOF && echo $* >>args -- cgit v1.2.3 From eba1ba9d32f14dd9c052b1e2eb50e5718421d10e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Sat, 4 Sep 2021 22:54:59 +0200 Subject: maintenance: `git maintenance run` learned `--scheduler=` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depending on the system, different schedulers can be used to schedule the hourly, daily and weekly executions of `git maintenance run`: * `launchctl` for MacOS, * `schtasks` for Windows and * `crontab` for everything else. `git maintenance run` now has an option to let the end-user explicitly choose which scheduler he wants to use: `--scheduler=auto|crontab|launchctl|schtasks`. When `git maintenance start --scheduler=XXX` is run, it not only registers `git maintenance run` tasks in the scheduler XXX, it also removes the `git maintenance run` tasks from all the other schedulers to ensure we cannot have two schedulers launching concurrent identical tasks. The default value is `auto` which chooses a suitable scheduler for the system. `git maintenance stop` doesn't have any `--scheduler` parameter because this command will try to remove the `git maintenance run` tasks from all the available schedulers. Signed-off-by: Lénaïc Huard Acked-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 55 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) (limited to 't/t7900-maintenance.sh') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index b93ae014ee..b36b7f5fb0 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -494,8 +494,21 @@ test_expect_success !MINGW 'register and unregister with regex metacharacters' ' maintenance.repo "$(pwd)/$META" ' +test_expect_success 'start --scheduler=' ' + test_expect_code 129 git maintenance start --scheduler=foo 2>err && + test_i18ngrep "unrecognized --scheduler argument" err && + + test_expect_code 129 git maintenance start --no-scheduler 2>err && + test_i18ngrep "unknown option" err && + + test_expect_code 128 \ + env GIT_TEST_MAINT_SCHEDULER="launchctl:true,schtasks:true" \ + git maintenance start --scheduler=crontab 2>err && + test_i18ngrep "fatal: crontab scheduler is not available" err +' + test_expect_success 'start from empty cron table' ' - GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start --scheduler=crontab && # start registers the repo git config --get --global --fixed-value maintenance.repo "$(pwd)" && @@ -518,7 +531,7 @@ test_expect_success 'stop from existing schedule' ' test_expect_success 'start preserves existing schedule' ' echo "Important information!" >cron.txt && - GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start --scheduler=crontab && grep "Important information!" cron.txt ' @@ -547,7 +560,7 @@ test_expect_success 'start and stop macOS maintenance' ' EOF rm -f args && - GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance start && + GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance start --scheduler=launchctl && # start registers the repo git config --get --global --fixed-value maintenance.repo "$(pwd)" && @@ -598,7 +611,7 @@ test_expect_success 'start and stop Windows maintenance' ' EOF rm -f args && - GIT_TEST_MAINT_SCHEDULER="schtasks:./print-args" git maintenance start && + GIT_TEST_MAINT_SCHEDULER="schtasks:./print-args" git maintenance start --scheduler=schtasks && # start registers the repo git config --get --global --fixed-value maintenance.repo "$(pwd)" && @@ -621,6 +634,40 @@ test_expect_success 'start and stop Windows maintenance' ' test_cmp expect args ' +test_expect_success 'start and stop when several schedulers are available' ' + write_script print-args <<-\EOF && + printf "%s\n" "$*" | sed "s:gui/[0-9][0-9]*:gui/[UID]:; s:\(schtasks /create .* /xml\).*:\1:;" >>args + EOF + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=launchctl && + printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ + hourly daily weekly >expect && + for frequency in hourly daily weekly + do + PLIST="$pfx/Library/LaunchAgents/org.git-scm.git.$frequency.plist" && + echo "launchctl bootout gui/[UID] $PLIST" >>expect && + echo "launchctl bootstrap gui/[UID] $PLIST" >>expect || return 1 + done && + test_cmp expect args && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=schtasks && + printf "launchctl bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ + hourly daily weekly >expect && + printf "schtasks /create /tn Git Maintenance (%s) /f /xml\n" \ + hourly daily weekly >>expect && + test_cmp expect args && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance stop && + printf "launchctl bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ + hourly daily weekly >expect && + printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ + hourly daily weekly >>expect && + test_cmp expect args +' + test_expect_success 'register preserves existing strategy' ' git config maintenance.strategy none && git maintenance register && -- cgit v1.2.3 From b681b191f923267aab80ae7f7ab2f85a692e8833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Sat, 4 Sep 2021 22:55:00 +0200 Subject: maintenance: add support for systemd timers on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing mechanism for scheduling background maintenance is done through cron. On Linux systems managed by systemd, systemd provides an alternative to schedule recurring tasks: systemd timers. The main motivations to implement systemd timers in addition to cron are: * cron is optional and Linux systems running systemd might not have it installed. * The execution of `crontab -l` can tell us if cron is installed but not if the daemon is actually running. * With systemd, each service is run in its own cgroup and its logs are tagged by the service inside journald. With cron, all scheduled tasks are running in the cron daemon cgroup and all the logs of the user-scheduled tasks are pretended to belong to the system cron service. Concretely, a user that doesn’t have access to the system logs won’t have access to the log of their own tasks scheduled by cron whereas they will have access to the log of their own tasks scheduled by systemd timer. Although `cron` attempts to send email, that email may go unseen by the user because these days, local mailboxes are not heavily used anymore. In order to schedule git maintenance, we need two unit template files: * ~/.config/systemd/user/git-maintenance@.service to define the command to be started by systemd and * ~/.config/systemd/user/git-maintenance@.timer to define the schedule at which the command should be run. Those units are templates that are parameterized by the frequency. Based on those templates, 3 timers are started: * git-maintenance@hourly.timer * git-maintenance@daily.timer * git-maintenance@weekly.timer The command launched by those three timers are the same as with the other scheduling methods: /path/to/git for-each-repo --exec-path=/path/to --config=maintenance.repo maintenance run --schedule=%i with the full path for git to ensure that the version of git launched for the scheduled maintenance is the same as the one used to run `maintenance start`. The timer unit contains `Persistent=true` so that, if the computer is powered down when a maintenance task should run, the task will be run when the computer is back powered on. Signed-off-by: Lénaïc Huard Acked-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 67 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 6 deletions(-) (limited to 't/t7900-maintenance.sh') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index b36b7f5fb0..b289cae6b9 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -20,6 +20,18 @@ test_xmllint () { fi } +test_lazy_prereq SYSTEMD_ANALYZE ' + systemd-analyze --help >out && + grep verify out +' + +test_systemd_analyze_verify () { + if test_have_prereq SYSTEMD_ANALYZE + then + systemd-analyze verify "$@" + fi +} + test_expect_success 'help text' ' test_expect_code 129 git maintenance -h 2>err && test_i18ngrep "usage: git maintenance " err && @@ -634,15 +646,56 @@ test_expect_success 'start and stop Windows maintenance' ' test_cmp expect args ' +test_expect_success 'start and stop Linux/systemd maintenance' ' + write_script print-args <<-\EOF && + printf "%s\n" "$*" >>args + EOF + + XDG_CONFIG_HOME="$PWD" && + export XDG_CONFIG_HOME && + rm -f args && + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args" git maintenance start --scheduler=systemd-timer && + + # start registers the repo + git config --get --global --fixed-value maintenance.repo "$(pwd)" && + + test_systemd_analyze_verify "systemd/user/git-maintenance@.service" && + + printf -- "--user enable --now git-maintenance@%s.timer\n" hourly daily weekly >expect && + test_cmp expect args && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args" git maintenance stop && + + # stop does not unregister the repo + git config --get --global --fixed-value maintenance.repo "$(pwd)" && + + test_path_is_missing "systemd/user/git-maintenance@.timer" && + test_path_is_missing "systemd/user/git-maintenance@.service" && + + printf -- "--user disable --now git-maintenance@%s.timer\n" hourly daily weekly >expect && + test_cmp expect args +' + test_expect_success 'start and stop when several schedulers are available' ' write_script print-args <<-\EOF && printf "%s\n" "$*" | sed "s:gui/[0-9][0-9]*:gui/[UID]:; s:\(schtasks /create .* /xml\).*:\1:;" >>args EOF rm -f args && - GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=launchctl && - printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args systemctl,launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=systemd-timer && + printf "launchctl bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ hourly daily weekly >expect && + printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ + hourly daily weekly >>expect && + printf -- "systemctl --user enable --now git-maintenance@%s.timer\n" hourly daily weekly >>expect && + test_cmp expect args && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args systemctl,launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=launchctl && + printf -- "systemctl --user disable --now git-maintenance@%s.timer\n" hourly daily weekly >expect && + printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ + hourly daily weekly >>expect && for frequency in hourly daily weekly do PLIST="$pfx/Library/LaunchAgents/org.git-scm.git.$frequency.plist" && @@ -652,17 +705,19 @@ test_expect_success 'start and stop when several schedulers are available' ' test_cmp expect args && rm -f args && - GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=schtasks && + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args systemctl,launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance start --scheduler=schtasks && + printf -- "systemctl --user disable --now git-maintenance@%s.timer\n" hourly daily weekly >expect && printf "launchctl bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ - hourly daily weekly >expect && + hourly daily weekly >>expect && printf "schtasks /create /tn Git Maintenance (%s) /f /xml\n" \ hourly daily weekly >>expect && test_cmp expect args && rm -f args && - GIT_TEST_MAINT_SCHEDULER="launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance stop && + GIT_TEST_MAINT_SCHEDULER="systemctl:./print-args systemctl,launchctl:./print-args launchctl,schtasks:./print-args schtasks" git maintenance stop && + printf -- "systemctl --user disable --now git-maintenance@%s.timer\n" hourly daily weekly >expect && printf "launchctl bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ - hourly daily weekly >expect && + hourly daily weekly >>expect && printf "schtasks /delete /tn Git Maintenance (%s) /f\n" \ hourly daily weekly >>expect && test_cmp expect args -- cgit v1.2.3 From b4724242fa3342d939e7a0c4102d3695091db1f6 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 24 Sep 2021 14:32:17 -0400 Subject: t7900: clean up some more broken refs The "incremental-repack task" test replaces the object directory with a known state. As a result, some of our refs point to objects that are not included in that state. Commit 3cf5f221be (t7900: clean up some broken refs, 2021-01-19) cleaned up some of those (that were causing warnings to stderr from the maintenance process). But there are a few more that were missed. These aren't hurting anything for now, but it's certainly an unexpected state to leave the test repository in, and it will become a problem if repack ever gets more picky about broken refs. Let's clean up those additional refs (which are all in refs/remotes, with nothing there that isn't broken), and add an extra "for-each-ref" call to assert that we've got everything. Signed-off-by: Jeff King Reviewed-by: Jonathan Tan Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 't/t7900-maintenance.sh') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 36a4218745..31245f6276 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -277,7 +277,7 @@ test_expect_success 'incremental-repack task' ' # Delete refs that have not been repacked in these packs. git for-each-ref --format="delete %(refname)" \ - refs/prefetch refs/tags >refs && + refs/prefetch refs/tags refs/remotes >refs && git update-ref --stdin packs-before && test_line_count = 3 packs-before && + # make sure we do not have any broken refs that were + # missed in the deletion above + git for-each-ref && + # the job repacks the two into a new pack, but does not # delete the old ones. git maintenance run --task=incremental-repack && -- cgit v1.2.3 From 670e5973992b6126aa0841c3c6bc183f0cec749f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Mon, 27 Sep 2021 23:30:16 +0200 Subject: maintenance: fix test t7900-maintenance.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit b681b191 introduced the support of systemd timers for git maintenance. A test is leveraging the `systemd-analyze verify` utility to verify the correctness of the systemd unit files generated by git. But on some systems, although the `systemd-analyze` tool is installed and supports the `verify` subcommand, it fails with some permission errors. So, instead of only checking if the `verify` subcommand exists, a more reliable way of detecting whether `systemd-analyze verify` can be used is to try to use it. The SYSTEMD_ANALYZE prerequisite is now trying to run `systemd-analyze verify` on a systemd unit file which is shipped by systemd itself. We can reasonably think that, on systemd hosts, this file is present and valid. Signed-off-by: Lénaïc Huard Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 't/t7900-maintenance.sh') diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index b289cae6b9..1386756f0d 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -21,8 +21,7 @@ test_xmllint () { } test_lazy_prereq SYSTEMD_ANALYZE ' - systemd-analyze --help >out && - grep verify out + systemd-analyze verify /lib/systemd/system/basic.target ' test_systemd_analyze_verify () { -- cgit v1.2.3