From 1e0ea5c4316d2241dd76ef430a2779db9a097dfb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:02:46 +0000 Subject: fsmonitor: config settings are repository-specific Move fsmonitor config settings to a new and opaque `struct fsmonitor_settings` structure. Add a lazily-loaded pointer to this into `struct repo_settings` Create an `enum fsmonitor_mode` type in `struct fsmonitor_settings` to represent the state of fsmonitor. This lets us represent which, if any, fsmonitor provider (hook or IPC) is enabled. Create `fsm_settings__get_*()` getters to lazily look up fsmonitor- related config settings. Get rid of the `core_fsmonitor` global variable. Move the code to lookup the existing `core.fsmonitor` config value into the fsmonitor settings. Create a hook pathname variable in `struct fsmonitor-settings` and only set it when in hook mode. Extend the definition of `core.fsmonitor` to be either a boolean or a hook pathname. When true, the builtin FSMonitor is used. When false or unset, no FSMonitor (neither builtin nor hook) is used. The existing `core_fsmonitor` global variable was used to store the pathname to the fsmonitor hook *and* it was used as a boolean to see if fsmonitor was enabled. This dual usage and global visibility leads to confusion when we add the IPC-based provider. So lets hide the details in fsmonitor-settings.c and let it decide which provider to use in the case of multiple settings. This avoids cluttering up repo-settings.c with these private details. A future commit in builtin-fsmonitor series will add the ability to disqualify worktrees for various reasons, such as being mounted from a remote volume, where fsmonitor should not be started. Having the config settings hidden in fsmonitor-settings.c allows such worktree restrictions to override the config values used. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 't') diff --git a/t/README b/t/README index f48e0542cd..9ffea1d314 100644 --- a/t/README +++ b/t/README @@ -405,8 +405,8 @@ every 'git commit-graph write', as if the `--changed-paths` option was passed in. GIT_TEST_FSMONITOR=$PWD/t7519/fsmonitor-all exercises the fsmonitor -code path for utilizing a file system monitor to speed up detecting -new or changed files. +code paths for utilizing a (hook based) file system monitor to speed up +detecting new or changed files. GIT_TEST_INDEX_VERSION= exercises the index read/write code path for the index version specified. Can be set to any valid version -- cgit v1.2.3 From dd77cf61a1a2fbf52c94d0cd986d555ad2ba8a4b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:02 +0000 Subject: help: include fsmonitor--daemon feature flag in version info Add the "feature: fsmonitor--daemon" message to the output of `git version --build-options`. The builtin FSMonitor is only available on certain platforms and even then only when certain Makefile flags are enabled, so print a message in the verbose version output when it is available. This can be used by test scripts for prereq testing. Granted, tests could just try `git fsmonitor--daemon status` and look for a 128 exit code or grep for a "not supported" message on stderr, but these methods are rather obscure. The main advantage is that the feature message will automatically appear in bug reports and other support requests. This concept was also used during the development of Scalar for similar reasons. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/test-lib.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 't') diff --git a/t/test-lib.sh b/t/test-lib.sh index e4716b0b86..5d819c1bc1 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1799,3 +1799,10 @@ test_lazy_prereq SHA1 ' # Tests that verify the scheduler integration must set this locally # to avoid errors. GIT_TEST_MAINT_SCHEDULER="none:exit 1" + +# Does this platform support `git fsmonitor--daemon` +# +test_lazy_prereq FSMONITOR_DAEMON ' + git version --build-options >output && + grep "feature: fsmonitor--daemon" output +' -- cgit v1.2.3 From 148405fb27980a63bb532d11f9ab242280052355 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:03 +0000 Subject: t/helper/fsmonitor-client: create IPC client to talk to FSMonitor Daemon Create an IPC client to send query and flush commands to the daemon. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/helper/test-fsmonitor-client.c | 116 +++++++++++++++++++++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 3 files changed, 118 insertions(+) create mode 100644 t/helper/test-fsmonitor-client.c (limited to 't') diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c new file mode 100644 index 0000000000..3062c8a3c2 --- /dev/null +++ b/t/helper/test-fsmonitor-client.c @@ -0,0 +1,116 @@ +/* + * test-fsmonitor-client.c: client code to send commands/requests to + * a `git fsmonitor--daemon` daemon. + */ + +#include "test-tool.h" +#include "cache.h" +#include "parse-options.h" +#include "fsmonitor-ipc.h" + +#ifndef HAVE_FSMONITOR_DAEMON_BACKEND +int cmd__fsmonitor_client(int argc, const char **argv) +{ + die("fsmonitor--daemon not available on this platform"); +} +#else + +/* + * Read the `.git/index` to get the last token written to the + * FSMonitor Index Extension. + */ +static const char *get_token_from_index(void) +{ + struct index_state *istate = the_repository->index; + + if (do_read_index(istate, the_repository->index_file, 0) < 0) + die("unable to read index file"); + if (!istate->fsmonitor_last_update) + die("index file does not have fsmonitor extension"); + + return istate->fsmonitor_last_update; +} + +/* + * Send an IPC query to a `git-fsmonitor--daemon` daemon and + * ask for the changes since the given token or from the last + * token in the index extension. + * + * This will implicitly start a daemon process if necessary. The + * daemon process will persist after we exit. + */ +static int do_send_query(const char *token) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + if (!token || !*token) + token = get_token_from_index(); + + ret = fsmonitor_ipc__send_query(token, &answer); + if (ret < 0) + die("could not query fsmonitor--daemon"); + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +/* + * Send a "flush" command to the `git-fsmonitor--daemon` (if running) + * and tell it to flush its cache. + * + * This feature is primarily used by the test suite to simulate a loss of + * sync with the filesystem where we miss kernel events. + */ +static int do_send_flush(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = fsmonitor_ipc__send_command("flush", &answer); + if (ret) + return ret; + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +int cmd__fsmonitor_client(int argc, const char **argv) +{ + const char *subcmd; + const char *token = NULL; + + const char * const fsmonitor_client_usage[] = { + "test-tool fsmonitor-client query []", + "test-tool fsmonitor-client flush", + NULL, + }; + + struct option options[] = { + OPT_STRING(0, "token", &token, "token", + "command token to send to the server"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, fsmonitor_client_usage, 0); + + if (argc != 1) + usage_with_options(fsmonitor_client_usage, options); + + subcmd = argv[0]; + + setup_git_directory(); + + if (!strcmp(subcmd, "query")) + return !!do_send_query(token); + + if (!strcmp(subcmd, "flush")) + return !!do_send_flush(); + + die("Unhandled subcommand: '%s'", subcmd); +} +#endif diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index e6ec69cf32..0424f7adf5 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -32,6 +32,7 @@ static struct test_cmd cmds[] = { { "dump-untracked-cache", cmd__dump_untracked_cache }, { "example-decorate", cmd__example_decorate }, { "fast-rebase", cmd__fast_rebase }, + { "fsmonitor-client", cmd__fsmonitor_client }, { "genrandom", cmd__genrandom }, { "genzeros", cmd__genzeros }, { "getcwd", cmd__getcwd }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 20756eefdd..c876e8246f 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -23,6 +23,7 @@ int cmd__dump_untracked_cache(int argc, const char **argv); int cmd__dump_reftable(int argc, const char **argv); int cmd__example_decorate(int argc, const char **argv); int cmd__fast_rebase(int argc, const char **argv); +int cmd__fsmonitor_client(int argc, const char **argv); int cmd__genrandom(int argc, const char **argv); int cmd__genzeros(int argc, const char **argv); int cmd__getcwd(int argc, const char **argv); -- cgit v1.2.3 From a00cdff81a2d578405cf0f68072da07eee462b93 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:04 +0000 Subject: t7527: create test for fsmonitor--daemon Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/t7527-builtin-fsmonitor.sh | 494 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100755 t/t7527-builtin-fsmonitor.sh (limited to 't') diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh new file mode 100755 index 0000000000..062e01c0df --- /dev/null +++ b/t/t7527-builtin-fsmonitor.sh @@ -0,0 +1,494 @@ +#!/bin/sh + +test_description='built-in file system watcher' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all="fsmonitor--daemon is not supported on this platform" + test_done +fi + +stop_daemon_delete_repo () { + r=$1 && + test_might_fail git -C $r fsmonitor--daemon stop && + rm -rf $1 +} + +start_daemon () { + r= tf= t2= tk= && + + while test "$#" -ne 0 + do + case "$1" in + -C) + r="-C ${2?}" + shift + ;; + --tf) + tf="${2?}" + shift + ;; + --t2) + t2="${2?}" + shift + ;; + --tk) + tk="${2?}" + shift + ;; + -*) + BUG "error: unknown option: '$1'" + ;; + *) + BUG "error: unbound argument: '$1'" + ;; + esac + shift + done && + + ( + if test -n "$tf" + then + GIT_TRACE_FSMONITOR="$tf" + export GIT_TRACE_FSMONITOR + fi && + + if test -n "$t2" + then + GIT_TRACE2_PERF="$t2" + export GIT_TRACE2_PERF + fi && + + if test -n "$tk" + then + GIT_TEST_FSMONITOR_TOKEN="$tk" + export GIT_TEST_FSMONITOR_TOKEN + fi && + + git $r fsmonitor--daemon start && + git $r fsmonitor--daemon status + ) +} + +# Is a Trace2 data event present with the given catetory and key? +# We do not care what the value is. +# +have_t2_data_event () { + c=$1 && + k=$2 && + + grep -e '"event":"data".*"category":"'"$c"'".*"key":"'"$k"'"' +} + +test_expect_success 'explicit daemon start and stop' ' + test_when_finished "stop_daemon_delete_repo test_explicit" && + + git init test_explicit && + start_daemon -C test_explicit && + + git -C test_explicit fsmonitor--daemon stop && + test_must_fail git -C test_explicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon start' ' + test_when_finished "stop_daemon_delete_repo test_implicit" && + + git init test_implicit && + test_must_fail git -C test_implicit fsmonitor--daemon status && + + # query will implicitly start the daemon. + # + # for test-script simplicity, we send a V1 timestamp rather than + # a V2 token. either way, the daemon response to any query contains + # a new V2 token. (the daemon may complain that we sent a V1 request, + # but this test case is only concerned with whether the daemon was + # implicitly started.) + + GIT_TRACE2_EVENT="$PWD/.git/trace" \ + test-tool -C test_implicit fsmonitor-client query --token 0 >actual && + nul_to_q actual.filtered && + grep "builtin:" actual.filtered && + + # confirm that a daemon was started in the background. + # + # since the mechanism for starting the background daemon is platform + # dependent, just confirm that the foreground command received a + # response from the daemon. + + have_t2_data_event fsm_client query/response-length <.git/trace && + + git -C test_implicit fsmonitor--daemon status && + git -C test_implicit fsmonitor--daemon stop && + test_must_fail git -C test_implicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (delete .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1" && + + git init test_implicit_1 && + + start_daemon -C test_implicit_1 && + + # deleting the .git directory will implicitly stop the daemon. + rm -rf test_implicit_1/.git && + + # [1] Create an empty .git directory so that the following Git + # command will stay relative to the `-C` directory. + # + # Without this, the Git command will override the requested + # -C argument and crawl out to the containing Git source tree. + # This would make the test result dependent upon whether we + # were using fsmonitor on our development worktree. + # + sleep 1 && + mkdir test_implicit_1/.git && + + test_must_fail git -C test_implicit_1 fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (rename .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_2" && + + git init test_implicit_2 && + + start_daemon -C test_implicit_2 && + + # renaming the .git directory will implicitly stop the daemon. + mv test_implicit_2/.git test_implicit_2/.xxx && + + # See [1] above. + # + sleep 1 && + mkdir test_implicit_2/.git && + + test_must_fail git -C test_implicit_2 fsmonitor--daemon status +' + +test_expect_success 'cannot start multiple daemons' ' + test_when_finished "stop_daemon_delete_repo test_multiple" && + + git init test_multiple && + + start_daemon -C test_multiple && + + test_must_fail git -C test_multiple fsmonitor--daemon start 2>actual && + grep "fsmonitor--daemon is already running" actual && + + git -C test_multiple fsmonitor--daemon stop && + test_must_fail git -C test_multiple fsmonitor--daemon status +' + +# These tests use the main repo in the trash directory + +test_expect_success 'setup' ' + >tracked && + >modified && + >delete && + >rename && + mkdir dir1 && + >dir1/tracked && + >dir1/modified && + >dir1/delete && + >dir1/rename && + mkdir dir2 && + >dir2/tracked && + >dir2/modified && + >dir2/delete && + >dir2/rename && + mkdir dirtorename && + >dirtorename/a && + >dirtorename/b && + + cat >.gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + + git -c core.fsmonitor=false add . && + test_tick && + git -c core.fsmonitor=false commit -m initial && + + git config core.fsmonitor true +' + +# The test already explicitly stopped (or tried to stop) the daemon. +# This is here in case something else fails first. +# +redundant_stop_daemon () { + test_might_fail git fsmonitor--daemon stop +} + +test_expect_success 'update-index implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$PWD/.git/trace_implicit_1" \ + git update-index --fsmonitor && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 +' + +test_expect_success 'status implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$PWD/.git/trace_implicit_2" \ + git status >actual && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 +' + +edit_files () { + echo 1 >modified && + echo 2 >dir1/modified && + echo 3 >dir2/modified && + >dir1/untracked +} + +delete_files () { + rm -f delete && + rm -f dir1/delete && + rm -f dir2/delete +} + +create_files () { + echo 1 >new && + echo 2 >dir1/new && + echo 3 >dir2/new +} + +rename_files () { + mv rename renamed && + mv dir1/rename dir1/renamed && + mv dir2/rename dir2/renamed +} + +file_to_directory () { + rm -f delete && + mkdir delete && + echo 1 >delete/new +} + +directory_to_file () { + rm -rf dir1 && + echo 1 >dir1 +} + +# The next few test cases confirm that our fsmonitor daemon sees each type +# of OS filesystem notification that we care about. At this layer we just +# ensure we are getting the OS notifications and do not try to confirm what +# is reported by `git status`. +# +# We run a simple query after modifying the filesystem just to introduce +# a bit of a delay so that the trace logging from the daemon has time to +# get flushed to disk. +# +# We `reset` and `clean` at the bottom of each test (and before stopping the +# daemon) because these commands might implicitly restart the daemon. + +clean_up_repo_and_stop_daemon () { + git reset --hard HEAD && + git clean -fd && + test_might_fail git fsmonitor--daemon stop && + rm -f .git/trace +} + +test_expect_success 'edit some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + edit_files && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dir1/modified$" .git/trace && + grep "^event: dir2/modified$" .git/trace && + grep "^event: modified$" .git/trace && + grep "^event: dir1/untracked$" .git/trace +' + +test_expect_success 'create some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + create_files && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dir1/new$" .git/trace && + grep "^event: dir2/new$" .git/trace && + grep "^event: new$" .git/trace +' + +test_expect_success 'delete some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + delete_files && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dir1/delete$" .git/trace && + grep "^event: dir2/delete$" .git/trace && + grep "^event: delete$" .git/trace +' + +test_expect_success 'rename some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + rename_files && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dir1/rename$" .git/trace && + grep "^event: dir2/rename$" .git/trace && + grep "^event: rename$" .git/trace && + grep "^event: dir1/renamed$" .git/trace && + grep "^event: dir2/renamed$" .git/trace && + grep "^event: renamed$" .git/trace +' + +test_expect_success 'rename directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + mv dirtorename dirrenamed && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dirtorename/*$" .git/trace && + grep "^event: dirrenamed/*$" .git/trace +' + +test_expect_success 'file changes to directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + file_to_directory && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: delete$" .git/trace && + grep "^event: delete/new$" .git/trace +' + +test_expect_success 'directory changes to a file' ' + test_when_finished clean_up_repo_and_stop_daemon && + + start_daemon --tf "$PWD/.git/trace" && + + directory_to_file && + + test-tool fsmonitor-client query --token 0 && + + grep "^event: dir1$" .git/trace +' + +# The next few test cases exercise the token-resync code. When filesystem +# drops events (because of filesystem velocity or because the daemon isn't +# polling fast enough), we need to discard the cached data (relative to the +# current token) and start collecting events under a new token. +# +# the 'test-tool fsmonitor-client flush' command can be used to send a +# "flush" message to a running daemon and ask it to do a flush/resync. + +test_expect_success 'flush cached data' ' + test_when_finished "stop_daemon_delete_repo test_flush" && + + git init test_flush && + + start_daemon -C test_flush --tf "$PWD/.git/trace_daemon" --tk true && + + # The daemon should have an initial token with no events in _0 and + # then a few (probably platform-specific number of) events in _1. + # These should both have the same . + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_0 && + nul_to_q actual_q0 && + + >test_flush/file_1 && + >test_flush/file_2 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_1 && + nul_to_q actual_q1 && + + grep "file_1" actual_q1 && + + # Force a flush. This will change the , reset the , and + # flush the file data. Then create some events and ensure that the file + # again appears in the cache. It should have the new . + + test-tool -C test_flush fsmonitor-client flush >flush_0 && + nul_to_q flush_q0 && + grep "^builtin:test_00000002:0Q/Q$" flush_q0 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_2 && + nul_to_q actual_q2 && + + grep "^builtin:test_00000002:0Q$" actual_q2 && + + >test_flush/file_3 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_3 && + nul_to_q actual_q3 && + + grep "file_3" actual_q3 +' + +# The next few test cases create repos where the .git directory is NOT +# inside the one of the working directory. That is, where .git is a file +# that points to a directory elsewhere. This happens for submodules and +# non-primary worktrees. + +test_expect_success 'setup worktree base' ' + git init wt-base && + echo 1 >wt-base/file1 && + git -C wt-base add file1 && + git -C wt-base commit -m "c1" +' + +test_expect_success 'worktree with .git file' ' + git -C wt-base worktree add ../wt-secondary && + + start_daemon -C wt-secondary \ + --tf "$PWD/trace_wt_secondary" \ + --t2 "$PWD/trace2_wt_secondary" && + + git -C wt-secondary fsmonitor--daemon stop && + test_must_fail git -C wt-secondary fsmonitor--daemon status +' + +# NEEDSWORK: Repeat one of the "edit" tests on wt-secondary and +# confirm that we get the same events and behavior -- that is, that +# fsmonitor--daemon correctly watches BOTH the working directory and +# the external GITDIR directory and behaves the same as when ".git" +# is a directory inside the working directory. + +test_expect_success 'cleanup worktrees' ' + stop_daemon_delete_repo wt-secondary && + stop_daemon_delete_repo wt-base +' + +test_done -- cgit v1.2.3 From 08894d334969011394dee7ab1dde63176fc51830 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:05 +0000 Subject: t/perf: avoid copying builtin fsmonitor files into test repo Do not copy any of the various fsmonitor--daemon files from the .git directory of the (GIT_PREF_REPO or GIT_PERF_LARGE_REPO) source repo into the test's trash directory. When perf tests start, they copy the contents of the source repo into the test's trash directory. If fsmonitor is running in the source repo, there may be control files, such as the IPC socket and/or fsmonitor cookie files. These should not be copied into the test repo. Unix domain sockets cannot be copied in the manner used by the test setup, so if present, the test setup fails. Cookie files are harmless, but we should avoid them. The builtin fsmonitor keeps all such control files/sockets in .git/fsmonitor--daemon*, so it is simple to exclude them. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/perf/perf-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 't') diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index 407252bac7..932105cd12 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -78,7 +78,7 @@ test_perf_copy_repo_contents () { for stuff in "$1"/* do case "$stuff" in - */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees) + */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees|*/fsmonitor--daemon*) ;; *) cp -R "$stuff" "$repo/.git/" || exit 1 -- cgit v1.2.3 From 369f0f54ff92fe321e8c7c2d2372e0eb31f77303 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:06 +0000 Subject: t/helper/test-chmtime: skip directories on Windows Teach `test-tool.exe chmtime` to ignore errors when setting the mtime on a directory on Windows. NEEDSWORK: The Windows version of `utime()` (aka `mingw_utime()`) does not properly handle directories because it uses `_wopen()`. It should be converted to using `CreateFileW()` and backup semantics at a minimum. Since I'm already in the middle of a large patch series, I did not want to destabilize other callers of `utime()` right now. The problem has only been observed in the t/perf/p7519 test when the test repo contains an empty directory on disk. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/helper/test-chmtime.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 't') diff --git a/t/helper/test-chmtime.c b/t/helper/test-chmtime.c index 524b55ca49..dc28890a18 100644 --- a/t/helper/test-chmtime.c +++ b/t/helper/test-chmtime.c @@ -134,6 +134,21 @@ int cmd__chmtime(int argc, const char **argv) } if (utb.modtime != sb.st_mtime && utime(argv[i], &utb) < 0) { +#ifdef GIT_WINDOWS_NATIVE + if (S_ISDIR(sb.st_mode)) { + /* + * NEEDSWORK: The Windows version of `utime()` + * (aka `mingw_utime()`) does not correctly + * handle directory arguments, since it uses + * `_wopen()`. Ignore it for now since this + * is just a test. + */ + fprintf(stderr, + ("Failed to modify time on directory %s. " + "Skipping\n"), argv[i]); + continue; + } +#endif fprintf(stderr, "Failed to modify time on %s: %s\n", argv[i], strerror(errno)); return 1; -- cgit v1.2.3 From 8aa020970195f49c369eeff9bf22079002e3454b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:07 +0000 Subject: t/perf/p7519: fix coding style Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/perf/p7519-fsmonitor.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 't') diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index c8be58f3c7..5241eb6c4e 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -72,7 +72,7 @@ then fi fi -trace_start() { +trace_start () { if test -n "$GIT_PERF_7519_TRACE" then name="$1" @@ -91,7 +91,7 @@ trace_start() { fi } -trace_stop() { +trace_stop () { if test -n "$GIT_PERF_7519_TRACE" then unset GIT_TRACE2_PERF @@ -133,7 +133,7 @@ test_expect_success "one time repo setup" ' fi ' -setup_for_fsmonitor() { +setup_for_fsmonitor () { # set INTEGRATION_SCRIPT depending on the environment if test -n "$INTEGRATION_PATH" then @@ -173,7 +173,7 @@ test_perf_w_drop_caches () { test_perf "$@" } -test_fsmonitor_suite() { +test_fsmonitor_suite () { if test -n "$INTEGRATION_SCRIPT"; then DESC="fsmonitor=$(basename $INTEGRATION_SCRIPT)" else -- cgit v1.2.3 From 86f7433f9711012fcd9857974fc43dfd693b0aa9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:08 +0000 Subject: t/perf/p7519: speed up test on Windows Change p7519 to use `test_seq` and `xargs` rather than a `for` loop to touch thousands of files. This takes minutes off of test runs on Windows because of process creation overhead. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/perf/p7519-fsmonitor.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 't') diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index 5241eb6c4e..a6c2a910e7 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -98,6 +98,13 @@ trace_stop () { fi } +touch_files () { + n=$1 && + d="$n"_files && + + (cd $d && test_seq 1 $n | xargs touch ) +} + test_expect_success "one time repo setup" ' # set untrackedCache depending on the environment if test -n "$GIT_PERF_7519_UNTRACKED_CACHE" @@ -119,10 +126,11 @@ test_expect_success "one time repo setup" ' fi && mkdir 1_file 10_files 100_files 1000_files 10000_files && - for i in $(test_seq 1 10); do touch 10_files/$i || return 1; done && - for i in $(test_seq 1 100); do touch 100_files/$i || return 1; done && - for i in $(test_seq 1 1000); do touch 1000_files/$i || return 1; done && - for i in $(test_seq 1 10000); do touch 10000_files/$i || return 1; done && + : 1_file directory should be left empty && + touch_files 10 && + touch_files 100 && + touch_files 1000 && + touch_files 10000 && git add 1_file 10_files 100_files 1000_files 10000_files && git commit -qm "Add files" && @@ -199,15 +207,15 @@ test_fsmonitor_suite () { # Update the mtimes on upto 100k files to make status think # that they are dirty. For simplicity, omit any files with - # LFs (i.e. anything that ls-files thinks it needs to dquote). - # Then fully backslash-quote the paths to capture any - # whitespace so that they pass thru xargs properly. + # LFs (i.e. anything that ls-files thinks it needs to dquote) + # and any files with whitespace so that they pass thru xargs + # properly. # test_perf_w_drop_caches "status (dirty) ($DESC)" ' git ls-files | \ head -100000 | \ grep -v \" | \ - sed '\''s/\(.\)/\\\1/g'\'' | \ + grep -v " ." | \ xargs test-tool chmtime -300 && git status ' -- cgit v1.2.3 From ad2b54e3e89d715a283a3c3ab6262c9bed8cb3a1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:09 +0000 Subject: t/perf/p7519: add fsmonitor--daemon test cases Repeat all of the fsmonitor perf tests using `git fsmonitor--daemon` and the "Simple IPC" interface. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/perf/p7519-fsmonitor.sh | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) (limited to 't') diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index a6c2a910e7..0b9129ca7b 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -141,7 +141,7 @@ test_expect_success "one time repo setup" ' fi ' -setup_for_fsmonitor () { +setup_for_fsmonitor_hook () { # set INTEGRATION_SCRIPT depending on the environment if test -n "$INTEGRATION_PATH" then @@ -182,7 +182,11 @@ test_perf_w_drop_caches () { } test_fsmonitor_suite () { - if test -n "$INTEGRATION_SCRIPT"; then + if test -n "$USE_FSMONITOR_DAEMON" + then + DESC="builtin fsmonitor--daemon" + elif test -n "$INTEGRATION_SCRIPT" + then DESC="fsmonitor=$(basename $INTEGRATION_SCRIPT)" else DESC="fsmonitor=disabled" @@ -261,11 +265,11 @@ test_fsmonitor_suite () { trace_start fsmonitor-watchman if test -n "$GIT_PERF_7519_FSMONITOR"; then for INTEGRATION_PATH in $GIT_PERF_7519_FSMONITOR; do - test_expect_success "setup for fsmonitor $INTEGRATION_PATH" 'setup_for_fsmonitor' + test_expect_success "setup for fsmonitor $INTEGRATION_PATH" 'setup_for_fsmonitor_hook' test_fsmonitor_suite done else - test_expect_success "setup for fsmonitor" 'setup_for_fsmonitor' + test_expect_success "setup for fsmonitor hook" 'setup_for_fsmonitor_hook' test_fsmonitor_suite fi @@ -293,4 +297,30 @@ test_expect_success "setup without fsmonitor" ' test_fsmonitor_suite trace_stop +# +# Run a full set of perf tests using the built-in fsmonitor--daemon. +# It does not use the Hook API, so it has a different setup. +# Explicitly start the daemon here and before we start client commands +# so that we can later add custom tracing. +# +if test_have_prereq FSMONITOR_DAEMON +then + USE_FSMONITOR_DAEMON=t + + test_expect_success "setup for builtin fsmonitor" ' + trace_start fsmonitor--daemon--server && + git fsmonitor--daemon start && + + trace_start fsmonitor--daemon--client && + + git config core.fsmonitor true && + git update-index --fsmonitor + ' + + test_fsmonitor_suite + + git fsmonitor--daemon stop + trace_stop +fi + test_done -- cgit v1.2.3 From a3dfe97f418762ccf1d0601c5cce40c77046d4fc Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 18:03:13 +0000 Subject: t7527: test status with untracked-cache and fsmonitor--daemon Create 2x2 test matrix with the untracked-cache and fsmonitor--daemon features and a series of edits and verify that status output is identical. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- t/t7527-builtin-fsmonitor.sh | 115 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) (limited to 't') diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 062e01c0df..bd0c952a11 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -205,6 +205,8 @@ test_expect_success 'setup' ' .gitignore expect* actual* + flush* + trace* EOF git -c core.fsmonitor=false add . && @@ -491,4 +493,117 @@ test_expect_success 'cleanup worktrees' ' stop_daemon_delete_repo wt-base ' +# The next few tests perform arbitrary/contrived file operations and +# confirm that status is correct. That is, that the data (or lack of +# data) from fsmonitor doesn't cause incorrect results. And doesn't +# cause incorrect results when the untracked-cache is enabled. + +test_lazy_prereq UNTRACKED_CACHE ' + git update-index --test-untracked-cache +' + +test_expect_success 'Matrix: setup for untracked-cache,fsmonitor matrix' ' + test_unconfig core.fsmonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop +' + +matrix_clean_up_repo () { + git reset --hard HEAD && + git clean -fd +} + +matrix_try () { + uc=$1 && + fsm=$2 && + fn=$3 && + + if test $uc = true && test $fsm = false + then + # The untracked-cache is buggy when FSMonitor is + # DISABLED, so skip the tests for this matrix + # combination. + # + # We've observed random, occasional test failures on + # Windows and MacOS when the UC is turned on and FSM + # is turned off. These are rare, but they do happen + # indicating that it is probably a race condition within + # the untracked cache itself. + # + # It usually happens when a test does F/D trickery and + # then the NEXT test fails because of extra status + # output from stale UC data from the previous test. + # + # Since FSMonitor is not involved in the error, skip + # the tests for this matrix combination. + # + return 0 + fi && + + test_expect_success "Matrix[uc:$uc][fsm:$fsm] $fn" ' + matrix_clean_up_repo && + $fn && + if test $uc = false && test $fsm = false + then + git status --porcelain=v1 >.git/expect.$fn + else + git status --porcelain=v1 >.git/actual.$fn && + test_cmp .git/expect.$fn .git/actual.$fn + fi + ' +} + +uc_values="false" +test_have_prereq UNTRACKED_CACHE && uc_values="false true" +for uc_val in $uc_values +do + if test $uc_val = false + then + test_expect_success "Matrix[uc:$uc_val] disable untracked cache" ' + git config core.untrackedcache false && + git update-index --no-untracked-cache + ' + else + test_expect_success "Matrix[uc:$uc_val] enable untracked cache" ' + git config core.untrackedcache true && + git update-index --untracked-cache + ' + fi + + fsm_values="false true" + for fsm_val in $fsm_values + do + if test $fsm_val = false + then + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor" ' + test_unconfig core.fsmonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop + ' + else + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] enable fsmonitor" ' + git config core.fsmonitor true && + git fsmonitor--daemon start && + git update-index --fsmonitor + ' + fi + + matrix_try $uc_val $fsm_val edit_files + matrix_try $uc_val $fsm_val delete_files + matrix_try $uc_val $fsm_val create_files + matrix_try $uc_val $fsm_val rename_files + matrix_try $uc_val $fsm_val file_to_directory + matrix_try $uc_val $fsm_val directory_to_file + + if test $fsm_val = true + then + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor at end" ' + test_unconfig core.fsmonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop + ' + fi + done +done + test_done -- cgit v1.2.3