From 5c8cdcfd807311d4ec7165c71b20c5bde9566acd Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 22 Sep 2017 12:35:46 -0400 Subject: fsmonitor: add test cases for fsmonitor extension Test the ability to add/remove the fsmonitor index extension via update-index. Test that dirty files returned from the integration script are properly represented in the index extension and verify that ls-files correctly reports their state. Test that ensure status results are correct when using the new fsmonitor extension. Test untracked, modified, and new files by ensuring the results are identical to when not using the extension. Test that if the fsmonitor extension doesn't tell git about a change, it doesn't discover it on its own. This ensures git is honoring the extension and that we get the performance benefits desired. Three test integration scripts are provided: fsmonitor-all - marks all files as dirty fsmonitor-none - marks no files as dirty fsmonitor-watchman - integrates with Watchman with debug logging To run tests in the test suite while utilizing fsmonitor: First copy t/t7519/fsmonitor-all to a location in your path and then set GIT_FORCE_PRELOAD_TEST=true and GIT_FSMONITOR_TEST=fsmonitor-all and run your tests. Note: currently when using the test script fsmonitor-watchman on Windows, many tests fail due to a reported but not yet fixed bug in Watchman where it holds on to handles for directories and files which prevents the test directory from being cleaned up properly. Signed-off-by: Ben Peart Signed-off-by: Junio C Hamano --- t/t7519-status-fsmonitor.sh | 304 ++++++++++++++++++++++++++++++++++++++++++++ t/t7519/fsmonitor-all | 24 ++++ t/t7519/fsmonitor-none | 22 ++++ t/t7519/fsmonitor-watchman | 140 ++++++++++++++++++++ 4 files changed, 490 insertions(+) create mode 100755 t/t7519-status-fsmonitor.sh create mode 100755 t/t7519/fsmonitor-all create mode 100755 t/t7519/fsmonitor-none create mode 100755 t/t7519/fsmonitor-watchman diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh new file mode 100755 index 0000000000..c6df85af5e --- /dev/null +++ b/t/t7519-status-fsmonitor.sh @@ -0,0 +1,304 @@ +#!/bin/sh + +test_description='git status with file system watcher' + +. ./test-lib.sh + +# +# To run the entire git test suite using fsmonitor: +# +# copy t/t7519/fsmonitor-all to a location in your path and then set +# GIT_FSMONITOR_TEST=fsmonitor-all and run your tests. +# + +# Note, after "git reset --hard HEAD" no extensions exist other than 'TREE' +# "git update-index --fsmonitor" can be used to get the extension written +# before testing the results. + +clean_repo () { + git reset --hard HEAD && + git clean -fd +} + +dirty_repo () { + : >untracked && + : >dir1/untracked && + : >dir2/untracked && + echo 1 >modified && + echo 2 >dir1/modified && + echo 3 >dir2/modified && + echo 4 >new && + echo 5 >dir1/new && + echo 6 >dir2/new +} + +write_integration_script () { + write_script .git/hooks/fsmonitor-test<<-\EOF + if test "$#" -ne 2 + then + echo "$0: exactly 2 arguments expected" + exit 2 + fi + if test "$1" != 1 + then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 + fi + printf "untracked\0" + printf "dir1/untracked\0" + printf "dir2/untracked\0" + printf "modified\0" + printf "dir1/modified\0" + printf "dir2/modified\0" + printf "new\0" + printf "dir1/new\0" + printf "dir2/new\0" + EOF +} + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_expect_success 'setup' ' + mkdir -p .git/hooks && + : >tracked && + : >modified && + mkdir dir1 && + : >dir1/tracked && + : >dir1/modified && + mkdir dir2 && + : >dir2/tracked && + : >dir2/modified && + git -c core.fsmonitor= add . && + git -c core.fsmonitor= commit -m initial && + git config core.fsmonitor .git/hooks/fsmonitor-test && + cat >.gitignore <<-\EOF + .gitignore + expect* + actual* + marker* + EOF +' + +# test that the fsmonitor extension is off by default +test_expect_success 'fsmonitor extension is off by default' ' + test-dump-fsmonitor >actual && + grep "^no fsmonitor" actual +' + +# test that "update-index --fsmonitor" adds the fsmonitor extension +test_expect_success 'update-index --fsmonitor" adds the fsmonitor extension' ' + git update-index --fsmonitor && + test-dump-fsmonitor >actual && + grep "^fsmonitor last update" actual +' + +# test that "update-index --no-fsmonitor" removes the fsmonitor extension +test_expect_success 'update-index --no-fsmonitor" removes the fsmonitor extension' ' + git update-index --no-fsmonitor && + test-dump-fsmonitor >actual && + grep "^no fsmonitor" actual +' + +cat >expect <actual && + test_cmp expect actual +' + +cat >expect <actual && + test_cmp expect actual +' + +cat >expect <actual && + test_cmp expect actual +' + +cat >expect <actual && + test_cmp expect actual +' + +cat >expect <actual && + test_cmp expect actual +' + +cat >expect <modified && + echo 2 >dir1/modified && + echo 3 >dir2/modified && + test_must_fail git update-index --refresh --force-write-index && + git ls-files -f >actual && + test_cmp expect actual +' + +# Ensure commands that call refresh_index() to move the index back in time +# properly invalidate the fsmonitor cache +test_expect_success 'refresh_index() invalidates fsmonitor cache' ' + write_script .git/hooks/fsmonitor-test<<-\EOF && + EOF + clean_repo && + dirty_repo && + git add . && + git commit -m "to reset" && + git reset HEAD~1 && + git status >actual && + git -c core.fsmonitor= status >expect && + test_i18ncmp expect actual +' + +# test fsmonitor with and without preloadIndex +preload_values="false true" +for preload_val in $preload_values +do + test_expect_success "setup preloadIndex to $preload_val" ' + git config core.preloadIndex $preload_val && + if test $preload_val = true + then + GIT_FORCE_PRELOAD_TEST=$preload_val; export GIT_FORCE_PRELOAD_TEST + else + unset GIT_FORCE_PRELOAD_TEST + fi + ' + + # test fsmonitor with and without the untracked cache (if available) + uc_values="false" + test_have_prereq UNTRACKED_CACHE && uc_values="false true" + for uc_val in $uc_values + do + test_expect_success "setup untracked cache to $uc_val" ' + git config core.untrackedcache $uc_val + ' + + # Status is well tested elsewhere so we'll just ensure that the results are + # the same when using core.fsmonitor. + test_expect_success 'compare status with and without fsmonitor' ' + write_integration_script && + clean_repo && + dirty_repo && + git add new && + git add dir1/new && + git add dir2/new && + git status >actual && + git -c core.fsmonitor= status >expect && + test_i18ncmp expect actual + ' + + # Make sure it's actually skipping the check for modified and untracked + # (if enabled) files unless it is told about them. + test_expect_success "status doesn't detect unreported modifications" ' + write_script .git/hooks/fsmonitor-test<<-\EOF && + :>marker + EOF + clean_repo && + git status && + test_path_is_file marker && + dirty_repo && + rm -f marker && + git status >actual && + test_path_is_file marker && + test_i18ngrep ! "Changes not staged for commit:" actual && + if test $uc_val = true + then + test_i18ngrep ! "Untracked files:" actual + fi && + if test $uc_val = false + then + test_i18ngrep "Untracked files:" actual + fi && + rm -f marker + ' + done +done + +test_done diff --git a/t/t7519/fsmonitor-all b/t/t7519/fsmonitor-all new file mode 100755 index 0000000000..691bc94dc2 --- /dev/null +++ b/t/t7519/fsmonitor-all @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An test hook script to integrate with git to test fsmonitor. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +#echo "$0 $*" >&2 + +if test "$#" -ne 2 +then + echo "$0: exactly 2 arguments expected" >&2 + exit 2 +fi + +if test "$1" != 1 +then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 +fi + +echo "/" diff --git a/t/t7519/fsmonitor-none b/t/t7519/fsmonitor-none new file mode 100755 index 0000000000..ed9cf5a6a9 --- /dev/null +++ b/t/t7519/fsmonitor-none @@ -0,0 +1,22 @@ +#!/bin/sh +# +# An test hook script to integrate with git to test fsmonitor. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +#echo "$0 $*" >&2 + +if test "$#" -ne 2 +then + echo "$0: exactly 2 arguments expected" >&2 + exit 2 +fi + +if test "$1" != 1 +then + echo "Unsupported core.fsmonitor hook version." >&2 + exit 1 +fi diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman new file mode 100755 index 0000000000..7ceb32dc18 --- /dev/null +++ b/t/t7519/fsmonitor-watchman @@ -0,0 +1,140 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; +#print STDERR "$0 $version $time\n"; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + $time = int $time / 1000000000; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +# Convert unix style paths to escaped Windows style paths when running +# in Windows command prompt + +my $system = `uname -s`; +$system =~ s/[\r\n]+//g; +my $git_work_tree; + +if ($system =~ m/^MSYS_NT/) { + $git_work_tree = `cygpath -aw "\$PWD"`; + $git_work_tree =~ s/[\r\n]+//g; + $git_work_tree =~ s,\\,/,g; +} else { + $git_work_tree = $ENV{'PWD'}; +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + # Set input record separator + local $/ = 0666; + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + # + # The category of transient files that we want to ignore will have a + # creation clock (cclock) newer than $time_t value and will also not + # currently exist. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"], + "expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]] + }] + END + + open (my $fh, ">", ".git/watchman-query.json"); + print $fh $query; + close $fh; + + print CHLD_IN $query; + my $response = ; + + open ($fh, ">", ".git/watchman-response.json"); + print $fh $response; + close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + + open ($fh, ">", ".git/watchman-output.out"); + print "/\0"; + close $fh; + + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + open ($fh, ">", ".git/watchman-output.out"); + print $fh @{$o->{files}}; + close $fh; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} -- cgit v1.2.3