fsmonitor: add a performance test

Add a test utility (test-drop-caches) that flushes all changes to disk
then drops file system cache on Windows, Linux, and OSX.

Add a perf test (p7519-fsmonitor.sh) for fsmonitor.

By default, the performance test will utilize the Watchman file system
monitor if it is installed.  If Watchman is not installed, it will use a
dummy integration script that does not report any new or modified files.
The dummy script has very little overhead which provides optimistic results.

The performance test will also use the untracked cache feature if it is
available as fsmonitor uses it to speed up scanning for untracked files.

There are 4 environment variables that can be used to alter the default
behavior of the performance test:

GIT_PERF_7519_UNTRACKED_CACHE: used to configure core.untrackedCache
GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex
GIT_PERF_7519_FSMONITOR: used to configure core.fsmonitor
GIT_PERF_7519_DROP_CACHE: if set, the OS caches are dropped between tests

The big win for using fsmonitor is the elimination of the need to scan the
working directory looking for changed and untracked files. If the file
information is all cached in RAM, the benefits are reduced.

Signed-off-by: Ben Peart <benpeart@microsoft.com>
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

2 files changed, 165 insertions, 0 deletions

diff --git a/t/helper/.gitignore b/t/helper/.gitignore
index f97400b8cc..87a648a7cf 100644
--- a/t/helper/.gitignore
+++ b/t/helper/.gitignore
@@ -3,6 +3,7 @@
 /test-config
 /test-date
 /test-delta
+/test-drop-caches
 /test-dump-cache-tree
 /test-dump-fsmonitor
 /test-dump-split-index
diff --git a/t/helper/test-drop-caches.c b/t/helper/test-drop-caches.c
new file mode 100644
index 0000000000..bd1a857d52
--- /dev/null
+++ b/t/helper/test-drop-caches.c
@@ -0,0 +1,164 @@
+#include "git-compat-util.h"
+
+#if defined(GIT_WINDOWS_NATIVE)
+
+static int cmd_sync(void)
+{
+	char Buffer[MAX_PATH];
+	DWORD dwRet;
+	char szVolumeAccessPath[] = "\\\\.\\X:";
+	HANDLE hVolWrite;
+	int success = 0;
+
+	dwRet = GetCurrentDirectory(MAX_PATH, Buffer);
+	if ((0 == dwRet) || (dwRet > MAX_PATH))
+		return error("Error getting current directory");
+
+	if ((Buffer[0] < 'A') || (Buffer[0] > 'Z'))
+		return error("Invalid drive letter '%c'", Buffer[0]);
+
+	szVolumeAccessPath[4] = Buffer[0];
+	hVolWrite = CreateFile(szVolumeAccessPath, GENERIC_READ | GENERIC_WRITE,
+		FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+	if (INVALID_HANDLE_VALUE == hVolWrite)
+		return error("Unable to open volume for writing, need admin access");
+
+	success = FlushFileBuffers(hVolWrite);
+	if (!success)
+		error("Unable to flush volume");
+
+	CloseHandle(hVolWrite);
+
+	return !success;
+}
+
+#define STATUS_SUCCESS			(0x00000000L)
+#define STATUS_PRIVILEGE_NOT_HELD	(0xC0000061L)
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+	SystemMemoryListInformation = 80,
+} SYSTEM_INFORMATION_CLASS;
+
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND {
+	MemoryCaptureAccessedBits,
+	MemoryCaptureAndResetAccessedBits,
+	MemoryEmptyWorkingSets,
+	MemoryFlushModifiedList,
+	MemoryPurgeStandbyList,
+	MemoryPurgeLowPriorityStandbyList,
+	MemoryCommandMax
+} SYSTEM_MEMORY_LIST_COMMAND;
+
+static BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
+{
+	BOOL bResult;
+	DWORD dwBufferLength;
+	LUID luid;
+	TOKEN_PRIVILEGES tpPreviousState;
+	TOKEN_PRIVILEGES tpNewState;
+
+	dwBufferLength = 16;
+	bResult = LookupPrivilegeValueA(0, lpName, &luid);
+	if (bResult) {
+		tpNewState.PrivilegeCount = 1;
+		tpNewState.Privileges[0].Luid = luid;
+		tpNewState.Privileges[0].Attributes = 0;
+		bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState,
+			(DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState),
+			&tpPreviousState, &dwBufferLength);
+		if (bResult) {
+			tpPreviousState.PrivilegeCount = 1;
+			tpPreviousState.Privileges[0].Luid = luid;
+			tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0;
+			bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState,
+				dwBufferLength, 0, 0);
+		}
+	}
+	return bResult;
+}
+
+static int cmd_dropcaches(void)
+{
+	HANDLE hProcess = GetCurrentProcess();
+	HANDLE hToken;
+	HMODULE ntdll;
+	DWORD(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG);
+	SYSTEM_MEMORY_LIST_COMMAND command;
+	int status;
+
+	if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
+		return error("Can't open current process token");
+
+	if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
+		return error("Can't get SeProfileSingleProcessPrivilege");
+
+	CloseHandle(hToken);
+
+	ntdll = LoadLibrary("ntdll.dll");
+	if (!ntdll)
+		return error("Can't load ntdll.dll, wrong Windows version?");
+
+	NtSetSystemInformation =
+		(DWORD(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation");
+	if (!NtSetSystemInformation)
+		return error("Can't get function addresses, wrong Windows version?");
+
+	command = MemoryPurgeStandbyList;
+	status = NtSetSystemInformation(
+		SystemMemoryListInformation,
+		&command,
+		sizeof(SYSTEM_MEMORY_LIST_COMMAND)
+	);
+	if (status == STATUS_PRIVILEGE_NOT_HELD)
+		error("Insufficient privileges to purge the standby list, need admin access");
+	else if (status != STATUS_SUCCESS)
+		error("Unable to execute the memory list command %d", status);
+
+	FreeLibrary(ntdll);
+
+	return status;
+}
+
+#elif defined(__linux__)
+
+static int cmd_sync(void)
+{
+	return system("sync");
+}
+
+static int cmd_dropcaches(void)
+{
+	return system("echo 3 | sudo tee /proc/sys/vm/drop_caches");
+}
+
+#elif defined(__APPLE__)
+
+static int cmd_sync(void)
+{
+	return system("sync");
+}
+
+static int cmd_dropcaches(void)
+{
+	return system("sudo purge");
+}
+
+#else
+
+static int cmd_sync(void)
+{
+	return 0;
+}
+
+static int cmd_dropcaches(void)
+{
+	return error("drop caches not implemented on this platform");
+}
+
+#endif
+
+int cmd_main(int argc, const char **argv)
+{
+	cmd_sync();
+	return cmd_dropcaches();
+}