From 94c0956b60969bbbb3ead19638d52591a59d713c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:33 +0000 Subject: sparse-checkout: create builtin with 'list' subcommand The sparse-checkout feature is mostly hidden to users, as its only documentation is supplementary information in the docs for 'git read-tree'. In addition, users need to know how to edit the .git/info/sparse-checkout file with the right patterns, then run the appropriate 'git read-tree -mu HEAD' command. Keeping the working directory in sync with the sparse-checkout file requires care. Begin an effort to make the sparse-checkout feature a porcelain feature by creating a new 'git sparse-checkout' builtin. This builtin will be the preferred mechanism for manipulating the sparse-checkout file and syncing the working directory. The documentation provided is adapted from the "git read-tree" documentation with a few edits for clarity in the new context. Extra sections are added to hint toward a future change to a more restricted pattern set. Helped-by: Elijah Newren Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 t/t1091-sparse-checkout-builtin.sh (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh new file mode 100755 index 0000000000..9b73d44907 --- /dev/null +++ b/t/t1091-sparse-checkout-builtin.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +test_description='sparse checkout builtin tests' + +. ./test-lib.sh + +test_expect_success 'setup' ' + git init repo && + ( + cd repo && + echo "initial" >a && + mkdir folder1 folder2 deep && + mkdir deep/deeper1 deep/deeper2 && + mkdir deep/deeper1/deepest && + cp a folder1 && + cp a folder2 && + cp a deep && + cp a deep/deeper1 && + cp a deep/deeper2 && + cp a deep/deeper1/deepest && + git add . && + git commit -m "initial commit" + ) +' + +test_expect_success 'git sparse-checkout list (empty)' ' + git -C repo sparse-checkout list >list 2>err && + test_must_be_empty list && + test_i18ngrep "this worktree is not sparse (sparse-checkout file may not exist)" err +' + +test_expect_success 'git sparse-checkout list (populated)' ' + test_when_finished rm -f repo/.git/info/sparse-checkout && + cat >repo/.git/info/sparse-checkout <<-EOF && + /folder1/* + /deep/ + **/a + !*bin* + EOF + cp repo/.git/info/sparse-checkout expect && + git -C repo sparse-checkout list >list && + test_cmp expect list +' + +test_done -- cgit v1.2.3 From bab3c3590879c5680060a5d5e2b93b4f97519025 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:34 +0000 Subject: sparse-checkout: create 'init' subcommand Getting started with a sparse-checkout file can be daunting. Help users start their sparse enlistment using 'git sparse-checkout init'. This will set 'core.sparseCheckout=true' in their config, write an initial set of patterns to the sparse-checkout file, and update their working directory. Make sure to use the `extensions.worktreeConfig` setting and write the sparse checkout config to the worktree-specific config file. This avoids confusing interactions with other worktrees. The use of running another process for 'git read-tree' is sub- optimal. This will be removed in a later change. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 9b73d44907..21143c529c 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -42,4 +42,44 @@ test_expect_success 'git sparse-checkout list (populated)' ' test_cmp expect list ' +test_expect_success 'git sparse-checkout init' ' + git -C repo sparse-checkout init && + cat >expect <<-EOF && + /* + !/*/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + test_cmp_config -C repo true core.sparsecheckout && + ls repo >dir && + echo a >expect && + test_cmp expect dir +' + +test_expect_success 'git sparse-checkout list after init' ' + git -C repo sparse-checkout list >actual && + cat >expect <<-EOF && + /* + !/*/ + EOF + test_cmp expect actual +' + +test_expect_success 'init with existing sparse-checkout' ' + echo "*folder*" >> repo/.git/info/sparse-checkout && + git -C repo sparse-checkout init && + cat >expect <<-EOF && + /* + !/*/ + *folder* + EOF + test_cmp expect repo/.git/info/sparse-checkout && + ls repo >dir && + cat >expect <<-EOF && + a + folder1 + folder2 + EOF + test_cmp expect dir +' + test_done -- cgit v1.2.3 From d89f09c8289a764f0a974e02bd5b38cf60d1a7d7 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:35 +0000 Subject: clone: add --sparse mode When someone wants to clone a large repository, but plans to work using a sparse-checkout file, they either need to do a full checkout first and then reduce the patterns they included, or clone with --no-checkout, set up their patterns, and then run a checkout manually. This requires knowing a lot about the repo shape and how sparse-checkout works. Add a new '--sparse' option to 'git clone' that initializes the sparse-checkout file to include the following patterns: /* !/*/ These patterns include every file in the root directory, but no directories. This allows a repo to include files like a README or a bootstrapping script to grow enlistments from that point. During the 'git sparse-checkout init' call, we must first look to see if HEAD is valid, since 'git clone' does not have a valid HEAD at the point where it initializes the sparse-checkout. The following checkout within the clone command will create the HEAD ref and update the working directory correctly. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 21143c529c..78c20cb7e3 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -82,4 +82,17 @@ test_expect_success 'init with existing sparse-checkout' ' test_cmp expect dir ' +test_expect_success 'clone --sparse' ' + git clone --sparse repo clone && + git -C clone sparse-checkout list >actual && + cat >expect <<-EOF && + /* + !/*/ + EOF + test_cmp expect actual && + ls clone >dir && + echo a >expect && + test_cmp expect dir +' + test_done -- cgit v1.2.3 From f6039a9423d042d61fb4cfccb395bd04c4bd5322 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:36 +0000 Subject: sparse-checkout: 'set' subcommand The 'git sparse-checkout set' subcommand takes a list of patterns as arguments and writes them to the sparse-checkout file. Then, it updates the working directory using 'git read-tree -mu HEAD'. The 'set' subcommand will replace the entire contents of the sparse-checkout file. The write_patterns_and_update() method is extracted from cmd_sparse_checkout() to make it easier to implement 'add' and/or 'remove' subcommands in the future. If the core.sparseCheckout config setting is disabled, then enable the config setting in the worktree config. If we set the config this way and the sparse-checkout fails, then re-disable the config setting. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 78c20cb7e3..72d8bc5c25 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -95,4 +95,37 @@ test_expect_success 'clone --sparse' ' test_cmp expect dir ' +test_expect_success 'set enables config' ' + git init empty-config && + ( + cd empty-config && + test_commit test file && + test_path_is_missing .git/config.worktree && + test_must_fail git sparse-checkout set nothing && + test_path_is_file .git/config.worktree && + test_must_fail git config core.sparseCheckout && + git sparse-checkout set "/*" && + test_cmp_config true core.sparseCheckout + ) +' + +test_expect_success 'set sparse-checkout using builtin' ' + git -C repo sparse-checkout set "/*" "!/*/" "*folder*" && + cat >expect <<-EOF && + /* + !/*/ + *folder* + EOF + git -C repo sparse-checkout list >actual && + test_cmp expect actual && + test_cmp expect repo/.git/info/sparse-checkout && + ls repo >dir && + cat >expect <<-EOF && + a + folder1 + folder2 + EOF + test_cmp expect dir +' + test_done -- cgit v1.2.3 From 7bffca95ea1ca4f55663374ea9b929b9df5be04b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:37 +0000 Subject: sparse-checkout: add '--stdin' option to set subcommand The 'git sparse-checkout set' subcommand takes a list of patterns and places them in the sparse-checkout file. Then, it updates the working directory to match those patterns. For a large list of patterns, the command-line call can get very cumbersome. Add a '--stdin' option to instead read patterns over standard in. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 72d8bc5c25..07e73b4674 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -128,4 +128,24 @@ test_expect_success 'set sparse-checkout using builtin' ' test_cmp expect dir ' +test_expect_success 'set sparse-checkout using --stdin' ' + cat >expect <<-EOF && + /* + !/*/ + /folder1/ + /folder2/ + EOF + git -C repo sparse-checkout set --stdin actual && + test_cmp expect actual && + test_cmp expect repo/.git/info/sparse-checkout && + ls repo >dir && + cat >expect <<-EOF && + a + folder1 + folder2 + EOF + test_cmp expect dir +' + test_done -- cgit v1.2.3 From 72918c1ad91504f56c395cc91c5072651125662a Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:38 +0000 Subject: sparse-checkout: create 'disable' subcommand The instructions for disabling a sparse-checkout to a full working directory are complicated and non-intuitive. Add a subcommand, 'git sparse-checkout disable', to perform those steps for the user. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 07e73b4674..c385c62c92 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -148,4 +148,19 @@ test_expect_success 'set sparse-checkout using --stdin' ' test_cmp expect dir ' +test_expect_success 'sparse-checkout disable' ' + git -C repo sparse-checkout disable && + test_path_is_missing repo/.git/info/sparse-checkout && + git -C repo config --list >config && + test_must_fail git config core.sparseCheckout && + ls repo >dir && + cat >expect <<-EOF && + a + deep + folder1 + folder2 + EOF + test_cmp expect dir +' + test_done -- cgit v1.2.3 From 879321eb0bec25779386445d65242452825155be Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:40 +0000 Subject: sparse-checkout: add 'cone' mode The sparse-checkout feature can have quadratic performance as the number of patterns and number of entries in the index grow. If there are 1,000 patterns and 1,000,000 entries, this time can be very significant. Create a new Boolean config option, core.sparseCheckoutCone, to indicate that we expect the sparse-checkout file to contain a more limited set of patterns. This is a separate config setting from core.sparseCheckout to avoid breaking older clients by introducing a tri-state option. The config option does nothing right now, but will be expanded upon in a later commit. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index c385c62c92..0b2715db52 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -148,6 +148,20 @@ test_expect_success 'set sparse-checkout using --stdin' ' test_cmp expect dir ' +test_expect_success 'cone mode: match patterns' ' + git -C repo config --worktree core.sparseCheckoutCone true && + rm -rf repo/a repo/folder1 repo/folder2 && + git -C repo read-tree -mu HEAD && + git -C repo reset --hard && + ls repo >dir && + cat >expect <<-EOF && + a + folder1 + folder2 + EOF + test_cmp expect dir +' + test_expect_success 'sparse-checkout disable' ' git -C repo sparse-checkout disable && test_path_is_missing repo/.git/info/sparse-checkout && -- cgit v1.2.3 From 96cc8ab5318cd57c8bc203b8f064b35883b2386f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:41 +0000 Subject: sparse-checkout: use hashmaps for cone patterns The parent and recursive patterns allowed by the "cone mode" option in sparse-checkout are restrictive enough that we can avoid using the regex parsing. Everything is based on prefix matches, so we can use hashsets to store the prefixes from the sparse-checkout file. When checking a path, we can strip path entries from the path and check the hashset for an exact match. As a test, I created a cone-mode sparse-checkout file for the Linux repository that actually includes every file. This was constructed by taking every folder in the Linux repo and creating the pattern pairs here: /$folder/ !/$folder/*/ This resulted in a sparse-checkout file sith 8,296 patterns. Running 'git read-tree -mu HEAD' on this file had the following performance: core.sparseCheckout=false: 0.21 s (0.00 s) core.sparseCheckout=true: 3.75 s (3.50 s) core.sparseCheckoutCone=true: 0.23 s (0.01 s) The times in parentheses above correspond to the time spent in the first clear_ce_flags() call, according to the trace2 performance traces. While this example is contrived, it demonstrates how these patterns can slow the sparse-checkout feature. Helped-by: Eric Wong Helped-by: Johannes Schindelin Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 0b2715db52..1ed003ac1d 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -151,7 +151,8 @@ test_expect_success 'set sparse-checkout using --stdin' ' test_expect_success 'cone mode: match patterns' ' git -C repo config --worktree core.sparseCheckoutCone true && rm -rf repo/a repo/folder1 repo/folder2 && - git -C repo read-tree -mu HEAD && + git -C repo read-tree -mu HEAD 2>err && + test_i18ngrep ! "disabling cone patterns" err && git -C repo reset --hard && ls repo >dir && cat >expect <<-EOF && @@ -162,6 +163,14 @@ test_expect_success 'cone mode: match patterns' ' test_cmp expect dir ' +test_expect_success 'cone mode: warn on bad pattern' ' + test_when_finished mv sparse-checkout repo/.git/info/ && + cp repo/.git/info/sparse-checkout . && + echo "!/deep/deeper/*" >>repo/.git/info/sparse-checkout && + git -C repo read-tree -mu HEAD 2>err && + test_i18ngrep "unrecognized negative pattern" err +' + test_expect_success 'sparse-checkout disable' ' git -C repo sparse-checkout disable && test_path_is_missing repo/.git/info/sparse-checkout && -- cgit v1.2.3 From af09ce24a9c79f6efc12d1d8f1052e1d1dbe5016 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:42 +0000 Subject: sparse-checkout: init and set in cone mode To make the cone pattern set easy to use, update the behavior of 'git sparse-checkout (init|set)'. Add '--cone' flag to 'git sparse-checkout init' to set the config option 'core.sparseCheckoutCone=true'. When running 'git sparse-checkout set' in cone mode, a user only needs to supply a list of recursive folder matches. Git will automatically add the necessary parent matches for the leading directories. When testing 'git sparse-checkout set' in cone mode, check the error stream to ensure we do not see any errors. Specifically, we want to avoid the warning that the patterns do not match the cone-mode patterns. Helped-by: Eric Wong Helped-by: Johannes Schindelin Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 1ed003ac1d..fbd46c3f61 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -186,4 +186,55 @@ test_expect_success 'sparse-checkout disable' ' test_cmp expect dir ' +test_expect_success 'cone mode: init and set' ' + git -C repo sparse-checkout init --cone && + git -C repo config --list >config && + test_i18ngrep "core.sparsecheckoutcone=true" config && + ls repo >dir && + echo a >expect && + test_cmp expect dir && + git -C repo sparse-checkout set deep/deeper1/deepest/ 2>err && + test_must_be_empty err && + ls repo >dir && + cat >expect <<-EOF && + a + deep + EOF + test_cmp expect dir && + ls repo/deep >dir && + cat >expect <<-EOF && + a + deeper1 + EOF + test_cmp expect dir && + ls repo/deep/deeper1 >dir && + cat >expect <<-EOF && + a + deepest + EOF + test_cmp expect dir && + cat >expect <<-EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + !/deep/deeper1/*/ + /deep/deeper1/deepest/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + git -C repo sparse-checkout set --stdin 2>err <<-EOF && + folder1 + folder2 + EOF + test_must_be_empty err && + cat >expect <<-EOF && + a + folder1 + folder2 + EOF + ls repo >dir && + test_cmp expect dir +' + test_done -- cgit v1.2.3 From e9de487aa36aa75b5c9068c6bd07cfb8bf2ee955 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:45 +0000 Subject: sparse-checkout: sanitize for nested folders If a user provides folders A/ and A/B/ for inclusion in a cone-mode sparse-checkout file, the parsing logic will notice that A/ appears both as a "parent" type pattern and as a "recursive" type pattern. This is unexpected and hence will complain via a warning and revert to the old logic for checking sparse-checkout patterns. Prevent this from happening accidentally by sanitizing the folders for this type of inclusion in the 'git sparse-checkout' builtin. This happens in two ways: 1. Do not include any parent patterns that also appear as recursive patterns. 2. Do not include any recursive patterns deeper than other recursive patterns. In order to minimize duplicate code for scanning parents, create hashmap_contains_parent() method. It takes a strbuf buffer to avoid reallocating a buffer when calling in a tight loop. Helped-by: Eric Wong Helped-by: Johannes Schindelin Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index fbd46c3f61..b88d08da98 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -237,4 +237,15 @@ test_expect_success 'cone mode: init and set' ' test_cmp expect dir ' +test_expect_success 'cone mode: set with nested folders' ' + git -C repo sparse-checkout set deep deep/deeper1/deepest 2>err && + test_line_count = 0 err && + cat >expect <<-EOF && + /* + !/*/ + /deep/ + EOF + test_cmp repo/.git/info/sparse-checkout expect +' + test_done -- cgit v1.2.3 From e091228e17e88b1bc16cb50d5c3aff10dc5119d1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:46 +0000 Subject: sparse-checkout: update working directory in-process The sparse-checkout builtin used 'git read-tree -mu HEAD' to update the skip-worktree bits in the index and to update the working directory. This extra process is overly complex, and prone to failure. It also requires that we write our changes to the sparse-checkout file before trying to update the index. Remove this extra process call by creating a direct call to unpack_trees() in the same way 'git read-tree -mu HEAD' does. In addition, provide an in-memory list of patterns so we can avoid reading from the sparse-checkout file. This allows us to test a proposed change to the file before writing to it. An earlier version of this patch included a bug when the 'set' command failed due to the "Sparse checkout leaves no entry on working directory" error. It would not rollback the index.lock file, so the replay of the old sparse-checkout specification would fail. A test in t1091 now covers that scenario. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index b88d08da98..53aeb5980f 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -248,4 +248,32 @@ test_expect_success 'cone mode: set with nested folders' ' test_cmp repo/.git/info/sparse-checkout expect ' +test_expect_success 'revert to old sparse-checkout on bad update' ' + echo update >repo/deep/deeper2/a && + cp repo/.git/info/sparse-checkout expect && + test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && + test_i18ngrep "Cannot update sparse checkout" err && + test_cmp repo/.git/info/sparse-checkout expect && + ls repo/deep >dir && + cat >expect <<-EOF && + a + deeper1 + deeper2 + EOF + test_cmp dir expect +' + +test_expect_success 'revert to old sparse-checkout on empty update' ' + git init empty-test && + ( + echo >file && + git add file && + git commit -m "test" && + test_must_fail git sparse-checkout set nothing 2>err && + test_i18ngrep "Sparse checkout leaves no entry on working directory" err && + test_i18ngrep ! ".git/index.lock" err && + git sparse-checkout set file + ) +' + test_done -- cgit v1.2.3 From 99dfa6f9702ee81c44ef9382933e4e391ec5d6ee Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:47 +0000 Subject: sparse-checkout: use in-process update for disable subcommand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'git sparse-checkout disable' subcommand returns a user to a full working directory. The old process for doing this required updating the sparse-checkout file with the "/*" pattern and then updating the working directory with core.sparseCheckout enabled. Finally, the sparse-checkout file could be removed and the config setting disabled. However, it is valuable to keep a user's sparse-checkout file intact so they can re-enable the sparse-checkout they previously used with 'git sparse-checkout init'. This is now possible with the in-process mechanism for updating the working directory. Reported-by: Szeder Gábor Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 53aeb5980f..b8f18e2a09 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -172,8 +172,9 @@ test_expect_success 'cone mode: warn on bad pattern' ' ' test_expect_success 'sparse-checkout disable' ' + test_when_finished rm -rf repo/.git/info/sparse-checkout && git -C repo sparse-checkout disable && - test_path_is_missing repo/.git/info/sparse-checkout && + test_path_is_file repo/.git/info/sparse-checkout && git -C repo config --list >config && test_must_fail git config core.sparseCheckout && ls repo >dir && -- cgit v1.2.3 From fb10ca5b54362e6f860e1e9049e03924fcf5f05b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:48 +0000 Subject: sparse-checkout: write using lockfile If two 'git sparse-checkout set' subcommands are launched at the same time, the behavior can be unexpected as they compete to write the sparse-checkout file and update the working directory. Take a lockfile around the writes to the sparse-checkout file. In addition, acquire this lock around the working directory update to avoid two commands updating the working directory in different ways. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index b8f18e2a09..f074b7f3be 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -277,4 +277,11 @@ test_expect_success 'revert to old sparse-checkout on empty update' ' ) ' +test_expect_success 'fail when lock is taken' ' + test_when_finished rm -rf repo/.git/info/sparse-checkout.lock && + touch repo/.git/info/sparse-checkout.lock && + test_must_fail git -C repo sparse-checkout set deep 2>err && + test_i18ngrep "File exists" err +' + test_done -- cgit v1.2.3 From f75a69f88099689b510c7a7261d637bc7004a749 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:49 +0000 Subject: sparse-checkout: cone mode should not interact with .gitignore During the development of the sparse-checkout "cone mode" feature, an incorrect placement of the initializer for "use_cone_patterns = 1" caused warnings to show up when a .gitignore file was present with non-cone-mode patterns. This was fixed in the original commit introducing the cone mode, but now we should add a test to avoid hitting this problem again in the future. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index f074b7f3be..e61ddb4ad5 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -284,4 +284,11 @@ test_expect_success 'fail when lock is taken' ' test_i18ngrep "File exists" err ' +test_expect_success '.gitignore should not warn about cone mode' ' + git -C repo config --worktree core.sparseCheckoutCone true && + echo "**/bin/*" >repo/.gitignore && + git -C repo reset --hard 2>err && + test_i18ngrep ! "disabling cone patterns" err +' + test_done -- cgit v1.2.3 From cff4e9138d8df45e3b6199171092ee781cdadaeb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 21 Nov 2019 22:04:51 +0000 Subject: sparse-checkout: check for dirty status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index-merge performed by 'git sparse-checkout' will erase any staged changes, which can lead to data loss. Prevent these attempts by requiring a clean 'git status' output. Helped-by: Szeder Gábor Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e61ddb4ad5..d5e2892526 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -250,10 +250,11 @@ test_expect_success 'cone mode: set with nested folders' ' ' test_expect_success 'revert to old sparse-checkout on bad update' ' + test_when_finished git -C repo reset --hard && echo update >repo/deep/deeper2/a && cp repo/.git/info/sparse-checkout expect && test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && - test_i18ngrep "Cannot update sparse checkout" err && + test_i18ngrep "cannot set sparse-checkout patterns" err && test_cmp repo/.git/info/sparse-checkout expect && ls repo/deep >dir && cat >expect <<-EOF && @@ -291,4 +292,16 @@ test_expect_success '.gitignore should not warn about cone mode' ' test_i18ngrep ! "disabling cone patterns" err ' +test_expect_success 'sparse-checkout (init|set|disable) fails with dirty status' ' + git clone repo dirty && + echo dirty >dirty/folder1/a && + test_must_fail git -C dirty sparse-checkout init && + test_must_fail git -C dirty sparse-checkout set /folder2/* /deep/deeper1/* && + test_must_fail git -C dirty sparse-checkout disable && + git -C dirty reset --hard && + git -C dirty sparse-checkout init && + git -C dirty sparse-checkout set /folder2/* /deep/deeper1/* && + git -C dirty sparse-checkout disable +' + test_done -- cgit v1.2.3 From 190a65f9db8db9d87d54351429f7879fcb4ad608 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 13 Dec 2019 18:09:53 +0000 Subject: sparse-checkout: respect core.ignoreCase in cone mode When a user uses the sparse-checkout feature in cone mode, they add patterns using "git sparse-checkout set ..." or by using "--stdin" to provide the directories line-by-line over stdin. This behaviour naturally looks a lot like the way a user would type "git add ..." If core.ignoreCase is enabled, then "git add" will match the input using a case-insensitive match. Do the same for the sparse-checkout feature. Perform case-insensitive checks while updating the skip-worktree bits during unpack_trees(). This is done by changing the hash algorithm and hashmap comparison methods to optionally use case- insensitive methods. When this is enabled, there is a small performance cost in the hashing algorithm. To tease out the worst possible case, the following was run on a repo with a deep directory structure: git ls-tree -d -r --name-only HEAD | git sparse-checkout set --stdin The 'set' command was timed with core.ignoreCase disabled or enabled. For the repo with a deep history, the numbers were core.ignoreCase=false: 62s core.ignoreCase=true: 74s (+19.3%) For reproducibility, the equivalent test on the Linux kernel repository had these numbers: core.ignoreCase=false: 3.1s core.ignoreCase=true: 3.6s (+16%) Now, this is not an entirely fair comparison, as most users will define their sparse cone using more shallow directories, and the performance improvement from eb42feca97 ("unpack-trees: hash less in cone mode" 2019-11-21) can remove most of the hash cost. For a more realistic test, drop the "-r" from the ls-tree command to store only the first-level directories. In that case, the Linux kernel repository takes 0.2-0.25s in each case, and the deep repository takes one second, plus or minus 0.05s, in each case. Thus, we _can_ demonstrate a cost to this change, but it is unlikely to matter to any reasonable sparse-checkout cone. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index d5e2892526..cee98a1c8a 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -304,4 +304,21 @@ test_expect_success 'sparse-checkout (init|set|disable) fails with dirty status' git -C dirty sparse-checkout disable ' +test_expect_success 'cone mode: set with core.ignoreCase=true' ' + git -C repo sparse-checkout init --cone && + git -C repo -c core.ignoreCase=true sparse-checkout set folder1 && + cat >expect <<-EOF && + /* + !/*/ + /folder1/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + ls repo >dir && + cat >expect <<-EOF && + a + folder1 + EOF + test_cmp expect dir +' + test_done -- cgit v1.2.3 From 761e3d26bbe44c51f83c4f1ad198461f57029ebd Mon Sep 17 00:00:00 2001 From: Ed Maste Date: Fri, 20 Dec 2019 19:41:14 +0000 Subject: sparse-checkout: improve OS ls compatibility On FreeBSD, when executed by root ls enables the '-A' option: -A Include directory entries whose names begin with a dot (`.') except for . and ... Automatically set for the super-user unless -I is specified. As a result the .git directory appeared in the output when run as root. Simulate no-dotfile ls behaviour using a shell glob. Helped-by: Eric Wong Helped-by: Junio C Hamano Helped-by: Eric Sunshine Signed-off-by: Ed Maste Acked-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 't/t1091-sparse-checkout-builtin.sh') diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index cee98a1c8a..6f7e2d0c9e 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -4,6 +4,14 @@ test_description='sparse checkout builtin tests' . ./test-lib.sh +list_files() { + # Do not replace this with 'ls "$1"', as "ls" with BSD-lineage + # enables "-A" by default for root and ends up including ".git" and + # such in its output. (Note, though, that running the test suite as + # root is generally not recommended.) + (cd "$1" && printf '%s\n' *) +} + test_expect_success 'setup' ' git init repo && ( @@ -50,7 +58,7 @@ test_expect_success 'git sparse-checkout init' ' EOF test_cmp expect repo/.git/info/sparse-checkout && test_cmp_config -C repo true core.sparsecheckout && - ls repo >dir && + list_files repo >dir && echo a >expect && test_cmp expect dir ' @@ -73,7 +81,7 @@ test_expect_success 'init with existing sparse-checkout' ' *folder* EOF test_cmp expect repo/.git/info/sparse-checkout && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a folder1 @@ -90,7 +98,7 @@ test_expect_success 'clone --sparse' ' !/*/ EOF test_cmp expect actual && - ls clone >dir && + list_files clone >dir && echo a >expect && test_cmp expect dir ' @@ -119,7 +127,7 @@ test_expect_success 'set sparse-checkout using builtin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a folder1 @@ -139,7 +147,7 @@ test_expect_success 'set sparse-checkout using --stdin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a folder1 @@ -154,7 +162,7 @@ test_expect_success 'cone mode: match patterns' ' git -C repo read-tree -mu HEAD 2>err && test_i18ngrep ! "disabling cone patterns" err && git -C repo reset --hard && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a folder1 @@ -177,7 +185,7 @@ test_expect_success 'sparse-checkout disable' ' test_path_is_file repo/.git/info/sparse-checkout && git -C repo config --list >config && test_must_fail git config core.sparseCheckout && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a deep @@ -191,24 +199,24 @@ test_expect_success 'cone mode: init and set' ' git -C repo sparse-checkout init --cone && git -C repo config --list >config && test_i18ngrep "core.sparsecheckoutcone=true" config && - ls repo >dir && + list_files repo >dir && echo a >expect && test_cmp expect dir && git -C repo sparse-checkout set deep/deeper1/deepest/ 2>err && test_must_be_empty err && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a deep EOF test_cmp expect dir && - ls repo/deep >dir && + list_files repo/deep >dir && cat >expect <<-EOF && a deeper1 EOF test_cmp expect dir && - ls repo/deep/deeper1 >dir && + list_files repo/deep/deeper1 >dir && cat >expect <<-EOF && a deepest @@ -234,7 +242,7 @@ test_expect_success 'cone mode: init and set' ' folder1 folder2 EOF - ls repo >dir && + list_files repo >dir && test_cmp expect dir ' @@ -256,7 +264,7 @@ test_expect_success 'revert to old sparse-checkout on bad update' ' test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && test_i18ngrep "cannot set sparse-checkout patterns" err && test_cmp repo/.git/info/sparse-checkout expect && - ls repo/deep >dir && + list_files repo/deep >dir && cat >expect <<-EOF && a deeper1 @@ -313,7 +321,7 @@ test_expect_success 'cone mode: set with core.ignoreCase=true' ' /folder1/ EOF test_cmp expect repo/.git/info/sparse-checkout && - ls repo >dir && + list_files repo >dir && cat >expect <<-EOF && a folder1 -- cgit v1.2.3