Merge branch 'nd/shallow-clone'

Fetching from a shallow-cloned repository used to be forbidden,
primarily because the codepaths involved were not carefully vetted
and we did not bother supporting such usage. This attempts to allow
object transfer out of a shallow-cloned repository in a controlled
way (i.e. the receiver become a shallow repository with truncated
history).

* nd/shallow-clone: (31 commits)
  t5537: fix incorrect expectation in test case 10
  shallow: remove unused code
  send-pack.c: mark a file-local function static
  git-clone.txt: remove shallow clone limitations
  prune: clean .git/shallow after pruning objects
  clone: use git protocol for cloning shallow repo locally
  send-pack: support pushing from a shallow clone via http
  receive-pack: support pushing to a shallow clone via http
  smart-http: support shallow fetch/clone
  remote-curl: pass ref SHA-1 to fetch-pack as well
  send-pack: support pushing to a shallow clone
  receive-pack: allow pushes that update .git/shallow
  connected.c: add new variant that runs with --shallow-file
  add GIT_SHALLOW_FILE to propagate --shallow-file to subprocesses
  receive/send-pack: support pushing from a shallow clone
  receive-pack: reorder some code in unpack()
  fetch: add --update-shallow to accept refs that update .git/shallow
  upload-pack: make sure deepening preserves shallow roots
  fetch: support fetching from a shallow repository
  clone: support remote shallow repository
  ...

4 files changed, 404 insertions, 0 deletions

diff --git a/t/t5304-prune.sh b/t/t5304-prune.sh
index e4bb3a1457..66c9a41739 100755
--- a/t/t5304-prune.sh
+++ b/t/t5304-prune.sh
@@ -221,4 +221,14 @@ EOF
 	test_cmp expected actual
 '
 
+test_expect_success 'prune .git/shallow' '
+	SHA1=`echo hi|git commit-tree HEAD^{tree}` &&
+	echo $SHA1 >.git/shallow &&
+	git prune --dry-run >out &&
+	grep $SHA1 .git/shallow &&
+	grep $SHA1 out &&
+	git prune &&
+	! test -f .git/shallow
+'
+
 test_done
diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh
new file mode 100755
index 0000000000..b0fa7387cb
--- /dev/null
+++ b/t/t5537-fetch-shallow.sh
@@ -0,0 +1,204 @@
+#!/bin/sh
+
+test_description='fetch/clone from a shallow clone'
+
+. ./test-lib.sh
+
+commit() {
+	echo "$1" >tracked &&
+	git add tracked &&
+	git commit -m "$1"
+}
+
+test_expect_success 'setup' '
+	commit 1 &&
+	commit 2 &&
+	commit 3 &&
+	commit 4 &&
+	git config --global transfer.fsckObjects true
+'
+
+test_expect_success 'setup shallow clone' '
+	git clone --no-local --depth=2 .git shallow &&
+	git --git-dir=shallow/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+4
+3
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'clone from shallow clone' '
+	git clone --no-local shallow shallow2 &&
+	(
+	cd shallow2 &&
+	git fsck &&
+	git log --format=%s >actual &&
+	cat <<EOF >expect &&
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'fetch from shallow clone' '
+	(
+	cd shallow &&
+	commit 5
+	) &&
+	(
+	cd shallow2 &&
+	git fetch &&
+	git fsck &&
+	git log --format=%s origin/master >actual &&
+	cat <<EOF >expect &&
+5
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'fetch --depth from shallow clone' '
+	(
+	cd shallow &&
+	commit 6
+	) &&
+	(
+	cd shallow2 &&
+	git fetch --depth=2 &&
+	git fsck &&
+	git log --format=%s origin/master >actual &&
+	cat <<EOF >expect &&
+6
+5
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'fetch --unshallow from shallow clone' '
+	(
+	cd shallow2 &&
+	git fetch --unshallow &&
+	git fsck &&
+	git log --format=%s origin/master >actual &&
+	cat <<EOF >expect &&
+6
+5
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'fetch something upstream has but hidden by clients shallow boundaries' '
+	# the blob "1" is available in .git but hidden by the
+	# shallow2/.git/shallow and it should be resent
+	! git --git-dir=shallow2/.git cat-file blob `echo 1|git hash-object --stdin` >/dev/null &&
+	echo 1 >1.t &&
+	git add 1.t &&
+	git commit -m add-1-back &&
+	(
+	cd shallow2 &&
+	git fetch ../.git +refs/heads/master:refs/remotes/top/master &&
+	git fsck &&
+	git log --format=%s top/master >actual &&
+	cat <<EOF >expect &&
+add-1-back
+4
+3
+EOF
+	test_cmp expect actual
+	) &&
+	git --git-dir=shallow2/.git cat-file blob `echo 1|git hash-object --stdin` >/dev/null
+
+'
+
+test_expect_success 'fetch that requires changes in .git/shallow is filtered' '
+	(
+	cd shallow &&
+	git checkout --orphan no-shallow &&
+	commit no-shallow
+	) &&
+	git init notshallow &&
+	(
+	cd notshallow &&
+	git fetch ../shallow/.git refs/heads/*:refs/remotes/shallow/*&&
+	git for-each-ref --format="%(refname)" >actual.refs &&
+	cat <<EOF >expect.refs &&
+refs/remotes/shallow/no-shallow
+EOF
+	test_cmp expect.refs actual.refs &&
+	git log --format=%s shallow/no-shallow >actual &&
+	cat <<EOF >expect &&
+no-shallow
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'fetch --update-shallow' '
+	(
+	cd shallow &&
+	git checkout master &&
+	commit 7 &&
+	git tag -m foo heavy-tag HEAD^ &&
+	git tag light-tag HEAD^:tracked
+	) &&
+	(
+	cd notshallow &&
+	git fetch --update-shallow ../shallow/.git refs/heads/*:refs/remotes/shallow/* &&
+	git fsck &&
+	git for-each-ref --sort=refname --format="%(refname)" >actual.refs &&
+	cat <<EOF >expect.refs &&
+refs/remotes/shallow/master
+refs/remotes/shallow/no-shallow
+refs/tags/heavy-tag
+refs/tags/light-tag
+EOF
+	test_cmp expect.refs actual.refs &&
+	git log --format=%s shallow/master >actual &&
+	cat <<EOF >expect &&
+7
+6
+5
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+if test -n "$NO_CURL" -o -z "$GIT_TEST_HTTPD"; then
+	say 'skipping remaining tests, git built without http support'
+	test_done
+fi
+
+LIB_HTTPD_PORT=${LIB_HTTPD_PORT-'5537'}
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+test_expect_success 'clone http repository' '
+	git clone --bare --no-local shallow "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	git clone $HTTPD_URL/smart/repo.git clone &&
+	(
+	cd clone &&
+	git fsck &&
+	git log --format=%s origin/master >actual &&
+	cat <<EOF >expect &&
+7
+6
+5
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+stop_httpd
+test_done
diff --git a/t/t5538-push-shallow.sh b/t/t5538-push-shallow.sh
new file mode 100755
index 0000000000..0a6e40f144
--- /dev/null
+++ b/t/t5538-push-shallow.sh
@@ -0,0 +1,183 @@
+#!/bin/sh
+
+test_description='push from/to a shallow clone'
+
+. ./test-lib.sh
+
+commit() {
+	echo "$1" >tracked &&
+	git add tracked &&
+	git commit -m "$1"
+}
+
+test_expect_success 'setup' '
+	git config --global transfer.fsckObjects true &&
+	commit 1 &&
+	commit 2 &&
+	commit 3 &&
+	commit 4 &&
+	git clone . full &&
+	(
+	git init full-abc &&
+	cd full-abc &&
+	commit a &&
+	commit b &&
+	commit c
+	) &&
+	git clone --no-local --depth=2 .git shallow &&
+	git --git-dir=shallow/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+4
+3
+EOF
+	test_cmp expect actual &&
+	git clone --no-local --depth=2 full-abc/.git shallow2 &&
+	git --git-dir=shallow2/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone' '
+	(
+	cd shallow &&
+	commit 5 &&
+	git push ../.git +master:refs/remotes/shallow/master
+	) &&
+	git log --format=%s shallow/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+5
+4
+3
+2
+1
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone, with grafted roots' '
+	(
+	cd shallow2 &&
+	test_must_fail git push ../.git +master:refs/remotes/shallow2/master 2>err &&
+	grep "shallow2/master.*shallow update not allowed" err
+	) &&
+	test_must_fail git rev-parse shallow2/master &&
+	git fsck
+'
+
+test_expect_success 'add new shallow root with receive.updateshallow on' '
+	test_config receive.shallowupdate true &&
+	(
+	cd shallow2 &&
+	git push ../.git +master:refs/remotes/shallow2/master
+	) &&
+	git log --format=%s shallow2/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow to shallow' '
+	(
+	cd shallow &&
+	git --git-dir=../shallow2/.git config receive.shallowupdate true &&
+	git push ../shallow2/.git +master:refs/remotes/shallow/master &&
+	git --git-dir=../shallow2/.git config receive.shallowupdate false
+	) &&
+	(
+	cd shallow2 &&
+	git log --format=%s shallow/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+5
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'push from full to shallow' '
+	! git --git-dir=shallow2/.git cat-file blob `echo 1|git hash-object --stdin` &&
+	commit 1 &&
+	git push shallow2/.git +master:refs/remotes/top/master &&
+	(
+	cd shallow2 &&
+	git log --format=%s top/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+1
+4
+3
+EOF
+	test_cmp expect actual &&
+	git cat-file blob `echo 1|git hash-object --stdin` >/dev/null
+	)
+'
+
+if test -n "$NO_CURL" -o -z "$GIT_TEST_HTTPD"; then
+	say 'skipping remaining tests, git built without http support'
+	test_done
+fi
+
+LIB_HTTPD_PORT=${LIB_HTTPD_PORT-'5537'}
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+test_expect_success 'push to shallow repo via http' '
+	git clone --bare --no-local shallow "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	(
+	cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	git config http.receivepack true
+	) &&
+	(
+	cd full &&
+	commit 9 &&
+	git push $HTTPD_URL/smart/repo.git +master:refs/remotes/top/master
+	) &&
+	(
+	cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	git fsck &&
+	git log --format=%s top/master >actual &&
+	cat <<EOF >expect &&
+9
+4
+3
+EOF
+	test_cmp expect actual
+	)
+'
+
+test_expect_success 'push from shallow repo via http' '
+	mv "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" shallow-upstream.git &&
+	git clone --bare --no-local full "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	(
+	cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	git config http.receivepack true
+	) &&
+	commit 10 &&
+	git push $HTTPD_URL/smart/repo.git +master:refs/remotes/top/master &&
+	(
+	cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	git fsck &&
+	git log --format=%s top/master >actual &&
+	cat <<EOF >expect &&
+10
+1
+4
+3
+2
+1
+EOF
+	test_cmp expect actual
+	)
+'
+
+stop_httpd
+test_done
diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh
index 62fbd7e664..5e67035be8 100755
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@ -430,4 +430,11 @@ test_expect_success 'clone from a repository with two identical branches' '
 
 '
 
+test_expect_success 'shallow clone locally' '
+	git clone --depth=1 --no-local src ssrrcc &&
+	git clone ssrrcc ddsstt &&
+	test_cmp ssrrcc/.git/shallow ddsstt/.git/shallow &&
+	( cd ddsstt && git fsck )
+'
+
 test_done