From 1f05d07c456e23c0827efbbb3e738afc9f3152e7 Mon Sep 17 00:00:00 2001 From: David Barr Date: Wed, 17 Nov 2010 23:03:51 -0600 Subject: vcs-svn: Allow simple v3 dumps (no deltas yet) Since the dumpfile version 1 days, the Subversion dump format gained some new fields: - a unique identifier for the repository (version 2 format) - whether the text and properties for a node should be interpreted as deltas - checksums for a delta's preimage - SHA-1 sums as alternatives to the existing MD5 checksums for copy source and the payload (delta). For now what is relevant to us is the Text-delta and Prop-delta fields, since not noticing these causes a dump file to be misinterpreted (see the previous commit). [jn: with tests] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 350 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 347 insertions(+), 3 deletions(-) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index e9e46ea0fe..be5372ab3b 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -9,6 +9,30 @@ reinit_git () { git init } +properties () { + while test "$#" -ne 0 + do + property="$1" && + value="$2" && + printf "%s\n" "K ${#property}" && + printf "%s\n" "$property" && + printf "%s\n" "V ${#value}" && + printf "%s\n" "$value" && + shift 2 || + return 1 + done +} + +text_no_props () { + text="$1 +" && + printf "%s\n" "Prop-content-length: 10" && + printf "%s\n" "Text-content-length: ${#text}" && + printf "%s\n" "Content-length: $((${#text} + 10))" && + printf "%s\n" "" "PROPS-END" && + printf "%s\n" "$text" +} + >empty test_expect_success 'empty dump' ' @@ -18,13 +42,333 @@ test_expect_success 'empty dump' ' git fast-import input && - test_must_fail test-svn-fe input >stream && + echo "SVN-fs-dump-format-version: 4" >v4.dump && + test_must_fail test-svn-fe v4.dump >stream && test_cmp empty stream ' +test_expect_failure 'empty revision' ' + reinit_git && + printf "rev : %s\n" "" "" >expect && + cat >emptyrev.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 0 + Content-length: 0 + + Revision-number: 2 + Prop-content-length: 0 + Content-length: 0 + + EOF + test-svn-fe emptyrev.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'empty properties' ' + reinit_git && + printf "rev : %s\n" "" "" >expect && + cat >emptyprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test-svn-fe emptyprop.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'author name and commit message' ' + reinit_git && + echo "" >expect.author && + cat >message <<-\EOF && + A concise summary of the change + + A detailed description of the change, why it is needed, what + was broken and why applying this is the best course of action. + + * file.c + Details pertaining to an individual file. + EOF + { + properties \ + svn:author author@example.com \ + svn:log "$(cat message)" && + echo PROPS-END + } >props && + { + echo "SVN-fs-dump-format-version: 3" && + echo && + echo "Revision-number: 1" && + echo Prop-content-length: $(wc -c log.dump && + test-svn-fe log.dump >stream && + git fast-import actual.log && + git log --format="<%an, %ae>" >actual.author && + test_cmp message actual.log && + test_cmp expect.author actual.author +' + +test_expect_success 'unsupported properties are ignored' ' + reinit_git && + echo author >expect && + cat >extraprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 56 + Content-length: 56 + + K 8 + nonsense + V 1 + y + K 10 + svn:author + V 6 + author + PROPS-END + EOF + test-svn-fe extraprop.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_failure 'timestamp and empty file' ' + echo author@example.com >expect.author && + echo 1999-01-01 >expect.date && + echo file >expect.files && + reinit_git && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-01T00:01:002.000000Z" \ + svn:log "add empty file" && + echo PROPS-END + } >props && + { + cat <<-EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + EOF + echo Prop-content-length: $(wc -c emptyfile.dump && + test-svn-fe emptyfile.dump >stream && + git fast-import actual.author && + git log --date=short --format=%ad HEAD >actual.date && + git ls-tree -r --name-only HEAD >actual.files && + test_cmp expect.author actual.author && + test_cmp expect.date actual.date && + test_cmp expect.files actual.files && + git checkout HEAD empty-file && + test_cmp empty file +' + +test_expect_success 'directory with files' ' + reinit_git && + printf "%s\n" directory/file1 directory/file2 >expect.files && + echo hi >hi && + echo hello >hello && + { + properties \ + svn:author author@example.com \ + svn:date "1999-02-01T00:01:002.000000Z" \ + svn:log "add directory with some files in it" && + echo PROPS-END + } >props && + { + cat <<-EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + EOF + echo Prop-content-length: $(wc -c directory.dump && + test-svn-fe directory.dump >stream && + git fast-import actual.files && + git checkout HEAD directory && + test_cmp expect.files actual.files && + test_cmp hello directory/file1 && + test_cmp hi directory/file2 +' + +test_expect_success 'deltas not supported' ' + { + # (old) h + (inline) ello + (old) \n + printf "SVNQ%b%b%s" "Q\003\006\005\004" "\001Q\0204\001\002" "ello" | + q_to_nul + } >delta && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-05T00:01:002.000000Z" \ + svn:log "add greeting" && + echo PROPS-END + } >props && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-06T00:01:002.000000Z" \ + svn:log "change it" && + echo PROPS-END + } >props2 && + { + echo SVN-fs-dump-format-version: 3 && + echo && + echo Revision-number: 1 && + echo Prop-content-length: $(wc -c delta.dump && + test_must_fail test-svn-fe delta.dump +' + +test_expect_success 'property deltas not supported' ' + { + properties \ + svn:author author@example.com \ + svn:date "1999-03-06T00:01:002.000000Z" \ + svn:log "make an executable, or chmod -x it" && + echo PROPS-END + } >revprops && + { + echo SVN-fs-dump-format-version: 3 && + echo && + echo Revision-number: 1 && + echo Prop-content-length: $(wc -c propdelta.dump && + test_must_fail test-svn-fe propdelta.dump +' + test_expect_success 't9135/svn.dump' ' svnadmin create simple-svn && svnadmin load simple-svn <"$TEST_DIRECTORY/t9135/svn.dump" && -- cgit v1.2.3 From 1d13e9f600986b7ced8db37a9a9c4967ee7ff9d5 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:46:22 -0600 Subject: vcs-svn: Eliminate node_ctx.srcRev global The srcRev variable is only used in handle_node(); its purpose is to hold the old mode for a path, to only be used if properties are not being changed. Narrow its scope to make its meaningful lifetime more obvious. No functional change intended. Add some tests as a sanity-check for the simplest case (no renames). Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index be5372ab3b..729e73dddf 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -252,6 +252,164 @@ test_expect_success 'directory with files' ' test_cmp hi directory/file2 ' +test_expect_failure 'change file mode but keep old content' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T greeting + OBJID + :100644 120000 OBJID OBJID T greeting + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + echo "link hello" >expect.blob && + echo hello >hello && + cat >filemode.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 33 + Content-length: 33 + + K 11 + svn:special + V 1 + * + PROPS-END + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test-svn-fe filemode.dump >stream && + git fast-import actual && + git show HEAD:greeting >actual.blob && + git show HEAD^:greeting >actual.target && + test_cmp expect actual && + test_cmp expect.blob actual.blob && + test_cmp hello actual.target +' + +test_expect_success 'change file mode and reiterate content' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T greeting + OBJID + :100644 120000 OBJID OBJID T greeting + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + echo "link hello" >expect.blob && + echo hello >hello && + cat >filemode.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 33 + Text-content-length: 11 + Content-length: 44 + + K 11 + svn:special + V 1 + * + PROPS-END + link hello + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + EOF + test-svn-fe filemode.dump >stream && + git fast-import actual && + git show HEAD:greeting >actual.blob && + git show HEAD^:greeting >actual.target && + test_cmp expect actual && + test_cmp expect.blob actual.blob && + test_cmp hello actual.target +' + test_expect_success 'deltas not supported' ' { # (old) h + (inline) ello + (old) \n -- cgit v1.2.3 From 414e569e453a49171b1f3db613f88378324104e8 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:52:59 -0600 Subject: vcs-svn: Reject path nodes without Node-action It would be better to flag such errors and let the import proceed anyway, but for now it is simpler not to worry about recovery from such weird cases. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index 729e73dddf..cb9a236245 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -252,6 +252,26 @@ test_expect_success 'directory with files' ' test_cmp hi directory/file2 ' +test_expect_success 'node without action' ' + cat >inaction.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: directory + Node-kind: dir + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test_must_fail test-svn-fe inaction.dump +' + test_expect_failure 'change file mode but keep old content' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.3 From c7dbf35e91cffbc326078d0c0470662f6422150d Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:53:34 -0600 Subject: vcs-svn: More dump format sanity checks Node-action: change is not appropriate when switching between file and directory or adding a new file. Current svn-fe silently accepts such nodes and the resulting tree has missing files in the "changed when meant to add" case. Node-action: add requires some content (text or directory); there is no such thing as an "intent to add" node in svn dumps. Current svn-fe accepts such contentless adds but produces an invalid fast-import stream that refers to nonexistent mark :0 in response. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index cb9a236245..f1e8799bbd 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -272,6 +272,27 @@ test_expect_success 'node without action' ' test_must_fail test-svn-fe inaction.dump ' +test_expect_success 'action: add node without text' ' + cat >textless.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: textless + Node-kind: file + Node-action: add + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test_must_fail test-svn-fe textless.dump +' + test_expect_failure 'change file mode but keep old content' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.3 From 6b01b67658e2905b550739f1aee56a00911ca13c Mon Sep 17 00:00:00 2001 From: David Barr Date: Fri, 19 Nov 2010 18:57:46 -0600 Subject: vcs-svn: Implement Prop-delta handling The rules for what file is used as delta source for each file are not documented in dump-load-format.txt. Luckily, the Apache Software Foundation repository has rich enough examples to figure out most of the rules: Node-action: replace implies the empty property set and empty text as preimage for deltas. Otherwise, if a copyfrom source is given, that node is the preimage for deltas. Lastly, if none of the above applies and the node path exists in the current revision, then that version forms the basis. [jn: refactored, with tests] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 2 deletions(-) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index f1e8799bbd..7dc06707a1 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -514,7 +514,12 @@ test_expect_success 'deltas not supported' ' test_must_fail test-svn-fe delta.dump ' -test_expect_success 'property deltas not supported' ' +test_expect_success 'property deltas supported' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :100755 100644 OBJID OBJID M script.sh + EOF { properties \ svn:author author@example.com \ @@ -565,7 +570,100 @@ test_expect_success 'property deltas not supported' ' PROPS-END EOF } >propdelta.dump && - test_must_fail test-svn-fe propdelta.dump + test-svn-fe propdelta.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'deltas for typechange' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T test-file + OBJID + :100755 120000 OBJID OBJID T test-file + OBJID + :000000 100755 OBJID OBJID A test-file + EOF + cat >deleteprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: add + Prop-delta: true + Prop-content-length: 35 + Text-content-length: 17 + Content-length: 52 + + K 14 + svn:executable + V 0 + + PROPS-END + link testing 123 + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: change + Prop-delta: true + Prop-content-length: 53 + Text-content-length: 17 + Content-length: 70 + + K 11 + svn:special + V 1 + * + D 14 + svn:executable + PROPS-END + link testing 231 + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: change + Prop-delta: true + Prop-content-length: 27 + Text-content-length: 17 + Content-length: 44 + + D 11 + svn:special + PROPS-END + link testing 321 + EOF + test-svn-fe deleteprop.dump >stream && + git fast-import actual && + test_cmp expect actual ' test_expect_success 't9135/svn.dump' ' -- cgit v1.2.3 From 9e8c532108b9078812f23c53a2df3509e7ce71bf Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Mon, 6 Dec 2010 16:19:32 -0600 Subject: vcs-svn: Allow change nodes for root of tree (/) It is not uncommon for a svn repository to include change records for properties at the top level of the tracked tree: Node-path: Node-kind: dir Node-action: change Prop-delta: true Prop-content-length: 43 Content-length: 43 K 10 svn:ignore V 11 build-area PROPS-END Unfortunately a recent svn-fe change (vcs-svn: More dump format sanity checks, 2010-11-19) causes such nodes to be rejected with the error message fatal: invalid dump: path to be modified is missing The repo_tree module does not keep a dirent for the root of the tree. Add a block to the dump parser to take care of this case. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 't') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index 7dc06707a1..87945073b0 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -580,6 +580,61 @@ test_expect_success 'property deltas supported' ' test_cmp expect actual ' +test_expect_success 'properties on /' ' + reinit_git && + cat <<-\EOF >expect && + OBJID + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + sed -e "s/X$//" <<-\EOF >changeroot.dump && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Text-content-length: 0 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: X + Node-kind: dir + Node-action: change + Prop-delta: true + Prop-content-length: 43 + Content-length: 43 + + K 10 + svn:ignore + V 11 + build-area + + PROPS-END + EOF + test-svn-fe changeroot.dump >stream && + git fast-import actual && + test_cmp expect actual +' + test_expect_success 'deltas for typechange' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.3 From 8dc6a373d201839859fe7924b63e57526ee2fc22 Mon Sep 17 00:00:00 2001 From: David Barr Date: Thu, 2 Dec 2010 21:40:20 +1100 Subject: fast-import: add 'ls' command Lazy fast-import frontend authors that want to rely on the backend to keep track of the content of the imported trees _almost_ have what they need in the 'cat-blob' command (v1.7.4-rc0~30^2~3, 2010-11-28). But it is not quite enough, since (1) cat-blob can be used to retrieve the content of files, but not their mode, and (2) using cat-blob requires the frontend to keep track of a name (mark number or object id) for each blob to be retrieved Introduce an 'ls' command to complement cat-blob and take care of the remaining needs. The 'ls' command finds what is at a given path within a given tree-ish (tag, commit, or tree): 'ls' SP SP LF or in fast-import's active commit: 'ls' SP LF The response is a single line sent through the cat-blob channel, imitating ls-tree output. So for example: FE> ls :1 Documentation gfi> 040000 tree 9e6c2b599341d28a2a375f8207507e0a2a627fe9 Documentation FE> ls 9e6c2b599341d28a2a375f8207507e0a2a627fe9 git-fast-import.txt gfi> 100644 blob 4f92954396e3f0f97e75b6838a5635b583708870 git-fast-import.txt FE> ls :1 RelNotes gfi> 120000 blob b942e499449d97aeb50c73ca2bdc1c6e6d528743 RelNotes FE> cat-blob b942e499449d97aeb50c73ca2bdc1c6e6d528743 gfi> b942e499449d97aeb50c73ca2bdc1c6e6d528743 blob 32 gfi> Documentation/RelNotes/1.7.4.txt The most interesting parts of the reply are the first word, which is a 6-digit octal mode (regular file, executable, symlink, directory, or submodule), and the part from the second space to the tab, which is a that can be used in later cat-blob, ls, and filemodify (M) commands to refer to the content (blob, tree, or commit) at that path. If there is nothing there, the response is "missing some/path". The intent is for this command to be used to read files from the active commit, so a frontend can apply patches to them, and to copy files and directories from previous revisions. For example, proposed updates to svn-fe use this command in place of its internal representation of the repository directory structure. This simplifies the frontend a great deal and means support for resuming an import in a separate fast-import run (i.e., incremental import) is basically free. Signed-off-by: David Barr Signed-off-by: Jonathan Nieder Improved-by: Junio C Hamano Improved-by: Sverre Rabbelier --- t/t9300-fast-import.sh | 92 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 8 deletions(-) (limited to 't') diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index 52ac0e56dc..6b1ba6c858 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -42,6 +42,14 @@ echo "$@"' >empty +test_expect_success 'setup: have pipes?' ' + rm -f frob && + if mkfifo frob + then + test_set_prereq PIPE + fi +' + ### ### series A ### @@ -898,6 +906,77 @@ test_expect_success \ git diff-tree -C --find-copies-harder -r N4^ N4 >actual && compare_diff_raw expect actual' +test_expect_success PIPE 'N: read and copy directory' ' + cat >expect <<-\EOF + :100755 100755 f1fb5da718392694d0076d677d6d0e364c79b0bc f1fb5da718392694d0076d677d6d0e364c79b0bc C100 file2/newf file3/newf + :100644 100644 7123f7f44e39be127c5eb701e5968176ee9d78b1 7123f7f44e39be127c5eb701e5968176ee9d78b1 C100 file2/oldf file3/oldf + EOF + git update-ref -d refs/heads/N4 && + rm -f backflow && + mkfifo backflow && + ( + exec $GIT_COMMITTER_DATE + data <backflow && + git diff-tree -C --find-copies-harder -r N4^ N4 >actual && + compare_diff_raw expect actual +' + +test_expect_success PIPE 'N: empty directory reads as missing' ' + cat <<-\EOF >expect && + OBJNAME + :000000 100644 OBJNAME OBJNAME A unrelated + EOF + echo "missing src" >expect.response && + git update-ref -d refs/heads/read-empty && + rm -f backflow && + mkfifo backflow && + ( + exec $GIT_COMMITTER_DATE + data <response && + cat <<-\EOF + D dst1 + D dst2 + EOF + ) | + git fast-import --cat-blob-fd=3 3>backflow && + test_cmp expect.response response && + git rev-list read-empty | + git diff-tree -r --root --stdin | + sed "s/$_x40/OBJNAME/g" >actual && + test_cmp expect actual +' + test_expect_success \ 'N: copy root directory by tree hash' \ 'cat >expect <<-\EOF && @@ -1861,6 +1940,11 @@ test_expect_success 'R: feature no-relative-marks should be honoured' ' test_cmp marks.new non-relative.out ' +test_expect_success 'R: feature ls supported' ' + echo "feature ls" | + git fast-import +' + test_expect_success 'R: feature cat-blob supported' ' echo "feature cat-blob" | git fast-import @@ -1986,14 +2070,6 @@ test_expect_success 'R: print two blobs to stdout' ' test_cmp expect actual ' -test_expect_success 'setup: have pipes?' ' - rm -f frob && - if mkfifo frob - then - test_set_prereq PIPE - fi -' - test_expect_success PIPE 'R: copy using cat-file' ' expect_id=$(git hash-object big) && expect_len=$(wc -c Date: Sun, 2 Jan 2011 18:50:16 -0600 Subject: vcs-svn: make test-line-buffer input format more flexible Imitate the input format of test-obj-pool to support arbitrary sequences of commands rather than alternating read/copy. This should make it easier to add tests that exercise other line_buffer functions. Signed-off-by: Jonathan Nieder --- t/t0080-vcs-svn.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 't') diff --git a/t/t0080-vcs-svn.sh b/t/t0080-vcs-svn.sh index d3225ada68..8be97002ae 100755 --- a/t/t0080-vcs-svn.sh +++ b/t/t0080-vcs-svn.sh @@ -85,40 +85,40 @@ test_expect_success 'line buffer' ' printf "%s\n" "" foo >expected6 && test-line-buffer <<-\EOF >actual1 && - 5 + read 5 HELLO EOF test-line-buffer <<-\EOF >actual2 && - 0 + read 0 - 5 + copy 5 HELLO EOF q_to_nul <<-\EOF | - 1 + read 1 Q EOF test-line-buffer >actual3 && q_to_nul <<-\EOF | - 0 + read 0 - 1 + copy 1 Q EOF test-line-buffer >actual4 && test-line-buffer <<-\EOF >actual5 && - 5 + read 5 foo EOF test-line-buffer <<-\EOF >actual6 && - 0 + read 0 - 5 + copy 5 foo EOF -- cgit v1.2.3 From 232087fd99915abaa7d917fd181ad8477bb689f2 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sun, 2 Jan 2011 18:51:07 -0600 Subject: tests: give vcs-svn/line_buffer its own test script Split the line_buffer test into small pieces and move it to its own file as preparation for adding more tests. Signed-off-by: Jonathan Nieder --- t/t0080-vcs-svn.sh | 54 ---------------------------------------- t/t0081-line-buffer.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 54 deletions(-) create mode 100755 t/t0081-line-buffer.sh (limited to 't') diff --git a/t/t0080-vcs-svn.sh b/t/t0080-vcs-svn.sh index 8be97002ae..99a314b080 100755 --- a/t/t0080-vcs-svn.sh +++ b/t/t0080-vcs-svn.sh @@ -76,60 +76,6 @@ test_expect_success 'obj pool: high-water mark' ' test_cmp expected actual ' -test_expect_success 'line buffer' ' - echo HELLO >expected1 && - printf "%s\n" "" HELLO >expected2 && - echo >expected3 && - printf "%s\n" "" Q | q_to_nul >expected4 && - printf "%s\n" foo "" >expected5 && - printf "%s\n" "" foo >expected6 && - - test-line-buffer <<-\EOF >actual1 && - read 5 - HELLO - EOF - - test-line-buffer <<-\EOF >actual2 && - read 0 - - copy 5 - HELLO - EOF - - q_to_nul <<-\EOF | - read 1 - Q - EOF - test-line-buffer >actual3 && - - q_to_nul <<-\EOF | - read 0 - - copy 1 - Q - EOF - test-line-buffer >actual4 && - - test-line-buffer <<-\EOF >actual5 && - read 5 - foo - EOF - - test-line-buffer <<-\EOF >actual6 && - read 0 - - copy 5 - foo - EOF - - test_cmp expected1 actual1 && - test_cmp expected2 actual2 && - test_cmp expected3 actual3 && - test_cmp expected4 actual4 && - test_cmp expected5 actual5 && - test_cmp expected6 actual6 -' - test_expect_success 'string pool' ' echo a does not equal b >expected.differ && echo a equals a >expected.match && diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh new file mode 100755 index 0000000000..13ac735b55 --- /dev/null +++ b/t/t0081-line-buffer.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +test_description="Test the svn importer's input handling routines. +" +. ./test-lib.sh + +test_expect_success 'read greeting' ' + echo HELLO >expect && + test-line-buffer <<-\EOF >actual && + read 5 + HELLO + EOF + test_cmp expect actual +' + +test_expect_success '0-length read, send along greeting' ' + printf "%s\n" "" HELLO >expect && + test-line-buffer <<-\EOF >actual && + read 0 + + copy 5 + HELLO + EOF + test_cmp expect actual +' + +test_expect_success 'buffer_read_string copes with trailing null byte' ' + echo >expect && + q_to_nul <<-\EOF | test-line-buffer >actual && + read 1 + Q + EOF + test_cmp expect actual +' + +test_expect_success '0-length read, copy null byte' ' + printf "%s\n" "" Q | q_to_nul >expect && + q_to_nul <<-\EOF | test-line-buffer >actual && + read 0 + + copy 1 + Q + EOF + test_cmp expect actual +' + +test_expect_success 'long reads are truncated' ' + printf "%s\n" foo "" >expect && + test-line-buffer <<-\EOF >actual && + read 5 + foo + EOF + test_cmp expect actual +' + +test_expect_success 'long copies are truncated' ' + printf "%s\n" "" foo >expect && + test-line-buffer <<-\EOF >actual && + read 0 + + copy 5 + foo + EOF + test_cmp expect actual +' + +test_done -- cgit v1.2.3 From 7b990c90514b24097ee71edbc02cb3a497a9476b Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sun, 2 Jan 2011 18:52:28 -0600 Subject: vcs-svn: tweak test-line-buffer to not assume line-oriented input Do not expect an implicit newline after each input record. Use a separate command to exercise buffer_skip_bytes. Signed-off-by: Jonathan Nieder --- t/t0081-line-buffer.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 't') diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh index 13ac735b55..68d6163995 100755 --- a/t/t0081-line-buffer.sh +++ b/t/t0081-line-buffer.sh @@ -7,45 +7,44 @@ test_description="Test the svn importer's input handling routines. test_expect_success 'read greeting' ' echo HELLO >expect && test-line-buffer <<-\EOF >actual && - read 5 + read 6 HELLO EOF test_cmp expect actual ' test_expect_success '0-length read, send along greeting' ' - printf "%s\n" "" HELLO >expect && + echo HELLO >expect && test-line-buffer <<-\EOF >actual && read 0 - - copy 5 + copy 6 HELLO EOF test_cmp expect actual ' -test_expect_success 'buffer_read_string copes with trailing null byte' ' - echo >expect && +test_expect_success 'buffer_read_string copes with null byte' ' + >expect && q_to_nul <<-\EOF | test-line-buffer >actual && - read 1 + read 2 Q EOF test_cmp expect actual ' -test_expect_success '0-length read, copy null byte' ' - printf "%s\n" "" Q | q_to_nul >expect && +test_expect_success 'skip, copy null byte' ' + echo Q | q_to_nul >expect && q_to_nul <<-\EOF | test-line-buffer >actual && - read 0 - - copy 1 + skip 2 + Q + copy 2 Q EOF test_cmp expect actual ' test_expect_success 'long reads are truncated' ' - printf "%s\n" foo "" >expect && + echo foo >expect && test-line-buffer <<-\EOF >actual && read 5 foo @@ -56,7 +55,7 @@ test_expect_success 'long reads are truncated' ' test_expect_success 'long copies are truncated' ' printf "%s\n" "" foo >expect && test-line-buffer <<-\EOF >actual && - read 0 + read 1 copy 5 foo -- cgit v1.2.3 From d280f68313eecb8b3838c70641a246382d5e5343 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sun, 2 Jan 2011 19:07:16 -0600 Subject: t0081 (line-buffer): add buffering tests POSIX makes the behavior of read(2) from a pipe fairly clear: a read from an empty pipe will block until there is data available and any other read will not block, prefering to return a partial result. Likewise, fread(3) and fgets(3) are clearly specified to act as though implemented by calling fgetc(3) in a simple loop. But the buffering behavior of fgetc is less clear. Luckily, no sane platform is going to implement fgetc by calling the equivalent of read(2) more than once. fgetc has to be able to return without filling its buffer to preserve errno when errors are encountered anyway. So let's assume the simpler behavior (trust) but add some tests to catch insane platforms that violate that when they come (verify). First check that fread can handle a 0-length read from an empty fifo. Because open(O_RDONLY) blocks until the writing end is open, open the writing end of the fifo in advance in a subshell. Next try short inputs from a pipe that is not filled all the way. Lastly (two tests) try very large inputs from a pipe that will not fit in the relevant buffers. The first of these tests reads a little more than 8192 bytes, which is BUFSIZ (the size of stdio's buffers) on this Linux machine. The second reads a little over 64 KiB (the pipe capacity on Linux) and is not run unless requested by setting the GIT_REMOTE_SVN_TEST_BIG_FILES environment variable. Signed-off-by: Jonathan Nieder --- t/t0081-line-buffer.sh | 110 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) (limited to 't') diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh index 68d6163995..33a728ed98 100755 --- a/t/t0081-line-buffer.sh +++ b/t/t0081-line-buffer.sh @@ -1,10 +1,76 @@ #!/bin/sh test_description="Test the svn importer's input handling routines. + +These tests exercise the line_buffer library, but their real purpose +is to check the assumptions that library makes of the platform's input +routines. Processes engaged in bi-directional communication would +hang if fread or fgets is too greedy. + +While at it, check that input of newlines and null bytes are handled +correctly. " . ./test-lib.sh -test_expect_success 'read greeting' ' +test -n "$GIT_REMOTE_SVN_TEST_BIG_FILES" && test_set_prereq EXPENSIVE + +generate_tens_of_lines () { + tens=$1 && + line=$2 && + + i=0 && + while test $i -lt "$tens" + do + for j in a b c d e f g h i j + do + echo "$line" + done && + : $((i = $i + 1)) || + return + done +} + +long_read_test () { + : each line is 10 bytes, including newline && + line=abcdefghi && + echo "$line" >expect && + + if ! test_declared_prereq PIPE + then + echo >&4 "long_read_test: need to declare PIPE prerequisite" + return 127 + fi && + tens_of_lines=$(($1 / 100 + 1)) && + lines=$(($tens_of_lines * 10)) && + readsize=$((($lines - 1) * 10 + 3)) && + copysize=7 && + rm -f input && + mkfifo input && + { + { + generate_tens_of_lines $tens_of_lines "$line" && + sleep 100 + } >input & + } && + test-line-buffer input <<-EOF >output && + read $readsize + copy $copysize + EOF + kill $! && + test_line_count = $lines output && + tail -n 1 actual && + test_cmp expect actual +} + +test_expect_success 'setup: have pipes?' ' + rm -f frob && + if mkfifo frob + then + test_set_prereq PIPE + fi +' + +test_expect_success 'hello world' ' echo HELLO >expect && test-line-buffer <<-\EOF >actual && read 6 @@ -13,6 +79,21 @@ test_expect_success 'read greeting' ' test_cmp expect actual ' +test_expect_success PIPE '0-length read, no input available' ' + >expect && + rm -f input && + mkfifo input && + { + sleep 100 >input & + } && + test-line-buffer input <<-\EOF >actual && + read 0 + copy 0 + EOF + kill $! && + test_cmp expect actual +' + test_expect_success '0-length read, send along greeting' ' echo HELLO >expect && test-line-buffer <<-\EOF >actual && @@ -23,6 +104,33 @@ test_expect_success '0-length read, send along greeting' ' test_cmp expect actual ' +test_expect_success PIPE '1-byte read, no input available' ' + printf "%s" ab >expect && + rm -f input && + mkfifo input && + { + { + printf "%s" a && + printf "%s" b && + sleep 100 + } >input & + } && + test-line-buffer input <<-\EOF >actual && + read 1 + copy 1 + EOF + kill $! && + test_cmp expect actual +' + +test_expect_success PIPE 'long read (around 8192 bytes)' ' + long_read_test 8192 +' + +test_expect_success PIPE,EXPENSIVE 'longer read (around 65536 bytes)' ' + long_read_test 65536 +' + test_expect_success 'buffer_read_string copes with null byte' ' >expect && q_to_nul <<-\EOF | test-line-buffer >actual && -- cgit v1.2.3 From e832f43c1d26bf70611d98b62d95870a99292add Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sun, 2 Jan 2011 21:05:46 -0600 Subject: vcs-svn: add binary-safe read function buffer_read_string works well for non line-oriented input except for one problem: it does not tell the caller how many bytes were actually written. This means that unless one is very careful about checking for errors (and eof) the calling program cannot tell the difference between the string "foo" followed by an early end of file and the string "foo\0bar\0baz". So introduce a variant that reports the length, too, a thinner wrapper around strbuf_fread. Its result is written to a strbuf so the caller does not need to keep track of the number of bytes read. Signed-off-by: Jonathan Nieder --- t/t0081-line-buffer.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 't') diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh index 33a728ed98..a8eeb20645 100755 --- a/t/t0081-line-buffer.sh +++ b/t/t0081-line-buffer.sh @@ -151,6 +151,15 @@ test_expect_success 'skip, copy null byte' ' test_cmp expect actual ' +test_expect_success 'read null byte' ' + echo ">QhelloQ" | q_to_nul >expect && + q_to_nul <<-\EOF | test-line-buffer >actual && + binary 8 + QhelloQ + EOF + test_cmp expect actual +' + test_expect_success 'long reads are truncated' ' echo foo >expect && test-line-buffer <<-\EOF >actual && @@ -171,4 +180,13 @@ test_expect_success 'long copies are truncated' ' test_cmp expect actual ' +test_expect_success 'long binary reads are truncated' ' + echo ">foo" >expect && + test-line-buffer <<-\EOF >actual && + binary 5 + foo + EOF + test_cmp expect actual +' + test_done -- cgit v1.2.3 From cb3f87cf1ba90373fdc240d65a4d65434099d9a3 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sun, 2 Jan 2011 21:09:38 -0600 Subject: vcs-svn: allow input from file descriptor Based-on-patch-by: David Barr Signed-off-by: Jonathan Nieder --- t/t0081-line-buffer.sh | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 't') diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh index a8eeb20645..550fad0823 100755 --- a/t/t0081-line-buffer.sh +++ b/t/t0081-line-buffer.sh @@ -131,6 +131,15 @@ test_expect_success PIPE,EXPENSIVE 'longer read (around 65536 bytes)' ' long_read_test 65536 ' +test_expect_success 'read from file descriptor' ' + rm -f input && + echo hello >expect && + echo hello >input && + echo copy 6 | + test-line-buffer "&4" 4actual && + test_cmp expect actual +' + test_expect_success 'buffer_read_string copes with null byte' ' >expect && q_to_nul <<-\EOF | test-line-buffer >actual && -- cgit v1.2.3