From a894ba17e636cd3dc9c58aa34d5b8211f849ecef Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:31 +0000 Subject: git_remote_helpers: allow building with Python 3 Change inline Python to call "print" as a function not a statement. This is harmless because Python 2 will see the parentheses as redundant grouping but they are necessary to run this code with Python 3. Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git_remote_helpers/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git_remote_helpers/Makefile b/git_remote_helpers/Makefile index 74b05dc91e..f65f0645ee 100644 --- a/git_remote_helpers/Makefile +++ b/git_remote_helpers/Makefile @@ -23,7 +23,7 @@ endif PYLIBDIR=$(shell $(PYTHON_PATH) -c \ "import sys; \ - print 'lib/python%i.%i/site-packages' % sys.version_info[:2]") + print('lib/python%i.%i/site-packages' % sys.version_info[:2])") all: $(pysetupfile) $(QUIET)$(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) build -- cgit v1.2.3 From 29cf0d3873d78f8b94353b37d9d3e32e51766a90 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:32 +0000 Subject: git_remote_helpers: fix input when running under Python 3 Although 2to3 will fix most issues in Python 2 code to make it run under Python 3, it does not handle the new strict separation between byte strings and unicode strings. There is one instance in git_remote_helpers where we are caught by this, which is when reading refs from "git for-each-ref". Fix this by operating on the returned string as a byte string rather than a unicode string. As this method is currently only used internally by the class this does not affect code anywhere else. Note that we cannot use byte strings in the source as the 'b' prefix is not supported before Python 2.7 so in order to maintain compatibility with the maximum range of Python versions we use an explicit call to encode(). Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git_remote_helpers/git/importer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/git_remote_helpers/git/importer.py b/git_remote_helpers/git/importer.py index e28cc8f986..d3f90e1024 100644 --- a/git_remote_helpers/git/importer.py +++ b/git_remote_helpers/git/importer.py @@ -18,13 +18,16 @@ class GitImporter(object): def get_refs(self, gitdir): """Returns a dictionary with refs. + + Note that the keys in the returned dictionary are byte strings as + read from git. """ args = ["git", "--git-dir=" + gitdir, "for-each-ref", "refs/heads"] - lines = check_output(args).strip().split('\n') + lines = check_output(args).strip().split('\n'.encode('ascii')) refs = {} for line in lines: - value, name = line.split(' ') - name = name.strip('commit\t') + value, name = line.split(' '.encode('ascii')) + name = name.strip('commit\t'.encode('ascii')) refs[name] = value return refs -- cgit v1.2.3 From fadf8c71510e3ff142afd1ecbc65703d7ff27c8b Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:33 +0000 Subject: git_remote_helpers: force rebuild if python version changes When different version of python are used to build via distutils, the behaviour can change. Detect changes in version and pass --force in this case. Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git_remote_helpers/.gitignore | 1 + git_remote_helpers/Makefile | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/git_remote_helpers/.gitignore b/git_remote_helpers/.gitignore index 2247d5f95a..cf040af3f5 100644 --- a/git_remote_helpers/.gitignore +++ b/git_remote_helpers/.gitignore @@ -1,2 +1,3 @@ +/GIT-PYTHON-VERSION /build /dist diff --git a/git_remote_helpers/Makefile b/git_remote_helpers/Makefile index f65f0645ee..0d2ae740b3 100644 --- a/git_remote_helpers/Makefile +++ b/git_remote_helpers/Makefile @@ -25,8 +25,14 @@ PYLIBDIR=$(shell $(PYTHON_PATH) -c \ "import sys; \ print('lib/python%i.%i/site-packages' % sys.version_info[:2])") +py_version=$(shell $(PYTHON_PATH) -c \ + 'import sys; print("%i.%i" % sys.version_info[:2])') + all: $(pysetupfile) - $(QUIET)$(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) build + $(QUIET)test "$$(cat GIT-PYTHON-VERSION 2>/dev/null)" = "$(py_version)" || \ + flags=--force; \ + $(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) build $$flags + $(QUIET)echo "$(py_version)" >GIT-PYTHON-VERSION install: $(pysetupfile) $(PYTHON_PATH) $(pysetupfile) install --prefix $(DESTDIR_SQ)$(prefix) -- cgit v1.2.3 From 62c814b6b60f5b5605e34c9806d874fe1088d416 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:34 +0000 Subject: git_remote_helpers: use 2to3 if building with Python 3 Using the approach detailed in the Python documentation[1], run 2to3 on the code as part of the build if building with Python 3. The code itself requires no changes to convert cleanly. [1] http://docs.python.org/3.3/howto/pyporting.html#during-installation Signed-off-by: John Keeping Acked-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git_remote_helpers/setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/git_remote_helpers/setup.py b/git_remote_helpers/setup.py index 4d434b65cb..6de41deb44 100644 --- a/git_remote_helpers/setup.py +++ b/git_remote_helpers/setup.py @@ -4,6 +4,15 @@ from distutils.core import setup +# If building under Python3 we need to run 2to3 on the code, do this by +# trying to import distutils' 2to3 builder, which is only available in +# Python3. +try: + from distutils.command.build_py import build_py_2to3 as build_py +except ImportError: + # 2.x + from distutils.command.build_py import build_py + setup( name = 'git_remote_helpers', version = '0.1.0', @@ -14,4 +23,5 @@ setup( url = 'http://www.git-scm.com/', package_dir = {'git_remote_helpers': ''}, packages = ['git_remote_helpers', 'git_remote_helpers.git'], + cmdclass = {'build_py': build_py}, ) -- cgit v1.2.3 From cadbf5c7ed4ea60d2b776346586b224e774dca4b Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:35 +0000 Subject: svn-fe: allow svnrdump_sim.py to run with Python 3 The changes to allow this script to run with Python 3 are minimal and do not affect its functionality on the versions of Python 2 that are already supported (2.4 onwards). Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- contrib/svn-fe/svnrdump_sim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/svn-fe/svnrdump_sim.py b/contrib/svn-fe/svnrdump_sim.py index 17cf6f961f..4e78a1c3cd 100755 --- a/contrib/svn-fe/svnrdump_sim.py +++ b/contrib/svn-fe/svnrdump_sim.py @@ -14,7 +14,7 @@ if sys.hexversion < 0x02040000: def getrevlimit(): var = 'SVNRMAX' - if os.environ.has_key(var): + if var in os.environ: return os.environ[var] return None @@ -44,7 +44,7 @@ def writedump(url, lower, upper): if __name__ == "__main__": if not (len(sys.argv) in (3, 4, 5)): - print "usage: %s dump URL -rLOWER:UPPER" + print("usage: %s dump URL -rLOWER:UPPER") sys.exit(1) if not sys.argv[1] == 'dump': raise NotImplementedError('only "dump" is suppported.') url = sys.argv[2] -- cgit v1.2.3 From 0846b0c905a885720138e8194fb0289f121b5da1 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:36 +0000 Subject: git-remote-testpy: hash bytes explicitly Under Python 3 'hasher.update(...)' must take a byte string and not a unicode string. Explicitly encode the argument to this method to hex bytes so that we don't need to worry about failures to encode that might occur if we chose a textual encoding. This changes the directory used by git-remote-testpy for its git mirror of the remote repository, but this tool should not have any serious users as it is used primarily to test the Python remote helper framework. The use of encode() moves the required Python version forward to 2.0. Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git-remote-testpy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git-remote-testpy.py b/git-remote-testpy.py index d94a66a870..197b7be508 100644 --- a/git-remote-testpy.py +++ b/git-remote-testpy.py @@ -31,9 +31,9 @@ from git_remote_helpers.git.exporter import GitExporter from git_remote_helpers.git.importer import GitImporter from git_remote_helpers.git.non_local import NonLocalGit -if sys.hexversion < 0x01050200: - # os.makedirs() is the limiter - sys.stderr.write("git-remote-testgit: requires Python 1.5.2 or later.\n") +if sys.hexversion < 0x02000000: + # string.encode() is the limiter + sys.stderr.write("git-remote-testgit: requires Python 2.0 or later.\n") sys.exit(1) def get_repo(alias, url): @@ -45,7 +45,7 @@ def get_repo(alias, url): repo.get_head() hasher = _digest() - hasher.update(repo.path) + hasher.update(repo.path.encode('hex')) repo.hash = hasher.hexdigest() repo.get_base_path = lambda base: os.path.join( -- cgit v1.2.3 From d04c94a2ea21cb218b43be192d8569c0e669d080 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:37 +0000 Subject: git-remote-testpy: don't do unbuffered text I/O Python 3 forbids unbuffered I/O in text mode. Change the reading of stdin in git-remote-testpy so that we read the lines as bytes and then decode them a line at a time. This allows us to keep the I/O unbuffered in order to avoid reintroducing the bug fixed by commit 7fb8e16 (git-remote-testgit: fix race when spawning fast-import). Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git-remote-testpy.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/git-remote-testpy.py b/git-remote-testpy.py index 197b7be508..5dbf1cc066 100644 --- a/git-remote-testpy.py +++ b/git-remote-testpy.py @@ -154,7 +154,7 @@ def do_import(repo, args): refs = [ref] while True: - line = sys.stdin.readline() + line = sys.stdin.readline().decode() if line == '\n': break if not line.startswith('import '): @@ -225,7 +225,7 @@ def read_one_line(repo): line = sys.stdin.readline() - cmdline = line + cmdline = line.decode() if not cmdline: warn("Unexpected EOF") @@ -277,7 +277,11 @@ def main(args): more = True - sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0) + # Use binary mode since Python 3 does not permit unbuffered I/O in text + # mode. Unbuffered I/O is required to avoid data that should be going + # to git-fast-import after an "export" command getting caught in our + # stdin buffer instead. + sys.stdin = os.fdopen(sys.stdin.fileno(), 'rb', 0) while (more): more = read_one_line(repo) -- cgit v1.2.3 From f9640ac26ce0478695487f657718157bb7b4f804 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 20 Jan 2013 13:15:38 +0000 Subject: git-remote-testpy: call print as a function This is harmless in Python 2, which sees the parentheses as redundant grouping, but is required for Python 3. Since this is the only change required to make this script just run under Python 3 without needing 2to3 it seems worthwhile. The case of an empty print must be handled specially because in that case Python 2 will interpret '()' as an empty tuple and print it as '()'; inserting an empty string fixes this. Signed-off-by: John Keeping Acked-by: Sverre Rabbelier Signed-off-by: Junio C Hamano --- git-remote-testpy.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/git-remote-testpy.py b/git-remote-testpy.py index 5dbf1cc066..c7a04ecae2 100644 --- a/git-remote-testpy.py +++ b/git-remote-testpy.py @@ -87,9 +87,9 @@ def do_capabilities(repo, args): """Prints the supported capabilities. """ - print "import" - print "export" - print "refspec refs/heads/*:%s*" % repo.prefix + print("import") + print("export") + print("refspec refs/heads/*:%s*" % repo.prefix) dirname = repo.get_base_path(repo.gitdir) @@ -98,11 +98,11 @@ def do_capabilities(repo, args): path = os.path.join(dirname, 'git.marks') - print "*export-marks %s" % path + print("*export-marks %s" % path) if os.path.exists(path): - print "*import-marks %s" % path + print("*import-marks %s" % path) - print # end capabilities + print('') # end capabilities def do_list(repo, args): @@ -115,16 +115,16 @@ def do_list(repo, args): for ref in repo.revs: debug("? refs/heads/%s", ref) - print "? refs/heads/%s" % ref + print("? refs/heads/%s" % ref) if repo.head: debug("@refs/heads/%s HEAD" % repo.head) - print "@refs/heads/%s HEAD" % repo.head + print("@refs/heads/%s HEAD" % repo.head) else: debug("@refs/heads/master HEAD") - print "@refs/heads/master HEAD" + print("@refs/heads/master HEAD") - print # end list + print('') # end list def update_local_repo(repo): @@ -164,7 +164,7 @@ def do_import(repo, args): ref = line[7:].strip() refs.append(ref) - print "feature done" + print("feature done") if os.environ.get("GIT_REMOTE_TESTGIT_FAILURE"): die('Told to fail') @@ -172,7 +172,7 @@ def do_import(repo, args): repo = update_local_repo(repo) repo.exporter.export_repo(repo.gitdir, refs) - print "done" + print("done") def do_export(repo, args): @@ -192,8 +192,8 @@ def do_export(repo, args): repo.non_local.push(repo.gitdir) for ref in changed: - print "ok %s" % ref - print + print("ok %s" % ref) + print('') COMMANDS = { -- cgit v1.2.3 From 3ac221a78eba2f447a7990fc5c4709853dc32fee Mon Sep 17 00:00:00 2001 From: John Keeping Date: Sun, 27 Jan 2013 14:50:56 +0000 Subject: git-remote-testpy: fix path hashing on Python 3 When this change was originally made (0846b0c - git-remote-testpy: hash bytes explicitly , I didn't realise that the "hex" encoding we chose is a "bytes to bytes" encoding so it just fails with an error on Python 3 in the same way as the original code. It is not possible to provide a single code path that works on Python 2 and Python 3 since Python 2.x will attempt to decode the string before encoding it, which fails for strings that are not valid in the default encoding. Python 3.1 introduced the "surrogateescape" error handler which handles this correctly and permits a bytes -> unicode -> bytes round-trip to be lossless. As the original came from reading the filesystem path, we convert them back into the original bytes encoded in sys.getfilesystemencoding(). At this point Python 3.0 is unsupported so we don't go out of our way to try to support it. Helped-by: Michael Haggerty Signed-off-by: John Keeping Signed-off-by: Junio C Hamano --- git-remote-testpy.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/git-remote-testpy.py b/git-remote-testpy.py index c7a04ecae2..ca6789996a 100644 --- a/git-remote-testpy.py +++ b/git-remote-testpy.py @@ -36,6 +36,22 @@ if sys.hexversion < 0x02000000: sys.stderr.write("git-remote-testgit: requires Python 2.0 or later.\n") sys.exit(1) + +def encode_filepath(path): + """Encodes a Unicode file path to a byte string. + + On Python 2 this is a no-op; on Python 3 we encode the string as + suggested by [1] which allows an exact round-trip from the command line + to the filesystem. + + [1] http://docs.python.org/3/c-api/unicode.html#file-system-encoding + + """ + if sys.hexversion < 0x03000000: + return path + return path.encode(sys.getfilesystemencoding(), 'surrogateescape') + + def get_repo(alias, url): """Returns a git repository object initialized for usage. """ @@ -45,7 +61,7 @@ def get_repo(alias, url): repo.get_head() hasher = _digest() - hasher.update(repo.path.encode('hex')) + hasher.update(encode_filepath(repo.path)) repo.hash = hasher.hexdigest() repo.get_base_path = lambda base: os.path.join( -- cgit v1.2.3 From ae6037bc710b248946ed715fd659ef535ea78980 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 30 Jan 2013 11:30:10 -0800 Subject: git_remote_helpers: remove GIT-PYTHON-VERSION upon "clean" fadf8c7 (git_remote_helpers: force rebuild if python version changes, 2013-01-20) started using a marker file to keep track of the version of Python interpreter used for the last build, but forgot to remove it when asked to "make clean". Acked-by: John Keeping Signed-off-by: Junio C Hamano --- git_remote_helpers/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git_remote_helpers/Makefile b/git_remote_helpers/Makefile index 0d2ae740b3..3d122328c8 100644 --- a/git_remote_helpers/Makefile +++ b/git_remote_helpers/Makefile @@ -42,4 +42,4 @@ instlibdir: $(pysetupfile) clean: $(QUIET)$(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) clean -a - $(RM) *.pyo *.pyc + $(RM) *.pyo *.pyc GIT-PYTHON-VERSION -- cgit v1.2.3