diff options
Diffstat (limited to 'git-p4.py')
| -rwxr-xr-x | git-p4.py | 681 |
1 files changed, 396 insertions, 285 deletions
@@ -7,10 +7,36 @@ # 2007 Trolltech ASA # License: MIT <http://www.opensource.org/licenses/mit-license.php> # - -import optparse, sys, os, marshal, subprocess, shelve -import tempfile, getopt, os.path, time, platform -import re, shutil +import sys +if sys.hexversion < 0x02040000: + # The limiter is the subprocess module + sys.stderr.write("git-p4: requires Python 2.4 or later.\n") + sys.exit(1) +import os +import optparse +import marshal +import subprocess +import tempfile +import time +import platform +import re +import shutil +import stat + +try: + from subprocess import CalledProcessError +except ImportError: + # from python2.7:subprocess.py + # Exception classes used by this module. + class CalledProcessError(Exception): + """This exception is raised when a process run by check_call() returns + a non-zero exit status. The exit status will be stored in the + returncode attribute.""" + def __init__(self, returncode, cmd): + self.returncode = returncode + self.cmd = cmd + def __str__(self): + return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) verbose = False @@ -53,12 +79,27 @@ def p4_build_cmd(cmd): real_cmd += cmd return real_cmd -def chdir(dir): - # P4 uses the PWD environment variable rather than getcwd(). Since we're - # not using the shell, we have to set it ourselves. This path could - # be relative, so go there first, then figure out where we ended up. - os.chdir(dir) - os.environ['PWD'] = os.getcwd() +def chdir(path, is_client_path=False): + """Do chdir to the given path, and set the PWD environment + variable for use by P4. It does not look at getcwd() output. + Since we're not using the shell, it is necessary to set the + PWD environment variable explicitly. + + Normally, expand the path to force it to be absolute. This + addresses the use of relative path names inside P4 settings, + e.g. P4CONFIG=.p4config. P4 does not simply open the filename + as given; it looks for .p4config using PWD. + + If is_client_path, the path was handed to us directly by p4, + and may be a symbolic link. Do not call os.getcwd() in this + case, because it will cause p4 to think that PWD is not inside + the client path. + """ + + os.chdir(path) + if not is_client_path: + path = os.getcwd() + os.environ['PWD'] = path def die(msg): if verbose: @@ -129,17 +170,56 @@ def p4_has_command(cmd): p.communicate() return p.returncode == 0 +def p4_has_move_command(): + """See if the move command exists, that it supports -k, and that + it has not been administratively disabled. The arguments + must be correct, but the filenames do not have to exist. Use + ones with wildcards so even if they exist, it will fail.""" + + if not p4_has_command("move"): + return False + cmd = p4_build_cmd(["move", "-k", "@from", "@to"]) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = p.communicate() + # return code will be 1 in either case + if err.find("Invalid option") >= 0: + return False + if err.find("disabled") >= 0: + return False + # assume it failed because @... was invalid changelist + return True + def system(cmd): expand = isinstance(cmd,basestring) if verbose: sys.stderr.write("executing %s\n" % str(cmd)) - subprocess.check_call(cmd, shell=expand) + retcode = subprocess.call(cmd, shell=expand) + if retcode: + raise CalledProcessError(retcode, cmd) def p4_system(cmd): """Specifically invoke p4 as the system command. """ real_cmd = p4_build_cmd(cmd) expand = isinstance(real_cmd, basestring) - subprocess.check_call(real_cmd, shell=expand) + retcode = subprocess.call(real_cmd, shell=expand) + if retcode: + raise CalledProcessError(retcode, real_cmd) + +_p4_version_string = None +def p4_version_string(): + """Read the version string, showing just the last line, which + hopefully is the interesting version bit. + + $ p4 -V + Perforce - The Fast Software Configuration Management System. + Copyright 1995-2011 Perforce Software. All rights reserved. + Rev. P4/NTX86/2011.1/393975 (2011/12/16). + """ + global _p4_version_string + if not _p4_version_string: + a = p4_read_pipe_lines(["-V"]) + _p4_version_string = a[-1].rstrip() + return _p4_version_string def p4_integrate(src, dest): p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)]) @@ -169,6 +249,29 @@ def p4_reopen(type, f): def p4_move(src, dest): p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)]) +def p4_describe(change): + """Make sure it returns a valid result by checking for + the presence of field "time". Return a dict of the + results.""" + + ds = p4CmdList(["describe", "-s", str(change)]) + if len(ds) != 1: + die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds))) + + d = ds[0] + + if "p4ExitCode" in d: + die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"], + str(d))) + if "code" in d: + if d["code"] == "error": + die("p4 describe -s %d returned error code: %s" % (change, str(d))) + + if "time" not in d: + die("p4 describe -s %d returned no \"time\": %s" % (change, str(d))) + + return d + # # Canonicalize the p4 type and return a tuple of the # base type, plus any modifiers. See "p4 help filetypes" @@ -207,8 +310,8 @@ def split_p4_type(p4type): # # return the raw p4 type of a file (text, text+ko, etc) # -def p4_type(file): - results = p4CmdList(["fstat", "-T", "headType", file]) +def p4_type(f): + results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)]) return results[0]['headType'] # @@ -491,43 +594,75 @@ def gitBranchExists(branch): return proc.wait() == 0; _gitConfig = {} -def gitConfig(key, args = None): # set args to "--bool", for instance + +def gitConfig(key): if not _gitConfig.has_key(key): - argsFilter = "" - if args != None: - argsFilter = "%s " % args - cmd = "git config %s%s" % (argsFilter, key) - _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip() + cmd = [ "git", "config", key ] + s = read_pipe(cmd, ignore_error=True) + _gitConfig[key] = s.strip() + return _gitConfig[key] + +def gitConfigBool(key): + """Return a bool, using git config --bool. It is True only if the + variable is set to true, and False if set to false or not present + in the config.""" + + if not _gitConfig.has_key(key): + cmd = [ "git", "config", "--bool", key ] + s = read_pipe(cmd, ignore_error=True) + v = s.strip() + _gitConfig[key] = v == "true" return _gitConfig[key] def gitConfigList(key): if not _gitConfig.has_key(key): - _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep) + s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) + _gitConfig[key] = s.strip().split(os.linesep) return _gitConfig[key] -def p4BranchesInGit(branchesAreInRemotes = True): +def p4BranchesInGit(branchesAreInRemotes=True): + """Find all the branches whose names start with "p4/", looking + in remotes or heads as specified by the argument. Return + a dictionary of { branch: revision } for each one found. + The branch names are the short names, without any + "p4/" prefix.""" + branches = {} cmdline = "git rev-parse --symbolic " if branchesAreInRemotes: - cmdline += " --remotes" + cmdline += "--remotes" else: - cmdline += " --branches" + cmdline += "--branches" for line in read_pipe_lines(cmdline): line = line.strip() - ## only import to p4/ - if not line.startswith('p4/') or line == "p4/HEAD": + # only import to p4/ + if not line.startswith('p4/'): + continue + # special symbolic ref to p4/master + if line == "p4/HEAD": continue - branch = line - # strip off p4 - branch = re.sub ("^p4/", "", line) + # strip off p4/ prefix + branch = line[len("p4/"):] branches[branch] = parseRevision(line) + return branches +def branch_exists(branch): + """Make sure that the given ref name really exists.""" + + cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = p.communicate() + if p.returncode: + return False + # expect exactly one line of output: the branch name + return out.rstrip() == branch + def findUpstreamBranchPoint(head = "HEAD"): branches = p4BranchesInGit() # map from depot-path to branch name @@ -629,8 +764,7 @@ def p4PathStartsWith(path, prefix): # # we may or may not have a problem. If you have core.ignorecase=true, # we treat DirA and dira as the same directory - ignorecase = gitConfig("core.ignorecase", "--bool") == "true" - if ignorecase: + if gitConfigBool("core.ignorecase"): return path.lower().startswith(prefix.lower()) return path.startswith(prefix) @@ -646,11 +780,14 @@ def getClientSpec(): # dictionary of all client parameters entry = specList[0] + # the //client/ name + client_name = entry["Client"] + # just the keys that start with "View" view_keys = [ k for k in entry.keys() if k.startswith("View") ] # hold this new View - view = View() + view = View(client_name) # append the lines, in order, to the view for view_num in range(len(view_keys)): @@ -700,7 +837,8 @@ def wildcard_encode(path): return path def wildcard_present(path): - return path.translate(None, "*#@%") != path + m = re.search("[*#@%]", path) + return m is not None class Command: def __init__(self): @@ -859,19 +997,21 @@ class P4Submit(Command, P4UserMap): optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"), optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"), optparse.make_option("--conflict", dest="conflict_behavior", - choices=self.conflict_behavior_choices) + choices=self.conflict_behavior_choices), + optparse.make_option("--branch", dest="branch"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" self.origin = "" self.detectRenames = False - self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true" + self.preserveUser = gitConfigBool("git-p4.preserveUser") self.dry_run = False self.prepare_p4_only = False self.conflict_behavior = None self.isWindows = (platform.system() == "Windows") self.exportLabels = False - self.p4HasMoveCommand = p4_has_command("move") + self.p4HasMoveCommand = p4_has_move_command() + self.branch = None def check(self): if len(p4CmdList("opened ...")) > 0: @@ -959,7 +1099,8 @@ class P4Submit(Command, P4UserMap): def p4UserForCommit(self,id): # Return the tuple (perforce user,git email) for a given git commit id self.getUserMapFromPerforceServer() - gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id) + gitEmail = read_pipe(["git", "log", "--max-count=1", + "--format=%ae", id]) gitEmail = gitEmail.strip() if not self.emails.has_key(gitEmail): return (None,gitEmail) @@ -972,7 +1113,7 @@ class P4Submit(Command, P4UserMap): (user,email) = self.p4UserForCommit(id) if not user: msg = "Cannot find p4 user for email %s in commit %s." % (email, id) - if gitConfig('git-p4.allowMissingP4Users').lower() == "true": + if gitConfigBool("git-p4.allowMissingP4Users"): print "%s" % msg else: die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg) @@ -1067,7 +1208,7 @@ class P4Submit(Command, P4UserMap): message. Return true if okay to continue with the submit.""" # if configured to skip the editing part, just submit - if gitConfig("git-p4.skipSubmitEdit") == "true": + if gitConfigBool("git-p4.skipSubmitEdit"): return True # look at the modification time, to check later if the user saved @@ -1079,11 +1220,11 @@ class P4Submit(Command, P4UserMap): editor = os.environ.get("P4EDITOR") else: editor = read_pipe("git var GIT_EDITOR").strip() - system(editor + " " + template_file) + system([editor, template_file]) # If the file was not saved, prompt to see if this patch should # be skipped. But skip this verification step if configured so. - if gitConfig("git-p4.skipSubmitEditCheck") == "true": + if gitConfigBool("git-p4.skipSubmitEditCheck"): return True # modification time updated means user saved the file @@ -1141,6 +1282,9 @@ class P4Submit(Command, P4UserMap): p4_edit(dest) pureRenameCopy.discard(dest) filesToChangeExecBit[dest] = diff['dst_mode'] + if self.isWindows: + # turn off read-only attribute + os.chmod(dest, stat.S_IWRITE) os.unlink(dest) editedFiles.add(dest) elif modifier == "R": @@ -1159,13 +1303,15 @@ class P4Submit(Command, P4UserMap): p4_edit(dest) # with move: already open, writable filesToChangeExecBit[dest] = diff['dst_mode'] if not self.p4HasMoveCommand: + if self.isWindows: + os.chmod(dest, stat.S_IWRITE) os.unlink(dest) filesToDelete.add(src) editedFiles.add(dest) else: die("unknown modifier %s for %s" % (modifier, path)) - diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) + diffcmd = "git diff-tree -p \"%s\"" % (id) patchcmd = diffcmd + " | git apply " tryPatchCmd = patchcmd + "--check -" applyPatchCmd = patchcmd + "--check --apply -" @@ -1178,7 +1324,7 @@ class P4Submit(Command, P4UserMap): # Patch failed, maybe it's just RCS keyword woes. Look through # the patch to see if that's possible. - if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true": + if gitConfigBool("git-p4.attemptRCSCleanup"): file = None pattern = None kwfiles = {} @@ -1199,6 +1345,10 @@ class P4Submit(Command, P4UserMap): for file in kwfiles: if verbose: print "zapping %s with %s" % (line,pattern) + # File is being deleted, so not open in p4. Must + # disable the read-only bit on windows. + if self.isWindows and file not in editedFiles: + os.chmod(file, stat.S_IWRITE) self.patchRCSKeywords(file, kwfiles[file]) fixed_rcs_keywords = True @@ -1408,8 +1558,8 @@ class P4Submit(Command, P4UserMap): for b in body: labelTemplate += "\t" + b + "\n" labelTemplate += "View:\n" - for mapping in clientSpec.mappings: - labelTemplate += "\t%s\n" % mapping.depot_side.path + for depot_side in clientSpec.mappings: + labelTemplate += "\t%s\n" % depot_side if self.dry_run: print "Would create p4 label %s for tag" % name @@ -1421,7 +1571,7 @@ class P4Submit(Command, P4UserMap): # Use the label p4_system(["tag", "-l", name] + - ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings]) + ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings]) if verbose: print "created p4 label for tag %s" % name @@ -1469,7 +1619,7 @@ class P4Submit(Command, P4UserMap): sys.exit(128) self.useClientSpec = False - if gitConfig("git-p4.useclientspec", "--bool") == "true": + if gitConfigBool("git-p4.useclientspec"): self.useClientSpec = True if self.useClientSpec: self.clientSpecDirs = getClientSpec() @@ -1492,7 +1642,7 @@ class P4Submit(Command, P4UserMap): new_client_dir = True os.makedirs(self.clientPath) - chdir(self.clientPath) + chdir(self.clientPath, is_client_path=True) if self.dry_run: print "Would synchronize p4 checkout in %s" % self.clientPath else: @@ -1505,11 +1655,11 @@ class P4Submit(Command, P4UserMap): self.check() commits = [] - for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)): + for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]): commits.append(line.strip()) commits.reverse() - if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"): + if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"): self.checkAuthorship = False else: self.checkAuthorship = True @@ -1545,7 +1695,7 @@ class P4Submit(Command, P4UserMap): else: self.diffOpts += " -C%s" % detectCopies - if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true": + if gitConfigBool("git-p4.detectCopiesHarder"): self.diffOpts += " --find-copies-harder" # @@ -1608,6 +1758,8 @@ class P4Submit(Command, P4UserMap): print "All commits applied!" sync = P4Sync() + if self.branch: + sync.branch = self.branch sync.run([]) rebase = P4Rebase() @@ -1627,7 +1779,7 @@ class P4Submit(Command, P4UserMap): "--format=format:%h %s", c]) print "You will have to do 'git p4 sync' and rebase." - if gitConfig("git-p4.exportLabels", "--bool") == "true": + if gitConfigBool("git-p4.exportLabels"): self.exportLabels = True if self.exportLabels: @@ -1637,7 +1789,7 @@ class P4Submit(Command, P4UserMap): missingGitTags = gitTags - p4Labels self.exportGitTags(missingGitTags) - # exit with error unless everything applied perfecly + # exit with error unless everything applied perfectly if len(commits) != len(applied): sys.exit(1) @@ -1647,117 +1799,16 @@ class View(object): """Represent a p4 view ("p4 help views"), and map files in a repo according to the view.""" - class Path(object): - """A depot or client path, possibly containing wildcards. - The only one supported is ... at the end, currently. - Initialize with the full path, with //depot or //client.""" - - def __init__(self, path, is_depot): - self.path = path - self.is_depot = is_depot - self.find_wildcards() - # remember the prefix bit, useful for relative mappings - m = re.match("(//[^/]+/)", self.path) - if not m: - die("Path %s does not start with //prefix/" % self.path) - prefix = m.group(1) - if not self.is_depot: - # strip //client/ on client paths - self.path = self.path[len(prefix):] - - def find_wildcards(self): - """Make sure wildcards are valid, and set up internal - variables.""" - - self.ends_triple_dot = False - # There are three wildcards allowed in p4 views - # (see "p4 help views"). This code knows how to - # handle "..." (only at the end), but cannot deal with - # "%%n" or "*". Only check the depot_side, as p4 should - # validate that the client_side matches too. - if re.search(r'%%[1-9]', self.path): - die("Can't handle %%n wildcards in view: %s" % self.path) - if self.path.find("*") >= 0: - die("Can't handle * wildcards in view: %s" % self.path) - triple_dot_index = self.path.find("...") - if triple_dot_index >= 0: - if triple_dot_index != len(self.path) - 3: - die("Can handle only single ... wildcard, at end: %s" % - self.path) - self.ends_triple_dot = True - - def ensure_compatible(self, other_path): - """Make sure the wildcards agree.""" - if self.ends_triple_dot != other_path.ends_triple_dot: - die("Both paths must end with ... if either does;\n" + - "paths: %s %s" % (self.path, other_path.path)) - - def match_wildcards(self, test_path): - """See if this test_path matches us, and fill in the value - of the wildcards if so. Returns a tuple of - (True|False, wildcards[]). For now, only the ... at end - is supported, so at most one wildcard.""" - if self.ends_triple_dot: - dotless = self.path[:-3] - if test_path.startswith(dotless): - wildcard = test_path[len(dotless):] - return (True, [ wildcard ]) - else: - if test_path == self.path: - return (True, []) - return (False, []) - - def match(self, test_path): - """Just return if it matches; don't bother with the wildcards.""" - b, _ = self.match_wildcards(test_path) - return b - - def fill_in_wildcards(self, wildcards): - """Return the relative path, with the wildcards filled in - if there are any.""" - if self.ends_triple_dot: - return self.path[:-3] + wildcards[0] - else: - return self.path - - class Mapping(object): - def __init__(self, depot_side, client_side, overlay, exclude): - # depot_side is without the trailing /... if it had one - self.depot_side = View.Path(depot_side, is_depot=True) - self.client_side = View.Path(client_side, is_depot=False) - self.overlay = overlay # started with "+" - self.exclude = exclude # started with "-" - assert not (self.overlay and self.exclude) - self.depot_side.ensure_compatible(self.client_side) - - def __str__(self): - c = " " - if self.overlay: - c = "+" - if self.exclude: - c = "-" - return "View.Mapping: %s%s -> %s" % \ - (c, self.depot_side.path, self.client_side.path) - - def map_depot_to_client(self, depot_path): - """Calculate the client path if using this mapping on the - given depot path; does not consider the effect of other - mappings in a view. Even excluded mappings are returned.""" - matches, wildcards = self.depot_side.match_wildcards(depot_path) - if not matches: - return "" - client_path = self.client_side.fill_in_wildcards(wildcards) - return client_path - - # - # View methods - # - def __init__(self): + def __init__(self, client_name): self.mappings = [] + self.client_prefix = "//%s/" % client_name + # cache results of "p4 where" to lookup client file locations + self.client_spec_path_cache = {} def append(self, view_line): """Parse a view line, splitting it into depot and client - sides. Append to self.mappings, preserving order.""" + sides. Append to self.mappings, preserving order. This + is only needed for tag creation.""" # Split the view line into exactly two words. P4 enforces # structure on these lines that simplifies this quite a bit. @@ -1785,76 +1836,62 @@ class View(object): depot_side = view_line[0:space_index] rhs_index = space_index + 1 - if view_line[rhs_index] == '"': - # Second word is double quoted. Make sure there is a - # double quote at the end too. - if not view_line.endswith('"'): - die("View line with rhs quote should end with one: %s" % - view_line) - # skip the quotes - client_side = view_line[rhs_index+1:-1] - else: - client_side = view_line[rhs_index:] - # prefix + means overlay on previous mapping - overlay = False if depot_side.startswith("+"): - overlay = True depot_side = depot_side[1:] - # prefix - means exclude this path + # prefix - means exclude this path, leave out of mappings exclude = False if depot_side.startswith("-"): exclude = True depot_side = depot_side[1:] - m = View.Mapping(depot_side, client_side, overlay, exclude) - self.mappings.append(m) + if not exclude: + self.mappings.append(depot_side) - def map_in_client(self, depot_path): - """Return the relative location in the client where this - depot file should live. Returns "" if the file should - not be mapped in the client.""" + def convert_client_path(self, clientFile): + # chop off //client/ part to make it relative + if not clientFile.startswith(self.client_prefix): + die("No prefix '%s' on clientFile '%s'" % + (self.client_prefix, clientFile)) + return clientFile[len(self.client_prefix):] - paths_filled = [] - client_path = "" + def update_client_spec_path_cache(self, files): + """ Caching file paths by "p4 where" batch query """ - # look at later entries first - for m in self.mappings[::-1]: + # List depot file paths exclude that already cached + fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache] - # see where will this path end up in the client - p = m.map_depot_to_client(depot_path) + if len(fileArgs) == 0: + return # All files in cache + + where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs) + for res in where_result: + if "code" in res and res["code"] == "error": + # assume error is "... file(s) not in client view" + continue + if "clientFile" not in res: + die("No clientFile in 'p4 where' output") + if "unmap" in res: + # it will list all of them, but only one not unmap-ped + continue + self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"]) - if p == "": - # Depot path does not belong in client. Must remember - # this, as previous items should not cause files to - # exist in this path either. Remember that the list is - # being walked from the end, which has higher precedence. - # Overlap mappings do not exclude previous mappings. - if not m.overlay: - paths_filled.append(m.client_side) + # not found files or unmap files set to "" + for depotFile in fileArgs: + if depotFile not in self.client_spec_path_cache: + self.client_spec_path_cache[depotFile] = "" - else: - # This mapping matched; no need to search any further. - # But, the mapping could be rejected if the client path - # has already been claimed by an earlier mapping (i.e. - # one later in the list, which we are walking backwards). - already_mapped_in_client = False - for f in paths_filled: - # this is View.Path.match - if f.match(p): - already_mapped_in_client = True - break - if not already_mapped_in_client: - # Include this file, unless it is from a line that - # explicitly said to exclude it. - if not m.exclude: - client_path = p + def map_in_client(self, depot_path): + """Return the relative location in the client where this + depot file should live. Returns "" if the file should + not be mapped in the client.""" - # a match, even if rejected, always stops the search - break + if depot_path in self.client_spec_path_cache: + return self.client_spec_path_cache[depot_path] - return client_path + die( "Error: %s is not found in client spec path" % depot_path ) + return "" class P4Sync(Command, P4UserMap): delete_actions = ( "delete", "move/delete", "purge" ) @@ -1897,7 +1934,6 @@ class P4Sync(Command, P4UserMap): self.syncWithOrigin = True self.importIntoRemotes = True self.maxChanges = "" - self.isWindows = (platform.system() == "Windows") self.keepRepoPath = False self.depotPaths = None self.p4BranchesInGit = [] @@ -1982,6 +2018,10 @@ class P4Sync(Command, P4UserMap): """Look at each depotFile in the commit to figure out to what branch it belongs.""" + if self.clientSpecDirs: + files = self.extractFilesFromCommit(commit) + self.clientSpecDirs.update_client_spec_path_cache(files) + branches = {} fnum = 0 while commit.has_key("depotFile%s" % fnum): @@ -2032,9 +2072,20 @@ class P4Sync(Command, P4UserMap): git_mode = "100755" if type_base == "symlink": git_mode = "120000" - # p4 print on a symlink contains "target\n"; remove the newline + # p4 print on a symlink sometimes contains "target\n"; + # if it does, remove the newline data = ''.join(contents) - contents = [data[:-1]] + if not data: + # Some version of p4 allowed creating a symlink that pointed + # to nothing. This causes p4 errors when checking out such + # a change, and errors here too. Work around it by ignoring + # the bad symlink; hopefully a future change fixes it. + print "\nIgnoring empty symlink in %s" % file['depotFile'] + return + elif data[-1] == '\n': + contents = [data[:-1]] + else: + contents = [data] if type_base == "utf16": # p4 delivers different text in the python output to -G @@ -2042,7 +2093,14 @@ class P4Sync(Command, P4UserMap): # operations. utf16 is converted to ascii or utf8, perhaps. # But ascii text saved as -t utf16 is completely mangled. # Invoke print -o to get the real contents. + # + # On windows, the newlines will always be mangled by print, so put + # them back too. This is not needed to the cygwin windows version, + # just the native "NT" type. + # text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']]) + if p4_version_string().find("/NT") >= 0: + text = text.replace("\r\n", "\n") contents = [ text ] if type_base == "apple": @@ -2058,15 +2116,6 @@ class P4Sync(Command, P4UserMap): print "\nIgnoring apple filetype file %s" % file['depotFile'] return - # Perhaps windows wants unicode, utf16 newlines translated too; - # but this is not doing it. - if self.isWindows and type_base == "text": - mangled = [] - for data in contents: - data = data.replace("\r\n", "\n") - mangled.append(data) - contents = mangled - # Note that we do not try to de-mangle keywords on utf16 files, # even though in theory somebody may want that. pattern = p4_keywords_regexp_for_type(type_base, type_mods) @@ -2097,6 +2146,29 @@ class P4Sync(Command, P4UserMap): # handle another chunk of streaming data def streamP4FilesCb(self, marshalled): + # catch p4 errors and complain + err = None + if "code" in marshalled: + if marshalled["code"] == "error": + if "data" in marshalled: + err = marshalled["data"].rstrip() + if err: + f = None + if self.stream_have_file_info: + if "depotFile" in self.stream_file: + f = self.stream_file["depotFile"] + # force a failure in fast-import, else an empty + # commit will be made + self.gitStream.write("\n") + self.gitStream.write("die-now\n") + self.gitStream.close() + # ignore errors, but make sure it exits first + self.importProcess.wait() + if f: + die("Error from p4 print for %s: %s" % (f, err)) + else: + die("Error from p4 print: %s" % err) + if marshalled.has_key('depotFile') and self.stream_have_file_info: # start of a new file - output the old one first self.streamOneP4File(self.stream_file, self.stream_contents) @@ -2210,6 +2282,9 @@ class P4Sync(Command, P4UserMap): else: sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path']) + if self.clientSpecDirs: + self.clientSpecDirs.update_client_spec_path_cache(files) + self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) @@ -2341,7 +2416,7 @@ class P4Sync(Command, P4UserMap): try: tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S") except ValueError: - print "Could not convert label time %s" % labelDetail['Update'] + print "Could not convert label time %s" % labelDetails['Update'] tmwhen = 1 when = int(time.mktime(tmwhen)) @@ -2438,13 +2513,6 @@ class P4Sync(Command, P4UserMap): branch = branch[len(self.projectName):] self.knownBranches[branch] = branch - def listExistingP4GitBranches(self): - # branches holds mapping from name to commit - branches = p4BranchesInGit(self.importIntoRemotes) - self.p4BranchesInGit = branches.keys() - for branch in branches.keys(): - self.initialParents[self.refPrefix + branch] = branches[branch] - def updateOptionDict(self, d): option_keys = {} if self.keepRepoPath: @@ -2528,7 +2596,8 @@ class P4Sync(Command, P4UserMap): def searchParent(self, parent, branch, target): parentFound = False - for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]): + for blob in read_pipe_lines(["git", "rev-list", "--reverse", + "--no-merges", parent]): blob = blob.strip() if len(read_pipe(["git", "diff-tree", blob, target])) == 0: parentFound = True @@ -2543,7 +2612,7 @@ class P4Sync(Command, P4UserMap): def importChanges(self, changes): cnt = 1 for change in changes: - description = p4Cmd(["describe", str(change)]) + description = p4_describe(change) self.updateOptionDict(description) if not self.silent: @@ -2599,7 +2668,7 @@ class P4Sync(Command, P4UserMap): blob = None if len(parent) > 0: - tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change)) + tempBranch = "%s/%d" % (self.tempBranchLocation, change) if self.verbose: print "Creating temporary branch: " + tempBranch self.commit(description, filesForCommit, tempBranch) @@ -2616,6 +2685,7 @@ class P4Sync(Command, P4UserMap): files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.initialParent) + # only needed once, to connect to the previous commit self.initialParent = "" except IOError: print self.gitError.read() @@ -2667,14 +2737,8 @@ class P4Sync(Command, P4UserMap): # Use time from top-most change so that all git p4 clones of # the same p4 repo have the same commit SHA1s. - res = p4CmdList("describe -s %d" % newestRevision) - newestTime = None - for r in res: - if r.has_key('time'): - newestTime = int(r['time']) - if newestTime is None: - die("\"describe -s\" on newest change %d did not give a time") - details["time"] = newestTime + res = p4_describe(newestRevision) + details["time"] = res["time"] self.updateOptionDict(details) try: @@ -2687,41 +2751,38 @@ class P4Sync(Command, P4UserMap): def run(self, args): self.depotPaths = [] self.changeRange = "" - self.initialParent = "" self.previousDepotPaths = [] + self.hasOrigin = False # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} - self.hasOrigin = originP4BranchesExist() - if not self.syncWithOrigin: - self.hasOrigin = False if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: self.refPrefix = "refs/heads/p4/" - if self.syncWithOrigin and self.hasOrigin: - if not self.silent: - print "Syncing with origin first by calling git fetch origin" - system("git fetch origin") + if self.syncWithOrigin: + self.hasOrigin = originP4BranchesExist() + if self.hasOrigin: + if not self.silent: + print 'Syncing with origin first, using "git fetch origin"' + system("git fetch origin") + branch_arg_given = bool(self.branch) if len(self.branch) == 0: self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: system("git update-ref %s refs/heads/p4" % self.branch) - system("git branch -D p4"); - # create it /after/ importing, when master exists - if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch): - system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) + system("git branch -D p4") # accept either the command-line option, or the configuration variable if self.useClientSpec: # will use this after clone to set the variable self.useClientSpec_from_options = True else: - if gitConfig("git-p4.useclientspec", "--bool") == "true": + if gitConfigBool("git-p4.useclientspec"): self.useClientSpec = True if self.useClientSpec: self.clientSpecDirs = getClientSpec() @@ -2731,12 +2792,25 @@ class P4Sync(Command, P4UserMap): if args == []: if self.hasOrigin: createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent) - self.listExistingP4GitBranches() + + # branches holds mapping from branch name to sha1 + branches = p4BranchesInGit(self.importIntoRemotes) + + # restrict to just this one, disabling detect-branches + if branch_arg_given: + short = self.branch.split("/")[-1] + if short in branches: + self.p4BranchesInGit = [ short ] + else: + self.p4BranchesInGit = branches.keys() if len(self.p4BranchesInGit) > 1: if not self.silent: print "Importing from/into multiple branches" self.detectBranches = True + for branch in branches.keys(): + self.initialParents[self.refPrefix + branch] = \ + branches[branch] if self.verbose: print "branches: %s" % self.p4BranchesInGit @@ -2773,13 +2847,21 @@ class P4Sync(Command, P4UserMap): if p4Change > 0: self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change - if not self.detectBranches: - self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch + # accept multiple ref name abbreviations: + # refs/foo/bar/branch -> use it exactly + # p4/branch -> prepend refs/remotes/ or refs/heads/ + # branch -> prepend refs/remotes/p4/ or refs/heads/p4/ if not self.branch.startswith("refs/"): - self.branch = "refs/heads/" + self.branch + if self.importIntoRemotes: + prepend = "refs/remotes/" + else: + prepend = "refs/heads/" + if not self.branch.startswith("p4/"): + prepend += "p4/" + self.branch = prepend + self.branch if len(args) == 0 and self.depotPaths: if not self.silent: @@ -2864,12 +2946,13 @@ class P4Sync(Command, P4UserMap): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) - importProcess = subprocess.Popen(["git", "fast-import"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE); - self.gitOutput = importProcess.stdout - self.gitStream = importProcess.stdin - self.gitError = importProcess.stderr + self.importProcess = subprocess.Popen(["git", "fast-import"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE); + self.gitOutput = self.importProcess.stdout + self.gitStream = self.importProcess.stdin + self.gitError = self.importProcess.stderr if revision: self.importHeadRevision(revision) @@ -2889,8 +2972,21 @@ class P4Sync(Command, P4UserMap): else: # catch "git p4 sync" with no new branches, in a repo that # does not have any existing p4 branches - if len(args) == 0 and not self.p4BranchesInGit: - die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here."); + if len(args) == 0: + if not self.p4BranchesInGit: + die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.") + + # The default branch is master, unless --branch is used to + # specify something else. Make sure it exists, or complain + # nicely about how to use --branch. + if not self.detectBranches: + if not branch_exists(self.branch): + if branch_arg_given: + die("Error: branch %s does not exist." % self.branch) + else: + die("Error: no branch %s; perhaps specify one with --branch." % + self.branch) + if self.verbose: print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) @@ -2908,6 +3004,14 @@ class P4Sync(Command, P4UserMap): self.updatedBranches = set() + if not self.detectBranches: + if args: + # start a new branch + self.initialParent = "" + else: + # build on a previous revision + self.initialParent = parseRevision(self.branch) + self.importChanges(changes) if not self.silent: @@ -2918,7 +3022,7 @@ class P4Sync(Command, P4UserMap): sys.stdout.write("%s " % b) sys.stdout.write("\n") - if gitConfig("git-p4.importLabels", "--bool") == "true": + if gitConfigBool("git-p4.importLabels"): self.importLabels = True if self.importLabels: @@ -2929,7 +3033,7 @@ class P4Sync(Command, P4UserMap): self.importP4Labels(self.gitStream, missingP4Labels) self.gitStream.close() - if importProcess.wait() != 0: + if self.importProcess.wait() != 0: die("fast-import failed: %s" % self.gitError.read()) self.gitOutput.close() self.gitError.close() @@ -2940,6 +3044,13 @@ class P4Sync(Command, P4UserMap): read_pipe("git update-ref -d %s" % branch) os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation)) + # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow + # a convenient shortcut refname "p4". + if self.importIntoRemotes: + head_ref = self.refPrefix + "HEAD" + if not gitBranchExists(head_ref) and gitBranchExists(self.branch): + system(["git", "symbolic-ref", head_ref, self.branch]) + return True class P4Rebase(Command): @@ -2963,7 +3074,7 @@ class P4Rebase(Command): if os.system("git update-index --refresh") != 0: die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash."); if len(read_pipe("git diff-index HEAD --")) > 0: - die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash."); + die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash."); [upstream, settings] = findUpstreamBranchPoint() if len(upstream) == 0: @@ -3029,6 +3140,7 @@ class P4Clone(P4Sync): self.cloneExclude = ["/"+p for p in self.cloneExclude] for p in depotPaths: if not p.startswith("//"): + sys.stderr.write('Depot paths must start with "//": %s\n' % p) return False if not self.cloneDestination: @@ -3043,21 +3155,21 @@ class P4Clone(P4Sync): init_cmd = [ "git", "init" ] if self.cloneBare: init_cmd.append("--bare") - subprocess.check_call(init_cmd) + retcode = subprocess.call(init_cmd) + if retcode: + raise CalledProcessError(retcode, init_cmd) if not P4Sync.run(self, depotPaths): return False - if self.branch != "master": - if self.importIntoRemotes: - masterbranch = "refs/remotes/p4/master" - else: - masterbranch = "refs/heads/p4/master" - if gitBranchExists(masterbranch): - system("git branch master %s" % masterbranch) - if not self.cloneBare: - system("git checkout -f") - else: - print "Could not detect main branch. No checkout/master branch created." + + # create a master branch and check out a work tree + if gitBranchExists(self.branch): + system([ "git", "branch", "master", self.branch ]) + if not self.cloneBare: + system([ "git", "checkout", "-f" ]) + else: + print 'Not checking out any branch, use ' \ + '"git checkout -q -b master <branch>"' # auto-set this variable if invoked with --use-client-spec if self.useClientSpec_from_options: @@ -3128,7 +3240,6 @@ def main(): printUsage(commands.keys()) sys.exit(2) - cmd = "" cmdName = sys.argv[1] try: klass = commands[cmdName] |