diff options
Diffstat (limited to 'contrib/fast-import')
| -rwxr-xr-x | contrib/fast-import/git-p4 | 835 | ||||
| -rw-r--r-- | contrib/fast-import/git-p4.txt | 251 |
2 files changed, 624 insertions, 462 deletions
diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 98d2aee67f..9ccc87b205 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -22,37 +22,41 @@ def p4_build_cmd(cmd): location. It means that hooking into the environment, or other configuration can be done more easily. """ - real_cmd = "%s " % "p4" + real_cmd = ["p4"] user = gitConfig("git-p4.user") if len(user) > 0: - real_cmd += "-u %s " % user + real_cmd += ["-u",user] password = gitConfig("git-p4.password") if len(password) > 0: - real_cmd += "-P %s " % password + real_cmd += ["-P", password] port = gitConfig("git-p4.port") if len(port) > 0: - real_cmd += "-p %s " % port + real_cmd += ["-p", port] host = gitConfig("git-p4.host") if len(host) > 0: - real_cmd += "-h %s " % host + real_cmd += ["-h", host] client = gitConfig("git-p4.client") if len(client) > 0: - real_cmd += "-c %s " % client + real_cmd += ["-c", client] - real_cmd += "%s" % (cmd) - if verbose: - print real_cmd + + if isinstance(cmd,basestring): + real_cmd = ' '.join(real_cmd) + ' ' + cmd + else: + real_cmd += cmd return real_cmd def chdir(dir): - if os.name == 'nt': - os.environ['PWD']=dir + # P4 uses the PWD environment variable rather than getcwd(). Since we're + # not using the shell, we have to set it ourselves. This path could + # be relative, so go there first, then figure out where we ended up. os.chdir(dir) + os.environ['PWD'] = os.getcwd() def die(msg): if verbose: @@ -61,29 +65,34 @@ def die(msg): sys.stderr.write(msg + "\n") sys.exit(1) -def write_pipe(c, str): +def write_pipe(c, stdin): if verbose: - sys.stderr.write('Writing pipe: %s\n' % c) + sys.stderr.write('Writing pipe: %s\n' % str(c)) - pipe = os.popen(c, 'w') - val = pipe.write(str) - if pipe.close(): - die('Command failed: %s' % c) + expand = isinstance(c,basestring) + p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) + pipe = p.stdin + val = pipe.write(stdin) + pipe.close() + if p.wait(): + die('Command failed: %s' % str(c)) return val -def p4_write_pipe(c, str): +def p4_write_pipe(c, stdin): real_cmd = p4_build_cmd(c) - return write_pipe(real_cmd, str) + return write_pipe(real_cmd, stdin) def read_pipe(c, ignore_error=False): if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % str(c)) - pipe = os.popen(c, 'rb') + expand = isinstance(c,basestring) + p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) + pipe = p.stdout val = pipe.read() - if pipe.close() and not ignore_error: - die('Command failed: %s' % c) + if p.wait() and not ignore_error: + die('Command failed: %s' % str(c)) return val @@ -93,12 +102,14 @@ def p4_read_pipe(c, ignore_error=False): def read_pipe_lines(c): if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) - ## todo: check return status - pipe = os.popen(c, 'rb') + sys.stderr.write('Reading pipe: %s\n' % str(c)) + + expand = isinstance(c, basestring) + p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) + pipe = p.stdout val = pipe.readlines() - if pipe.close(): - die('Command failed: %s' % c) + if pipe.close() or p.wait(): + die('Command failed: %s' % str(c)) return val @@ -108,23 +119,73 @@ def p4_read_pipe_lines(c): return read_pipe_lines(real_cmd) def system(cmd): + expand = isinstance(cmd,basestring) if verbose: - sys.stderr.write("executing %s\n" % cmd) - if os.system(cmd) != 0: - die("command failed: %s" % cmd) + sys.stderr.write("executing %s\n" % str(cmd)) + subprocess.check_call(cmd, shell=expand) def p4_system(cmd): """Specifically invoke p4 as the system command. """ real_cmd = p4_build_cmd(cmd) - return system(real_cmd) + expand = isinstance(real_cmd, basestring) + subprocess.check_call(real_cmd, shell=expand) + +def p4_integrate(src, dest): + p4_system(["integrate", "-Dt", src, dest]) + +def p4_sync(path): + p4_system(["sync", path]) + +def p4_add(f): + p4_system(["add", f]) -def isP4Exec(kind): - """Determine if a Perforce 'kind' should have execute permission +def p4_delete(f): + p4_system(["delete", f]) + +def p4_edit(f): + p4_system(["edit", f]) + +def p4_revert(f): + p4_system(["revert", f]) + +def p4_reopen(type, file): + p4_system(["reopen", "-t", type, file]) + +# +# Canonicalize the p4 type and return a tuple of the +# base type, plus any modifiers. See "p4 help filetypes" +# for a list and explanation. +# +def split_p4_type(p4type): + + p4_filetypes_historical = { + "ctempobj": "binary+Sw", + "ctext": "text+C", + "cxtext": "text+Cx", + "ktext": "text+k", + "kxtext": "text+kx", + "ltext": "text+F", + "tempobj": "binary+FSw", + "ubinary": "binary+F", + "uresource": "resource+F", + "uxbinary": "binary+Fx", + "xbinary": "binary+x", + "xltext": "text+Fx", + "xtempobj": "binary+Swx", + "xtext": "text+x", + "xunicode": "unicode+x", + "xutf16": "utf16+x", + } + if p4type in p4_filetypes_historical: + p4type = p4_filetypes_historical[p4type] + mods = "" + s = p4type.split("+") + base = s[0] + mods = "" + if len(s) > 1: + mods = s[1] + return (base, mods) - 'p4 help filetypes' gives a list of the types. If it starts with 'x', - or x follows one of a few letters. Otherwise, if there is an 'x' after - a plus sign, it is also executable""" - return (re.search(r"(^[cku]?x)|\+.*x", kind) != None) def setP4ExecBit(file, mode): # Reopens an already open file and changes the execute bit to match @@ -139,12 +200,12 @@ def setP4ExecBit(file, mode): if p4Type[-1] == "+": p4Type = p4Type[0:-1] - p4_system("reopen -t %s %s" % (p4Type, file)) + p4_reopen(p4Type, file) def getP4OpenedType(file): # Returns the perforce file type for the given file. - result = p4_read_pipe("opened %s" % file) + result = p4_read_pipe(["opened", file]) match = re.match(".*\((.+)\)\r?$", result) if match: return match.group(1) @@ -200,9 +261,17 @@ def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): - cmd = p4_build_cmd("-G %s" % (cmd)) + + if isinstance(cmd,basestring): + cmd = "-G " + cmd + expand = True + else: + cmd = ["-G"] + cmd + expand = False + + cmd = p4_build_cmd(cmd) if verbose: - sys.stderr.write("Opening pipe: %s\n" % cmd) + sys.stderr.write("Opening pipe: %s\n" % str(cmd)) # Use a temporary file to avoid deadlocks without # subprocess.communicate(), which would put another copy @@ -210,11 +279,16 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): stdin_file = None if stdin is not None: stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) - stdin_file.write(stdin) + if isinstance(stdin,basestring): + stdin_file.write(stdin) + else: + for i in stdin: + stdin_file.write(i + '\n') stdin_file.flush() stdin_file.seek(0) - p4 = subprocess.Popen(cmd, shell=True, + p4 = subprocess.Popen(cmd, + shell=expand, stdin=stdin_file, stdout=subprocess.PIPE) @@ -247,7 +321,7 @@ def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" depotPath = depotPath + "..." - outputList = p4CmdList("where %s" % depotPath) + outputList = p4CmdList(["where", depotPath]) output = None for entry in outputList: if "depotFile" in entry: @@ -288,6 +362,11 @@ def isValidGitDir(path): def parseRevision(ref): return read_pipe("git rev-parse %s" % ref).strip() +def branchExists(ref): + rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref], + ignore_error=True) + return len(rev) > 0 + def extractLogMessageFromGitCommit(commit): logMessage = "" @@ -342,6 +421,11 @@ def gitConfig(key, args = None): # set args to "--bool", for instance _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip() return _gitConfig[key] +def gitConfigList(key): + if not _gitConfig.has_key(key): + _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep) + return _gitConfig[key] + def p4BranchesInGit(branchesAreInRemotes = True): branches = {} @@ -444,8 +528,10 @@ def originP4BranchesExist(): def p4ChangesForPaths(depotPaths, changeRange): assert depotPaths - output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange) - for p in depotPaths])) + cmd = ['changes'] + for p in depotPaths: + cmd += ["%s...%s" % (p, changeRange)] + output = p4_read_pipe_lines(cmd) changes = {} for line in output: @@ -469,6 +555,46 @@ def p4PathStartsWith(path, prefix): return path.lower().startswith(prefix.lower()) return path.startswith(prefix) +def getClientSpec(): + """Look at the p4 client spec, create a View() object that contains + all the mappings, and return it.""" + + specList = p4CmdList("client -o") + if len(specList) != 1: + die('Output from "client -o" is %d lines, expecting 1' % + len(specList)) + + # dictionary of all client parameters + entry = specList[0] + + # just the keys that start with "View" + view_keys = [ k for k in entry.keys() if k.startswith("View") ] + + # hold this new View + view = View() + + # append the lines, in order, to the view + for view_num in range(len(view_keys)): + k = "View%d" % view_num + if k not in view_keys: + die("Expected view key %s missing" % k) + view.append(entry[k]) + + return view + +def getClientRoot(): + """Grab the client directory.""" + + output = p4CmdList("client -o") + if len(output) != 1: + die('Output from "client -o" is %d lines, expecting 1' % len(output)) + + entry = output[0] + if "Root" not in entry: + die('Client has no "Root"') + + return entry["Root"] + class Command: def __init__(self): self.usage = "usage: %prog [options]" @@ -528,7 +654,7 @@ class P4Debug(Command): def run(self, args): j = 0 - for output in p4CmdList(" ".join(args)): + for output in p4CmdList(args): print 'Element: %d' % j j += 1 print output @@ -682,7 +808,7 @@ class P4Submit(Command, P4UserMap): break if not client: die("could not get client spec") - results = p4CmdList("changes -c %s -m 1" % client) + results = p4CmdList(["changes", "-c", client, "-m", "1"]) for r in results: if r.has_key('change'): return r['change'] @@ -745,7 +871,7 @@ class P4Submit(Command, P4UserMap): # remove lines in the Files section that show changes to files outside the depot path we're committing into template = "" inFilesSection = False - for line in p4_read_pipe_lines("change -o"): + for line in p4_read_pipe_lines(['change', '-o']): if line.endswith("\r\n"): line = line[:-2] + "\n" if inFilesSection: @@ -767,6 +893,41 @@ class P4Submit(Command, P4UserMap): return template + def edit_template(self, template_file): + """Invoke the editor to let the user change the submission + message. Return true if okay to continue with the submit.""" + + # if configured to skip the editing part, just submit + if gitConfig("git-p4.skipSubmitEdit") == "true": + return True + + # look at the modification time, to check later if the user saved + # the file + mtime = os.stat(template_file).st_mtime + + # invoke the editor + if os.environ.has_key("P4EDITOR"): + editor = os.environ.get("P4EDITOR") + else: + editor = read_pipe("git var GIT_EDITOR").strip() + system(editor + " " + template_file) + + # If the file was not saved, prompt to see if this patch should + # be skipped. But skip this verification step if configured so. + if gitConfig("git-p4.skipSubmitEditCheck") == "true": + return True + + # modification time updated means user saved the file + if os.stat(template_file).st_mtime > mtime: + return True + + while True: + response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") + if response == 'y': + return True + if response == 'n': + return False + def applyCommit(self, id): print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id)) @@ -774,17 +935,22 @@ class P4Submit(Command, P4UserMap): if not self.detectRenames: # If not explicitly set check the config variable - self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true" + self.detectRenames = gitConfig("git-p4.detectRenames") - if self.detectRenames: + if self.detectRenames.lower() == "false" or self.detectRenames == "": + diffOpts = "" + elif self.detectRenames.lower() == "true": diffOpts = "-M" else: - diffOpts = "" + diffOpts = "-M%s" % self.detectRenames - if gitConfig("git-p4.detectCopies").lower() == "true": + detectCopies = gitConfig("git-p4.detectCopies") + if detectCopies.lower() == "true": diffOpts += " -C" + elif detectCopies != "" and detectCopies.lower() != "false": + diffOpts += " -C%s" % detectCopies - if gitConfig("git-p4.detectCopiesHarder").lower() == "true": + if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true": diffOpts += " --find-copies-harder" diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id)) @@ -797,7 +963,7 @@ class P4Submit(Command, P4UserMap): modifier = diff['status'] path = diff['src'] if modifier == "M": - p4_system("edit \"%s\"" % path) + p4_edit(path) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): filesToChangeExecBit[path] = diff['dst_mode'] editedFiles.add(path) @@ -812,21 +978,21 @@ class P4Submit(Command, P4UserMap): filesToAdd.remove(path) elif modifier == "C": src, dest = diff['src'], diff['dst'] - p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) + p4_integrate(src, dest) if diff['src_sha1'] != diff['dst_sha1']: - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) filesToChangeExecBit[dest] = diff['dst_mode'] os.unlink(dest) editedFiles.add(dest) elif modifier == "R": src, dest = diff['src'], diff['dst'] - p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) + p4_integrate(src, dest) if diff['src_sha1'] != diff['dst_sha1']: - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) filesToChangeExecBit[dest] = diff['dst_mode'] os.unlink(dest) editedFiles.add(dest) @@ -849,9 +1015,9 @@ class P4Submit(Command, P4UserMap): if response == "s": print "Skipping! Good luck with the next patches..." for f in editedFiles: - p4_system("revert \"%s\"" % f); + p4_revert(f) for f in filesToAdd: - system("rm %s" %f) + os.remove(f) return elif response == "a": os.system(applyPatchCmd) @@ -872,10 +1038,10 @@ class P4Submit(Command, P4UserMap): system(applyPatchCmd) for f in filesToAdd: - p4_system("add \"%s\"" % f) + p4_add(f) for f in filesToDelete: - p4_system("revert \"%s\"" % f) - p4_system("delete \"%s\"" % f) + p4_revert(f) + p4_delete(f) # Set/clear executable bits for f in filesToChangeExecBit.keys(): @@ -897,7 +1063,7 @@ class P4Submit(Command, P4UserMap): del(os.environ["P4DIFF"]) diff = "" for editedFile in editedFiles: - diff += p4_read_pipe("diff -du %r" % editedFile) + diff += p4_read_pipe(['diff', '-du', editedFile]) newdiff = "" for newFile in filesToAdd: @@ -916,7 +1082,7 @@ class P4Submit(Command, P4UserMap): separatorLine = "######## everything below this line is just the diff #######\n" - [handle, fileName] = tempfile.mkstemp() + (handle, fileName) = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+") if self.isWindows: submitTemplate = submitTemplate.replace("\n", "\r\n") @@ -924,46 +1090,32 @@ class P4Submit(Command, P4UserMap): newdiff = newdiff.replace("\n", "\r\n") tmpFile.write(submitTemplate + separatorLine + diff + newdiff) tmpFile.close() - mtime = os.stat(fileName).st_mtime - if os.environ.has_key("P4EDITOR"): - editor = os.environ.get("P4EDITOR") - else: - editor = read_pipe("git var GIT_EDITOR").strip() - system(editor + " " + fileName) - - if gitConfig("git-p4.skipSubmitEditCheck") == "true": - checkModTime = False - else: - checkModTime = True - response = "y" - if checkModTime and (os.stat(fileName).st_mtime <= mtime): - response = "x" - while response != "y" and response != "n": - response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") - - if response == "y": + if self.edit_template(fileName): + # read the edited message and submit tmpFile = open(fileName, "rb") message = tmpFile.read() tmpFile.close() submitTemplate = message[:message.index(separatorLine)] if self.isWindows: submitTemplate = submitTemplate.replace("\r\n", "\n") - p4_write_pipe("submit -i", submitTemplate) + p4_write_pipe(['submit', '-i'], submitTemplate) if self.preserveUser: if p4User: # Get last changelist number. Cannot easily get it from - # the submit command output as the output is unmarshalled. + # the submit command output as the output is + # unmarshalled. changelist = self.lastP4Changelist() self.modifyChangelistUser(changelist, p4User) - else: + # skip this patch + print "Submission cancelled, undoing p4 changes." for f in editedFiles: - p4_system("revert \"%s\"" % f); + p4_revert(f) for f in filesToAdd: - p4_system("revert \"%s\"" % f); - system("rm %s" %f) + p4_revert(f) + os.remove(f) os.remove(fileName) else: @@ -982,6 +1134,8 @@ class P4Submit(Command, P4UserMap): die("Detecting current git branch failed!") elif len(args) == 1: self.master = args[0] + if not branchExists(self.master): + die("Branch %s does not exist" % self.master) else: return False @@ -1005,19 +1159,31 @@ class P4Submit(Command, P4UserMap): print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) - self.clientPath = p4Where(self.depotPath) + self.useClientSpec = False + if gitConfig("git-p4.useclientspec", "--bool") == "true": + self.useClientSpec = True + if self.useClientSpec: + self.clientSpecDirs = getClientSpec() + + if self.useClientSpec: + # all files are relative to the client spec + self.clientPath = getClientRoot() + else: + self.clientPath = p4Where(self.depotPath) - if len(self.clientPath) == 0: - print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath - sys.exit(128) + if self.clientPath == "": + die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath) print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath) self.oldWorkingDirectory = os.getcwd() + # ensure the clientPath exists + if not os.path.exists(self.clientPath): + os.makedirs(self.clientPath) + chdir(self.clientPath) print "Synchronizing p4 checkout..." - p4_system("sync ...") - + p4_sync("...") self.check() commits = [] @@ -1052,6 +1218,218 @@ class P4Submit(Command, P4UserMap): return True +class View(object): + """Represent a p4 view ("p4 help views"), and map files in a + repo according to the view.""" + + class Path(object): + """A depot or client path, possibly containing wildcards. + The only one supported is ... at the end, currently. + Initialize with the full path, with //depot or //client.""" + + def __init__(self, path, is_depot): + self.path = path + self.is_depot = is_depot + self.find_wildcards() + # remember the prefix bit, useful for relative mappings + m = re.match("(//[^/]+/)", self.path) + if not m: + die("Path %s does not start with //prefix/" % self.path) + prefix = m.group(1) + if not self.is_depot: + # strip //client/ on client paths + self.path = self.path[len(prefix):] + + def find_wildcards(self): + """Make sure wildcards are valid, and set up internal + variables.""" + + self.ends_triple_dot = False + # There are three wildcards allowed in p4 views + # (see "p4 help views"). This code knows how to + # handle "..." (only at the end), but cannot deal with + # "%%n" or "*". Only check the depot_side, as p4 should + # validate that the client_side matches too. + if re.search(r'%%[1-9]', self.path): + die("Can't handle %%n wildcards in view: %s" % self.path) + if self.path.find("*") >= 0: + die("Can't handle * wildcards in view: %s" % self.path) + triple_dot_index = self.path.find("...") + if triple_dot_index >= 0: + if not self.path.endswith("..."): + die("Can handle ... wildcard only at end of path: %s" % + self.path) + self.ends_triple_dot = True + + def ensure_compatible(self, other_path): + """Make sure the wildcards agree.""" + if self.ends_triple_dot != other_path.ends_triple_dot: + die("Both paths must end with ... if either does;\n" + + "paths: %s %s" % (self.path, other_path.path)) + + def match_wildcards(self, test_path): + """See if this test_path matches us, and fill in the value + of the wildcards if so. Returns a tuple of + (True|False, wildcards[]). For now, only the ... at end + is supported, so at most one wildcard.""" + if self.ends_triple_dot: + dotless = self.path[:-3] + if test_path.startswith(dotless): + wildcard = test_path[len(dotless):] + return (True, [ wildcard ]) + else: + if test_path == self.path: + return (True, []) + return (False, []) + + def match(self, test_path): + """Just return if it matches; don't bother with the wildcards.""" + b, _ = self.match_wildcards(test_path) + return b + + def fill_in_wildcards(self, wildcards): + """Return the relative path, with the wildcards filled in + if there are any.""" + if self.ends_triple_dot: + return self.path[:-3] + wildcards[0] + else: + return self.path + + class Mapping(object): + def __init__(self, depot_side, client_side, overlay, exclude): + # depot_side is without the trailing /... if it had one + self.depot_side = View.Path(depot_side, is_depot=True) + self.client_side = View.Path(client_side, is_depot=False) + self.overlay = overlay # started with "+" + self.exclude = exclude # started with "-" + assert not (self.overlay and self.exclude) + self.depot_side.ensure_compatible(self.client_side) + + def __str__(self): + c = " " + if self.overlay: + c = "+" + if self.exclude: + c = "-" + return "View.Mapping: %s%s -> %s" % \ + (c, self.depot_side, self.client_side) + + def map_depot_to_client(self, depot_path): + """Calculate the client path if using this mapping on the + given depot path; does not consider the effect of other + mappings in a view. Even excluded mappings are returned.""" + matches, wildcards = self.depot_side.match_wildcards(depot_path) + if not matches: + return "" + client_path = self.client_side.fill_in_wildcards(wildcards) + return client_path + + # + # View methods + # + def __init__(self): + self.mappings = [] + + def append(self, view_line): + """Parse a view line, splitting it into depot and client + sides. Append to self.mappings, preserving order.""" + + # Split the view line into exactly two words. P4 enforces + # structure on these lines that simplifies this quite a bit. + # + # Either or both words may be double-quoted. + # Single quotes do not matter. + # Double-quote marks cannot occur inside the words. + # A + or - prefix is also inside the quotes. + # There are no quotes unless they contain a space. + # The line is already white-space stripped. + # The two words are separated by a single space. + # + if view_line[0] == '"': + # First word is double quoted. Find its end. + close_quote_index = view_line.find('"', 1) + if close_quote_index <= 0: + die("No first-word closing quote found: %s" % view_line) + depot_side = view_line[1:close_quote_index] + # skip closing quote and space + rhs_index = close_quote_index + 1 + 1 + else: + space_index = view_line.find(" ") + if space_index <= 0: + die("No word-splitting space found: %s" % view_line) + depot_side = view_line[0:space_index] + rhs_index = space_index + 1 + + if view_line[rhs_index] == '"': + # Second word is double quoted. Make sure there is a + # double quote at the end too. + if not view_line.endswith('"'): + die("View line with rhs quote should end with one: %s" % + view_line) + # skip the quotes + client_side = view_line[rhs_index+1:-1] + else: + client_side = view_line[rhs_index:] + + # prefix + means overlay on previous mapping + overlay = False + if depot_side.startswith("+"): + overlay = True + depot_side = depot_side[1:] + + # prefix - means exclude this path + exclude = False + if depot_side.startswith("-"): + exclude = True + depot_side = depot_side[1:] + + m = View.Mapping(depot_side, client_side, overlay, exclude) + self.mappings.append(m) + + def map_in_client(self, depot_path): + """Return the relative location in the client where this + depot file should live. Returns "" if the file should + not be mapped in the client.""" + + paths_filled = [] + client_path = "" + + # look at later entries first + for m in self.mappings[::-1]: + + # see where will this path end up in the client + p = m.map_depot_to_client(depot_path) + + if p == "": + # Depot path does not belong in client. Must remember + # this, as previous items should not cause files to + # exist in this path either. Remember that the list is + # being walked from the end, which has higher precedence. + # Overlap mappings do not exclude previous mappings. + if not m.overlay: + paths_filled.append(m.client_side) + + else: + # This mapping matched; no need to search any further. + # But, the mapping could be rejected if the client path + # has already been claimed by an earlier mapping. + already_mapped_in_client = False + for f in paths_filled: + # this is View.Path.match + if f.match(p): + already_mapped_in_client = True + break + if not already_mapped_in_client: + # Include this file, unless it is from a line that + # explicitly said to exclude it. + if not m.exclude: + client_path = p + + # a match, even if rejected, always stops the search + break + + return client_path + class P4Sync(Command, P4UserMap): delete_actions = ( "delete", "move/delete", "purge" ) @@ -1099,7 +1477,8 @@ class P4Sync(Command, P4UserMap): self.p4BranchesInGit = [] self.cloneExclude = [] self.useClientSpec = False - self.clientSpecDirs = [] + self.useClientSpec_from_options = False + self.clientSpecDirs = None if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -1150,20 +1529,7 @@ class P4Sync(Command, P4UserMap): def stripRepoPath(self, path, prefixes): if self.useClientSpec: - - # if using the client spec, we use the output directory - # specified in the client. For example, a view - # //depot/foo/branch/... //client/branch/foo/... - # will end up putting all foo/branch files into - # branch/foo/ - for val in self.clientSpecDirs: - if path.startswith(val[0]): - # replace the depot path with the client path - path = path.replace(val[0], val[1][1]) - # now strip out the client (//client/...) - path = re.sub("^(//[^/]+/)", '', path) - # the rest is all path - return path + return self.clientSpecDirs.map_in_client(path) if self.keepRepoPath: prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])] @@ -1209,38 +1575,66 @@ class P4Sync(Command, P4UserMap): # - helper for streamP4Files def streamOneP4File(self, file, contents): - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring" % \ - file['depotFile'] - return - relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) relPath = self.wildcard_decode(relPath) if verbose: sys.stderr.write("%s\n" % relPath) - mode = "644" - if isP4Exec(file["type"]): - mode = "755" - elif file["type"] == "symlink": - mode = "120000" - # p4 print on a symlink contains "target\n", so strip it off + (type_base, type_mods) = split_p4_type(file["type"]) + + git_mode = "100644" + if "x" in type_mods: + git_mode = "100755" + if type_base == "symlink": + git_mode = "120000" + # p4 print on a symlink contains "target\n"; remove the newline data = ''.join(contents) contents = [data[:-1]] - if self.isWindows and file["type"].endswith("text"): + if type_base == "utf16": + # p4 delivers different text in the python output to -G + # than it does when using "print -o", or normal p4 client + # operations. utf16 is converted to ascii or utf8, perhaps. + # But ascii text saved as -t utf16 is completely mangled. + # Invoke print -o to get the real contents. + text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']]) + contents = [ text ] + + if type_base == "apple": + # Apple filetype files will be streamed as a concatenation of + # its appledouble header and the contents. This is useless + # on both macs and non-macs. If using "print -q -o xx", it + # will create "xx" with the data, and "%xx" with the header. + # This is also not very useful. + # + # Ideally, someday, this script can learn how to generate + # appledouble files directly and import those to git, but + # non-mac machines can never find a use for apple filetype. + print "\nIgnoring apple filetype file %s" % file['depotFile'] + return + + # Perhaps windows wants unicode, utf16 newlines translated too; + # but this is not doing it. + if self.isWindows and type_base == "text": mangled = [] for data in contents: data = data.replace("\r\n", "\n") mangled.append(data) contents = mangled - if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): - contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents) - elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): - contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents) + # Note that we do not try to de-mangle keywords on utf16 files, + # even though in theory somebody may want that. + if type_base in ("text", "unicode", "binary"): + if "ko" in type_mods: + text = ''.join(contents) + text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text) + contents = [ text ] + elif "k" in type_mods: + text = ''.join(contents) + text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text) + contents = [ text ] - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("M %s inline %s\n" % (git_mode, relPath)) # total length... length = 0 @@ -1285,19 +1679,17 @@ class P4Sync(Command, P4UserMap): filesToDelete = [] for f in files: - includeFile = True - for val in self.clientSpecDirs: - if f['path'].startswith(val[0]): - if val[1][0] <= 0: - includeFile = False - break + # if using a client spec, only add the files that have + # a path in the client + if self.clientSpecDirs: + if self.clientSpecDirs.map_in_client(f['path']) == "": + continue - if includeFile: - filesForCommit.append(f) - if f['action'] in self.delete_actions: - filesToDelete.append(f) - else: - filesToRead.append(f) + filesForCommit.append(f) + if f['action'] in self.delete_actions: + filesToDelete.append(f) + else: + filesToRead.append(f) # deleted files... for f in filesToDelete: @@ -1312,10 +1704,11 @@ class P4Sync(Command, P4UserMap): def streamP4FilesCbSelf(entry): self.streamP4FilesCb(entry) - p4CmdList("-x - print", - '\n'.join(['%s#%s' % (f['path'], f['rev']) - for f in filesToRead]), - cb=streamP4FilesCbSelf) + fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead] + + p4CmdList(["-x", "-", "print"], + stdin=fileArgs, + cb=streamP4FilesCbSelf) # do the last chunk if self.stream_file.has_key('depotFile'): @@ -1376,8 +1769,8 @@ class P4Sync(Command, P4UserMap): if self.verbose: print "Change %s is labelled %s" % (change, labelDetails) - files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change) - for p in branchPrefixes])) + files = p4CmdList(["files"] + ["%s...@%s" % (p, change) + for p in branchPrefixes]) if len(files) == len(labelRevisions): @@ -1425,9 +1818,9 @@ class P4Sync(Command, P4UserMap): newestChange = 0 if self.verbose: print "Querying files for label %s" % label - for file in p4CmdList("files " - + ' '.join (["%s...@%s" % (p, label) - for p in self.depotPaths])): + for file in p4CmdList(["files"] + + ["%s...@%s" % (p, label) + for p in self.depotPaths]): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: @@ -1450,7 +1843,13 @@ class P4Sync(Command, P4UserMap): def getBranchMapping(self): lostAndFoundBranches = set() - for info in p4CmdList("branches"): + user = gitConfig("git-p4.branchUser") + if len(user) > 0: + command = "branches -u %s" % user + else: + command = "branches" + + for info in p4CmdList(command): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 while details.has_key("View%s" % viewIdx): @@ -1479,6 +1878,25 @@ class P4Sync(Command, P4UserMap): if source not in self.knownBranches: lostAndFoundBranches.add(source) + # Perforce does not strictly require branches to be defined, so we also + # check git config for a branch list. + # + # Example of branch definition in git config file: + # [git-p4] + # branchList=main:branchA + # branchList=main:branchB + # branchList=branchA:branchC + configBranches = gitConfigList("git-p4.branchList") + for branch in configBranches: + if branch: + (source, destination) = branch.split(":") + self.knownBranches[destination] = source + + lostAndFoundBranches.discard(destination) + + if source not in self.knownBranches: + lostAndFoundBranches.add(source) + for branch in lostAndFoundBranches: self.knownBranches[branch] = branch @@ -1649,17 +2067,17 @@ class P4Sync(Command, P4UserMap): def importHeadRevision(self, revision): print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch) - details = { "user" : "git perforce import user", "time" : int(time.time()) } + details = {} + details["user"] = "git perforce import user" details["desc"] = ("Initial import of %s from the state at revision %s\n" % (' '.join(self.depotPaths), revision)) details["change"] = revision newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files " - + ' '.join(["%s...%s" - % (p, revision) - for p in self.depotPaths])): + fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths] + + for info in p4CmdList(["files"] + fileArgs): if 'code' in info and info['code'] == 'error': sys.stderr.write("p4 returned an error: %s\n" @@ -1689,6 +2107,18 @@ class P4Sync(Command, P4UserMap): fileCnt = fileCnt + 1 details["change"] = newestRevision + + # Use time from top-most change so that all git-p4 clones of + # the same p4 repo have the same commit SHA1s. + res = p4CmdList("describe -s %d" % newestRevision) + newestTime = None + for r in res: + if r.has_key('time'): + newestTime = int(r['time']) + if newestTime is None: + die("\"describe -s\" on newest change %d did not give a time") + details["time"] = newestTime + self.updateOptionDict(details) try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths) @@ -1697,52 +2127,6 @@ class P4Sync(Command, P4UserMap): print self.gitError.read() - def getClientSpec(self): - specList = p4CmdList( "client -o" ) - temp = {} - for entry in specList: - for k,v in entry.iteritems(): - if k.startswith("View"): - - # p4 has these %%1 to %%9 arguments in specs to - # reorder paths; which we can't handle (yet :) - if re.match('%%\d', v) != None: - print "Sorry, can't handle %%n arguments in client specs" - sys.exit(1) - - if v.startswith('"'): - start = 1 - else: - start = 0 - index = v.find("...") - - # save the "client view"; i.e the RHS of the view - # line that tells the client where to put the - # files for this view. - cv = v[index+3:].strip() # +3 to remove previous '...' - - # if the client view doesn't end with a - # ... wildcard, then we're going to mess up the - # output directory, so fail gracefully. - if not cv.endswith('...'): - print 'Sorry, client view in "%s" needs to end with wildcard' % (k) - sys.exit(1) - cv=cv[:-3] - - # now save the view; +index means included, -index - # means it should be filtered out. - v = v[start:index] - if v.startswith("-"): - v = v[1:] - include = -len(v) - else: - include = len(v) - - temp[v] = (include, cv) - - self.clientSpecDirs = temp.items() - self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) ) - def run(self, args): self.depotPaths = [] self.changeRange = "" @@ -1775,8 +2159,15 @@ class P4Sync(Command, P4UserMap): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch): system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true": - self.getClientSpec() + # accept either the command-line option, or the configuration variable + if self.useClientSpec: + # will use this after clone to set the variable + self.useClientSpec_from_options = True + else: + if gitConfig("git-p4.useclientspec", "--bool") == "true": + self.useClientSpec = True + if self.useClientSpec: + self.clientSpecDirs = getClientSpec() # TODO: should always look at previous commits, # merge with previous imports, if possible. @@ -1811,12 +2202,14 @@ class P4Sync(Command, P4UserMap): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, min(len(cur), len(prev))): - if cur[i] <> prev[i]: + prev_list = prev.split("/") + cur_list = cur.split("/") + for i in range(0, min(len(cur_list), len(prev_list))): + if cur_list[i] <> prev_list[i]: i = i - 1 break - paths.append (cur[:i + 1]) + paths.append ("/".join(cur_list[:i + 1])) self.previousDepotPaths = paths @@ -1846,6 +2239,17 @@ class P4Sync(Command, P4UserMap): revision = "" self.users = {} + # Make sure no revision specifiers are used when --changesfile + # is specified. + bad_changesfile = False + if len(self.changesFile) > 0: + for p in self.depotPaths: + if p.find("@") >= 0 or p.find("#") >= 0: + bad_changesfile = True + break + if bad_changesfile: + die("Option --changesfile is incompatible with revision specifiers") + newPaths = [] for p in self.depotPaths: if p.find("@") != -1: @@ -1862,7 +2266,10 @@ class P4Sync(Command, P4UserMap): revision = p[hashIdx:] p = p[:hashIdx] elif self.previousDepotPaths == []: - revision = "#head" + # pay attention to changesfile, if given, else import + # the entire p4 tree at the head revision + if len(self.changesFile) == 0: + revision = "#head" p = re.sub ("\.\.\.$", "", p) if not p.endswith("/"): @@ -2075,6 +2482,10 @@ class P4Clone(P4Sync): else: print "Could not detect main branch. No checkout/master branch created." + # auto-set this variable if invoked with --use-client-spec + if self.useClientSpec_from_options: + system("git config --bool git-p4.useclientspec true") + return True class P4Branches(Command): @@ -2157,7 +2568,8 @@ def main(): args = sys.argv[2:] if len(options) > 0: - options.append(optparse.make_option("--git-dir", dest="gitdir")) + if cmd.needsGit: + options.append(optparse.make_option("--git-dir", dest="gitdir")) parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), options, @@ -2187,6 +2599,7 @@ def main(): if not cmd.run(args): parser.print_help() + sys.exit(2) if __name__ == '__main__': diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt deleted file mode 100644 index caa4bb3e30..0000000000 --- a/contrib/fast-import/git-p4.txt +++ /dev/null @@ -1,251 +0,0 @@ -git-p4 - Perforce <-> Git converter using git-fast-import - -Usage -===== - -git-p4 can be used in two different ways: - -1) To import changes from Perforce to a Git repository, using "git-p4 sync". - -2) To submit changes from Git back to Perforce, using "git-p4 submit". - -Importing -========= - -Simply start with - - git-p4 clone //depot/path/project - -or - - git-p4 clone //depot/path/project myproject - -This will: - -1) Create an empty git repository in a subdirectory called "project" (or -"myproject" with the second command) - -2) Import the head revision from the given Perforce path into a git branch -called "p4" (remotes/p4 actually) - -3) Create a master branch based on it and check it out. - -If you want the entire history (not just the head revision) then you can simply -append a "@all" to the depot path: - - git-p4 clone //depot/project/main@all myproject - - - -If you want more control you can also use the git-p4 sync command directly: - - mkdir repo-git - cd repo-git - git init - git-p4 sync //path/in/your/perforce/depot - -This will import the current head revision of the specified depot path into a -"remotes/p4/master" branch of your git repository. You can use the ---branch=mybranch option to import into a different branch. - -If you want to import the entire history of a given depot path simply use: - - git-p4 sync //path/in/depot@all - - -Note: - -To achieve optimal compression you may want to run 'git repack -a -d -f' after -a big import. This may take a while. - -Incremental Imports -=================== - -After an initial import you can continue to synchronize your git repository -with newer changes from the Perforce depot by just calling - - git-p4 sync - -in your git repository. By default the "remotes/p4/master" branch is updated. - -Advanced Setup -============== - -Suppose you have a periodically updated git repository somewhere, containing a -complete import of a Perforce project. This repository can be cloned and used -with git-p4. When updating the cloned repository with the "sync" command, -git-p4 will try to fetch changes from the original repository first. The git -protocol used with this is usually faster than importing from Perforce -directly. - -This behaviour can be disabled by setting the "git-p4.syncFromOrigin" git -configuration variable to "false". - -Updating -======== - -A common working pattern is to fetch the latest changes from the Perforce depot -and merge them with local uncommitted changes. The recommended way is to use -git's rebase mechanism to preserve linear history. git-p4 provides a convenient - - git-p4 rebase - -command that calls git-p4 sync followed by git rebase to rebase the current -working branch. - -Submitting -========== - -git-p4 has support for submitting changes from a git repository back to the -Perforce depot. This requires a Perforce checkout separate from your git -repository. To submit all changes that are in the current git branch but not in -the "p4" branch (or "origin" if "p4" doesn't exist) simply call - - git-p4 submit - -in your git repository. If you want to submit changes in a specific branch that -is not your current git branch you can also pass that as an argument: - - git-p4 submit mytopicbranch - -You can override the reference branch with the --origin=mysourcebranch option. - -The Perforce changelists will be created with the user who ran git-p4. If you -use --preserve-user then git-p4 will attempt to create Perforce changelists -with the Perforce user corresponding to the git commit author. You need to -have sufficient permissions within Perforce, and the git users need to have -Perforce accounts. Permissions can be granted using 'p4 protect'. - -If a submit fails you may have to "p4 resolve" and submit manually. You can -continue importing the remaining changes with - - git-p4 submit --continue - -Example -======= - -# Clone a repository - git-p4 clone //depot/path/project -# Enter the newly cloned directory - cd project -# Do some work... - vi foo.h -# ... and commit locally to gi - git commit foo.h -# In the meantime somebody submitted changes to the Perforce depot. Rebase your latest -# changes against the latest changes in Perforce: - git-p4 rebase -# Submit your locally committed changes back to Perforce - git-p4 submit -# ... and synchronize with Perforce - git-p4 rebase - - -Configuration parameters -======================== - -git-p4.user ($P4USER) - -Allows you to specify the username to use to connect to the Perforce repository. - - git config [--global] git-p4.user public - -git-p4.password ($P4PASS) - -Allows you to specify the password to use to connect to the Perforce repository. -Warning this password will be visible on the command-line invocation of the p4 binary. - - git config [--global] git-p4.password public1234 - -git-p4.port ($P4PORT) - -Specify the port to be used to contact the Perforce server. As this will be passed -directly to the p4 binary, it may be in the format host:port as well. - - git config [--global] git-p4.port codes.zimbra.com:2666 - -git-p4.host ($P4HOST) - -Specify the host to contact for a Perforce repository. - - git config [--global] git-p4.host perforce.example.com - -git-p4.client ($P4CLIENT) - -Specify the client name to use - - git config [--global] git-p4.client public-view - -git-p4.allowSubmit - - git config [--global] git-p4.allowSubmit false - -git-p4.syncFromOrigin - -A useful setup may be that you have a periodically updated git repository -somewhere that contains a complete import of a Perforce project. That git -repository can be used to clone the working repository from and one would -import from Perforce directly after cloning using git-p4. If the connection to -the Perforce server is slow and the working repository hasn't been synced for a -while it may be desirable to fetch changes from the origin git repository using -the efficient git protocol. git-p4 supports this setup by calling "git fetch origin" -by default if there is an origin branch. You can disable this using: - - git config [--global] git-p4.syncFromOrigin false - -git-p4.useclientspec - - git config [--global] git-p4.useclientspec false - -The P4CLIENT environment variable should be correctly set for p4 to be -able to find the relevant client. This client spec will be used to -both filter the files cloned by git and set the directory layout as -specified in the client (this implies --keep-path style semantics). - -git-p4.skipSubmitModTimeCheck - - git config [--global] git-p4.skipSubmitModTimeCheck false - -If true, submit will not check if the p4 change template has been modified. - -git-p4.preserveUser - - git config [--global] git-p4.preserveUser false - -If true, attempt to preserve user names by modifying the p4 changelists. See -the "--preserve-user" submit option. - -git-p4.allowMissingPerforceUsers - - git config [--global] git-p4.allowMissingP4Users false - -If git-p4 is setting the perforce user for a commit (--preserve-user) then -if there is no perforce user corresponding to the git author, git-p4 will -stop. With allowMissingPerforceUsers set to true, git-p4 will use the -current user (i.e. the behavior without --preserve-user) and carry on with -the perforce commit. - -git-p4.skipUserNameCheck - - git config [--global] git-p4.skipUserNameCheck false - -When submitting, git-p4 checks that the git commits are authored by the current -p4 user, and warns if they are not. This disables the check. - -Implementation Details... -========================= - -* Changesets from Perforce are imported using git fast-import. -* The import does not require anything from the Perforce client view as it just uses - "p4 print //depot/path/file#revision" to get the actual file contents. -* Every imported changeset has a special [git-p4...] line at the - end of the log message that gives information about the corresponding - Perforce change number and is also used by git-p4 itself to find out - where to continue importing when doing incremental imports. - Basically when syncing it extracts the perforce change number of the - latest commit in the "p4" branch and uses "p4 changes //depot/path/...@changenum,#head" - to find out which changes need to be imported. -* git-p4 submit uses "git rev-list" to pick the commits between the "p4" branch - and the current branch. - The commits themselves are applied using git diff/format-patch ... | git apply - |