diff options
Diffstat (limited to 'contrib/fast-import')
| -rwxr-xr-x | contrib/fast-import/git-import.perl | 2 | ||||
| -rwxr-xr-x | contrib/fast-import/git-import.sh | 2 | ||||
| -rwxr-xr-x | contrib/fast-import/git-p4 | 1926 | ||||
| -rw-r--r-- | contrib/fast-import/git-p4.README | 12 | ||||
| -rw-r--r-- | contrib/fast-import/git-p4.bat | 1 | ||||
| -rw-r--r-- | contrib/fast-import/git-p4.txt | 210 | ||||
| -rwxr-xr-x | contrib/fast-import/import-directories.perl | 4 | ||||
| -rwxr-xr-x | contrib/fast-import/import-tars.perl | 31 | ||||
| -rwxr-xr-x | contrib/fast-import/import-zips.py | 99 |
9 files changed, 88 insertions, 2199 deletions
diff --git a/contrib/fast-import/git-import.perl b/contrib/fast-import/git-import.perl index f9fef6db28..0891b9e366 100755 --- a/contrib/fast-import/git-import.perl +++ b/contrib/fast-import/git-import.perl @@ -7,7 +7,7 @@ use strict; use File::Find; -my $USAGE = 'Usage: git-import branch import-message'; +my $USAGE = 'usage: git-import branch import-message'; my $branch = shift or die "$USAGE\n"; my $message = shift or die "$USAGE\n"; diff --git a/contrib/fast-import/git-import.sh b/contrib/fast-import/git-import.sh index 0ca7718d05..f8d803c5e2 100755 --- a/contrib/fast-import/git-import.sh +++ b/contrib/fast-import/git-import.sh @@ -5,7 +5,7 @@ # but is meant to be a simple fast-import example. if [ -z "$1" -o -z "$2" ]; then - echo "Usage: git-import branch import-message" + echo "usage: git-import branch import-message" exit 1 fi diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 deleted file mode 100755 index 04ce7e3b02..0000000000 --- a/contrib/fast-import/git-p4 +++ /dev/null @@ -1,1926 +0,0 @@ -#!/usr/bin/env python -# -# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. -# -# Author: Simon Hausmann <simon@lst.de> -# Copyright: 2007 Simon Hausmann <simon@lst.de> -# 2007 Trolltech ASA -# License: MIT <http://www.opensource.org/licenses/mit-license.php> -# - -import optparse, sys, os, marshal, subprocess, shelve -import tempfile, getopt, os.path, time, platform -import re - -verbose = False - - -def p4_build_cmd(cmd): - """Build a suitable p4 command line. - - This consolidates building and returning a p4 command line into one - location. It means that hooking into the environment, or other configuration - can be done more easily. - """ - real_cmd = "%s " % "p4" - - user = gitConfig("git-p4.user") - if len(user) > 0: - real_cmd += "-u %s " % user - - password = gitConfig("git-p4.password") - if len(password) > 0: - real_cmd += "-P %s " % password - - port = gitConfig("git-p4.port") - if len(port) > 0: - real_cmd += "-p %s " % port - - host = gitConfig("git-p4.host") - if len(host) > 0: - real_cmd += "-h %s " % host - - client = gitConfig("git-p4.client") - if len(client) > 0: - real_cmd += "-c %s " % client - - real_cmd += "%s" % (cmd) - if verbose: - print real_cmd - return real_cmd - -def chdir(dir): - if os.name == 'nt': - os.environ['PWD']=dir - os.chdir(dir) - -def die(msg): - if verbose: - raise Exception(msg) - else: - sys.stderr.write(msg + "\n") - sys.exit(1) - -def write_pipe(c, str): - if verbose: - sys.stderr.write('Writing pipe: %s\n' % c) - - pipe = os.popen(c, 'w') - val = pipe.write(str) - if pipe.close(): - die('Command failed: %s' % c) - - return val - -def p4_write_pipe(c, str): - real_cmd = p4_build_cmd(c) - return write_pipe(real_cmd, str) - -def read_pipe(c, ignore_error=False): - if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) - - pipe = os.popen(c, 'rb') - val = pipe.read() - if pipe.close() and not ignore_error: - die('Command failed: %s' % c) - - return val - -def p4_read_pipe(c, ignore_error=False): - real_cmd = p4_build_cmd(c) - return read_pipe(real_cmd, ignore_error) - -def read_pipe_lines(c): - if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) - ## todo: check return status - pipe = os.popen(c, 'rb') - val = pipe.readlines() - if pipe.close(): - die('Command failed: %s' % c) - - return val - -def p4_read_pipe_lines(c): - """Specifically invoke p4 on the command supplied. """ - real_cmd = p4_build_cmd(c) - return read_pipe_lines(real_cmd) - -def system(cmd): - if verbose: - sys.stderr.write("executing %s\n" % cmd) - if os.system(cmd) != 0: - die("command failed: %s" % cmd) - -def p4_system(cmd): - """Specifically invoke p4 as the system command. """ - real_cmd = p4_build_cmd(cmd) - return system(real_cmd) - -def isP4Exec(kind): - """Determine if a Perforce 'kind' should have execute permission - - 'p4 help filetypes' gives a list of the types. If it starts with 'x', - or x follows one of a few letters. Otherwise, if there is an 'x' after - a plus sign, it is also executable""" - return (re.search(r"(^[cku]?x)|\+.*x", kind) != None) - -def setP4ExecBit(file, mode): - # Reopens an already open file and changes the execute bit to match - # the execute bit setting in the passed in mode. - - p4Type = "+x" - - if not isModeExec(mode): - p4Type = getP4OpenedType(file) - p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type) - p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type) - if p4Type[-1] == "+": - p4Type = p4Type[0:-1] - - p4_system("reopen -t %s %s" % (p4Type, file)) - -def getP4OpenedType(file): - # Returns the perforce file type for the given file. - - result = p4_read_pipe("opened %s" % file) - match = re.match(".*\((.+)\)\r?$", result) - if match: - return match.group(1) - else: - die("Could not determine file type for %s (result: '%s')" % (file, result)) - -def diffTreePattern(): - # This is a simple generator for the diff tree regex pattern. This could be - # a class variable if this and parseDiffTreeEntry were a part of a class. - pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') - while True: - yield pattern - -def parseDiffTreeEntry(entry): - """Parses a single diff tree entry into its component elements. - - See git-diff-tree(1) manpage for details about the format of the diff - output. This method returns a dictionary with the following elements: - - src_mode - The mode of the source file - dst_mode - The mode of the destination file - src_sha1 - The sha1 for the source file - dst_sha1 - The sha1 fr the destination file - status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) - status_score - The score for the status (applicable for 'C' and 'R' - statuses). This is None if there is no score. - src - The path for the source file. - dst - The path for the destination file. This is only present for - copy or renames. If it is not present, this is None. - - If the pattern is not matched, None is returned.""" - - match = diffTreePattern().next().match(entry) - if match: - return { - 'src_mode': match.group(1), - 'dst_mode': match.group(2), - 'src_sha1': match.group(3), - 'dst_sha1': match.group(4), - 'status': match.group(5), - 'status_score': match.group(6), - 'src': match.group(7), - 'dst': match.group(10) - } - return None - -def isModeExec(mode): - # Returns True if the given git mode represents an executable file, - # otherwise False. - return mode[-3:] == "755" - -def isModeExecChanged(src_mode, dst_mode): - return isModeExec(src_mode) != isModeExec(dst_mode) - -def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): - cmd = p4_build_cmd("-G %s" % (cmd)) - if verbose: - sys.stderr.write("Opening pipe: %s\n" % cmd) - - # Use a temporary file to avoid deadlocks without - # subprocess.communicate(), which would put another copy - # of stdout into memory. - stdin_file = None - if stdin is not None: - stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) - stdin_file.write(stdin) - stdin_file.flush() - stdin_file.seek(0) - - p4 = subprocess.Popen(cmd, shell=True, - stdin=stdin_file, - stdout=subprocess.PIPE) - - result = [] - try: - while True: - entry = marshal.load(p4.stdout) - if cb is not None: - cb(entry) - else: - result.append(entry) - except EOFError: - pass - exitCode = p4.wait() - if exitCode != 0: - entry = {} - entry["p4ExitCode"] = exitCode - result.append(entry) - - return result - -def p4Cmd(cmd): - list = p4CmdList(cmd) - result = {} - for entry in list: - result.update(entry) - return result; - -def p4Where(depotPath): - if not depotPath.endswith("/"): - depotPath += "/" - depotPath = depotPath + "..." - outputList = p4CmdList("where %s" % depotPath) - output = None - for entry in outputList: - if "depotFile" in entry: - if entry["depotFile"] == depotPath: - output = entry - break - elif "data" in entry: - data = entry.get("data") - space = data.find(" ") - if data[:space] == depotPath: - output = entry - break - if output == None: - return "" - if output["code"] == "error": - return "" - clientPath = "" - if "path" in output: - clientPath = output.get("path") - elif "data" in output: - data = output.get("data") - lastSpace = data.rfind(" ") - clientPath = data[lastSpace + 1:] - - if clientPath.endswith("..."): - clientPath = clientPath[:-3] - return clientPath - -def currentGitBranch(): - return read_pipe("git name-rev HEAD").split(" ")[1].strip() - -def isValidGitDir(path): - if (os.path.exists(path + "/HEAD") - and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")): - return True; - return False - -def parseRevision(ref): - return read_pipe("git rev-parse %s" % ref).strip() - -def extractLogMessageFromGitCommit(commit): - logMessage = "" - - ## fixme: title is first line of commit, not 1st paragraph. - foundTitle = False - for log in read_pipe_lines("git cat-file commit %s" % commit): - if not foundTitle: - if len(log) == 1: - foundTitle = True - continue - - logMessage += log - return logMessage - -def extractSettingsGitLog(log): - values = {} - for line in log.split("\n"): - line = line.strip() - m = re.search (r"^ *\[git-p4: (.*)\]$", line) - if not m: - continue - - assignments = m.group(1).split (':') - for a in assignments: - vals = a.split ('=') - key = vals[0].strip() - val = ('='.join (vals[1:])).strip() - if val.endswith ('\"') and val.startswith('"'): - val = val[1:-1] - - values[key] = val - - paths = values.get("depot-paths") - if not paths: - paths = values.get("depot-path") - if paths: - values['depot-paths'] = paths.split(',') - return values - -def gitBranchExists(branch): - proc = subprocess.Popen(["git", "rev-parse", branch], - stderr=subprocess.PIPE, stdout=subprocess.PIPE); - return proc.wait() == 0; - -_gitConfig = {} -def gitConfig(key): - if not _gitConfig.has_key(key): - _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip() - return _gitConfig[key] - -def p4BranchesInGit(branchesAreInRemotes = True): - branches = {} - - cmdline = "git rev-parse --symbolic " - if branchesAreInRemotes: - cmdline += " --remotes" - else: - cmdline += " --branches" - - for line in read_pipe_lines(cmdline): - line = line.strip() - - ## only import to p4/ - if not line.startswith('p4/') or line == "p4/HEAD": - continue - branch = line - - # strip off p4 - branch = re.sub ("^p4/", "", line) - - branches[branch] = parseRevision(line) - return branches - -def findUpstreamBranchPoint(head = "HEAD"): - branches = p4BranchesInGit() - # map from depot-path to branch name - branchByDepotPath = {} - for branch in branches.keys(): - tip = branches[branch] - log = extractLogMessageFromGitCommit(tip) - settings = extractSettingsGitLog(log) - if settings.has_key("depot-paths"): - paths = ",".join(settings["depot-paths"]) - branchByDepotPath[paths] = "remotes/p4/" + branch - - settings = None - parent = 0 - while parent < 65535: - commit = head + "~%s" % parent - log = extractLogMessageFromGitCommit(commit) - settings = extractSettingsGitLog(log) - if settings.has_key("depot-paths"): - paths = ",".join(settings["depot-paths"]) - if branchByDepotPath.has_key(paths): - return [branchByDepotPath[paths], settings] - - parent = parent + 1 - - return ["", settings] - -def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True): - if not silent: - print ("Creating/updating branch(es) in %s based on origin branch(es)" - % localRefPrefix) - - originPrefix = "origin/p4/" - - for line in read_pipe_lines("git rev-parse --symbolic --remotes"): - line = line.strip() - if (not line.startswith(originPrefix)) or line.endswith("HEAD"): - continue - - headName = line[len(originPrefix):] - remoteHead = localRefPrefix + headName - originHead = line - - original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) - if (not original.has_key('depot-paths') - or not original.has_key('change')): - continue - - update = False - if not gitBranchExists(remoteHead): - if verbose: - print "creating %s" % remoteHead - update = True - else: - settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) - if settings.has_key('change') > 0: - if settings['depot-paths'] == original['depot-paths']: - originP4Change = int(original['change']) - p4Change = int(settings['change']) - if originP4Change > p4Change: - print ("%s (%s) is newer than %s (%s). " - "Updating p4 branch from origin." - % (originHead, originP4Change, - remoteHead, p4Change)) - update = True - else: - print ("Ignoring: %s was imported from %s while " - "%s was imported from %s" - % (originHead, ','.join(original['depot-paths']), - remoteHead, ','.join(settings['depot-paths']))) - - if update: - system("git update-ref %s %s" % (remoteHead, originHead)) - -def originP4BranchesExist(): - return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") - -def p4ChangesForPaths(depotPaths, changeRange): - assert depotPaths - output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange) - for p in depotPaths])) - - changes = {} - for line in output: - changeNum = int(line.split(" ")[1]) - changes[changeNum] = True - - changelist = changes.keys() - changelist.sort() - return changelist - -class Command: - def __init__(self): - self.usage = "usage: %prog [options]" - self.needsGit = True - -class P4Debug(Command): - def __init__(self): - Command.__init__(self) - self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true", - default=False), - ] - self.description = "A tool to debug the output of p4 -G." - self.needsGit = False - self.verbose = False - - def run(self, args): - j = 0 - for output in p4CmdList(" ".join(args)): - print 'Element: %d' % j - j += 1 - print output - return True - -class P4RollBack(Command): - def __init__(self): - Command.__init__(self) - self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") - ] - self.description = "A tool to debug the multi-branch import. Don't use :)" - self.verbose = False - self.rollbackLocalBranches = False - - def run(self, args): - if len(args) != 1: - return False - maxChange = int(args[0]) - - if "p4ExitCode" in p4Cmd("changes -m 1"): - die("Problems executing p4"); - - if self.rollbackLocalBranches: - refPrefix = "refs/heads/" - lines = read_pipe_lines("git rev-parse --symbolic --branches") - else: - refPrefix = "refs/remotes/" - lines = read_pipe_lines("git rev-parse --symbolic --remotes") - - for line in lines: - if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): - line = line.strip() - ref = refPrefix + line - log = extractLogMessageFromGitCommit(ref) - settings = extractSettingsGitLog(log) - - depotPaths = settings['depot-paths'] - change = settings['change'] - - changed = False - - if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange) - for p in depotPaths]))) == 0: - print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) - system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) - continue - - while change and int(change) > maxChange: - changed = True - if self.verbose: - print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) - system("git update-ref %s \"%s^\"" % (ref, ref)) - log = extractLogMessageFromGitCommit(ref) - settings = extractSettingsGitLog(log) - - - depotPaths = settings['depot-paths'] - change = settings['change'] - - if changed: - print "%s rewound to %s" % (ref, change) - - return True - -class P4Submit(Command): - def __init__(self): - Command.__init__(self) - self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--origin", dest="origin"), - optparse.make_option("-M", dest="detectRename", action="store_true"), - ] - self.description = "Submit changes from git to the perforce depot." - self.usage += " [name of git branch to submit into perforce depot]" - self.interactive = True - self.origin = "" - self.detectRename = False - self.verbose = False - self.isWindows = (platform.system() == "Windows") - - def check(self): - if len(p4CmdList("opened ...")) > 0: - die("You have files opened with perforce! Close them before starting the sync.") - - # replaces everything between 'Description:' and the next P4 submit template field with the - # commit message - def prepareLogMessage(self, template, message): - result = "" - - inDescriptionSection = False - - for line in template.split("\n"): - if line.startswith("#"): - result += line + "\n" - continue - - if inDescriptionSection: - if line.startswith("Files:"): - inDescriptionSection = False - else: - continue - else: - if line.startswith("Description:"): - inDescriptionSection = True - line += "\n" - for messageLine in message.split("\n"): - line += "\t" + messageLine + "\n" - - result += line + "\n" - - return result - - def prepareSubmitTemplate(self): - # remove lines in the Files section that show changes to files outside the depot path we're committing into - template = "" - inFilesSection = False - for line in p4_read_pipe_lines("change -o"): - if line.endswith("\r\n"): - line = line[:-2] + "\n" - if inFilesSection: - if line.startswith("\t"): - # path starts and ends with a tab - path = line[1:] - lastTab = path.rfind("\t") - if lastTab != -1: - path = path[:lastTab] - if not path.startswith(self.depotPath): - continue - else: - inFilesSection = False - else: - if line.startswith("Files:"): - inFilesSection = True - - template += line - - return template - - def applyCommit(self, id): - print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id)) - diffOpts = ("", "-M")[self.detectRename] - diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id)) - filesToAdd = set() - filesToDelete = set() - editedFiles = set() - filesToChangeExecBit = {} - for line in diff: - diff = parseDiffTreeEntry(line) - modifier = diff['status'] - path = diff['src'] - if modifier == "M": - p4_system("edit \"%s\"" % path) - if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - filesToChangeExecBit[path] = diff['dst_mode'] - editedFiles.add(path) - elif modifier == "A": - filesToAdd.add(path) - filesToChangeExecBit[path] = diff['dst_mode'] - if path in filesToDelete: - filesToDelete.remove(path) - elif modifier == "D": - filesToDelete.add(path) - if path in filesToAdd: - filesToAdd.remove(path) - elif modifier == "R": - src, dest = diff['src'], diff['dst'] - p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) - p4_system("edit \"%s\"" % (dest)) - if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - filesToChangeExecBit[dest] = diff['dst_mode'] - os.unlink(dest) - editedFiles.add(dest) - filesToDelete.add(src) - else: - die("unknown modifier %s for %s" % (modifier, path)) - - diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) - patchcmd = diffcmd + " | git apply " - tryPatchCmd = patchcmd + "--check -" - applyPatchCmd = patchcmd + "--check --apply -" - - if os.system(tryPatchCmd) != 0: - print "Unfortunately applying the change failed!" - print "What do you want to do?" - response = "x" - while response != "s" and response != "a" and response != "w": - response = raw_input("[s]kip this patch / [a]pply the patch forcibly " - "and with .rej files / [w]rite the patch to a file (patch.txt) ") - if response == "s": - print "Skipping! Good luck with the next patches..." - for f in editedFiles: - p4_system("revert \"%s\"" % f); - for f in filesToAdd: - system("rm %s" %f) - return - elif response == "a": - os.system(applyPatchCmd) - if len(filesToAdd) > 0: - print "You may also want to call p4 add on the following files:" - print " ".join(filesToAdd) - if len(filesToDelete): - print "The following files should be scheduled for deletion with p4 delete:" - print " ".join(filesToDelete) - die("Please resolve and submit the conflict manually and " - + "continue afterwards with git-p4 submit --continue") - elif response == "w": - system(diffcmd + " > patch.txt") - print "Patch saved to patch.txt in %s !" % self.clientPath - die("Please resolve and submit the conflict manually and " - "continue afterwards with git-p4 submit --continue") - - system(applyPatchCmd) - - for f in filesToAdd: - p4_system("add \"%s\"" % f) - for f in filesToDelete: - p4_system("revert \"%s\"" % f) - p4_system("delete \"%s\"" % f) - - # Set/clear executable bits - for f in filesToChangeExecBit.keys(): - mode = filesToChangeExecBit[f] - setP4ExecBit(f, mode) - - logMessage = extractLogMessageFromGitCommit(id) - logMessage = logMessage.strip() - - template = self.prepareSubmitTemplate() - - if self.interactive: - submitTemplate = self.prepareLogMessage(template, logMessage) - if os.environ.has_key("P4DIFF"): - del(os.environ["P4DIFF"]) - diff = "" - for editedFile in editedFiles: - diff += p4_read_pipe("diff -du %r" % editedFile) - - newdiff = "" - for newFile in filesToAdd: - newdiff += "==== new file ====\n" - newdiff += "--- /dev/null\n" - newdiff += "+++ %s\n" % newFile - f = open(newFile, "r") - for line in f.readlines(): - newdiff += "+" + line - f.close() - - separatorLine = "######## everything below this line is just the diff #######\n" - - [handle, fileName] = tempfile.mkstemp() - tmpFile = os.fdopen(handle, "w+") - if self.isWindows: - submitTemplate = submitTemplate.replace("\n", "\r\n") - separatorLine = separatorLine.replace("\n", "\r\n") - newdiff = newdiff.replace("\n", "\r\n") - tmpFile.write(submitTemplate + separatorLine + diff + newdiff) - tmpFile.close() - mtime = os.stat(fileName).st_mtime - if os.environ.has_key("P4EDITOR"): - editor = os.environ.get("P4EDITOR") - else: - editor = read_pipe("git var GIT_EDITOR").strip() - system(editor + " " + fileName) - - response = "y" - if os.stat(fileName).st_mtime <= mtime: - response = "x" - while response != "y" and response != "n": - response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") - - if response == "y": - tmpFile = open(fileName, "rb") - message = tmpFile.read() - tmpFile.close() - submitTemplate = message[:message.index(separatorLine)] - if self.isWindows: - submitTemplate = submitTemplate.replace("\r\n", "\n") - p4_write_pipe("submit -i", submitTemplate) - else: - for f in editedFiles: - p4_system("revert \"%s\"" % f); - for f in filesToAdd: - p4_system("revert \"%s\"" % f); - system("rm %s" %f) - - os.remove(fileName) - else: - fileName = "submit.txt" - file = open(fileName, "w+") - file.write(self.prepareLogMessage(template, logMessage)) - file.close() - print ("Perforce submit template written as %s. " - + "Please review/edit and then use p4 submit -i < %s to submit directly!" - % (fileName, fileName)) - - def run(self, args): - if len(args) == 0: - self.master = currentGitBranch() - if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): - die("Detecting current git branch failed!") - elif len(args) == 1: - self.master = args[0] - else: - return False - - allowSubmit = gitConfig("git-p4.allowSubmit") - if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","): - die("%s is not in git-p4.allowSubmit" % self.master) - - [upstream, settings] = findUpstreamBranchPoint() - self.depotPath = settings['depot-paths'][0] - if len(self.origin) == 0: - self.origin = upstream - - if self.verbose: - print "Origin branch is " + self.origin - - if len(self.depotPath) == 0: - print "Internal error: cannot locate perforce depot path from existing branches" - sys.exit(128) - - self.clientPath = p4Where(self.depotPath) - - if len(self.clientPath) == 0: - print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath - sys.exit(128) - - print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath) - self.oldWorkingDirectory = os.getcwd() - - chdir(self.clientPath) - print "Synchronizing p4 checkout..." - p4_system("sync ...") - - self.check() - - commits = [] - for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)): - commits.append(line.strip()) - commits.reverse() - - while len(commits) > 0: - commit = commits[0] - commits = commits[1:] - self.applyCommit(commit) - if not self.interactive: - break - - if len(commits) == 0: - print "All changes applied!" - chdir(self.oldWorkingDirectory) - - sync = P4Sync() - sync.run([]) - - rebase = P4Rebase() - rebase.rebase() - - return True - -class P4Sync(Command): - def __init__(self): - Command.__init__(self) - self.options = [ - optparse.make_option("--branch", dest="branch"), - optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), - optparse.make_option("--changesfile", dest="changesFile"), - optparse.make_option("--silent", dest="silent", action="store_true"), - optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), - optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", - help="Import into refs/heads/ , not refs/remotes"), - optparse.make_option("--max-changes", dest="maxChanges"), - optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true', - help="Keep entire BRANCH/DIR/SUBDIR prefix during import"), - optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true', - help="Only sync files that are included in the Perforce Client Spec") - ] - self.description = """Imports from Perforce into a git repository.\n - example: - //depot/my/project/ -- to import the current head - //depot/my/project/@all -- to import everything - //depot/my/project/@1,6 -- to import only from revision 1 to 6 - - (a ... is not needed in the path p4 specification, it's added implicitly)""" - - self.usage += " //depot/path[@revRange]" - self.silent = False - self.createdBranches = set() - self.committedChanges = set() - self.branch = "" - self.detectBranches = False - self.detectLabels = False - self.changesFile = "" - self.syncWithOrigin = True - self.verbose = False - self.importIntoRemotes = True - self.maxChanges = "" - self.isWindows = (platform.system() == "Windows") - self.keepRepoPath = False - self.depotPaths = None - self.p4BranchesInGit = [] - self.cloneExclude = [] - self.useClientSpec = False - self.clientSpecDirs = [] - - if gitConfig("git-p4.syncFromOrigin") == "false": - self.syncWithOrigin = False - - def extractFilesFromCommit(self, commit): - self.cloneExclude = [re.sub(r"\.\.\.$", "", path) - for path in self.cloneExclude] - files = [] - fnum = 0 - while commit.has_key("depotFile%s" % fnum): - path = commit["depotFile%s" % fnum] - - if [p for p in self.cloneExclude - if path.startswith (p)]: - found = False - else: - found = [p for p in self.depotPaths - if path.startswith (p)] - if not found: - fnum = fnum + 1 - continue - - file = {} - file["path"] = path - file["rev"] = commit["rev%s" % fnum] - file["action"] = commit["action%s" % fnum] - file["type"] = commit["type%s" % fnum] - files.append(file) - fnum = fnum + 1 - return files - - def stripRepoPath(self, path, prefixes): - if self.keepRepoPath: - prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])] - - for p in prefixes: - if path.startswith(p): - path = path[len(p):] - - return path - - def splitFilesIntoBranches(self, commit): - branches = {} - fnum = 0 - while commit.has_key("depotFile%s" % fnum): - path = commit["depotFile%s" % fnum] - found = [p for p in self.depotPaths - if path.startswith (p)] - if not found: - fnum = fnum + 1 - continue - - file = {} - file["path"] = path - file["rev"] = commit["rev%s" % fnum] - file["action"] = commit["action%s" % fnum] - file["type"] = commit["type%s" % fnum] - fnum = fnum + 1 - - relPath = self.stripRepoPath(path, self.depotPaths) - - for branch in self.knownBranches.keys(): - - # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 - if relPath.startswith(branch + "/"): - if branch not in branches: - branches[branch] = [] - branches[branch].append(file) - break - - return branches - - # output one file from the P4 stream - # - helper for streamP4Files - - def streamOneP4File(self, file, contents): - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring" % \ - file['depotFile'] - return - - relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) - if verbose: - sys.stderr.write("%s\n" % relPath) - - mode = "644" - if isP4Exec(file["type"]): - mode = "755" - elif file["type"] == "symlink": - mode = "120000" - # p4 print on a symlink contains "target\n", so strip it off - data = ''.join(contents) - contents = [data[:-1]] - - if self.isWindows and file["type"].endswith("text"): - mangled = [] - for data in contents: - data = data.replace("\r\n", "\n") - mangled.append(data) - contents = mangled - - if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): - contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents) - elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): - contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents) - - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) - - # total length... - length = 0 - for d in contents: - length = length + len(d) - - self.gitStream.write("data %d\n" % length) - for d in contents: - self.gitStream.write(d) - self.gitStream.write("\n") - - def streamOneP4Deletion(self, file): - relPath = self.stripRepoPath(file['path'], self.branchPrefixes) - if verbose: - sys.stderr.write("delete %s\n" % relPath) - self.gitStream.write("D %s\n" % relPath) - - # handle another chunk of streaming data - def streamP4FilesCb(self, marshalled): - - if marshalled.has_key('depotFile') and self.stream_have_file_info: - # start of a new file - output the old one first - self.streamOneP4File(self.stream_file, self.stream_contents) - self.stream_file = {} - self.stream_contents = [] - self.stream_have_file_info = False - - # pick up the new file information... for the - # 'data' field we need to append to our array - for k in marshalled.keys(): - if k == 'data': - self.stream_contents.append(marshalled['data']) - else: - self.stream_file[k] = marshalled[k] - - self.stream_have_file_info = True - - # Stream directly from "p4 files" into "git fast-import" - def streamP4Files(self, files): - filesForCommit = [] - filesToRead = [] - filesToDelete = [] - - for f in files: - includeFile = True - for val in self.clientSpecDirs: - if f['path'].startswith(val[0]): - if val[1] <= 0: - includeFile = False - break - - if includeFile: - filesForCommit.append(f) - if f['action'] not in ('delete', 'move/delete', 'purge'): - filesToRead.append(f) - else: - filesToDelete.append(f) - - # deleted files... - for f in filesToDelete: - self.streamOneP4Deletion(f) - - if len(filesToRead) > 0: - self.stream_file = {} - self.stream_contents = [] - self.stream_have_file_info = False - - # curry self argument - def streamP4FilesCbSelf(entry): - self.streamP4FilesCb(entry) - - p4CmdList("-x - print", - '\n'.join(['%s#%s' % (f['path'], f['rev']) - for f in filesToRead]), - cb=streamP4FilesCbSelf) - - # do the last chunk - if self.stream_file.has_key('depotFile'): - self.streamOneP4File(self.stream_file, self.stream_contents) - - def commit(self, details, files, branch, branchPrefixes, parent = ""): - epoch = details["time"] - author = details["user"] - self.branchPrefixes = branchPrefixes - - if self.verbose: - print "commit into %s" % branch - - # start with reading files; if that fails, we should not - # create a commit. - new_files = [] - for f in files: - if [p for p in branchPrefixes if f['path'].startswith(p)]: - new_files.append (f) - else: - sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) - - self.gitStream.write("commit %s\n" % branch) -# gitStream.write("mark :%s\n" % details["change"]) - self.committedChanges.add(int(details["change"])) - committer = "" - if author not in self.users: - self.getUserMapFromPerforceServer() - if author in self.users: - committer = "%s %s %s" % (self.users[author], epoch, self.tz) - else: - committer = "%s <a@b> %s %s" % (author, epoch, self.tz) - - self.gitStream.write("committer %s\n" % committer) - - self.gitStream.write("data <<EOT\n") - self.gitStream.write(details["desc"]) - self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" - % (','.join (branchPrefixes), details["change"])) - if len(details['options']) > 0: - self.gitStream.write(": options = %s" % details['options']) - self.gitStream.write("]\nEOT\n\n") - - if len(parent) > 0: - if self.verbose: - print "parent %s" % parent - self.gitStream.write("from %s\n" % parent) - - self.streamP4Files(new_files) - self.gitStream.write("\n") - - change = int(details["change"]) - - if self.labels.has_key(change): - label = self.labels[change] - labelDetails = label[0] - labelRevisions = label[1] - if self.verbose: - print "Change %s is labelled %s" % (change, labelDetails) - - files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change) - for p in branchPrefixes])) - - if len(files) == len(labelRevisions): - - cleanedFiles = {} - for info in files: - if info["action"] in ("delete", "purge"): - continue - cleanedFiles[info["depotFile"]] = info["rev"] - - if cleanedFiles == labelRevisions: - self.gitStream.write("tag tag_%s\n" % labelDetails["label"]) - self.gitStream.write("from %s\n" % branch) - - owner = labelDetails["Owner"] - tagger = "" - if author in self.users: - tagger = "%s %s %s" % (self.users[owner], epoch, self.tz) - else: - tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz) - self.gitStream.write("tagger %s\n" % tagger) - self.gitStream.write("data <<EOT\n") - self.gitStream.write(labelDetails["Description"]) - self.gitStream.write("EOT\n\n") - - else: - if not self.silent: - print ("Tag %s does not match with change %s: files do not match." - % (labelDetails["label"], change)) - - else: - if not self.silent: - print ("Tag %s does not match with change %s: file count is different." - % (labelDetails["label"], change)) - - def getUserCacheFilename(self): - home = os.environ.get("HOME", os.environ.get("USERPROFILE")) - return home + "/.gitp4-usercache.txt" - - def getUserMapFromPerforceServer(self): - if self.userMapFromPerforceServer: - return - self.users = {} - - for output in p4CmdList("users"): - if not output.has_key("User"): - continue - self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - - - s = '' - for (key, val) in self.users.items(): - s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) - - open(self.getUserCacheFilename(), "wb").write(s) - self.userMapFromPerforceServer = True - - def loadUserMapFromCache(self): - self.users = {} - self.userMapFromPerforceServer = False - try: - cache = open(self.getUserCacheFilename(), "rb") - lines = cache.readlines() - cache.close() - for line in lines: - entry = line.strip().split("\t") - self.users[entry[0]] = entry[1] - except IOError: - self.getUserMapFromPerforceServer() - - def getLabels(self): - self.labels = {} - - l = p4CmdList("labels %s..." % ' '.join (self.depotPaths)) - if len(l) > 0 and not self.silent: - print "Finding files belonging to labels in %s" % `self.depotPaths` - - for output in l: - label = output["label"] - revisions = {} - newestChange = 0 - if self.verbose: - print "Querying files for label %s" % label - for file in p4CmdList("files " - + ' '.join (["%s...@%s" % (p, label) - for p in self.depotPaths])): - revisions[file["depotFile"]] = file["rev"] - change = int(file["change"]) - if change > newestChange: - newestChange = change - - self.labels[newestChange] = [output, revisions] - - if self.verbose: - print "Label changes: %s" % self.labels.keys() - - def guessProjectName(self): - for p in self.depotPaths: - if p.endswith("/"): - p = p[:-1] - p = p[p.strip().rfind("/") + 1:] - if not p.endswith("/"): - p += "/" - return p - - def getBranchMapping(self): - lostAndFoundBranches = set() - - for info in p4CmdList("branches"): - details = p4Cmd("branch -o %s" % info["branch"]) - viewIdx = 0 - while details.has_key("View%s" % viewIdx): - paths = details["View%s" % viewIdx].split(" ") - viewIdx = viewIdx + 1 - # require standard //depot/foo/... //depot/bar/... mapping - if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."): - continue - source = paths[0] - destination = paths[1] - ## HACK - if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): - source = source[len(self.depotPaths[0]):-4] - destination = destination[len(self.depotPaths[0]):-4] - - if destination in self.knownBranches: - if not self.silent: - print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination) - print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination) - continue - - self.knownBranches[destination] = source - - lostAndFoundBranches.discard(destination) - - if source not in self.knownBranches: - lostAndFoundBranches.add(source) - - - for branch in lostAndFoundBranches: - self.knownBranches[branch] = branch - - def getBranchMappingFromGitBranches(self): - branches = p4BranchesInGit(self.importIntoRemotes) - for branch in branches.keys(): - if branch == "master": - branch = "main" - else: - branch = branch[len(self.projectName):] - self.knownBranches[branch] = branch - - def listExistingP4GitBranches(self): - # branches holds mapping from name to commit - branches = p4BranchesInGit(self.importIntoRemotes) - self.p4BranchesInGit = branches.keys() - for branch in branches.keys(): - self.initialParents[self.refPrefix + branch] = branches[branch] - - def updateOptionDict(self, d): - option_keys = {} - if self.keepRepoPath: - option_keys['keepRepoPath'] = 1 - - d["options"] = ' '.join(sorted(option_keys.keys())) - - def readOptions(self, d): - self.keepRepoPath = (d.has_key('options') - and ('keepRepoPath' in d['options'])) - - def gitRefForBranch(self, branch): - if branch == "main": - return self.refPrefix + "master" - - if len(branch) <= 0: - return branch - - return self.refPrefix + self.projectName + branch - - def gitCommitByP4Change(self, ref, change): - if self.verbose: - print "looking in ref " + ref + " for change %s using bisect..." % change - - earliestCommit = "" - latestCommit = parseRevision(ref) - - while True: - if self.verbose: - print "trying: earliest %s latest %s" % (earliestCommit, latestCommit) - next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip() - if len(next) == 0: - if self.verbose: - print "argh" - return "" - log = extractLogMessageFromGitCommit(next) - settings = extractSettingsGitLog(log) - currentChange = int(settings['change']) - if self.verbose: - print "current change %s" % currentChange - - if currentChange == change: - if self.verbose: - print "found %s" % next - return next - - if currentChange < change: - earliestCommit = "^%s" % next - else: - latestCommit = "%s" % next - - return "" - - def importNewBranch(self, branch, maxChange): - # make fast-import flush all changes to disk and update the refs using the checkpoint - # command so that we can try to find the branch parent in the git history - self.gitStream.write("checkpoint\n\n"); - self.gitStream.flush(); - branchPrefix = self.depotPaths[0] + branch + "/" - range = "@1,%s" % maxChange - #print "prefix" + branchPrefix - changes = p4ChangesForPaths([branchPrefix], range) - if len(changes) <= 0: - return False - firstChange = changes[0] - #print "first change in branch: %s" % firstChange - sourceBranch = self.knownBranches[branch] - sourceDepotPath = self.depotPaths[0] + sourceBranch - sourceRef = self.gitRefForBranch(sourceBranch) - #print "source " + sourceBranch - - branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"]) - #print "branch parent: %s" % branchParentChange - gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange) - if len(gitParent) > 0: - self.initialParents[self.gitRefForBranch(branch)] = gitParent - #print "parent git commit: %s" % gitParent - - self.importChanges(changes) - return True - - def importChanges(self, changes): - cnt = 1 - for change in changes: - description = p4Cmd("describe %s" % change) - self.updateOptionDict(description) - - if not self.silent: - sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) - sys.stdout.flush() - cnt = cnt + 1 - - try: - if self.detectBranches: - branches = self.splitFilesIntoBranches(description) - for branch in branches.keys(): - ## HACK --hwn - branchPrefix = self.depotPaths[0] + branch + "/" - - parent = "" - - filesForCommit = branches[branch] - - if self.verbose: - print "branch is %s" % branch - - self.updatedBranches.add(branch) - - if branch not in self.createdBranches: - self.createdBranches.add(branch) - parent = self.knownBranches[branch] - if parent == branch: - parent = "" - else: - fullBranch = self.projectName + branch - if fullBranch not in self.p4BranchesInGit: - if not self.silent: - print("\n Importing new branch %s" % fullBranch); - if self.importNewBranch(branch, change - 1): - parent = "" - self.p4BranchesInGit.append(fullBranch) - if not self.silent: - print("\n Resuming with change %s" % change); - - if self.verbose: - print "parent determined through known branches: %s" % parent - - branch = self.gitRefForBranch(branch) - parent = self.gitRefForBranch(parent) - - if self.verbose: - print "looking for initial parent for %s; current parent is %s" % (branch, parent) - - if len(parent) == 0 and branch in self.initialParents: - parent = self.initialParents[branch] - del self.initialParents[branch] - - self.commit(description, filesForCommit, branch, [branchPrefix], parent) - else: - files = self.extractFilesFromCommit(description) - self.commit(description, files, self.branch, self.depotPaths, - self.initialParent) - self.initialParent = "" - except IOError: - print self.gitError.read() - sys.exit(1) - - def importHeadRevision(self, revision): - print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch) - - details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = ("Initial import of %s from the state at revision %s" - % (' '.join(self.depotPaths), revision)) - details["change"] = revision - newestRevision = 0 - - fileCnt = 0 - for info in p4CmdList("files " - + ' '.join(["%s...%s" - % (p, revision) - for p in self.depotPaths])): - - if info['code'] == 'error': - sys.stderr.write("p4 returned an error: %s\n" - % info['data']) - sys.exit(1) - - - change = int(info["change"]) - if change > newestRevision: - newestRevision = change - - if info["action"] in ("delete", "purge"): - # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! - #fileCnt = fileCnt + 1 - continue - - for prop in ["depotFile", "rev", "action", "type" ]: - details["%s%s" % (prop, fileCnt)] = info[prop] - - fileCnt = fileCnt + 1 - - details["change"] = newestRevision - self.updateOptionDict(details) - try: - self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths) - except IOError: - print "IO error with git fast-import. Is your git version recent enough?" - print self.gitError.read() - - - def getClientSpec(self): - specList = p4CmdList( "client -o" ) - temp = {} - for entry in specList: - for k,v in entry.iteritems(): - if k.startswith("View"): - if v.startswith('"'): - start = 1 - else: - start = 0 - index = v.find("...") - v = v[start:index] - if v.startswith("-"): - v = v[1:] - temp[v] = -len(v) - else: - temp[v] = len(v) - self.clientSpecDirs = temp.items() - self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) ) - - def run(self, args): - self.depotPaths = [] - self.changeRange = "" - self.initialParent = "" - self.previousDepotPaths = [] - - # map from branch depot path to parent branch - self.knownBranches = {} - self.initialParents = {} - self.hasOrigin = originP4BranchesExist() - if not self.syncWithOrigin: - self.hasOrigin = False - - if self.importIntoRemotes: - self.refPrefix = "refs/remotes/p4/" - else: - self.refPrefix = "refs/heads/p4/" - - if self.syncWithOrigin and self.hasOrigin: - if not self.silent: - print "Syncing with origin first by calling git fetch origin" - system("git fetch origin") - - if len(self.branch) == 0: - self.branch = self.refPrefix + "master" - if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: - system("git update-ref %s refs/heads/p4" % self.branch) - system("git branch -D p4"); - # create it /after/ importing, when master exists - if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch): - system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - - if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true": - self.getClientSpec() - - # TODO: should always look at previous commits, - # merge with previous imports, if possible. - if args == []: - if self.hasOrigin: - createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent) - self.listExistingP4GitBranches() - - if len(self.p4BranchesInGit) > 1: - if not self.silent: - print "Importing from/into multiple branches" - self.detectBranches = True - - if self.verbose: - print "branches: %s" % self.p4BranchesInGit - - p4Change = 0 - for branch in self.p4BranchesInGit: - logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) - - settings = extractSettingsGitLog(logMsg) - - self.readOptions(settings) - if (settings.has_key('depot-paths') - and settings.has_key ('change')): - change = int(settings['change']) + 1 - p4Change = max(p4Change, change) - - depotPaths = sorted(settings['depot-paths']) - if self.previousDepotPaths == []: - self.previousDepotPaths = depotPaths - else: - paths = [] - for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, min(len(cur), len(prev))): - if cur[i] <> prev[i]: - i = i - 1 - break - - paths.append (cur[:i + 1]) - - self.previousDepotPaths = paths - - if p4Change > 0: - self.depotPaths = sorted(self.previousDepotPaths) - self.changeRange = "@%s,#head" % p4Change - if not self.detectBranches: - self.initialParent = parseRevision(self.branch) - if not self.silent and not self.detectBranches: - print "Performing incremental import into %s git branch" % self.branch - - if not self.branch.startswith("refs/"): - self.branch = "refs/heads/" + self.branch - - if len(args) == 0 and self.depotPaths: - if not self.silent: - print "Depot paths: %s" % ' '.join(self.depotPaths) - else: - if self.depotPaths and self.depotPaths != args: - print ("previous import used depot path %s and now %s was specified. " - "This doesn't work!" % (' '.join (self.depotPaths), - ' '.join (args))) - sys.exit(1) - - self.depotPaths = sorted(args) - - revision = "" - self.users = {} - - newPaths = [] - for p in self.depotPaths: - if p.find("@") != -1: - atIdx = p.index("@") - self.changeRange = p[atIdx:] - if self.changeRange == "@all": - self.changeRange = "" - elif ',' not in self.changeRange: - revision = self.changeRange - self.changeRange = "" - p = p[:atIdx] - elif p.find("#") != -1: - hashIdx = p.index("#") - revision = p[hashIdx:] - p = p[:hashIdx] - elif self.previousDepotPaths == []: - revision = "#head" - - p = re.sub ("\.\.\.$", "", p) - if not p.endswith("/"): - p += "/" - - newPaths.append(p) - - self.depotPaths = newPaths - - - self.loadUserMapFromCache() - self.labels = {} - if self.detectLabels: - self.getLabels(); - - if self.detectBranches: - ## FIXME - what's a P4 projectName ? - self.projectName = self.guessProjectName() - - if self.hasOrigin: - self.getBranchMappingFromGitBranches() - else: - self.getBranchMapping() - if self.verbose: - print "p4-git branches: %s" % self.p4BranchesInGit - print "initial parents: %s" % self.initialParents - for b in self.p4BranchesInGit: - if b != "master": - - ## FIXME - b = b[len(self.projectName):] - self.createdBranches.add(b) - - self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) - - importProcess = subprocess.Popen(["git", "fast-import"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE); - self.gitOutput = importProcess.stdout - self.gitStream = importProcess.stdin - self.gitError = importProcess.stderr - - if revision: - self.importHeadRevision(revision) - else: - changes = [] - - if len(self.changesFile) > 0: - output = open(self.changesFile).readlines() - changeSet = set() - for line in output: - changeSet.add(int(line)) - - for change in changeSet: - changes.append(change) - - changes.sort() - else: - if self.verbose: - print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), - self.changeRange) - changes = p4ChangesForPaths(self.depotPaths, self.changeRange) - - if len(self.maxChanges) > 0: - changes = changes[:min(int(self.maxChanges), len(changes))] - - if len(changes) == 0: - if not self.silent: - print "No changes to import!" - return True - - if not self.silent and not self.detectBranches: - print "Import destination: %s" % self.branch - - self.updatedBranches = set() - - self.importChanges(changes) - - if not self.silent: - print "" - if len(self.updatedBranches) > 0: - sys.stdout.write("Updated branches: ") - for b in self.updatedBranches: - sys.stdout.write("%s " % b) - sys.stdout.write("\n") - - self.gitStream.close() - if importProcess.wait() != 0: - die("fast-import failed: %s" % self.gitError.read()) - self.gitOutput.close() - self.gitError.close() - - return True - -class P4Rebase(Command): - def __init__(self): - Command.__init__(self) - self.options = [ ] - self.description = ("Fetches the latest revision from perforce and " - + "rebases the current work (branch) against it") - self.verbose = False - - def run(self, args): - sync = P4Sync() - sync.run([]) - - return self.rebase() - - def rebase(self): - if os.system("git update-index --refresh") != 0: - die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash."); - if len(read_pipe("git diff-index HEAD --")) > 0: - die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash."); - - [upstream, settings] = findUpstreamBranchPoint() - if len(upstream) == 0: - die("Cannot find upstream branchpoint for rebase") - - # the branchpoint may be p4/foo~3, so strip off the parent - upstream = re.sub("~[0-9]+$", "", upstream) - - print "Rebasing the current branch onto %s" % upstream - oldHead = read_pipe("git rev-parse HEAD").strip() - system("git rebase %s" % upstream) - system("git diff-tree --stat --summary -M %s HEAD" % oldHead) - return True - -class P4Clone(P4Sync): - def __init__(self): - P4Sync.__init__(self) - self.description = "Creates a new git repository and imports from Perforce into it" - self.usage = "usage: %prog [options] //depot/path[@revRange]" - self.options += [ - optparse.make_option("--destination", dest="cloneDestination", - action='store', default=None, - help="where to leave result of the clone"), - optparse.make_option("-/", dest="cloneExclude", - action="append", type="string", - help="exclude depot path") - ] - self.cloneDestination = None - self.needsGit = False - - # This is required for the "append" cloneExclude action - def ensure_value(self, attr, value): - if not hasattr(self, attr) or getattr(self, attr) is None: - setattr(self, attr, value) - return getattr(self, attr) - - def defaultDestination(self, args): - ## TODO: use common prefix of args? - depotPath = args[0] - depotDir = re.sub("(@[^@]*)$", "", depotPath) - depotDir = re.sub("(#[^#]*)$", "", depotDir) - depotDir = re.sub(r"\.\.\.$", "", depotDir) - depotDir = re.sub(r"/$", "", depotDir) - return os.path.split(depotDir)[1] - - def run(self, args): - if len(args) < 1: - return False - - if self.keepRepoPath and not self.cloneDestination: - sys.stderr.write("Must specify destination for --keep-path\n") - sys.exit(1) - - depotPaths = args - - if not self.cloneDestination and len(depotPaths) > 1: - self.cloneDestination = depotPaths[-1] - depotPaths = depotPaths[:-1] - - self.cloneExclude = ["/"+p for p in self.cloneExclude] - for p in depotPaths: - if not p.startswith("//"): - return False - - if not self.cloneDestination: - self.cloneDestination = self.defaultDestination(args) - - print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) - if not os.path.exists(self.cloneDestination): - os.makedirs(self.cloneDestination) - chdir(self.cloneDestination) - system("git init") - self.gitdir = os.getcwd() + "/.git" - if not P4Sync.run(self, depotPaths): - return False - if self.branch != "master": - if self.importIntoRemotes: - masterbranch = "refs/remotes/p4/master" - else: - masterbranch = "refs/heads/p4/master" - if gitBranchExists(masterbranch): - system("git branch master %s" % masterbranch) - system("git checkout -f") - else: - print "Could not detect main branch. No checkout/master branch created." - - return True - -class P4Branches(Command): - def __init__(self): - Command.__init__(self) - self.options = [ ] - self.description = ("Shows the git branches that hold imports and their " - + "corresponding perforce depot paths") - self.verbose = False - - def run(self, args): - if originP4BranchesExist(): - createOrUpdateBranchesFromOrigin() - - cmdline = "git rev-parse --symbolic " - cmdline += " --remotes" - - for line in read_pipe_lines(cmdline): - line = line.strip() - - if not line.startswith('p4/') or line == "p4/HEAD": - continue - branch = line - - log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch) - settings = extractSettingsGitLog(log) - - print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]) - return True - -class HelpFormatter(optparse.IndentedHelpFormatter): - def __init__(self): - optparse.IndentedHelpFormatter.__init__(self) - - def format_description(self, description): - if description: - return description + "\n" - else: - return "" - -def printUsage(commands): - print "usage: %s <command> [options]" % sys.argv[0] - print "" - print "valid commands: %s" % ", ".join(commands) - print "" - print "Try %s <command> --help for command specific help." % sys.argv[0] - print "" - -commands = { - "debug" : P4Debug, - "submit" : P4Submit, - "commit" : P4Submit, - "sync" : P4Sync, - "rebase" : P4Rebase, - "clone" : P4Clone, - "rollback" : P4RollBack, - "branches" : P4Branches -} - - -def main(): - if len(sys.argv[1:]) == 0: - printUsage(commands.keys()) - sys.exit(2) - - cmd = "" - cmdName = sys.argv[1] - try: - klass = commands[cmdName] - cmd = klass() - except KeyError: - print "unknown command %s" % cmdName - print "" - printUsage(commands.keys()) - sys.exit(2) - - options = cmd.options - cmd.gitdir = os.environ.get("GIT_DIR", None) - - args = sys.argv[2:] - - if len(options) > 0: - options.append(optparse.make_option("--git-dir", dest="gitdir")) - - parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), - options, - description = cmd.description, - formatter = HelpFormatter()) - - (cmd, args) = parser.parse_args(sys.argv[2:], cmd); - global verbose - verbose = cmd.verbose - if cmd.needsGit: - if cmd.gitdir == None: - cmd.gitdir = os.path.abspath(".git") - if not isValidGitDir(cmd.gitdir): - cmd.gitdir = read_pipe("git rev-parse --git-dir").strip() - if os.path.exists(cmd.gitdir): - cdup = read_pipe("git rev-parse --show-cdup").strip() - if len(cdup) > 0: - chdir(cdup); - - if not isValidGitDir(cmd.gitdir): - if isValidGitDir(cmd.gitdir + "/.git"): - cmd.gitdir += "/.git" - else: - die("fatal: cannot locate git repository at %s" % cmd.gitdir) - - os.environ["GIT_DIR"] = cmd.gitdir - - if not cmd.run(args): - parser.print_help() - - -if __name__ == '__main__': - main() diff --git a/contrib/fast-import/git-p4.README b/contrib/fast-import/git-p4.README new file mode 100644 index 0000000000..cec5ecfa7c --- /dev/null +++ b/contrib/fast-import/git-p4.README @@ -0,0 +1,12 @@ +The git-p4 script moved to the top-level of the git source directory. + +Invoke it as any other git command, like "git p4 clone", for instance. + +Note that the top-level git-p4.py script is now the source. It is +built using make to git-p4, which will be installed. + +Windows users can copy the git-p4.py source script directly, possibly +invoking it through a batch file called "git-p4.bat" in the same folder. +It should contain just one line: + + @python "%~d0%~p0git-p4.py" %* diff --git a/contrib/fast-import/git-p4.bat b/contrib/fast-import/git-p4.bat deleted file mode 100644 index 9f97e884f5..0000000000 --- a/contrib/fast-import/git-p4.bat +++ /dev/null @@ -1 +0,0 @@ -@python "%~d0%~p0git-p4" %* diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt deleted file mode 100644 index 49b335921a..0000000000 --- a/contrib/fast-import/git-p4.txt +++ /dev/null @@ -1,210 +0,0 @@ -git-p4 - Perforce <-> Git converter using git-fast-import - -Usage -===== - -git-p4 can be used in two different ways: - -1) To import changes from Perforce to a Git repository, using "git-p4 sync". - -2) To submit changes from Git back to Perforce, using "git-p4 submit". - -Importing -========= - -Simply start with - - git-p4 clone //depot/path/project - -or - - git-p4 clone //depot/path/project myproject - -This will: - -1) Create an empty git repository in a subdirectory called "project" (or -"myproject" with the second command) - -2) Import the head revision from the given Perforce path into a git branch -called "p4" (remotes/p4 actually) - -3) Create a master branch based on it and check it out. - -If you want the entire history (not just the head revision) then you can simply -append a "@all" to the depot path: - - git-p4 clone //depot/project/main@all myproject - - - -If you want more control you can also use the git-p4 sync command directly: - - mkdir repo-git - cd repo-git - git init - git-p4 sync //path/in/your/perforce/depot - -This will import the current head revision of the specified depot path into a -"remotes/p4/master" branch of your git repository. You can use the ---branch=mybranch option to import into a different branch. - -If you want to import the entire history of a given depot path simply use: - - git-p4 sync //path/in/depot@all - - -Note: - -To achieve optimal compression you may want to run 'git repack -a -d -f' after -a big import. This may take a while. - -Incremental Imports -=================== - -After an initial import you can continue to synchronize your git repository -with newer changes from the Perforce depot by just calling - - git-p4 sync - -in your git repository. By default the "remotes/p4/master" branch is updated. - -Advanced Setup -============== - -Suppose you have a periodically updated git repository somewhere, containing a -complete import of a Perforce project. This repository can be cloned and used -with git-p4. When updating the cloned repository with the "sync" command, -git-p4 will try to fetch changes from the original repository first. The git -protocol used with this is usually faster than importing from Perforce -directly. - -This behaviour can be disabled by setting the "git-p4.syncFromOrigin" git -configuration variable to "false". - -Updating -======== - -A common working pattern is to fetch the latest changes from the Perforce depot -and merge them with local uncommitted changes. The recommended way is to use -git's rebase mechanism to preserve linear history. git-p4 provides a convenient - - git-p4 rebase - -command that calls git-p4 sync followed by git rebase to rebase the current -working branch. - -Submitting -========== - -git-p4 has support for submitting changes from a git repository back to the -Perforce depot. This requires a Perforce checkout separate from your git -repository. To submit all changes that are in the current git branch but not in -the "p4" branch (or "origin" if "p4" doesn't exist) simply call - - git-p4 submit - -in your git repository. If you want to submit changes in a specific branch that -is not your current git branch you can also pass that as an argument: - - git-p4 submit mytopicbranch - -You can override the reference branch with the --origin=mysourcebranch option. - -If a submit fails you may have to "p4 resolve" and submit manually. You can -continue importing the remaining changes with - - git-p4 submit --continue - -Example -======= - -# Clone a repository - git-p4 clone //depot/path/project -# Enter the newly cloned directory - cd project -# Do some work... - vi foo.h -# ... and commit locally to gi - git commit foo.h -# In the meantime somebody submitted changes to the Perforce depot. Rebase your latest -# changes against the latest changes in Perforce: - git-p4 rebase -# Submit your locally committed changes back to Perforce - git-p4 submit -# ... and synchronize with Perforce - git-p4 rebase - - -Configuration parameters -======================== - -git-p4.user ($P4USER) - -Allows you to specify the username to use to connect to the Perforce repository. - - git config [--global] git-p4.user public - -git-p4.password ($P4PASS) - -Allows you to specify the password to use to connect to the Perforce repository. -Warning this password will be visible on the command-line invocation of the p4 binary. - - git config [--global] git-p4.password public1234 - -git-p4.port ($P4PORT) - -Specify the port to be used to contact the Perforce server. As this will be passed -directly to the p4 binary, it may be in the format host:port as well. - - git config [--global] git-p4.port codes.zimbra.com:2666 - -git-p4.host ($P4HOST) - -Specify the host to contact for a Perforce repository. - - git config [--global] git-p4.host perforce.example.com - -git-p4.client ($P4CLIENT) - -Specify the client name to use - - git config [--global] git-p4.client public-view - -git-p4.allowSubmit - - git config [--global] git-p4.allowSubmit false - -git-p4.syncFromOrigin - -A useful setup may be that you have a periodically updated git repository -somewhere that contains a complete import of a Perforce project. That git -repository can be used to clone the working repository from and one would -import from Perforce directly after cloning using git-p4. If the connection to -the Perforce server is slow and the working repository hasn't been synced for a -while it may be desirable to fetch changes from the origin git repository using -the efficient git protocol. git-p4 supports this setup by calling "git fetch origin" -by default if there is an origin branch. You can disable this using: - - git config [--global] git-p4.syncFromOrigin false - -git-p4.useclientspec - - git config [--global] git-p4.useclientspec false - -Implementation Details... -========================= - -* Changesets from Perforce are imported using git fast-import. -* The import does not require anything from the Perforce client view as it just uses - "p4 print //depot/path/file#revision" to get the actual file contents. -* Every imported changeset has a special [git-p4...] line at the - end of the log message that gives information about the corresponding - Perforce change number and is also used by git-p4 itself to find out - where to continue importing when doing incremental imports. - Basically when syncing it extracts the perforce change number of the - latest commit in the "p4" branch and uses "p4 changes //depot/path/...@changenum,#head" - to find out which changes need to be imported. -* git-p4 submit uses "git rev-list" to pick the commits between the "p4" branch - and the current branch. - The commits themselves are applied using git diff/format-patch ... | git apply - diff --git a/contrib/fast-import/import-directories.perl b/contrib/fast-import/import-directories.perl index 7f3afa5ac4..4dec1f18e4 100755 --- a/contrib/fast-import/import-directories.perl +++ b/contrib/fast-import/import-directories.perl @@ -109,8 +109,8 @@ was available previously is not included in this revision, it will be removed. If an on-disk revision is incomplete, you can point to files from -a previous revision. There are no restriction as to where the source -files are located, nor to the names of them. +a previous revision. There are no restrictions on where the source +files are located, nor on their names. [3.files] ; the key is the path inside the repository, the value is the path diff --git a/contrib/fast-import/import-tars.perl b/contrib/fast-import/import-tars.perl index 95438e1ed4..d60b4315ed 100755 --- a/contrib/fast-import/import-tars.perl +++ b/contrib/fast-import/import-tars.perl @@ -96,18 +96,21 @@ foreach my $tar_file (@ARGV) $mtime = oct $mtime; next if $typeflag == 5; # directory - print FI "blob\n", "mark :$next_mark\n"; - if ($typeflag == 2) { # symbolic link - print FI "data ", length($linkname), "\n", $linkname; - $mode = 0120000; - } else { - print FI "data $size\n"; - while ($size > 0 && read(I, $_, 512) == 512) { - print FI substr($_, 0, $size); - $size -= 512; + if ($typeflag != 1) { # handle hard links later + print FI "blob\n", "mark :$next_mark\n"; + if ($typeflag == 2) { # symbolic link + print FI "data ", length($linkname), "\n", + $linkname; + $mode = 0120000; + } else { + print FI "data $size\n"; + while ($size > 0 && read(I, $_, 512) == 512) { + print FI substr($_, 0, $size); + $size -= 512; + } } + print FI "\n"; } - print FI "\n"; my $path; if ($prefix) { @@ -115,7 +118,13 @@ foreach my $tar_file (@ARGV) } else { $path = "$name"; } - $files{$path} = [$next_mark++, $mode]; + + if ($typeflag == 1) { # hard link + $linkname = "$prefix/$linkname" if $prefix; + $files{$path} = [ $files{$linkname}->[0], $mode ]; + } else { + $files{$path} = [$next_mark++, $mode]; + } $author_time = $mtime if $mtime > $author_time; $path =~ m,^([^/]+)/,; diff --git a/contrib/fast-import/import-zips.py b/contrib/fast-import/import-zips.py index 82f5ed3ddc..d12c296223 100755 --- a/contrib/fast-import/import-zips.py +++ b/contrib/fast-import/import-zips.py @@ -9,13 +9,18 @@ ## git log --stat import-zips from os import popen, path -from sys import argv, exit +from sys import argv, exit, hexversion, stderr from time import mktime from zipfile import ZipFile +if hexversion < 0x01060000: + # The limiter is the zipfile module + stderr.write("import-zips.py: requires Python 1.6.0 or later.\n") + exit(1) + if len(argv) < 2: - print 'Usage:', argv[0], '<zipfile>...' - exit(1) + print 'usage:', argv[0], '<zipfile>...' + exit(1) branch_ref = 'refs/heads/import-zips' committer_name = 'Z Ip Creator' @@ -23,51 +28,51 @@ committer_email = 'zip@example.com' fast_import = popen('git fast-import --quiet', 'w') def printlines(list): - for str in list: - fast_import.write(str + "\n") + for str in list: + fast_import.write(str + "\n") for zipfile in argv[1:]: - commit_time = 0 - next_mark = 1 - common_prefix = None - mark = dict() - - zip = ZipFile(zipfile, 'r') - for name in zip.namelist(): - if name.endswith('/'): - continue - info = zip.getinfo(name) - - if commit_time < info.date_time: - commit_time = info.date_time - if common_prefix == None: - common_prefix = name[:name.rfind('/') + 1] - else: - while not name.startswith(common_prefix): - last_slash = common_prefix[:-1].rfind('/') + 1 - common_prefix = common_prefix[:last_slash] - - mark[name] = ':' + str(next_mark) - next_mark += 1 - - printlines(('blob', 'mark ' + mark[name], \ - 'data ' + str(info.file_size))) - fast_import.write(zip.read(name) + "\n") - - committer = committer_name + ' <' + committer_email + '> %d +0000' % \ - mktime(commit_time + (0, 0, 0)) - - printlines(('commit ' + branch_ref, 'committer ' + committer, \ - 'data <<EOM', 'Imported from ' + zipfile + '.', 'EOM', \ - '', 'deleteall')) - - for name in mark.keys(): - fast_import.write('M 100644 ' + mark[name] + ' ' + - name[len(common_prefix):] + "\n") - - printlines(('', 'tag ' + path.basename(zipfile), \ - 'from ' + branch_ref, 'tagger ' + committer, \ - 'data <<EOM', 'Package ' + zipfile, 'EOM', '')) + commit_time = 0 + next_mark = 1 + common_prefix = None + mark = dict() + + zip = ZipFile(zipfile, 'r') + for name in zip.namelist(): + if name.endswith('/'): + continue + info = zip.getinfo(name) + + if commit_time < info.date_time: + commit_time = info.date_time + if common_prefix == None: + common_prefix = name[:name.rfind('/') + 1] + else: + while not name.startswith(common_prefix): + last_slash = common_prefix[:-1].rfind('/') + 1 + common_prefix = common_prefix[:last_slash] + + mark[name] = ':' + str(next_mark) + next_mark += 1 + + printlines(('blob', 'mark ' + mark[name], \ + 'data ' + str(info.file_size))) + fast_import.write(zip.read(name) + "\n") + + committer = committer_name + ' <' + committer_email + '> %d +0000' % \ + mktime(commit_time + (0, 0, 0)) + + printlines(('commit ' + branch_ref, 'committer ' + committer, \ + 'data <<EOM', 'Imported from ' + zipfile + '.', 'EOM', \ + '', 'deleteall')) + + for name in mark.keys(): + fast_import.write('M 100644 ' + mark[name] + ' ' + + name[len(common_prefix):] + "\n") + + printlines(('', 'tag ' + path.basename(zipfile), \ + 'from ' + branch_ref, 'tagger ' + committer, \ + 'data <<EOM', 'Package ' + zipfile, 'EOM', '')) if fast_import.close(): - exit(1) + exit(1) |