diff options
Diffstat (limited to 'contrib/fast-import')
| -rwxr-xr-x | contrib/fast-import/git-p4 | 654 | ||||
| -rw-r--r-- | contrib/fast-import/git-p4.txt | 79 | ||||
| -rwxr-xr-x | contrib/fast-import/import-directories.perl | 417 | ||||
| -rwxr-xr-x | contrib/fast-import/import-tars.perl | 70 | ||||
| -rwxr-xr-x | contrib/fast-import/import-zips.py | 2 |
5 files changed, 1059 insertions, 163 deletions
diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 342529db30..2f7b270566 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -8,12 +8,10 @@ # License: MIT <http://www.opensource.org/licenses/mit-license.php> # -import optparse, sys, os, marshal, popen2, subprocess, shelve -import tempfile, getopt, sha, os.path, time, platform +import optparse, sys, os, marshal, subprocess, shelve +import tempfile, getopt, os.path, time, platform import re -from sets import Set; - verbose = False @@ -201,7 +199,7 @@ def isModeExec(mode): def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) -def p4CmdList(cmd, stdin=None, stdin_mode='w+b'): +def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): cmd = p4_build_cmd("-G %s" % (cmd)) if verbose: sys.stderr.write("Opening pipe: %s\n" % cmd) @@ -224,7 +222,10 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b'): try: while True: entry = marshal.load(p4.stdout) - result.append(entry) + if cb is not None: + cb(entry) + else: + result.append(entry) except EOFError: pass exitCode = p4.wait() @@ -332,9 +333,18 @@ def gitBranchExists(branch): return proc.wait() == 0; _gitConfig = {} -def gitConfig(key): +def gitConfig(key, args = None): # set args to "--bool", for instance + if not _gitConfig.has_key(key): + argsFilter = "" + if args != None: + argsFilter = "%s " % args + cmd = "git config %s%s" % (argsFilter, key) + _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip() + return _gitConfig[key] + +def gitConfigList(key): if not _gitConfig.has_key(key): - _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip() + _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep) return _gitConfig[key] def p4BranchesInGit(branchesAreInRemotes = True): @@ -444,18 +454,72 @@ def p4ChangesForPaths(depotPaths, changeRange): changes = {} for line in output: - changeNum = int(line.split(" ")[1]) - changes[changeNum] = True + changeNum = int(line.split(" ")[1]) + changes[changeNum] = True changelist = changes.keys() changelist.sort() return changelist +def p4PathStartsWith(path, prefix): + # This method tries to remedy a potential mixed-case issue: + # + # If UserA adds //depot/DirA/file1 + # and UserB adds //depot/dira/file2 + # + # we may or may not have a problem. If you have core.ignorecase=true, + # we treat DirA and dira as the same directory + ignorecase = gitConfig("core.ignorecase", "--bool") == "true" + if ignorecase: + return path.lower().startswith(prefix.lower()) + return path.startswith(prefix) + class Command: def __init__(self): self.usage = "usage: %prog [options]" self.needsGit = True +class P4UserMap: + def __init__(self): + self.userMapFromPerforceServer = False + + def getUserCacheFilename(self): + home = os.environ.get("HOME", os.environ.get("USERPROFILE")) + return home + "/.gitp4-usercache.txt" + + def getUserMapFromPerforceServer(self): + if self.userMapFromPerforceServer: + return + self.users = {} + self.emails = {} + + for output in p4CmdList("users"): + if not output.has_key("User"): + continue + self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + self.emails[output["Email"]] = output["User"] + + + s = '' + for (key, val) in self.users.items(): + s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) + + open(self.getUserCacheFilename(), "wb").write(s) + self.userMapFromPerforceServer = True + + def loadUserMapFromCache(self): + self.users = {} + self.userMapFromPerforceServer = False + try: + cache = open(self.getUserCacheFilename(), "rb") + lines = cache.readlines() + cache.close() + for line in lines: + entry = line.strip().split("\t") + self.users[entry[0]] = entry[1] + except IOError: + self.getUserMapFromPerforceServer() + class P4Debug(Command): def __init__(self): Command.__init__(self) @@ -536,21 +600,26 @@ class P4RollBack(Command): return True -class P4Submit(Command): +class P4Submit(Command, P4UserMap): def __init__(self): Command.__init__(self) + P4UserMap.__init__(self) self.options = [ optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--origin", dest="origin"), - optparse.make_option("-M", dest="detectRename", action="store_true"), + optparse.make_option("-M", dest="detectRenames", action="store_true"), + # preserve the user, requires relevant p4 permissions + optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" self.interactive = True self.origin = "" - self.detectRename = False + self.detectRenames = False self.verbose = False + self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true" self.isWindows = (platform.system() == "Windows") + self.myP4UserId = None def check(self): if len(p4CmdList("opened ...")) > 0: @@ -569,7 +638,7 @@ class P4Submit(Command): continue if inDescriptionSection: - if line.startswith("Files:"): + if line.startswith("Files:") or line.startswith("Jobs:"): inDescriptionSection = False else: continue @@ -584,6 +653,99 @@ class P4Submit(Command): return result + def p4UserForCommit(self,id): + # Return the tuple (perforce user,git email) for a given git commit id + self.getUserMapFromPerforceServer() + gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id) + gitEmail = gitEmail.strip() + if not self.emails.has_key(gitEmail): + return (None,gitEmail) + else: + return (self.emails[gitEmail],gitEmail) + + def checkValidP4Users(self,commits): + # check if any git authors cannot be mapped to p4 users + for id in commits: + (user,email) = self.p4UserForCommit(id) + if not user: + msg = "Cannot find p4 user for email %s in commit %s." % (email, id) + if gitConfig('git-p4.allowMissingP4Users').lower() == "true": + print "%s" % msg + else: + die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg) + + def lastP4Changelist(self): + # Get back the last changelist number submitted in this client spec. This + # then gets used to patch up the username in the change. If the same + # client spec is being used by multiple processes then this might go + # wrong. + results = p4CmdList("client -o") # find the current client + client = None + for r in results: + if r.has_key('Client'): + client = r['Client'] + break + if not client: + die("could not get client spec") + results = p4CmdList("changes -c %s -m 1" % client) + for r in results: + if r.has_key('change'): + return r['change'] + die("Could not get changelist number for last submit - cannot patch up user details") + + def modifyChangelistUser(self, changelist, newUser): + # fixup the user field of a changelist after it has been submitted. + changes = p4CmdList("change -o %s" % changelist) + if len(changes) != 1: + die("Bad output from p4 change modifying %s to user %s" % + (changelist, newUser)) + + c = changes[0] + if c['User'] == newUser: return # nothing to do + c['User'] = newUser + input = marshal.dumps(c) + + result = p4CmdList("change -f -i", stdin=input) + for r in result: + if r.has_key('code'): + if r['code'] == 'error': + die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data'])) + if r.has_key('data'): + print("Updated user field for changelist %s to %s" % (changelist, newUser)) + return + die("Could not modify user field of changelist %s to %s" % (changelist, newUser)) + + def canChangeChangelists(self): + # check to see if we have p4 admin or super-user permissions, either of + # which are required to modify changelists. + results = p4CmdList("protects %s" % self.depotPath) + for r in results: + if r.has_key('perm'): + if r['perm'] == 'admin': + return 1 + if r['perm'] == 'super': + return 1 + return 0 + + def p4UserId(self): + if self.myP4UserId: + return self.myP4UserId + + results = p4CmdList("user -o") + for r in results: + if r.has_key('User'): + self.myP4UserId = r['User'] + return r['User'] + die("Could not find your p4 user id") + + def p4UserIsMe(self, p4User): + # return True if the given p4 user is actually me + me = self.p4UserId() + if not p4User or p4User != me: + return False + else: + return True + def prepareSubmitTemplate(self): # remove lines in the Files section that show changes to files outside the depot path we're committing into template = "" @@ -598,7 +760,7 @@ class P4Submit(Command): lastTab = path.rfind("\t") if lastTab != -1: path = path[:lastTab] - if not path.startswith(self.depotPath): + if not p4PathStartsWith(path, self.depotPath): continue else: inFilesSection = False @@ -612,7 +774,29 @@ class P4Submit(Command): def applyCommit(self, id): print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id)) - diffOpts = ("", "-M")[self.detectRename] + + (p4User, gitEmail) = self.p4UserForCommit(id) + + if not self.detectRenames: + # If not explicitly set check the config variable + self.detectRenames = gitConfig("git-p4.detectRenames") + + if self.detectRenames.lower() == "false" or self.detectRenames == "": + diffOpts = "" + elif self.detectRenames.lower() == "true": + diffOpts = "-M" + else: + diffOpts = "-M%s" % self.detectRenames + + detectCopies = gitConfig("git-p4.detectCopies") + if detectCopies.lower() == "true": + diffOpts += " -C" + elif detectCopies != "" and detectCopies.lower() != "false": + diffOpts += " -C%s" % detectCopies + + if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true": + diffOpts += " --find-copies-harder" + diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id)) filesToAdd = set() filesToDelete = set() @@ -636,11 +820,23 @@ class P4Submit(Command): filesToDelete.add(path) if path in filesToAdd: filesToAdd.remove(path) + elif modifier == "C": + src, dest = diff['src'], diff['dst'] + p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) + if diff['src_sha1'] != diff['dst_sha1']: + p4_system("edit \"%s\"" % (dest)) + if isModeExecChanged(diff['src_mode'], diff['dst_mode']): + p4_system("edit \"%s\"" % (dest)) + filesToChangeExecBit[dest] = diff['dst_mode'] + os.unlink(dest) + editedFiles.add(dest) elif modifier == "R": src, dest = diff['src'], diff['dst'] p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) - p4_system("edit \"%s\"" % (dest)) + if diff['src_sha1'] != diff['dst_sha1']: + p4_system("edit \"%s\"" % (dest)) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): + p4_system("edit \"%s\"" % (dest)) filesToChangeExecBit[dest] = diff['dst_mode'] os.unlink(dest) editedFiles.add(dest) @@ -703,9 +899,15 @@ class P4Submit(Command): if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) + + if self.preserveUser: + submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User) + if os.environ.has_key("P4DIFF"): del(os.environ["P4DIFF"]) - diff = p4_read_pipe("diff -du ...") + diff = "" + for editedFile in editedFiles: + diff += p4_read_pipe("diff -du %r" % editedFile) newdiff = "" for newFile in filesToAdd: @@ -717,6 +919,11 @@ class P4Submit(Command): newdiff += "+" + line f.close() + if self.checkAuthorship and not self.p4UserIsMe(p4User): + submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail + submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n" + submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n" + separatorLine = "######## everything below this line is just the diff #######\n" [handle, fileName] = tempfile.mkstemp() @@ -728,17 +935,19 @@ class P4Submit(Command): tmpFile.write(submitTemplate + separatorLine + diff + newdiff) tmpFile.close() mtime = os.stat(fileName).st_mtime - defaultEditor = "vi" - if platform.system() == "Windows": - defaultEditor = "notepad" if os.environ.has_key("P4EDITOR"): editor = os.environ.get("P4EDITOR") else: - editor = os.environ.get("EDITOR", defaultEditor); + editor = read_pipe("git var GIT_EDITOR").strip() system(editor + " " + fileName) + if gitConfig("git-p4.skipSubmitEditCheck") == "true": + checkModTime = False + else: + checkModTime = True + response = "y" - if os.stat(fileName).st_mtime <= mtime: + if checkModTime and (os.stat(fileName).st_mtime <= mtime): response = "x" while response != "y" and response != "n": response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") @@ -751,6 +960,14 @@ class P4Submit(Command): if self.isWindows: submitTemplate = submitTemplate.replace("\r\n", "\n") p4_write_pipe("submit -i", submitTemplate) + + if self.preserveUser: + if p4User: + # Get last changelist number. Cannot easily get it from + # the submit command output as the output is unmarshalled. + changelist = self.lastP4Changelist() + self.modifyChangelistUser(changelist, p4User) + else: for f in editedFiles: p4_system("revert \"%s\"" % f); @@ -787,6 +1004,10 @@ class P4Submit(Command): if len(self.origin) == 0: self.origin = upstream + if self.preserveUser: + if not self.canChangeChangelists(): + die("Cannot preserve user names without p4 super-user or admin permissions") + if self.verbose: print "Origin branch is " + self.origin @@ -804,7 +1025,7 @@ class P4Submit(Command): self.oldWorkingDirectory = os.getcwd() chdir(self.clientPath) - print "Syncronizing p4 checkout..." + print "Synchronizing p4 checkout..." p4_system("sync ...") self.check() @@ -814,6 +1035,14 @@ class P4Submit(Command): commits.append(line.strip()) commits.reverse() + if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"): + self.checkAuthorship = False + else: + self.checkAuthorship = True + + if self.preserveUser: + self.checkValidP4Users(commits) + while len(commits) > 0: commit = commits[0] commits = commits[1:] @@ -833,9 +1062,12 @@ class P4Submit(Command): return True -class P4Sync(Command): +class P4Sync(Command, P4UserMap): + delete_actions = ( "delete", "move/delete", "purge" ) + def __init__(self): Command.__init__(self) + P4UserMap.__init__(self) self.options = [ optparse.make_option("--branch", dest="branch"), optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), @@ -861,8 +1093,8 @@ class P4Sync(Command): self.usage += " //depot/path[@revRange]" self.silent = False - self.createdBranches = Set() - self.committedChanges = Set() + self.createdBranches = set() + self.committedChanges = set() self.branch = "" self.detectBranches = False self.detectLabels = False @@ -882,6 +1114,23 @@ class P4Sync(Command): if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False + # + # P4 wildcards are not allowed in filenames. P4 complains + # if you simply add them, but you can force it with "-f", in + # which case it translates them into %xx encoding internally. + # Search for and fix just these four characters. Do % last so + # that fixing it does not inadvertently create new %-escapes. + # + def wildcard_decode(self, path): + # Cannot have * in a filename in windows; untested as to + # what p4 would do in such a case. + if not self.isWindows: + path = path.replace("%2A", "*") + path = path.replace("%23", "#") \ + .replace("%40", "@") \ + .replace("%25", "%") + return path + def extractFilesFromCommit(self, commit): self.cloneExclude = [re.sub(r"\.\.\.$", "", path) for path in self.cloneExclude] @@ -891,11 +1140,11 @@ class P4Sync(Command): path = commit["depotFile%s" % fnum] if [p for p in self.cloneExclude - if path.startswith (p)]: + if p4PathStartsWith(path, p)]: found = False else: found = [p for p in self.depotPaths - if path.startswith (p)] + if p4PathStartsWith(path, p)] if not found: fnum = fnum + 1 continue @@ -910,11 +1159,27 @@ class P4Sync(Command): return files def stripRepoPath(self, path, prefixes): + if self.useClientSpec: + + # if using the client spec, we use the output directory + # specified in the client. For example, a view + # //depot/foo/branch/... //client/branch/foo/... + # will end up putting all foo/branch files into + # branch/foo/ + for val in self.clientSpecDirs: + if path.startswith(val[0]): + # replace the depot path with the client path + path = path.replace(val[0], val[1][1]) + # now strip out the client (//client/...) + path = re.sub("^(//[^/]+/)", '', path) + # the rest is all path + return path + if self.keepRepoPath: prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])] for p in prefixes: - if path.startswith(p): + if p4PathStartsWith(path, p): path = path[len(p):] return path @@ -925,7 +1190,7 @@ class P4Sync(Command): while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] found = [p for p in self.depotPaths - if path.startswith (p)] + if p4PathStartsWith(path, p)] if not found: fnum = fnum + 1 continue @@ -950,67 +1215,126 @@ class P4Sync(Command): return branches - ## Should move this out, doesn't use SELF. - def readP4Files(self, files): + # output one file from the P4 stream + # - helper for streamP4Files + + def streamOneP4File(self, file, contents): + if file["type"] == "apple": + print "\nfile %s is a strange apple file that forks. Ignoring" % \ + file['depotFile'] + return + + relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) + relPath = self.wildcard_decode(relPath) + if verbose: + sys.stderr.write("%s\n" % relPath) + + mode = "644" + if isP4Exec(file["type"]): + mode = "755" + elif file["type"] == "symlink": + mode = "120000" + # p4 print on a symlink contains "target\n", so strip it off + data = ''.join(contents) + contents = [data[:-1]] + + if self.isWindows and file["type"].endswith("text"): + mangled = [] + for data in contents: + data = data.replace("\r\n", "\n") + mangled.append(data) + contents = mangled + + if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): + contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents) + elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): + contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents) + + self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + + # total length... + length = 0 + for d in contents: + length = length + len(d) + + self.gitStream.write("data %d\n" % length) + for d in contents: + self.gitStream.write(d) + self.gitStream.write("\n") + + def streamOneP4Deletion(self, file): + relPath = self.stripRepoPath(file['path'], self.branchPrefixes) + if verbose: + sys.stderr.write("delete %s\n" % relPath) + self.gitStream.write("D %s\n" % relPath) + + # handle another chunk of streaming data + def streamP4FilesCb(self, marshalled): + + if marshalled.has_key('depotFile') and self.stream_have_file_info: + # start of a new file - output the old one first + self.streamOneP4File(self.stream_file, self.stream_contents) + self.stream_file = {} + self.stream_contents = [] + self.stream_have_file_info = False + + # pick up the new file information... for the + # 'data' field we need to append to our array + for k in marshalled.keys(): + if k == 'data': + self.stream_contents.append(marshalled['data']) + else: + self.stream_file[k] = marshalled[k] + + self.stream_have_file_info = True + + # Stream directly from "p4 files" into "git fast-import" + def streamP4Files(self, files): filesForCommit = [] filesToRead = [] + filesToDelete = [] for f in files: includeFile = True for val in self.clientSpecDirs: if f['path'].startswith(val[0]): - if val[1] <= 0: + if val[1][0] <= 0: includeFile = False break if includeFile: filesForCommit.append(f) - if f['action'] not in ('delete', 'purge'): + if f['action'] in self.delete_actions: + filesToDelete.append(f) + else: filesToRead.append(f) - filedata = [] - if len(filesToRead) > 0: - filedata = p4CmdList('-x - print', - stdin='\n'.join(['%s#%s' % (f['path'], f['rev']) - for f in filesToRead]), - stdin_mode='w+') - - if "p4ExitCode" in filedata[0]: - die("Problems executing p4. Error: [%d]." - % (filedata[0]['p4ExitCode'])); - - j = 0; - contents = {} - while j < len(filedata): - stat = filedata[j] - j += 1 - text = '' - while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'): - text += filedata[j]['data'] - del filedata[j]['data'] - j += 1 - - if not stat.has_key('depotFile'): - sys.stderr.write("p4 print fails with: %s\n" % repr(stat)) - continue + # deleted files... + for f in filesToDelete: + self.streamOneP4Deletion(f) - if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): - text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text) - elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): - text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text) + if len(filesToRead) > 0: + self.stream_file = {} + self.stream_contents = [] + self.stream_have_file_info = False - contents[stat['depotFile']] = text + # curry self argument + def streamP4FilesCbSelf(entry): + self.streamP4FilesCb(entry) - for f in filesForCommit: - path = f['path'] - if contents.has_key(path): - f['data'] = contents[path] + p4CmdList("-x - print", + '\n'.join(['%s#%s' % (f['path'], f['rev']) + for f in filesToRead]), + cb=streamP4FilesCbSelf) - return filesForCommit + # do the last chunk + if self.stream_file.has_key('depotFile'): + self.streamOneP4File(self.stream_file, self.stream_contents) def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] author = details["user"] + self.branchPrefixes = branchPrefixes if self.verbose: print "commit into %s" % branch @@ -1019,11 +1343,10 @@ class P4Sync(Command): # create a commit. new_files = [] for f in files: - if [p for p in branchPrefixes if f['path'].startswith(p)]: + if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]: new_files.append (f) else: - sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) - files = self.readP4Files(new_files) + sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path']) self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) @@ -1051,33 +1374,7 @@ class P4Sync(Command): print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - for file in files: - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path'] - continue - - relPath = self.stripRepoPath(file['path'], branchPrefixes) - if file["action"] in ("delete", "purge"): - self.gitStream.write("D %s\n" % relPath) - else: - data = file['data'] - - mode = "644" - if isP4Exec(file["type"]): - mode = "755" - elif file["type"] == "symlink": - mode = "120000" - # p4 print on a symlink contains "target\n", so strip it off - data = data[:-1] - - if self.isWindows and file["type"].endswith("text"): - data = data.replace("\r\n", "\n") - - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) - self.gitStream.write("data %s\n" % len(data)) - self.gitStream.write(data) - self.gitStream.write("\n") - + self.streamP4Files(new_files) self.gitStream.write("\n") change = int(details["change"]) @@ -1096,7 +1393,7 @@ class P4Sync(Command): cleanedFiles = {} for info in files: - if info["action"] in ("delete", "purge"): + if info["action"] in self.delete_actions: continue cleanedFiles[info["depotFile"]] = info["rev"] @@ -1125,41 +1422,6 @@ class P4Sync(Command): print ("Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)) - def getUserCacheFilename(self): - home = os.environ.get("HOME", os.environ.get("USERPROFILE")) - return home + "/.gitp4-usercache.txt" - - def getUserMapFromPerforceServer(self): - if self.userMapFromPerforceServer: - return - self.users = {} - - for output in p4CmdList("users"): - if not output.has_key("User"): - continue - self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - - - s = '' - for (key, val) in self.users.items(): - s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) - - open(self.getUserCacheFilename(), "wb").write(s) - self.userMapFromPerforceServer = True - - def loadUserMapFromCache(self): - self.users = {} - self.userMapFromPerforceServer = False - try: - cache = open(self.getUserCacheFilename(), "rb") - lines = cache.readlines() - cache.close() - for line in lines: - entry = line.strip().split("\t") - self.users[entry[0]] = entry[1] - except IOError: - self.getUserMapFromPerforceServer() - def getLabels(self): self.labels = {} @@ -1198,7 +1460,13 @@ class P4Sync(Command): def getBranchMapping(self): lostAndFoundBranches = set() - for info in p4CmdList("branches"): + user = gitConfig("git-p4.branchUser") + if len(user) > 0: + command = "branches -u %s" % user + else: + command = "branches" + + for info in p4CmdList(command): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 while details.has_key("View%s" % viewIdx): @@ -1210,7 +1478,7 @@ class P4Sync(Command): source = paths[0] destination = paths[1] ## HACK - if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): + if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]): source = source[len(self.depotPaths[0]):-4] destination = destination[len(self.depotPaths[0]):-4] @@ -1227,6 +1495,25 @@ class P4Sync(Command): if source not in self.knownBranches: lostAndFoundBranches.add(source) + # Perforce does not strictly require branches to be defined, so we also + # check git config for a branch list. + # + # Example of branch definition in git config file: + # [git-p4] + # branchList=main:branchA + # branchList=main:branchB + # branchList=branchA:branchC + configBranches = gitConfigList("git-p4.branchList") + for branch in configBranches: + if branch: + (source, destination) = branch.split(":") + self.knownBranches[destination] = source + + lostAndFoundBranches.discard(destination) + + if source not in self.knownBranches: + lostAndFoundBranches.add(source) + for branch in lostAndFoundBranches: self.knownBranches[branch] = branch @@ -1397,8 +1684,9 @@ class P4Sync(Command): def importHeadRevision(self, revision): print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch) - details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = ("Initial import of %s from the state at revision %s" + details = {} + details["user"] = "git perforce import user" + details["desc"] = ("Initial import of %s from the state at revision %s\n" % (' '.join(self.depotPaths), revision)) details["change"] = revision newestRevision = 0 @@ -1409,9 +1697,16 @@ class P4Sync(Command): % (p, revision) for p in self.depotPaths])): - if info['code'] == 'error': + if 'code' in info and info['code'] == 'error': sys.stderr.write("p4 returned an error: %s\n" % info['data']) + if info['data'].find("must refer to client") >= 0: + sys.stderr.write("This particular p4 error is misleading.\n") + sys.stderr.write("Perhaps the depot path was misspelled.\n"); + sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths)) + sys.exit(1) + if 'p4ExitCode' in info: + sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode']) sys.exit(1) @@ -1419,7 +1714,7 @@ class P4Sync(Command): if change > newestRevision: newestRevision = change - if info["action"] in ("delete", "purge"): + if info["action"] in self.delete_actions: # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! #fileCnt = fileCnt + 1 continue @@ -1430,6 +1725,18 @@ class P4Sync(Command): fileCnt = fileCnt + 1 details["change"] = newestRevision + + # Use time from top-most change so that all git-p4 clones of + # the same p4 repo have the same commit SHA1s. + res = p4CmdList("describe -s %d" % newestRevision) + newestTime = None + for r in res: + if r.has_key('time'): + newestTime = int(r['time']) + if newestTime is None: + die("\"describe -s\" on newest change %d did not give a time") + details["time"] = newestTime + self.updateOptionDict(details) try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths) @@ -1444,19 +1751,45 @@ class P4Sync(Command): for entry in specList: for k,v in entry.iteritems(): if k.startswith("View"): + + # p4 has these %%1 to %%9 arguments in specs to + # reorder paths; which we can't handle (yet :) + if re.match('%%\d', v) != None: + print "Sorry, can't handle %%n arguments in client specs" + sys.exit(1) + if v.startswith('"'): start = 1 else: start = 0 index = v.find("...") + + # save the "client view"; i.e the RHS of the view + # line that tells the client where to put the + # files for this view. + cv = v[index+3:].strip() # +3 to remove previous '...' + + # if the client view doesn't end with a + # ... wildcard, then we're going to mess up the + # output directory, so fail gracefully. + if not cv.endswith('...'): + print 'Sorry, client view in "%s" needs to end with wildcard' % (k) + sys.exit(1) + cv=cv[:-3] + + # now save the view; +index means included, -index + # means it should be filtered out. v = v[start:index] if v.startswith("-"): v = v[1:] - temp[v] = -len(v) + include = -len(v) else: - temp[v] = len(v) + include = len(v) + + temp[v] = (include, cv) + self.clientSpecDirs = temp.items() - self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) ) + self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) ) def run(self, args): self.depotPaths = [] @@ -1526,12 +1859,14 @@ class P4Sync(Command): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, min(len(cur), len(prev))): - if cur[i] <> prev[i]: + prev_list = prev.split("/") + cur_list = cur.split("/") + for i in range(0, min(len(cur_list), len(prev_list))): + if cur_list[i] <> prev_list[i]: i = i - 1 break - paths.append (cur[:i + 1]) + paths.append ("/".join(cur_list[:i + 1])) self.previousDepotPaths = paths @@ -1627,7 +1962,7 @@ class P4Sync(Command): if len(self.changesFile) > 0: output = open(self.changesFile).readlines() - changeSet = Set() + changeSet = set() for line in output: changeSet.add(int(line)) @@ -1636,6 +1971,10 @@ class P4Sync(Command): changes.sort() else: + # catch "git-p4 sync" with no new branches, in a repo that + # does not have any existing git-p4 branches + if len(args) == 0 and not self.p4BranchesInGit: + die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here."); if self.verbose: print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) @@ -1716,10 +2055,13 @@ class P4Clone(P4Sync): help="where to leave result of the clone"), optparse.make_option("-/", dest="cloneExclude", action="append", type="string", - help="exclude depot path") + help="exclude depot path"), + optparse.make_option("--bare", dest="cloneBare", + action="store_true", default=False), ] self.cloneDestination = None self.needsGit = False + self.cloneBare = False # This is required for the "append" cloneExclude action def ensure_value(self, attr, value): @@ -1759,11 +2101,16 @@ class P4Clone(P4Sync): self.cloneDestination = self.defaultDestination(args) print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) + if not os.path.exists(self.cloneDestination): os.makedirs(self.cloneDestination) chdir(self.cloneDestination) - system("git init") - self.gitdir = os.getcwd() + "/.git" + + init_cmd = [ "git", "init" ] + if self.cloneBare: + init_cmd.append("--bare") + subprocess.check_call(init_cmd) + if not P4Sync.run(self, depotPaths): return False if self.branch != "master": @@ -1773,7 +2120,8 @@ class P4Clone(P4Sync): masterbranch = "refs/heads/p4/master" if gitBranchExists(masterbranch): system("git branch master %s" % masterbranch) - system("git checkout -f") + if not self.cloneBare: + system("git checkout -f") else: print "Could not detect main branch. No checkout/master branch created." diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 49b335921a..52003ae904 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -110,6 +110,12 @@ is not your current git branch you can also pass that as an argument: You can override the reference branch with the --origin=mysourcebranch option. +The Perforce changelists will be created with the user who ran git-p4. If you +use --preserve-user then git-p4 will attempt to create Perforce changelists +with the Perforce user corresponding to the git commit author. You need to +have sufficient permissions within Perforce, and the git users need to have +Perforce accounts. Permissions can be granted using 'p4 protect'. + If a submit fails you may have to "p4 resolve" and submit manually. You can continue importing the remaining changes with @@ -191,6 +197,79 @@ git-p4.useclientspec git config [--global] git-p4.useclientspec false +The P4CLIENT environment variable should be correctly set for p4 to be +able to find the relevant client. This client spec will be used to +both filter the files cloned by git and set the directory layout as +specified in the client (this implies --keep-path style semantics). + +git-p4.skipSubmitModTimeCheck + + git config [--global] git-p4.skipSubmitModTimeCheck false + +If true, submit will not check if the p4 change template has been modified. + +git-p4.preserveUser + + git config [--global] git-p4.preserveUser false + +If true, attempt to preserve user names by modifying the p4 changelists. See +the "--preserve-user" submit option. + +git-p4.allowMissingPerforceUsers + + git config [--global] git-p4.allowMissingP4Users false + +If git-p4 is setting the perforce user for a commit (--preserve-user) then +if there is no perforce user corresponding to the git author, git-p4 will +stop. With allowMissingPerforceUsers set to true, git-p4 will use the +current user (i.e. the behavior without --preserve-user) and carry on with +the perforce commit. + +git-p4.skipUserNameCheck + + git config [--global] git-p4.skipUserNameCheck false + +When submitting, git-p4 checks that the git commits are authored by the current +p4 user, and warns if they are not. This disables the check. + +git-p4.detectRenames + +Detect renames when submitting changes to Perforce server. Will enable -M git +argument. Can be optionally set to a number representing the threshold +percentage value of the rename detection. + + git config [--global] git-p4.detectRenames true + git config [--global] git-p4.detectRenames 50 + +git-p4.detectCopies + +Detect copies when submitting changes to Perforce server. Will enable -C git +argument. Can be optionally set to a number representing the threshold +percentage value of the copy detection. + + git config [--global] git-p4.detectCopies true + git config [--global] git-p4.detectCopies 80 + +git-p4.detectCopiesHarder + +Detect copies even between files that did not change when submitting changes to +Perforce server. Will enable --find-copies-harder git argument. + + git config [--global] git-p4.detectCopies true + +git-p4.branchUser + +Only use branch specifications defined by the selected username. + + git config [--global] git-p4.branchUser username + +git-p4.branchList + +List of branches to be imported when branch detection is enabled. + + git config [--global] git-p4.branchList main:branchA + git config [--global] --add git-p4.branchList main:branchB + Implementation Details... ========================= diff --git a/contrib/fast-import/import-directories.perl b/contrib/fast-import/import-directories.perl new file mode 100755 index 0000000000..7f3afa5ac4 --- /dev/null +++ b/contrib/fast-import/import-directories.perl @@ -0,0 +1,417 @@ +#!/usr/bin/perl +# +# Copyright 2008-2009 Peter Krefting <peter@softwolves.pp.se> +# +# ------------------------------------------------------------------------ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# ------------------------------------------------------------------------ + +=pod + +=head1 NAME + +import-directories - Import bits and pieces to Git. + +=head1 SYNOPSIS + +B<import-directories.perl> F<configfile> F<outputfile> + +=head1 DESCRIPTION + +Script to import arbitrary projects version controlled by the "copy the +source directory to a new location and edit it there"-version controlled +projects into version control. Handles projects with arbitrary branching +and version trees, taking a file describing the inputs and generating a +file compatible with the L<git-fast-import(1)> format. + +=head1 CONFIGURATION FILE + +=head2 Format + +The configuration file is based on the standard I<.ini> format. + + ; Comments start with semi-colons + [section] + key=value + +Please see below for information on how to escape special characters. + +=head2 Global configuration + +Global configuration is done in the B<[config]> section, which should be +the first section in the file. Configuration can be changed by +repeating configuration sections later on. + + [config] + ; configure conversion of CRLFs. "convert" means that all CRLFs + ; should be converted into LFs (suitable for the core.autocrlf + ; setting set to true in Git). "none" means that all data is + ; treated as binary. + crlf=convert + +=head2 Revision configuration + +Each revision that is to be imported is described in three +sections. Revisions should be defined in topological order, so +that a revision's parent has always been defined when a new revision +is introduced. All the sections for one revision must be defined +before defining the next revision. + +Each revision is assigned a unique numerical identifier. The +numbers do not need to be consecutive, nor monotonically +increasing. + +For instance, if your configuration file contains only the two +revisions 4711 and 42, where 4711 is the initial commit, the +only requirement is that 4711 is completely defined before 42. + +=pod + +=head3 Revision description section + +A section whose section name is just an integer gives meta-data +about the revision. + + [3] + ; author sets the author of the revisions + author=Peter Krefting <peter@softwolves.pp.se> + ; branch sets the branch that the revision should be committed to + branch=master + ; parent describes the revision that is the parent of this commit + ; (optional) + parent=1 + ; merges describes a revision that is merged into this commit + ; (optional; can be repeated) + merges=2 + ; selects one file to take the timestamp from + ; (optional; if unspecified, the most recent file from the .files + ; section is used) + timestamp=3/source.c + +=head3 Revision contents section + +A section whose section name is an integer followed by B<.files> +describe all the files included in this revision. If a file that +was available previously is not included in this revision, it will +be removed. + +If an on-disk revision is incomplete, you can point to files from +a previous revision. There are no restriction as to where the source +files are located, nor to the names of them. + + [3.files] + ; the key is the path inside the repository, the value is the path + ; as seen from the importer script. + source.c=ver-3.00/source.c + source.h=ver-2.99/source.h + readme.txt=ver-3.00/introduction to the project.txt + +File names are treated as byte strings (but please see below on +quoting rules), and should be stored in the configuration file in +the encoding that should be used in the generated repository. + +=head3 Revision commit message section + +A section whose section name is an integer followed by B<.message> +gives the commit message. This section is read verbatim, up until +the beginning of the next section. As such, a commit message may not +contain a line that begins with an opening square bracket ("[") and +ends with a closing square bracket ("]"), unless they are surrounded +by whitespace or other characters. + + [3.message] + Implement foobar. + ; trailing blank lines are ignored. + +=cut + +# Globals +use strict; +use warnings; +use integer; +my $crlfmode = 0; +my @revs; +my (%revmap, %message, %files, %author, %branch, %parent, %merges, %time, %timesource); +my $sectiontype = 0; +my $rev = 0; +my $mark = 1; + +# Check command line +if ($#ARGV < 1 || $ARGV[0] =~ /^--?h/) +{ + exec('perldoc', $0); + exit 1; +} + +# Open configuration +my $config = $ARGV[0]; +open CFG, '<', $config or die "Cannot open configuration file \"$config\": "; + +# Open output +my $output = $ARGV[1]; +open OUT, '>', $output or die "Cannot create output file \"$output\": "; +binmode OUT; + +LINE: while (my $line = <CFG>) +{ + $line =~ s/\r?\n$//; + next LINE if $sectiontype != 4 && $line eq ''; + next LINE if $line =~ /^;/; + my $oldsectiontype = $sectiontype; + my $oldrev = $rev; + + # Sections + if ($line =~ m"^\[(config|(\d+)(|\.files|\.message))\]$") + { + if ($1 eq 'config') + { + $sectiontype = 1; + } + elsif ($3 eq '') + { + $sectiontype = 2; + $rev = $2; + # Create a new revision + die "Duplicate rev: $line\n " if defined $revmap{$rev}; + print "Reading revision $rev\n"; + push @revs, $rev; + $revmap{$rev} = $mark ++; + $time{$revmap{$rev}} = 0; + } + elsif ($3 eq '.files') + { + $sectiontype = 3; + $rev = $2; + die "Revision mismatch: $line\n " unless $rev == $oldrev; + } + elsif ($3 eq '.message') + { + $sectiontype = 4; + $rev = $2; + die "Revision mismatch: $line\n " unless $rev == $oldrev; + } + else + { + die "Internal parse error: $line\n "; + } + next LINE; + } + + # Parse data + if ($sectiontype != 4) + { + # Key and value + if ($line =~ m"^\s*([^\s].*=.*[^\s])\s*$") + { + my ($key, $value) = &parsekeyvaluepair($1); + # Global configuration + if (1 == $sectiontype) + { + if ($key eq 'crlf') + { + $crlfmode = 1, next LINE if $value eq 'convert'; + $crlfmode = 0, next LINE if $value eq 'none'; + } + die "Unknown configuration option: $line\n "; + } + # Revision specification + if (2 == $sectiontype) + { + my $current = $revmap{$rev}; + $author{$current} = $value, next LINE if $key eq 'author'; + $branch{$current} = $value, next LINE if $key eq 'branch'; + $parent{$current} = $value, next LINE if $key eq 'parent'; + $timesource{$current} = $value, next LINE if $key eq 'timestamp'; + push(@{$merges{$current}}, $value), next LINE if $key eq 'merges'; + die "Unknown revision option: $line\n "; + } + # Filespecs + if (3 == $sectiontype) + { + # Add the file and create a marker + die "File not found: $line\n " unless -f $value; + my $current = $revmap{$rev}; + ${$files{$current}}{$key} = $mark; + my $time = &fileblob($value, $crlfmode, $mark ++); + + # Update revision timestamp if more recent than other + # files seen, or if this is the file we have selected + # to take the time stamp from using the "timestamp" + # directive. + if ((defined $timesource{$current} && $timesource{$current} eq $value) + || $time > $time{$current}) + { + $time{$current} = $time; + } + } + } + else + { + die "Parse error: $line\n "; + } + } + else + { + # Commit message + my $current = $revmap{$rev}; + if (defined $message{$current}) + { + $message{$current} .= "\n"; + } + $message{$current} .= $line; + } +} +close CFG; + +# Start spewing out data for git-fast-import +foreach my $commit (@revs) +{ + # Progress + print OUT "progress Creating revision $commit\n"; + + # Create commit header + my $mark = $revmap{$commit}; + + # Branch and commit id + print OUT "commit refs/heads/", $branch{$mark}, "\nmark :", $mark, "\n"; + + # Author and timestamp + die "No timestamp defined for $commit (no files?)\n" unless defined $time{$mark}; + print OUT "committer ", $author{$mark}, " ", $time{$mark}, " +0100\n"; + + # Commit message + die "No message defined for $commit\n" unless defined $message{$mark}; + my $message = $message{$mark}; + $message =~ s/\n$//; # Kill trailing empty line + print OUT "data ", length($message), "\n", $message, "\n"; + + # Parent and any merges + print OUT "from :", $revmap{$parent{$mark}}, "\n" if defined $parent{$mark}; + if (defined $merges{$mark}) + { + foreach my $merge (@{$merges{$mark}}) + { + print OUT "merge :", $revmap{$merge}, "\n"; + } + } + + # Output file marks + print OUT "deleteall\n"; # start from scratch + foreach my $file (sort keys %{$files{$mark}}) + { + print OUT "M 644 :", ${$files{$mark}}{$file}, " $file\n"; + } + print OUT "\n"; +} + +# Create one file blob +sub fileblob +{ + my ($filename, $crlfmode, $mark) = @_; + + # Import the file + print OUT "progress Importing $filename\nblob\nmark :$mark\n"; + open FILE, '<', $filename or die "Cannot read $filename\n "; + binmode FILE; + my ($size, $mtime) = (stat(FILE))[7,9]; + my $file; + read FILE, $file, $size; + close FILE; + $file =~ s/\r\n/\n/g if $crlfmode; + print OUT "data ", length($file), "\n", $file, "\n"; + + return $mtime; +} + +# Parse a key=value pair +sub parsekeyvaluepair +{ +=pod + +=head2 Escaping special characters + +Key and value strings may be enclosed in quotes, in which case +whitespace inside the quotes is preserved. Additionally, an equal +sign may be included in the key by preceding it with a backslash. +For example: + + "key1 "=value1 + key2=" value2" + key\=3=value3 + key4=value=4 + "key5""=value5 + +Here the first key is "key1 " (note the trailing white-space) and the +second value is " value2" (note the leading white-space). The third +key contains an equal sign "key=3" and so does the fourth value, which +does not need to be escaped. The fifth key contains a trailing quote, +which does not need to be escaped since it is inside a surrounding +quote. + +=cut + my $pair = shift; + + # Separate key and value by the first non-quoted equal sign + my ($key, $value); + if ($pair =~ /^(.*[^\\])=(.*)$/) + { + ($key, $value) = ($1, $2) + } + else + { + die "Parse error: $pair\n "; + } + + # Unquote and unescape the key and value separately + return (&unescape($key), &unescape($value)); +} + +# Unquote and unescape +sub unescape +{ + my $string = shift; + + # First remove enclosing quotes. Backslash before the trailing + # quote leaves both. + if ($string =~ /^"(.*[^\\])"$/) + { + $string = $1; + } + + # Second remove any backslashes inside the unquoted string. + # For later: Handle special sequences like \t ? + $string =~ s/\\(.)/$1/g; + + return $string; +} + +__END__ + +=pod + +=head1 EXAMPLES + +B<import-directories.perl> F<project.import> + +=head1 AUTHOR + +Copyright 2008-2009 Peter Krefting E<lt>peter@softwolves.pp.se> + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation. + +=cut diff --git a/contrib/fast-import/import-tars.perl b/contrib/fast-import/import-tars.perl index 6309d146e7..95438e1ed4 100755 --- a/contrib/fast-import/import-tars.perl +++ b/contrib/fast-import/import-tars.perl @@ -8,9 +8,20 @@ ## perl import-tars.perl *.tar.bz2 ## git whatchanged import-tars ## +## Use --metainfo to specify the extension for a meta data file, where +## import-tars can read the commit message and optionally author and +## committer information. +## +## echo 'This is the commit message' > myfile.tar.bz2.msg +## perl import-tars.perl --metainfo=msg myfile.tar.bz2 use strict; -die "usage: import-tars *.tar.{gz,bz2,Z}\n" unless @ARGV; +use Getopt::Long; + +my $metaext = ''; + +die "usage: import-tars [--metainfo=extension] *.tar.{gz,bz2,lzma,xz,Z}\n" + unless GetOptions('metainfo=s' => \$metaext) && @ARGV; my $branch_name = 'import-tars'; my $branch_ref = "refs/heads/$branch_name"; @@ -38,6 +49,9 @@ foreach my $tar_file (@ARGV) } elsif ($tar_name =~ s/\.tar\.Z$//) { open(I, '-|', 'uncompress', '-c', $tar_file) or die "Unable to uncompress -c $tar_file: $!\n"; + } elsif ($tar_name =~ s/\.(tar\.(lzma|xz)|(tlz|txz))$//) { + open(I, '-|', 'xz', '-dc', $tar_file) + or die "Unable to xz -dc $tar_file: $!\n"; } elsif ($tar_name =~ s/\.tar$//) { open(I, $tar_file) or die "Unable to open $tar_file: $!\n"; } else { @@ -82,10 +96,16 @@ foreach my $tar_file (@ARGV) $mtime = oct $mtime; next if $typeflag == 5; # directory - print FI "blob\n", "mark :$next_mark\n", "data $size\n"; - while ($size > 0 && read(I, $_, 512) == 512) { - print FI substr($_, 0, $size); - $size -= 512; + print FI "blob\n", "mark :$next_mark\n"; + if ($typeflag == 2) { # symbolic link + print FI "data ", length($linkname), "\n", $linkname; + $mode = 0120000; + } else { + print FI "data $size\n"; + while ($size > 0 && read(I, $_, 512) == 512) { + print FI substr($_, 0, $size); + $size -= 512; + } } print FI "\n"; @@ -103,12 +123,43 @@ foreach my $tar_file (@ARGV) $have_top_dir = 0 if $top_dir ne $1; } + my $commit_msg = "Imported from $tar_file."; + my $this_committer_name = $committer_name; + my $this_committer_email = $committer_email; + my $this_author_name = $author_name; + my $this_author_email = $author_email; + if ($metaext ne '') { + # Optionally read a commit message from <filename.tar>.msg + # Add a line on the form "Committer: name <e-mail>" to override + # the committer and "Author: name <e-mail>" to override the + # author for this tar ball. + if (open MSG, '<', "${tar_file}.${metaext}") { + my $header_done = 0; + $commit_msg = ''; + while (<MSG>) { + if (!$header_done && /^Committer:\s+([^<>]*)\s+<(.*)>\s*$/i) { + $this_committer_name = $1; + $this_committer_email = $2; + } elsif (!$header_done && /^Author:\s+([^<>]*)\s+<(.*)>\s*$/i) { + $this_author_name = $1; + $this_author_email = $2; + } elsif (!$header_done && /^$/) { # empty line ends header. + $header_done = 1; + } else { + $commit_msg .= $_; + $header_done = 1; + } + } + close MSG; + } + } + print FI <<EOF; commit $branch_ref -author $author_name <$author_email> $author_time +0000 -committer $committer_name <$committer_email> $commit_time +0000 +author $this_author_name <$this_author_email> $author_time +0000 +committer $this_committer_name <$this_committer_email> $commit_time +0000 data <<END_OF_COMMIT_MESSAGE -Imported from $tar_file. +$commit_msg END_OF_COMMIT_MESSAGE deleteall @@ -118,7 +169,8 @@ EOF { my ($mark, $mode) = @{$files{$path}}; $path =~ s,^([^/]+)/,, if $have_top_dir; - printf FI "M %o :%i %s\n", $mode & 0111 ? 0755 : 0644, $mark, $path; + $mode = $mode & 0111 ? 0755 : 0644 unless $mode == 0120000; + printf FI "M %o :%i %s\n", $mode, $mark, $path; } print FI "\n"; diff --git a/contrib/fast-import/import-zips.py b/contrib/fast-import/import-zips.py index 7051a83a59..82f5ed3ddc 100755 --- a/contrib/fast-import/import-zips.py +++ b/contrib/fast-import/import-zips.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python ## zip archive frontend for git-fast-import ## |