diff options
Diffstat (limited to 'git-p4.py')
| -rwxr-xr-x | git-p4.py | 618 |
1 files changed, 499 insertions, 119 deletions
@@ -22,6 +22,9 @@ import platform import re import shutil import stat +import zipfile +import zlib +import ctypes try: from subprocess import CalledProcessError @@ -43,6 +46,9 @@ verbose = False # Only labels/tags matching this will be imported/exported defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' +# Grab changes in blocks of this many revisions, unless otherwise requested +defaultBlockSize = 512 + def p4_build_cmd(cmd): """Build a suitable p4 command line. @@ -101,6 +107,16 @@ def chdir(path, is_client_path=False): path = os.getcwd() os.environ['PWD'] = path +def calcDiskFree(): + """Return free space in bytes on the disk of the given dirname.""" + if platform.system() == 'Windows': + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + st = os.statvfs(os.getcwd()) + return st.f_bavail * st.f_frsize + def die(msg): if verbose: raise Exception(msg) @@ -131,13 +147,11 @@ def read_pipe(c, ignore_error=False): sys.stderr.write('Reading pipe: %s\n' % str(c)) expand = isinstance(c,basestring) - p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) - pipe = p.stdout - val = pipe.read() - if p.wait() and not ignore_error: - die('Command failed: %s' % str(c)) - - return val + p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) + (out, err) = p.communicate() + if p.returncode != 0 and not ignore_error: + die('Command failed: %s\nError: %s' % (str(c), err)) + return out def p4_read_pipe(c, ignore_error=False): real_cmd = p4_build_cmd(c) @@ -189,14 +203,16 @@ def p4_has_move_command(): # assume it failed because @... was invalid changelist return True -def system(cmd): +def system(cmd, ignore_error=False): expand = isinstance(cmd,basestring) if verbose: sys.stderr.write("executing %s\n" % str(cmd)) retcode = subprocess.call(cmd, shell=expand) - if retcode: + if retcode and not ignore_error: raise CalledProcessError(retcode, cmd) + return retcode + def p4_system(cmd): """Specifically invoke p4 as the system command. """ real_cmd = p4_build_cmd(cmd) @@ -237,8 +253,8 @@ def p4_add(f): def p4_delete(f): p4_system(["delete", wildcard_encode(f)]) -def p4_edit(f): - p4_system(["edit", wildcard_encode(f)]) +def p4_edit(f, *options): + p4_system(["edit"] + list(options) + [wildcard_encode(f)]) def p4_revert(f): p4_system(["revert", wildcard_encode(f)]) @@ -249,6 +265,10 @@ def p4_reopen(type, f): def p4_move(src, dest): p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)]) +def p4_last_change(): + results = p4CmdList(["changes", "-m", "1"]) + return int(results[0]['change']) + def p4_describe(change): """Make sure it returns a valid result by checking for the presence of field "time". Return a dict of the @@ -368,7 +388,7 @@ def getP4OpenedType(file): # Returns the perforce file type for the given file. result = p4_read_pipe(["opened", wildcard_encode(file)]) - match = re.match(".*\((.+)\)\r?$", result) + match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) if match: return match.group(1) else: @@ -502,12 +522,14 @@ def p4Cmd(cmd): def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" - depotPath = depotPath + "..." - outputList = p4CmdList(["where", depotPath]) + depotPathLong = depotPath + "..." + outputList = p4CmdList(["where", depotPathLong]) output = None for entry in outputList: if "depotFile" in entry: - if entry["depotFile"] == depotPath: + # Search for the base client side depot path, as long as it starts with the branch's P4 path. + # The base path always ends with "/...". + if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...": output = entry break elif "data" in entry: @@ -533,7 +555,12 @@ def p4Where(depotPath): return clientPath def currentGitBranch(): - return read_pipe("git name-rev HEAD").split(" ")[1].strip() + retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True) + if retcode != 0: + # on a detached head + return None + else: + return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip() def isValidGitDir(path): if (os.path.exists(path + "/HEAD") @@ -595,9 +622,12 @@ def gitBranchExists(branch): _gitConfig = {} -def gitConfig(key): +def gitConfig(key, typeSpecifier=None): if not _gitConfig.has_key(key): - cmd = [ "git", "config", key ] + cmd = [ "git", "config" ] + if typeSpecifier: + cmd += [ typeSpecifier ] + cmd += [ key ] s = read_pipe(cmd, ignore_error=True) _gitConfig[key] = s.strip() return _gitConfig[key] @@ -608,16 +638,26 @@ def gitConfigBool(key): in the config.""" if not _gitConfig.has_key(key): - cmd = [ "git", "config", "--bool", key ] + _gitConfig[key] = gitConfig(key, '--bool') == "true" + return _gitConfig[key] + +def gitConfigInt(key): + if not _gitConfig.has_key(key): + cmd = [ "git", "config", "--int", key ] s = read_pipe(cmd, ignore_error=True) v = s.strip() - _gitConfig[key] = v == "true" + try: + _gitConfig[key] = int(gitConfig(key, '--int')) + except ValueError: + _gitConfig[key] = None return _gitConfig[key] def gitConfigList(key): if not _gitConfig.has_key(key): s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) _gitConfig[key] = s.strip().split(os.linesep) + if _gitConfig[key] == ['']: + _gitConfig[key] = [] return _gitConfig[key] def p4BranchesInGit(branchesAreInRemotes=True): @@ -740,21 +780,79 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent def originP4BranchesExist(): return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") -def p4ChangesForPaths(depotPaths, changeRange): + +def p4ParseNumericChangeRange(parts): + changeStart = int(parts[0][1:]) + if parts[1] == '#head': + changeEnd = p4_last_change() + else: + changeEnd = int(parts[1]) + + return (changeStart, changeEnd) + +def chooseBlockSize(blockSize): + if blockSize: + return blockSize + else: + return defaultBlockSize + +def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): assert depotPaths - cmd = ['changes'] - for p in depotPaths: - cmd += ["%s...%s" % (p, changeRange)] - output = p4_read_pipe_lines(cmd) - changes = {} - for line in output: - changeNum = int(line.split(" ")[1]) - changes[changeNum] = True + # Parse the change range into start and end. Try to find integer + # revision ranges as these can be broken up into blocks to avoid + # hitting server-side limits (maxrows, maxscanresults). But if + # that doesn't work, fall back to using the raw revision specifier + # strings, without using block mode. + + if changeRange is None or changeRange == '': + changeStart = 1 + changeEnd = p4_last_change() + block_size = chooseBlockSize(requestedBlockSize) + else: + parts = changeRange.split(',') + assert len(parts) == 2 + try: + (changeStart, changeEnd) = p4ParseNumericChangeRange(parts) + block_size = chooseBlockSize(requestedBlockSize) + except: + changeStart = parts[0][1:] + changeEnd = parts[1] + if requestedBlockSize: + die("cannot use --changes-block-size with non-numeric revisions") + block_size = None + + changes = [] + + # Retrieve changes a block at a time, to prevent running + # into a MaxResults/MaxScanRows error from the server. + + while True: + cmd = ['changes'] + + if block_size: + end = min(changeEnd, changeStart + block_size) + revisionRange = "%d,%d" % (changeStart, end) + else: + revisionRange = "%s,%s" % (changeStart, changeEnd) + + for p in depotPaths: + cmd += ["%s...@%s" % (p, revisionRange)] + + # Insert changes in chronological order + for line in reversed(p4_read_pipe_lines(cmd)): + changes.append(int(line.split(" ")[1])) + + if not block_size: + break + + if end >= changeEnd: + break - changelist = changes.keys() - changelist.sort() - return changelist + changeStart = end + 1 + + changes = sorted(changes) + return changes def p4PathStartsWith(path, prefix): # This method tries to remedy a potential mixed-case issue: @@ -840,6 +938,182 @@ def wildcard_present(path): m = re.search("[*#@%]", path) return m is not None +class LargeFileSystem(object): + """Base class for large file system support.""" + + def __init__(self, writeToGitStream): + self.largeFiles = set() + self.writeToGitStream = writeToGitStream + + def generatePointer(self, cloneDestination, contentFile): + """Return the content of a pointer file that is stored in Git instead of + the actual content.""" + assert False, "Method 'generatePointer' required in " + self.__class__.__name__ + + def pushFile(self, localLargeFile): + """Push the actual content which is not stored in the Git repository to + a server.""" + assert False, "Method 'pushFile' required in " + self.__class__.__name__ + + def hasLargeFileExtension(self, relPath): + return reduce( + lambda a, b: a or b, + [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')], + False + ) + + def generateTempFile(self, contents): + contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) + for d in contents: + contentFile.write(d) + contentFile.close() + return contentFile.name + + def exceedsLargeFileThreshold(self, relPath, contents): + if gitConfigInt('git-p4.largeFileThreshold'): + contentsSize = sum(len(d) for d in contents) + if contentsSize > gitConfigInt('git-p4.largeFileThreshold'): + return True + if gitConfigInt('git-p4.largeFileCompressedThreshold'): + contentsSize = sum(len(d) for d in contents) + if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'): + return False + contentTempFile = self.generateTempFile(contents) + compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) + zf = zipfile.ZipFile(compressedContentFile.name, mode='w') + zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) + zf.close() + compressedContentsSize = zf.infolist()[0].compress_size + os.remove(contentTempFile) + os.remove(compressedContentFile.name) + if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'): + return True + return False + + def addLargeFile(self, relPath): + self.largeFiles.add(relPath) + + def removeLargeFile(self, relPath): + self.largeFiles.remove(relPath) + + def isLargeFile(self, relPath): + return relPath in self.largeFiles + + def processContent(self, git_mode, relPath, contents): + """Processes the content of git fast import. This method decides if a + file is stored in the large file system and handles all necessary + steps.""" + if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath): + contentTempFile = self.generateTempFile(contents) + (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile) + + # Move temp file to final location in large file system + largeFileDir = os.path.dirname(localLargeFile) + if not os.path.isdir(largeFileDir): + os.makedirs(largeFileDir) + shutil.move(contentTempFile, localLargeFile) + self.addLargeFile(relPath) + if gitConfigBool('git-p4.largeFilePush'): + self.pushFile(localLargeFile) + if verbose: + sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile)) + return (git_mode, contents) + +class MockLFS(LargeFileSystem): + """Mock large file system for testing.""" + + def generatePointer(self, contentFile): + """The pointer content is the original content prefixed with "pointer-". + The local filename of the large file storage is derived from the file content. + """ + with open(contentFile, 'r') as f: + content = next(f) + gitMode = '100644' + pointerContents = 'pointer-' + content + localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1]) + return (gitMode, pointerContents, localLargeFile) + + def pushFile(self, localLargeFile): + """The remote filename of the large file storage is the same as the local + one but in a different directory. + """ + remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote') + if not os.path.exists(remotePath): + os.makedirs(remotePath) + shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile))) + +class GitLFS(LargeFileSystem): + """Git LFS as backend for the git-p4 large file system. + See https://git-lfs.github.com/ for details.""" + + def __init__(self, *args): + LargeFileSystem.__init__(self, *args) + self.baseGitAttributes = [] + + def generatePointer(self, contentFile): + """Generate a Git LFS pointer for the content. Return LFS Pointer file + mode and content which is stored in the Git repository instead of + the actual content. Return also the new location of the actual + content. + """ + pointerProcess = subprocess.Popen( + ['git', 'lfs', 'pointer', '--file=' + contentFile], + stdout=subprocess.PIPE + ) + pointerFile = pointerProcess.stdout.read() + if pointerProcess.wait(): + os.remove(contentFile) + die('git-lfs pointer command failed. Did you install the extension?') + pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]] + oid = pointerContents[1].split(' ')[1].split(':')[1][:-1] + localLargeFile = os.path.join( + os.getcwd(), + '.git', 'lfs', 'objects', oid[:2], oid[2:4], + oid, + ) + # LFS Spec states that pointer files should not have the executable bit set. + gitMode = '100644' + return (gitMode, pointerContents, localLargeFile) + + def pushFile(self, localLargeFile): + uploadProcess = subprocess.Popen( + ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)] + ) + if uploadProcess.wait(): + die('git-lfs push command failed. Did you define a remote?') + + def generateGitAttributes(self): + return ( + self.baseGitAttributes + + [ + '\n', + '#\n', + '# Git LFS (see https://git-lfs.github.com/)\n', + '#\n', + ] + + ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n' + for f in sorted(gitConfigList('git-p4.largeFileExtensions')) + ] + + ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n' + for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f) + ] + ) + + def addLargeFile(self, relPath): + LargeFileSystem.addLargeFile(self, relPath) + self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes()) + + def removeLargeFile(self, relPath): + LargeFileSystem.removeLargeFile(self, relPath) + self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes()) + + def processContent(self, git_mode, relPath, contents): + if relPath == '.gitattributes': + self.baseGitAttributes = contents + return (git_mode, self.generateGitAttributes()) + else: + return LargeFileSystem.processContent(self, git_mode, relPath, contents) + class Command: def __init__(self): self.usage = "usage: %prog [options]" @@ -1013,6 +1287,9 @@ class P4Submit(Command, P4UserMap): self.p4HasMoveCommand = p4_has_move_command() self.branch = None + if gitConfig('git-p4.largeFileSystem'): + die("Large file system not supported for git-p4 submit command. Please remove it from config.") + def check(self): if len(p4CmdList("opened ...")) > 0: die("You have files opened with perforce! Close them before starting the sync.") @@ -1179,6 +1456,8 @@ class P4Submit(Command, P4UserMap): Remove lines in the Files section that show changes to files outside the depot path we're committing into.""" + [upstream, settings] = findUpstreamBranchPoint() + template = "" inFilesSection = False for line in p4_read_pipe_lines(['change', '-o']): @@ -1191,8 +1470,13 @@ class P4Submit(Command, P4UserMap): lastTab = path.rfind("\t") if lastTab != -1: path = path[:lastTab] - if not p4PathStartsWith(path, self.depotPath): - continue + if settings.has_key('depot-paths'): + if not [p for p in settings['depot-paths'] + if p4PathStartsWith(path, p)]: + continue + else: + if not p4PathStartsWith(path, self.depotPath): + continue else: inFilesSection = False else: @@ -1220,7 +1504,7 @@ class P4Submit(Command, P4UserMap): editor = os.environ.get("P4EDITOR") else: editor = read_pipe("git var GIT_EDITOR").strip() - system([editor, template_file]) + system(["sh", "-c", ('%s "$@"' % editor), editor, template_file]) # If the file was not saved, prompt to see if this patch should # be skipped. But skip this verification step if configured so. @@ -1270,6 +1554,7 @@ class P4Submit(Command, P4UserMap): diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id)) filesToAdd = set() + filesToChangeType = set() filesToDelete = set() editedFiles = set() pureRenameCopy = set() @@ -1330,6 +1615,8 @@ class P4Submit(Command, P4UserMap): os.unlink(dest) filesToDelete.add(src) editedFiles.add(dest) + elif modifier == "T": + filesToChangeType.add(path) else: die("unknown modifier %s for %s" % (modifier, path)) @@ -1389,6 +1676,8 @@ class P4Submit(Command, P4UserMap): # system(applyPatchCmd) + for f in filesToChangeType: + p4_edit(f, "-t", "auto") for f in filesToAdd: p4_add(f) for f in filesToDelete: @@ -1469,44 +1758,47 @@ class P4Submit(Command, P4UserMap): # # Let the user edit the change description, then submit it. # - if self.edit_template(fileName): - # read the edited message and submit - ret = True - tmpFile = open(fileName, "rb") - message = tmpFile.read() - tmpFile.close() - if self.isWindows: - message = message.replace("\r\n", "\n") - submitTemplate = message[:message.index(separatorLine)] - p4_write_pipe(['submit', '-i'], submitTemplate) - - if self.preserveUser: - if p4User: - # Get last changelist number. Cannot easily get it from - # the submit command output as the output is - # unmarshalled. - changelist = self.lastP4Changelist() - self.modifyChangelistUser(changelist, p4User) - - # The rename/copy happened by applying a patch that created a - # new file. This leaves it writable, which confuses p4. - for f in pureRenameCopy: - p4_sync(f, "-f") + submitted = False - else: + try: + if self.edit_template(fileName): + # read the edited message and submit + tmpFile = open(fileName, "rb") + message = tmpFile.read() + tmpFile.close() + if self.isWindows: + message = message.replace("\r\n", "\n") + submitTemplate = message[:message.index(separatorLine)] + p4_write_pipe(['submit', '-i'], submitTemplate) + + if self.preserveUser: + if p4User: + # Get last changelist number. Cannot easily get it from + # the submit command output as the output is + # unmarshalled. + changelist = self.lastP4Changelist() + self.modifyChangelistUser(changelist, p4User) + + # The rename/copy happened by applying a patch that created a + # new file. This leaves it writable, which confuses p4. + for f in pureRenameCopy: + p4_sync(f, "-f") + submitted = True + + finally: # skip this patch - ret = False - print "Submission cancelled, undoing p4 changes." - for f in editedFiles: - p4_revert(f) - for f in filesToAdd: - p4_revert(f) - os.remove(f) - for f in filesToDelete: - p4_revert(f) + if not submitted: + print "Submission cancelled, undoing p4 changes." + for f in editedFiles: + p4_revert(f) + for f in filesToAdd: + p4_revert(f) + os.remove(f) + for f in filesToDelete: + p4_revert(f) os.remove(fileName) - return ret + return submitted # Export git tags as p4 labels. Create a p4 label and then tag # with that. @@ -1582,8 +1874,6 @@ class P4Submit(Command, P4UserMap): def run(self, args): if len(args) == 0: self.master = currentGitBranch() - if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): - die("Detecting current git branch failed!") elif len(args) == 1: self.master = args[0] if not branchExists(self.master): @@ -1591,9 +1881,10 @@ class P4Submit(Command, P4UserMap): else: return False - allowSubmit = gitConfig("git-p4.allowSubmit") - if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","): - die("%s is not in git-p4.allowSubmit" % self.master) + if self.master: + allowSubmit = gitConfig("git-p4.allowSubmit") + if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","): + die("%s is not in git-p4.allowSubmit" % self.master) [upstream, settings] = findUpstreamBranchPoint() self.depotPath = settings['depot-paths'][0] @@ -1627,7 +1918,10 @@ class P4Submit(Command, P4UserMap): if self.useClientSpec: self.clientSpecDirs = getClientSpec() - if self.useClientSpec: + # Check for the existance of P4 branches + branchesDetected = (len(p4BranchesInGit().keys()) > 1) + + if self.useClientSpec and not branchesDetected: # all files are relative to the client spec self.clientPath = getClientRoot() else: @@ -1658,7 +1952,12 @@ class P4Submit(Command, P4UserMap): self.check() commits = [] - for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]): + if self.master: + commitish = self.master + else: + commitish = 'HEAD' + + for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]): commits.append(line.strip()) commits.reverse() @@ -1878,10 +2177,14 @@ class View(object): if "unmap" in res: # it will list all of them, but only one not unmap-ped continue + if gitConfigBool("core.ignorecase"): + res['depotFile'] = res['depotFile'].lower() self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"]) # not found files or unmap files set to "" for depotFile in fileArgs: + if gitConfigBool("core.ignorecase"): + depotFile = depotFile.lower() if depotFile not in self.client_spec_path_cache: self.client_spec_path_cache[depotFile] = "" @@ -1890,6 +2193,9 @@ class View(object): depot file should live. Returns "" if the file should not be mapped in the client.""" + if gitConfigBool("core.ignorecase"): + depot_path = depot_path.lower() + if depot_path in self.client_spec_path_cache: return self.client_spec_path_cache[depot_path] @@ -1911,7 +2217,10 @@ class P4Sync(Command, P4UserMap): optparse.make_option("--import-labels", dest="importLabels", action="store_true"), optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", help="Import into refs/heads/ , not refs/remotes"), - optparse.make_option("--max-changes", dest="maxChanges"), + optparse.make_option("--max-changes", dest="maxChanges", + help="Maximum number of changes to import"), + optparse.make_option("--changes-block-size", dest="changes_block_size", type="int", + help="Internal block size to use when iteratively calling p4 changes"), optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true', help="Keep entire BRANCH/DIR/SUBDIR prefix during import"), optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true', @@ -1940,6 +2249,7 @@ class P4Sync(Command, P4UserMap): self.syncWithOrigin = True self.importIntoRemotes = True self.maxChanges = "" + self.changes_block_size = None self.keepRepoPath = False self.depotPaths = None self.p4BranchesInGit = [] @@ -1949,6 +2259,13 @@ class P4Sync(Command, P4UserMap): self.clientSpecDirs = None self.tempBranches = [] self.tempBranchLocation = "git-p4-tmp" + self.largeFileSystem = None + + if gitConfig('git-p4.largeFileSystem'): + largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')] + self.largeFileSystem = largeFileSystemConstructor( + lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents) + ) if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -2069,13 +2386,22 @@ class P4Sync(Command, P4UserMap): return branches + def writeToGitStream(self, gitMode, relPath, contents): + self.gitStream.write('M %s inline %s\n' % (gitMode, relPath)) + self.gitStream.write('data %d\n' % sum(len(d) for d in contents)) + for d in contents: + self.gitStream.write(d) + self.gitStream.write('\n') + # output one file from the P4 stream # - helper for streamP4Files def streamOneP4File(self, file, contents): relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) if verbose: - sys.stderr.write("%s\n" % relPath) + size = int(self.stream_file['fileSize']) + sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024)) + sys.stdout.flush() (type_base, type_mods) = split_p4_type(file["type"]) @@ -2110,10 +2436,17 @@ class P4Sync(Command, P4UserMap): # them back too. This is not needed to the cygwin windows version, # just the native "NT" type. # - text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']]) - if p4_version_string().find("/NT") >= 0: - text = text.replace("\r\n", "\n") - contents = [ text ] + try: + text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])]) + except Exception as e: + if 'Translation of file content failed' in str(e): + type_base = 'binary' + else: + raise e + else: + if p4_version_string().find('/NT') >= 0: + text = text.replace('\r\n', '\n') + contents = [ text ] if type_base == "apple": # Apple filetype files will be streamed as a concatenation of @@ -2137,24 +2470,31 @@ class P4Sync(Command, P4UserMap): text = regexp.sub(r'$\1$', text) contents = [ text ] - self.gitStream.write("M %s inline %s\n" % (git_mode, relPath)) + try: + relPath.decode('ascii') + except: + encoding = 'utf8' + if gitConfig('git-p4.pathEncoding'): + encoding = gitConfig('git-p4.pathEncoding') + relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace') + if self.verbose: + print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath) - # total length... - length = 0 - for d in contents: - length = length + len(d) + if self.largeFileSystem: + (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents) - self.gitStream.write("data %d\n" % length) - for d in contents: - self.gitStream.write(d) - self.gitStream.write("\n") + self.writeToGitStream(git_mode, relPath, contents) def streamOneP4Deletion(self, file): relPath = self.stripRepoPath(file['path'], self.branchPrefixes) if verbose: - sys.stderr.write("delete %s\n" % relPath) + sys.stdout.write("delete %s\n" % relPath) + sys.stdout.flush() self.gitStream.write("D %s\n" % relPath) + if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath): + self.largeFileSystem.removeLargeFile(relPath) + # handle another chunk of streaming data def streamP4FilesCb(self, marshalled): @@ -2164,6 +2504,14 @@ class P4Sync(Command, P4UserMap): if marshalled["code"] == "error": if "data" in marshalled: err = marshalled["data"].rstrip() + + if not err and 'fileSize' in self.stream_file: + required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree()) + if required_bytes > 0: + err = 'Not enough space left on %s! Free at least %i MB.' % ( + os.getcwd(), required_bytes/1024/1024 + ) + if err: f = None if self.stream_have_file_info: @@ -2192,10 +2540,23 @@ class P4Sync(Command, P4UserMap): # 'data' field we need to append to our array for k in marshalled.keys(): if k == 'data': + if 'streamContentSize' not in self.stream_file: + self.stream_file['streamContentSize'] = 0 + self.stream_file['streamContentSize'] += len(marshalled['data']) self.stream_contents.append(marshalled['data']) else: self.stream_file[k] = marshalled[k] + if (verbose and + 'streamContentSize' in self.stream_file and + 'fileSize' in self.stream_file and + 'depotFile' in self.stream_file): + size = int(self.stream_file["fileSize"]) + if size > 0: + progress = 100*self.stream_file['streamContentSize']/size + sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024))) + sys.stdout.flush() + self.stream_have_file_info = True # Stream directly from "p4 files" into "git fast-import" @@ -2205,12 +2566,6 @@ class P4Sync(Command, P4UserMap): filesToDelete = [] for f in files: - # if using a client spec, only add the files that have - # a path in the client - if self.clientSpecDirs: - if self.clientSpecDirs.map_in_client(f['path']) == "": - continue - filesForCommit.append(f) if f['action'] in self.delete_actions: filesToDelete.append(f) @@ -2246,8 +2601,11 @@ class P4Sync(Command, P4UserMap): else: return "%s <a@b>" % userid - # Stream a p4 tag def streamTag(self, gitStream, labelName, labelDetails, commit, epoch): + """ Stream a p4 tag. + commit is either a git commit, or a fast-import mark, ":<p4commit>" + """ + if verbose: print "writing tag %s for commit %s" % (labelName, commit) gitStream.write("tag %s\n" % labelName) @@ -2278,27 +2636,43 @@ class P4Sync(Command, P4UserMap): gitStream.write(description) gitStream.write("\n") + def inClientSpec(self, path): + if not self.clientSpecDirs: + return True + inClientSpec = self.clientSpecDirs.map_in_client(path) + if not inClientSpec and self.verbose: + print('Ignoring file outside of client spec: {0}'.format(path)) + return inClientSpec + + def hasBranchPrefix(self, path): + if not self.branchPrefixes: + return True + hasPrefix = [p for p in self.branchPrefixes + if p4PathStartsWith(path, p)] + if hasPrefix and self.verbose: + print('Ignoring file outside of prefix: {0}'.format(path)) + return hasPrefix + def commit(self, details, files, branch, parent = ""): epoch = details["time"] author = details["user"] if self.verbose: - print "commit into %s" % branch - - # start with reading files; if that fails, we should not - # create a commit. - new_files = [] - for f in files: - if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]: - new_files.append (f) - else: - sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path']) + print('commit into {0}'.format(branch)) if self.clientSpecDirs: self.clientSpecDirs.update_client_spec_path_cache(files) + files = [f for f in files + if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])] + + if not files and not gitConfigBool('git-p4.keepEmptyCommits'): + print('Ignoring revision {0} as it would produce an empty commit.' + .format(details['change'])) + return + self.gitStream.write("commit %s\n" % branch) -# gitStream.write("mark :%s\n" % details["change"]) + self.gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" if author not in self.users: @@ -2320,7 +2694,7 @@ class P4Sync(Command, P4UserMap): print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - self.streamP4Files(new_files) + self.streamP4Files(files) self.gitStream.write("\n") change = int(details["change"]) @@ -2417,13 +2791,19 @@ class P4Sync(Command, P4UserMap): if change.has_key('change'): # find the corresponding git commit; take the oldest commit changelist = int(change['change']) - gitCommit = read_pipe(["git", "rev-list", "--max-count=1", - "--reverse", ":/\[git-p4:.*change = %d\]" % changelist]) - if len(gitCommit) == 0: - print "could not find git commit for changelist %d" % changelist - else: - gitCommit = gitCommit.strip() + if changelist in self.committedChanges: + gitCommit = ":%d" % changelist # use a fast-import mark commitFound = True + else: + gitCommit = read_pipe(["git", "rev-list", "--max-count=1", + "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True) + if len(gitCommit) == 0: + print "importing label %s: could not find git commit for changelist %d" % (name, changelist) + else: + commitFound = True + gitCommit = gitCommit.strip() + + if commitFound: # Convert from p4 time format try: tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S") @@ -2586,7 +2966,7 @@ class P4Sync(Command, P4UserMap): branchPrefix = self.depotPaths[0] + branch + "/" range = "@1,%s" % maxChange #print "prefix" + branchPrefix - changes = p4ChangesForPaths([branchPrefix], range) + changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size) if len(changes) <= 0: return False firstChange = changes[0] @@ -3002,7 +3382,7 @@ class P4Sync(Command, P4UserMap): if self.verbose: print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) - changes = p4ChangesForPaths(self.depotPaths, self.changeRange) + changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size) if len(self.maxChanges) > 0: changes = changes[:min(int(self.maxChanges), len(changes))] |