diff options
author | Junio C Hamano <gitster@pobox.com> | 2013-05-06 22:16:26 -0700 |
---|---|---|
committer | Junio C Hamano <gitster@pobox.com> | 2013-05-06 22:16:26 -0700 |
commit | bba536718381705507ecc4f944c340312f93e739 (patch) | |
tree | 4bdd3ee50e3fca1231a8604303592236512dd745 /contrib | |
parent | Merge branch 'fc/push-with-export-reporting-result' (diff) | |
parent | remote-bzr: avoid bad refs (diff) | |
download | tgif-bba536718381705507ecc4f944c340312f93e739.tar.xz |
Merge branch 'fc/remote-bzr'
* fc/remote-bzr:
remote-bzr: avoid bad refs
remote-bzr: convert all unicode keys to str
remote-bzr: access branches only when needed
remote-bzr: delay peer branch usage
remote-bzr: iterate revisions properly
remote-bzr: improve progress reporting
remote-bzr: add option to specify branches
remote-bzr: add custom method to find branches
remote-bzr: improve author sanitazion
remote-bzr: add support for shared repo
remote-bzr: fix branch names
remote-bzr: add support for bzr repos
remote-bzr: use branch variable when appropriate
remote-bzr: fix partially pushed merge
remote-bzr: fixes for branch diverge
remote-bzr: add support to push merges
remote-bzr: always try to update the worktree
remote-bzr: fix order of locking in CustomTree
remote-bzr: delay blob fetching until the very end
remote-bzr: cleanup CustomTree
Diffstat (limited to 'contrib')
-rwxr-xr-x | contrib/remote-helpers/git-remote-bzr | 307 | ||||
-rwxr-xr-x | contrib/remote-helpers/test-bzr.sh | 72 |
2 files changed, 298 insertions, 81 deletions
diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index c19ed0e26b..0ef30f8d55 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -13,6 +13,9 @@ # or # % git clone bzr::lp:myrepo # +# If you want to specify which branches you want track (per repo): +# git config remote-bzr.branches 'trunk, devel, test' +# import sys @@ -27,6 +30,7 @@ import bzrlib.generate_ids import bzrlib.transport import bzrlib.errors import bzrlib.ui +import bzrlib.urlutils import sys import os @@ -37,6 +41,7 @@ import atexit, shutil, hashlib, urlparse, subprocess NAME_RE = re.compile('^([^<>]+)') AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$') +EMAIL_RE = re.compile('^([^<>]+[^ \\\t<>])?\\b(?:[ \\t<>]*?)\\b([^ \\t<>]+@[^ \\t<>]+)') RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)') def die(msg, *args): @@ -49,6 +54,12 @@ def warn(msg, *args): def gittz(tz): return '%+03d%02d' % (tz / 3600, tz % 3600 / 60) +def get_config(config): + cmd = ['git', 'config', '--get', config] + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + output, _ = process.communicate() + return output + class Marks: def __init__(self, path): @@ -84,7 +95,7 @@ class Marks: return self.marks[rev] def to_rev(self, mark): - return self.rev_marks[mark] + return str(self.rev_marks[mark]) def next_mark(self): self.last_mark += 1 @@ -174,9 +185,19 @@ def fixup_user(user): name = m.group(1) mail = m.group(2).strip() else: - m = NAME_RE.match(user) + m = EMAIL_RE.match(user) if m: - name = m.group(1).strip() + name = m.group(1) + mail = m.group(2) + else: + m = NAME_RE.match(user) + if m: + name = m.group(1).strip() + + if not name: + name = 'unknown' + if not mail: + mail = 'Unknown' return '%s <%s>' % (name, mail) @@ -250,22 +271,28 @@ def export_files(tree, files): return final -def export_branch(branch, name): +def export_branch(repo, name): global prefix ref = '%s/heads/%s' % (prefix, name) tip = marks.get_tip(name) + branch = bzrlib.branch.Branch.open(branches[name]) repo = branch.repository - repo.lock_read() + + branch.lock_read() revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward') - count = 0 + tip_revno = branch.revision_id_to_revno(tip) + last_revno, _ = branch.last_revision_info() + total = last_revno - tip_revno - revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)] + for revid, _, seq, _ in revs: - for revid in revs: + if marks.is_marked(revid): + continue rev = repo.get_revision(revid) + revno = seq[0] parents = rev.parent_ids time = rev.timestamp @@ -320,12 +347,15 @@ def export_branch(branch, name): print "M %s :%u %s" % f print - count += 1 - if (count % 100 == 0): - print "progress revision %s (%d/%d)" % (revid, count, len(revs)) - print "#############################################################" + if len(seq) > 1: + # let's skip branch revisions from the progress report + continue - repo.unlock() + progress = (revno - tip_revno) + if (progress % 100 == 0): + print "progress revision %d '%s' (%d/%d)" % (revno, name, progress, total) + + branch.unlock() revid = branch.last_revision() @@ -347,23 +377,24 @@ def export_tag(repo, name): def do_import(parser): global dirname - branch = parser.repo + repo = parser.repo path = os.path.join(dirname, 'marks-git') print "feature done" if os.path.exists(path): print "feature import-marks=%s" % path print "feature export-marks=%s" % path + print "feature force" sys.stdout.flush() while parser.check('import'): ref = parser[1] if ref.startswith('refs/heads/'): name = ref[len('refs/heads/'):] - export_branch(branch, name) + export_branch(repo, name) if ref.startswith('refs/tags/'): name = ref[len('refs/tags/'):] - export_tag(branch, name) + export_tag(repo, name) parser.next() print 'done' @@ -382,23 +413,21 @@ def parse_blob(parser): class CustomTree(): - def __init__(self, repo, revid, parents, files): + def __init__(self, branch, revid, parents, files): global files_cache - self.repo = repo - self.revid = revid - self.parents = parents self.updates = {} + self.branch = branch def copy_tree(revid): files = files_cache[revid] = {} - tree = repo.repository.revision_tree(revid) - repo.lock_read() + branch.lock_read() + tree = branch.repository.revision_tree(revid) try: for path, entry in tree.iter_entries_by_dir(): - files[path] = entry.file_id + files[path] = [entry.file_id, None] finally: - repo.unlock() + branch.unlock() return files if len(parents) == 0: @@ -411,12 +440,18 @@ class CustomTree(): self.base_files = copy_tree(self.base_id) self.files = files_cache[revid] = self.base_files.copy() + self.rev_files = {} + + for path, data in self.files.iteritems(): + fid, mark = data + self.rev_files[fid] = [path, mark] for path, f in files.iteritems(): - fid = self.files.get(path, None) + fid, mark = self.files.get(path, [None, None]) if not fid: fid = bzrlib.generate_ids.gen_file_id(path) f['path'] = path + self.rev_files[fid] = [path, mark] self.updates[fid] = f def last_revision(self): @@ -426,16 +461,16 @@ class CustomTree(): changes = [] def get_parent(dirname, basename): - parent_fid = self.base_files.get(dirname, None) + parent_fid, mark = self.base_files.get(dirname, [None, None]) if parent_fid: return parent_fid - parent_fid = self.files.get(dirname, None) + parent_fid, mark = self.files.get(dirname, [None, None]) if parent_fid: return parent_fid if basename == '': return None fid = bzrlib.generate_ids.gen_file_id(path) - d = add_entry(fid, dirname, 'directory') + add_entry(fid, dirname, 'directory') return fid def add_entry(fid, path, kind, mode = None): @@ -456,9 +491,8 @@ class CustomTree(): (None, basename), (None, kind), (None, executable)) - self.files[path] = change[0] + self.files[path] = [change[0], None] changes.append(change) - return change def update_entry(fid, path, kind, mode = None): dirname, basename = os.path.split(path) @@ -478,9 +512,8 @@ class CustomTree(): (None, basename), (None, kind), (None, executable)) - self.files[path] = change[0] + self.files[path] = [change[0], None] changes.append(change) - return change def remove_entry(fid, path, kind): dirname, basename = os.path.split(path) @@ -495,7 +528,6 @@ class CustomTree(): (None, None)) del self.files[path] changes.append(change) - return change for fid, f in self.updates.iteritems(): path = f['path'] @@ -509,13 +541,30 @@ class CustomTree(): else: add_entry(fid, path, 'file', f['mode']) + self.files[path][1] = f['mark'] + self.rev_files[fid][1] = f['mark'] + return changes + def get_content(self, file_id): + path, mark = self.rev_files[file_id] + if mark: + return blob_marks[mark] + + # last resort + tree = self.branch.repository.revision_tree(self.base_id) + return tree.get_file_text(file_id) + def get_file_with_stat(self, file_id, path=None): - return (StringIO.StringIO(self.updates[file_id]['data']), None) + content = self.get_content(file_id) + return (StringIO.StringIO(content), None) def get_symlink_target(self, file_id): - return self.updates[file_id]['data'] + return self.get_content(file_id) + + def id2path(self, file_id): + path, mark = self.rev_files[file_id] + return path def c_style_unescape(string): if string[0] == string[-1] == '"': @@ -531,8 +580,11 @@ def parse_commit(parser): ref = parser[1] parser.next() - if ref != 'refs/heads/master': - die("bzr doesn't support multiple branches; use 'master'") + if ref.startswith('refs/heads/'): + name = ref[len('refs/heads/'):] + branch = bzrlib.branch.Branch.open(branches[name]) + else: + die('unknown ref') commit_mark = parser.get_mark() parser.next() @@ -559,7 +611,7 @@ def parse_commit(parser): if parser.check('M'): t, m, mark_ref, path = line.split(' ', 3) mark = int(mark_ref[1:]) - f = { 'mode' : m, 'data' : blob_marks[mark] } + f = { 'mode' : m, 'mark' : mark } elif parser.check('D'): t, path = line.split(' ') f = { 'deleted' : True } @@ -568,20 +620,18 @@ def parse_commit(parser): path = c_style_unescape(path).decode('utf-8') files[path] = f - repo = parser.repo - committer, date, tz = committer - parents = [str(mark_to_rev(p)) for p in parents] + parents = [mark_to_rev(p) for p in parents] revid = bzrlib.generate_ids.gen_revision_id(committer, date) props = {} - props['branch-nick'] = repo.nick + props['branch-nick'] = branch.nick - mtree = CustomTree(repo, revid, parents, files) + mtree = CustomTree(branch, revid, parents, files) changes = mtree.iter_changes() - repo.lock_write() + branch.lock_write() try: - builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid) + builder = branch.get_commit_builder(parents, None, date, tz, committer, props, revid) try: list(builder.record_iter_changes(mtree, mtree.last_revision(), changes)) builder.finish_inventory() @@ -590,7 +640,7 @@ def parse_commit(parser): builder.abort() raise finally: - repo.unlock() + branch.unlock() parsed_refs[ref] = revid marks.new_mark(revid, commit_mark) @@ -601,9 +651,6 @@ def parse_reset(parser): ref = parser[1] parser.next() - if ref != 'refs/heads/master': - die("bzr doesn't support multiple branches; use 'master'") - # ugh if parser.check('commit'): parse_commit(parser) @@ -616,7 +663,7 @@ def parse_reset(parser): parsed_refs[ref] = mark_to_rev(from_mark) def do_export(parser): - global parsed_refs, dirname, peer + global parsed_refs, dirname parser.next() @@ -634,20 +681,33 @@ def do_export(parser): else: die('unhandled export command: %s' % line) - repo = parser.repo - for ref, revid in parsed_refs.iteritems(): - if ref == 'refs/heads/master': - repo.generate_revision_history(revid, marks.get_tip('master')) - if peer: + if ref.startswith('refs/heads/'): + name = ref[len('refs/heads/'):] + branch = bzrlib.branch.Branch.open(branches[name]) + branch.generate_revision_history(revid, marks.get_tip(name)) + + if name in peers: + peer = bzrlib.branch.Branch.open(peers[name]) try: - repo.push(peer, stop_revision=revid) + peer.bzrdir.push_branch(branch, revision_id=revid) except bzrlib.errors.DivergedBranches: print "error %s non-fast forward" % ref continue - else: - wt = repo.bzrdir.open_workingtree() + + try: + wt = branch.bzrdir.open_workingtree() wt.update() + except bzrlib.errors.NoWorkingTree: + pass + elif ref.startswith('refs/tags/'): + # TODO: implement tag push + print "error %s pushing tags not supported" % ref + continue + else: + # transport-helper/fast-export bugs + continue + print "ok %s" % ref print @@ -673,9 +733,15 @@ def ref_is_valid(name): def do_list(parser): global tags - print "? refs/heads/%s" % 'master' - branch = parser.repo + master_branch = None + + for name in branches: + if not master_branch: + master_branch = name + print "? refs/heads/%s" % name + + branch = bzrlib.branch.Branch.open(branches[master_branch]) branch.lock_read() for tag, revid in branch.tags.get_tag_dict().items(): try: @@ -687,36 +753,112 @@ def do_list(parser): print "? refs/tags/%s" % tag tags[tag] = revid branch.unlock() - print "@refs/heads/%s HEAD" % 'master' + + print "@refs/heads/%s HEAD" % master_branch print +def get_remote_branch(origin, remote_branch, name): + global dirname, peers + + branch_path = os.path.join(dirname, 'clone', name) + if os.path.exists(branch_path): + # pull + d = bzrlib.bzrdir.BzrDir.open(branch_path) + branch = d.open_branch() + try: + branch.pull(remote_branch, [], None, False) + except bzrlib.errors.DivergedBranches: + # use remote branch for now + return remote_branch + else: + # clone + d = origin.sprout(branch_path, None, + hardlink=True, create_tree_if_local=False, + force_new_repo=False, + source_branch=remote_branch) + branch = d.open_branch() + + return branch + +def find_branches(repo, wanted): + transport = repo.user_transport + + for fn in transport.iter_files_recursive(): + if not fn.endswith('.bzr/branch-format'): + continue + + name = subdir = fn[:-len('/.bzr/branch-format')] + name = name if name != '' else 'master' + name = name.replace('/', '+') + + if wanted and not name in wanted: + continue + + try: + cur = transport.clone(subdir) + branch = bzrlib.branch.Branch.open_from_transport(cur) + except bzrlib.errors.NotBranchError: + continue + else: + yield name, branch + def get_repo(url, alias): - global dirname, peer + global dirname, peer, branches + normal_url = bzrlib.urlutils.normalize_url(url) origin = bzrlib.bzrdir.BzrDir.open(url) - branch = origin.open_branch() + is_local = isinstance(origin.transport, bzrlib.transport.local.LocalTransport) + + shared_path = os.path.join(gitdir, 'bzr') + try: + shared_dir = bzrlib.bzrdir.BzrDir.open(shared_path) + except bzrlib.errors.NotBranchError: + shared_dir = bzrlib.bzrdir.BzrDir.create(shared_path) + try: + shared_repo = shared_dir.open_repository() + except bzrlib.errors.NoRepositoryPresent: + shared_repo = shared_dir.create_repository(shared=True) - if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport): + if not is_local: clone_path = os.path.join(dirname, 'clone') - remote_branch = branch - if os.path.exists(clone_path): - # pull - d = bzrlib.bzrdir.BzrDir.open(clone_path) - branch = d.open_branch() - result = branch.pull(remote_branch, [], None, False) + if not os.path.exists(clone_path): + os.mkdir(clone_path) + + try: + repo = origin.open_repository() + except bzrlib.errors.NoRepositoryPresent: + # branch + + name = 'master' + remote_branch = origin.open_branch() + + if not is_local: + peers[name] = remote_branch.base + branch = get_remote_branch(origin, remote_branch, name) else: - # clone - d = origin.sprout(clone_path, None, - hardlink=True, create_tree_if_local=False, - source_branch=remote_branch) - branch = d.open_branch() - branch.bind(remote_branch) - - peer = remote_branch + branch = remote_branch + + branches[name] = branch.base + + return branch.repository else: - peer = None + # repository - return branch + wanted = get_config('remote-bzr.branches').rstrip().split(', ') + # stupid python + wanted = [e for e in wanted if e] + + for name, remote_branch in find_branches(repo, wanted): + + if not is_local: + peers[name] = remote_branch.base + branch = get_remote_branch(origin, remote_branch, name) + else: + branch = remote_branch + + branches[name] = branch.base + + return repo def fix_path(alias, orig_url): url = urlparse.urlparse(orig_url, 'file') @@ -727,12 +869,13 @@ def fix_path(alias, orig_url): subprocess.call(cmd) def main(args): - global marks, prefix, dirname + global marks, prefix, gitdir, dirname global tags, filenodes global blob_marks global parsed_refs global files_cache global is_tmp + global branches, peers alias = args[1] url = args[2] @@ -743,6 +886,8 @@ def main(args): parsed_refs = {} files_cache = {} marks = None + branches = {} + peers = {} if alias[5:] == url: is_tmp = True diff --git a/contrib/remote-helpers/test-bzr.sh b/contrib/remote-helpers/test-bzr.sh index 34666e1d0f..cec55f132e 100755 --- a/contrib/remote-helpers/test-bzr.sh +++ b/contrib/remote-helpers/test-bzr.sh @@ -228,4 +228,76 @@ test_expect_success 'push utf-8 filenames' ' test_cmp expected actual ' +test_expect_success 'pushing a merge' ' + mkdir -p tmp && cd tmp && + test_when_finished "cd .. && rm -rf tmp" && + + ( + bzr init bzrrepo && + cd bzrrepo && + echo one > content && + bzr add content && + bzr commit -m one + ) && + + git clone "bzr::$PWD/bzrrepo" gitrepo && + + ( + cd bzrrepo && + echo two > content && + bzr commit -m two + ) && + + ( + cd gitrepo && + echo three > content && + git commit -a -m three && + git fetch && + git merge origin/master || true && + echo three > content && + git commit -a --no-edit && + git push + ) && + + echo three > expected && + cat bzrrepo/content > actual && + test_cmp expected actual +' + +cat > expected <<EOF +origin/HEAD +origin/branch +origin/trunk +EOF + +test_expect_success 'proper bzr repo' ' + mkdir -p tmp && cd tmp && + test_when_finished "cd .. && rm -rf tmp" && + + bzr init-repo bzrrepo && + + bzr init bzrrepo/trunk && + ( + cd bzrrepo/trunk && + echo one >> content && + bzr add content && + bzr commit -m one + ) && + + bzr branch bzrrepo/trunk bzrrepo/branch && + ( + cd bzrrepo/branch && + echo two >> content && + bzr commit -m one + ) && + + git clone "bzr::$PWD/bzrrepo" gitrepo && + ( + cd gitrepo && + git for-each-ref --format "%(refname:short)" refs/remotes/origin > ../actual + ) && + + test_cmp ../expected actual +' + test_done |