diff options
Diffstat (limited to 'contrib')
| -rwxr-xr-x | contrib/completion/git-completion.bash | 53 | ||||
| -rw-r--r-- | contrib/diff-highlight/README | 57 | ||||
| -rwxr-xr-x | contrib/diff-highlight/diff-highlight | 124 | ||||
| -rwxr-xr-x | contrib/fast-import/git-p4 | 300 | ||||
| -rw-r--r-- | contrib/git-jump/README | 92 | ||||
| -rwxr-xr-x | contrib/git-jump/git-jump | 69 | ||||
| -rwxr-xr-x | contrib/mw-to-git/git-remote-mediawiki | 827 | ||||
| -rw-r--r-- | contrib/mw-to-git/git-remote-mediawiki.txt | 7 |
8 files changed, 1403 insertions, 126 deletions
diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 8648a36e7b..b7c1edf1cc 100755 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -110,6 +110,7 @@ __git_ps1_show_upstream () local upstream=git legacy="" verbose="" # get some config options from git-config + local output="$(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ')" while read key value; do case "$key" in bash.showupstream) @@ -125,7 +126,7 @@ __git_ps1_show_upstream () upstream=svn+git # default upstream is SVN if available, else git ;; esac - done < <(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ') + done <<< "$output" # parse configuration values for option in ${GIT_PS1_SHOWUPSTREAM}; do @@ -1259,12 +1260,9 @@ _git_commit () " "" "${cur##--cleanup=}" return ;; - --reuse-message=*) - __gitcomp "$(__git_refs)" "" "${cur##--reuse-message=}" - return - ;; - --reedit-message=*) - __gitcomp "$(__git_refs)" "" "${cur##--reedit-message=}" + --reuse-message=*|--reedit-message=*|\ + --fixup=*|--squash=*) + __gitcomp "$(__git_refs)" "" "${cur#*=}" return ;; --untracked-files=*) @@ -1278,7 +1276,7 @@ _git_commit () --dry-run --reuse-message= --reedit-message= --reset-author --file= --message= --template= --cleanup= --untracked-files --untracked-files= - --verbose --quiet + --verbose --quiet --fixup= --squash= " return esac @@ -1432,6 +1430,10 @@ _git_gitk () _gitk } +__git_match_ctag() { + awk "/^${1////\\/}/ { print \$1 }" "$2" +} + _git_grep () { __git_has_doubledash && return @@ -1454,6 +1456,15 @@ _git_grep () ;; esac + case "$cword,$prev" in + 2,*|*,-*) + if test -r tags; then + __gitcomp "$(__git_match_ctag "$cur" tags)" + return + fi + ;; + esac + __gitcomp "$(__git_refs)" } @@ -1556,14 +1567,9 @@ _git_log () merge="--merge" fi case "$cur" in - --pretty=*) + --pretty=*|--format=*) __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) - " "" "${cur##--pretty=}" - return - ;; - --format=*) - __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) - " "" "${cur##--format=}" + " "" "${cur#*=}" return ;; --date=*) @@ -1671,11 +1677,9 @@ _git_notes () ;; esac ;; - add,--reuse-message=*|append,--reuse-message=*) - __gitcomp "$(__git_refs)" "" "${cur##--reuse-message=}" - ;; + add,--reuse-message=*|append,--reuse-message=*|\ add,--reedit-message=*|append,--reedit-message=*) - __gitcomp "$(__git_refs)" "" "${cur##--reedit-message=}" + __gitcomp "$(__git_refs)" "" "${cur#*=}" ;; add,--*|append,--*) __gitcomp '--file= --message= --reedit-message= @@ -1733,7 +1737,7 @@ _git_push () --*) __gitcomp " --all --mirror --tags --dry-run --force --verbose - --receive-pack= --repo= + --receive-pack= --repo= --set-upstream " return ;; @@ -2370,14 +2374,9 @@ _git_show () __git_has_doubledash && return case "$cur" in - --pretty=*) - __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) - " "" "${cur##--pretty=}" - return - ;; - --format=*) + --pretty=*|--format=*) __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) - " "" "${cur##--format=}" + " "" "${cur#*=}" return ;; --*) diff --git a/contrib/diff-highlight/README b/contrib/diff-highlight/README new file mode 100644 index 0000000000..1b7b6df8eb --- /dev/null +++ b/contrib/diff-highlight/README @@ -0,0 +1,57 @@ +diff-highlight +============== + +Line oriented diffs are great for reviewing code, because for most +hunks, you want to see the old and the new segments of code next to each +other. Sometimes, though, when an old line and a new line are very +similar, it's hard to immediately see the difference. + +You can use "--color-words" to highlight only the changed portions of +lines. However, this can often be hard to read for code, as it loses +the line structure, and you end up with oddly formatted bits. + +Instead, this script post-processes the line-oriented diff, finds pairs +of lines, and highlights the differing segments. It's currently very +simple and stupid about doing these tasks. In particular: + + 1. It will only highlight a pair of lines if they are the only two + lines in a hunk. It could instead try to match up "before" and + "after" lines for a given hunk into pairs of similar lines. + However, this may end up visually distracting, as the paired + lines would have other highlighted lines in between them. And in + practice, the lines which most need attention called to their + small, hard-to-see changes are touching only a single line. + + 2. It will find the common prefix and suffix of two lines, and + consider everything in the middle to be "different". It could + instead do a real diff of the characters between the two lines and + find common subsequences. However, the point of the highlight is to + call attention to a certain area. Even if some small subset of the + highlighted area actually didn't change, that's OK. In practice it + ends up being more readable to just have a single blob on the line + showing the interesting bit. + +The goal of the script is therefore not to be exact about highlighting +changes, but to call attention to areas of interest without being +visually distracting. Non-diff lines and existing diff coloration is +preserved; the intent is that the output should look exactly the same as +the input, except for the occasional highlight. + +Use +--- + +You can try out the diff-highlight program with: + +--------------------------------------------- +git log -p --color | /path/to/diff-highlight +--------------------------------------------- + +If you want to use it all the time, drop it in your $PATH and put the +following in your git configuration: + +--------------------------------------------- +[pager] + log = diff-highlight | less + show = diff-highlight | less + diff = diff-highlight | less +--------------------------------------------- diff --git a/contrib/diff-highlight/diff-highlight b/contrib/diff-highlight/diff-highlight new file mode 100755 index 0000000000..d8938982e4 --- /dev/null +++ b/contrib/diff-highlight/diff-highlight @@ -0,0 +1,124 @@ +#!/usr/bin/perl + +# Highlight by reversing foreground and background. You could do +# other things like bold or underline if you prefer. +my $HIGHLIGHT = "\x1b[7m"; +my $UNHIGHLIGHT = "\x1b[27m"; +my $COLOR = qr/\x1b\[[0-9;]*m/; + +my @window; + +while (<>) { + # We highlight only single-line changes, so we need + # a 4-line window to make a decision on whether + # to highlight. + push @window, $_; + next if @window < 4; + if ($window[0] =~ /^$COLOR*(\@| )/ && + $window[1] =~ /^$COLOR*-/ && + $window[2] =~ /^$COLOR*\+/ && + $window[3] !~ /^$COLOR*\+/) { + print shift @window; + show_pair(shift @window, shift @window); + } + else { + print shift @window; + } + + # Most of the time there is enough output to keep things streaming, + # but for something like "git log -Sfoo", you can get one early + # commit and then many seconds of nothing. We want to show + # that one commit as soon as possible. + # + # Since we can receive arbitrary input, there's no optimal + # place to flush. Flushing on a blank line is a heuristic that + # happens to match git-log output. + if (!length) { + local $| = 1; + } +} + +# Special case a single-line hunk at the end of file. +if (@window == 3 && + $window[0] =~ /^$COLOR*(\@| )/ && + $window[1] =~ /^$COLOR*-/ && + $window[2] =~ /^$COLOR*\+/) { + print shift @window; + show_pair(shift @window, shift @window); +} + +# And then flush any remaining lines. +while (@window) { + print shift @window; +} + +exit 0; + +sub show_pair { + my @a = split_line(shift); + my @b = split_line(shift); + + # Find common prefix, taking care to skip any ansi + # color codes. + my $seen_plusminus; + my ($pa, $pb) = (0, 0); + while ($pa < @a && $pb < @b) { + if ($a[$pa] =~ /$COLOR/) { + $pa++; + } + elsif ($b[$pb] =~ /$COLOR/) { + $pb++; + } + elsif ($a[$pa] eq $b[$pb]) { + $pa++; + $pb++; + } + elsif (!$seen_plusminus && $a[$pa] eq '-' && $b[$pb] eq '+') { + $seen_plusminus = 1; + $pa++; + $pb++; + } + else { + last; + } + } + + # Find common suffix, ignoring colors. + my ($sa, $sb) = ($#a, $#b); + while ($sa >= $pa && $sb >= $pb) { + if ($a[$sa] =~ /$COLOR/) { + $sa--; + } + elsif ($b[$sb] =~ /$COLOR/) { + $sb--; + } + elsif ($a[$sa] eq $b[$sb]) { + $sa--; + $sb--; + } + else { + last; + } + } + + print highlight(\@a, $pa, $sa); + print highlight(\@b, $pb, $sb); +} + +sub split_line { + local $_ = shift; + return map { /$COLOR/ ? $_ : (split //) } + split /($COLOR*)/; +} + +sub highlight { + my ($line, $prefix, $suffix) = @_; + + return join('', + @{$line}[0..($prefix-1)], + $HIGHLIGHT, + @{$line}[$prefix..$suffix], + $UNHIGHLIGHT, + @{$line}[($suffix+1)..$#$line] + ); +} diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2f7b270566..b975d67fca 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -22,36 +22,39 @@ def p4_build_cmd(cmd): location. It means that hooking into the environment, or other configuration can be done more easily. """ - real_cmd = "%s " % "p4" + real_cmd = ["p4"] user = gitConfig("git-p4.user") if len(user) > 0: - real_cmd += "-u %s " % user + real_cmd += ["-u",user] password = gitConfig("git-p4.password") if len(password) > 0: - real_cmd += "-P %s " % password + real_cmd += ["-P", password] port = gitConfig("git-p4.port") if len(port) > 0: - real_cmd += "-p %s " % port + real_cmd += ["-p", port] host = gitConfig("git-p4.host") if len(host) > 0: - real_cmd += "-h %s " % host + real_cmd += ["-h", host] client = gitConfig("git-p4.client") if len(client) > 0: - real_cmd += "-c %s " % client + real_cmd += ["-c", client] - real_cmd += "%s" % (cmd) - if verbose: - print real_cmd + + if isinstance(cmd,basestring): + real_cmd = ' '.join(real_cmd) + ' ' + cmd + else: + real_cmd += cmd return real_cmd def chdir(dir): - if os.name == 'nt': - os.environ['PWD']=dir + # P4 uses the PWD environment variable rather than getcwd(). Since we're + # not using the shell, we have to set it ourselves. + os.environ['PWD']=dir os.chdir(dir) def die(msg): @@ -61,29 +64,34 @@ def die(msg): sys.stderr.write(msg + "\n") sys.exit(1) -def write_pipe(c, str): +def write_pipe(c, stdin): if verbose: - sys.stderr.write('Writing pipe: %s\n' % c) + sys.stderr.write('Writing pipe: %s\n' % str(c)) - pipe = os.popen(c, 'w') - val = pipe.write(str) - if pipe.close(): - die('Command failed: %s' % c) + expand = isinstance(c,basestring) + p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) + pipe = p.stdin + val = pipe.write(stdin) + pipe.close() + if p.wait(): + die('Command failed: %s' % str(c)) return val -def p4_write_pipe(c, str): +def p4_write_pipe(c, stdin): real_cmd = p4_build_cmd(c) - return write_pipe(real_cmd, str) + return write_pipe(real_cmd, stdin) def read_pipe(c, ignore_error=False): if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % str(c)) - pipe = os.popen(c, 'rb') + expand = isinstance(c,basestring) + p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) + pipe = p.stdout val = pipe.read() - if pipe.close() and not ignore_error: - die('Command failed: %s' % c) + if p.wait() and not ignore_error: + die('Command failed: %s' % str(c)) return val @@ -93,12 +101,14 @@ def p4_read_pipe(c, ignore_error=False): def read_pipe_lines(c): if verbose: - sys.stderr.write('Reading pipe: %s\n' % c) - ## todo: check return status - pipe = os.popen(c, 'rb') + sys.stderr.write('Reading pipe: %s\n' % str(c)) + + expand = isinstance(c, basestring) + p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) + pipe = p.stdout val = pipe.readlines() - if pipe.close(): - die('Command failed: %s' % c) + if pipe.close() or p.wait(): + die('Command failed: %s' % str(c)) return val @@ -108,23 +118,73 @@ def p4_read_pipe_lines(c): return read_pipe_lines(real_cmd) def system(cmd): + expand = isinstance(cmd,basestring) if verbose: - sys.stderr.write("executing %s\n" % cmd) - if os.system(cmd) != 0: - die("command failed: %s" % cmd) + sys.stderr.write("executing %s\n" % str(cmd)) + subprocess.check_call(cmd, shell=expand) def p4_system(cmd): """Specifically invoke p4 as the system command. """ real_cmd = p4_build_cmd(cmd) - return system(real_cmd) + expand = isinstance(real_cmd, basestring) + subprocess.check_call(real_cmd, shell=expand) + +def p4_integrate(src, dest): + p4_system(["integrate", "-Dt", src, dest]) + +def p4_sync(path): + p4_system(["sync", path]) + +def p4_add(f): + p4_system(["add", f]) -def isP4Exec(kind): - """Determine if a Perforce 'kind' should have execute permission +def p4_delete(f): + p4_system(["delete", f]) + +def p4_edit(f): + p4_system(["edit", f]) + +def p4_revert(f): + p4_system(["revert", f]) + +def p4_reopen(type, file): + p4_system(["reopen", "-t", type, file]) + +# +# Canonicalize the p4 type and return a tuple of the +# base type, plus any modifiers. See "p4 help filetypes" +# for a list and explanation. +# +def split_p4_type(p4type): + + p4_filetypes_historical = { + "ctempobj": "binary+Sw", + "ctext": "text+C", + "cxtext": "text+Cx", + "ktext": "text+k", + "kxtext": "text+kx", + "ltext": "text+F", + "tempobj": "binary+FSw", + "ubinary": "binary+F", + "uresource": "resource+F", + "uxbinary": "binary+Fx", + "xbinary": "binary+x", + "xltext": "text+Fx", + "xtempobj": "binary+Swx", + "xtext": "text+x", + "xunicode": "unicode+x", + "xutf16": "utf16+x", + } + if p4type in p4_filetypes_historical: + p4type = p4_filetypes_historical[p4type] + mods = "" + s = p4type.split("+") + base = s[0] + mods = "" + if len(s) > 1: + mods = s[1] + return (base, mods) - 'p4 help filetypes' gives a list of the types. If it starts with 'x', - or x follows one of a few letters. Otherwise, if there is an 'x' after - a plus sign, it is also executable""" - return (re.search(r"(^[cku]?x)|\+.*x", kind) != None) def setP4ExecBit(file, mode): # Reopens an already open file and changes the execute bit to match @@ -139,12 +199,12 @@ def setP4ExecBit(file, mode): if p4Type[-1] == "+": p4Type = p4Type[0:-1] - p4_system("reopen -t %s %s" % (p4Type, file)) + p4_reopen(p4Type, file) def getP4OpenedType(file): # Returns the perforce file type for the given file. - result = p4_read_pipe("opened %s" % file) + result = p4_read_pipe(["opened", file]) match = re.match(".*\((.+)\)\r?$", result) if match: return match.group(1) @@ -200,9 +260,17 @@ def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): - cmd = p4_build_cmd("-G %s" % (cmd)) + + if isinstance(cmd,basestring): + cmd = "-G " + cmd + expand = True + else: + cmd = ["-G"] + cmd + expand = False + + cmd = p4_build_cmd(cmd) if verbose: - sys.stderr.write("Opening pipe: %s\n" % cmd) + sys.stderr.write("Opening pipe: %s\n" % str(cmd)) # Use a temporary file to avoid deadlocks without # subprocess.communicate(), which would put another copy @@ -210,11 +278,16 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): stdin_file = None if stdin is not None: stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) - stdin_file.write(stdin) + if isinstance(stdin,basestring): + stdin_file.write(stdin) + else: + for i in stdin: + stdin_file.write(i + '\n') stdin_file.flush() stdin_file.seek(0) - p4 = subprocess.Popen(cmd, shell=True, + p4 = subprocess.Popen(cmd, + shell=expand, stdin=stdin_file, stdout=subprocess.PIPE) @@ -247,7 +320,7 @@ def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" depotPath = depotPath + "..." - outputList = p4CmdList("where %s" % depotPath) + outputList = p4CmdList(["where", depotPath]) output = None for entry in outputList: if "depotFile" in entry: @@ -449,8 +522,10 @@ def originP4BranchesExist(): def p4ChangesForPaths(depotPaths, changeRange): assert depotPaths - output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange) - for p in depotPaths])) + cmd = ['changes'] + for p in depotPaths: + cmd += ["%s...%s" % (p, changeRange)] + output = p4_read_pipe_lines(cmd) changes = {} for line in output: @@ -533,7 +608,7 @@ class P4Debug(Command): def run(self, args): j = 0 - for output in p4CmdList(" ".join(args)): + for output in p4CmdList(args): print 'Element: %d' % j j += 1 print output @@ -687,7 +762,7 @@ class P4Submit(Command, P4UserMap): break if not client: die("could not get client spec") - results = p4CmdList("changes -c %s -m 1" % client) + results = p4CmdList(["changes", "-c", client, "-m", "1"]) for r in results: if r.has_key('change'): return r['change'] @@ -750,7 +825,7 @@ class P4Submit(Command, P4UserMap): # remove lines in the Files section that show changes to files outside the depot path we're committing into template = "" inFilesSection = False - for line in p4_read_pipe_lines("change -o"): + for line in p4_read_pipe_lines(['change', '-o']): if line.endswith("\r\n"): line = line[:-2] + "\n" if inFilesSection: @@ -807,7 +882,7 @@ class P4Submit(Command, P4UserMap): modifier = diff['status'] path = diff['src'] if modifier == "M": - p4_system("edit \"%s\"" % path) + p4_edit(path) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): filesToChangeExecBit[path] = diff['dst_mode'] editedFiles.add(path) @@ -822,21 +897,21 @@ class P4Submit(Command, P4UserMap): filesToAdd.remove(path) elif modifier == "C": src, dest = diff['src'], diff['dst'] - p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) + p4_integrate(src, dest) if diff['src_sha1'] != diff['dst_sha1']: - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) filesToChangeExecBit[dest] = diff['dst_mode'] os.unlink(dest) editedFiles.add(dest) elif modifier == "R": src, dest = diff['src'], diff['dst'] - p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest)) + p4_integrate(src, dest) if diff['src_sha1'] != diff['dst_sha1']: - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) if isModeExecChanged(diff['src_mode'], diff['dst_mode']): - p4_system("edit \"%s\"" % (dest)) + p4_edit(dest) filesToChangeExecBit[dest] = diff['dst_mode'] os.unlink(dest) editedFiles.add(dest) @@ -859,9 +934,9 @@ class P4Submit(Command, P4UserMap): if response == "s": print "Skipping! Good luck with the next patches..." for f in editedFiles: - p4_system("revert \"%s\"" % f); + p4_revert(f) for f in filesToAdd: - system("rm %s" %f) + os.remove(f) return elif response == "a": os.system(applyPatchCmd) @@ -882,10 +957,10 @@ class P4Submit(Command, P4UserMap): system(applyPatchCmd) for f in filesToAdd: - p4_system("add \"%s\"" % f) + p4_add(f) for f in filesToDelete: - p4_system("revert \"%s\"" % f) - p4_system("delete \"%s\"" % f) + p4_revert(f) + p4_delete(f) # Set/clear executable bits for f in filesToChangeExecBit.keys(): @@ -907,7 +982,7 @@ class P4Submit(Command, P4UserMap): del(os.environ["P4DIFF"]) diff = "" for editedFile in editedFiles: - diff += p4_read_pipe("diff -du %r" % editedFile) + diff += p4_read_pipe(['diff', '-du', editedFile]) newdiff = "" for newFile in filesToAdd: @@ -959,7 +1034,7 @@ class P4Submit(Command, P4UserMap): submitTemplate = message[:message.index(separatorLine)] if self.isWindows: submitTemplate = submitTemplate.replace("\r\n", "\n") - p4_write_pipe("submit -i", submitTemplate) + p4_write_pipe(['submit', '-i'], submitTemplate) if self.preserveUser: if p4User: @@ -970,10 +1045,10 @@ class P4Submit(Command, P4UserMap): else: for f in editedFiles: - p4_system("revert \"%s\"" % f); + p4_revert(f) for f in filesToAdd: - p4_system("revert \"%s\"" % f); - system("rm %s" %f) + p4_revert(f) + os.remove(f) os.remove(fileName) else: @@ -1026,8 +1101,7 @@ class P4Submit(Command, P4UserMap): chdir(self.clientPath) print "Synchronizing p4 checkout..." - p4_system("sync ...") - + p4_sync("...") self.check() commits = [] @@ -1219,38 +1293,66 @@ class P4Sync(Command, P4UserMap): # - helper for streamP4Files def streamOneP4File(self, file, contents): - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring" % \ - file['depotFile'] - return - relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) relPath = self.wildcard_decode(relPath) if verbose: sys.stderr.write("%s\n" % relPath) - mode = "644" - if isP4Exec(file["type"]): - mode = "755" - elif file["type"] == "symlink": - mode = "120000" - # p4 print on a symlink contains "target\n", so strip it off + (type_base, type_mods) = split_p4_type(file["type"]) + + git_mode = "100644" + if "x" in type_mods: + git_mode = "100755" + if type_base == "symlink": + git_mode = "120000" + # p4 print on a symlink contains "target\n"; remove the newline data = ''.join(contents) contents = [data[:-1]] - if self.isWindows and file["type"].endswith("text"): + if type_base == "utf16": + # p4 delivers different text in the python output to -G + # than it does when using "print -o", or normal p4 client + # operations. utf16 is converted to ascii or utf8, perhaps. + # But ascii text saved as -t utf16 is completely mangled. + # Invoke print -o to get the real contents. + text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']]) + contents = [ text ] + + if type_base == "apple": + # Apple filetype files will be streamed as a concatenation of + # its appledouble header and the contents. This is useless + # on both macs and non-macs. If using "print -q -o xx", it + # will create "xx" with the data, and "%xx" with the header. + # This is also not very useful. + # + # Ideally, someday, this script can learn how to generate + # appledouble files directly and import those to git, but + # non-mac machines can never find a use for apple filetype. + print "\nIgnoring apple filetype file %s" % file['depotFile'] + return + + # Perhaps windows wants unicode, utf16 newlines translated too; + # but this is not doing it. + if self.isWindows and type_base == "text": mangled = [] for data in contents: data = data.replace("\r\n", "\n") mangled.append(data) contents = mangled - if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'): - contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents) - elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'): - contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents) + # Note that we do not try to de-mangle keywords on utf16 files, + # even though in theory somebody may want that. + if type_base in ("text", "unicode", "binary"): + if "ko" in type_mods: + text = ''.join(contents) + text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text) + contents = [ text ] + elif "k" in type_mods: + text = ''.join(contents) + text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text) + contents = [ text ] - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("M %s inline %s\n" % (git_mode, relPath)) # total length... length = 0 @@ -1322,10 +1424,11 @@ class P4Sync(Command, P4UserMap): def streamP4FilesCbSelf(entry): self.streamP4FilesCb(entry) - p4CmdList("-x - print", - '\n'.join(['%s#%s' % (f['path'], f['rev']) - for f in filesToRead]), - cb=streamP4FilesCbSelf) + fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead] + + p4CmdList(["-x", "-", "print"], + stdin=fileArgs, + cb=streamP4FilesCbSelf) # do the last chunk if self.stream_file.has_key('depotFile'): @@ -1386,8 +1489,8 @@ class P4Sync(Command, P4UserMap): if self.verbose: print "Change %s is labelled %s" % (change, labelDetails) - files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change) - for p in branchPrefixes])) + files = p4CmdList(["files"] + ["%s...@%s" % (p, change) + for p in branchPrefixes]) if len(files) == len(labelRevisions): @@ -1435,9 +1538,9 @@ class P4Sync(Command, P4UserMap): newestChange = 0 if self.verbose: print "Querying files for label %s" % label - for file in p4CmdList("files " - + ' '.join (["%s...@%s" % (p, label) - for p in self.depotPaths])): + for file in p4CmdList(["files"] + + ["%s...@%s" % (p, label) + for p in self.depotPaths]): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: @@ -1692,10 +1795,9 @@ class P4Sync(Command, P4UserMap): newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files " - + ' '.join(["%s...%s" - % (p, revision) - for p in self.depotPaths])): + fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths] + + for info in p4CmdList(["files"] + fileArgs): if 'code' in info and info['code'] == 'error': sys.stderr.write("p4 returned an error: %s\n" diff --git a/contrib/git-jump/README b/contrib/git-jump/README new file mode 100644 index 0000000000..1cebc328cb --- /dev/null +++ b/contrib/git-jump/README @@ -0,0 +1,92 @@ +git-jump +======== + +Git-jump is a script for helping you jump to "interesting" parts of your +project in your editor. It works by outputting a set of interesting +spots in the "quickfix" format, which editors like vim can use as a +queue of places to visit (this feature is usually used to jump to errors +produced by a compiler). For example, given a diff like this: + +------------------------------------ +diff --git a/foo.c b/foo.c +index a655540..5a59044 100644 +--- a/foo.c ++++ b/foo.c +@@ -1,3 +1,3 @@ + int main(void) { +- printf("hello word!\n"); ++ printf("hello world!\n"); + } +----------------------------------- + +git-jump will feed this to the editor: + +----------------------------------- +foo.c:2: printf("hello word!\n"); +----------------------------------- + +Obviously this trivial case isn't that interesting; you could just open +`foo.c` yourself. But when you have many changes scattered across a +project, you can use the editor's support to "jump" from point to point. + +Git-jump can generate three types of interesting lists: + + 1. The beginning of any diff hunks. + + 2. The beginning of any merge conflict markers. + + 3. Any grep matches. + + +Using git-jump +-------------- + +To use it, just drop git-jump in your PATH, and then invoke it like +this: + +-------------------------------------------------- +# jump to changes not yet staged for commit +git jump diff + +# jump to changes that are staged for commit; you can give +# arbitrary diff options +git jump diff --cached + +# jump to merge conflicts +git jump merge + +# jump to all instances of foo_bar +git jump grep foo_bar + +# same as above, but case-insensitive; you can give +# arbitrary grep options +git jump grep -i foo_bar +-------------------------------------------------- + + +Related Programs +---------------- + +You can accomplish some of the same things with individual tools. For +example, you can use `git mergetool` to start vimdiff on each unmerged +file. `git jump merge` is for the vim-wielding luddite who just wants to +jump straight to the conflict text with no fanfare. + +As of git v1.7.2, `git grep` knows the `--open-files-in-pager` option, +which does something similar to `git jump grep`. However, it is limited +to positioning the cursor to the correct line in only the first file, +leaving you to locate subsequent hits in that file or other files using +the editor or pager. By contrast, git-jump provides the editor with a +complete list of files and line numbers for each match. + + +Limitations +----------- + +This scripts was written and tested with vim. Given that the quickfix +format is the same as what gcc produces, I expect emacs users have a +similar feature for iterating through the list, but I know nothing about +how to activate it. + +The shell snippets to generate the quickfix lines will almost certainly +choke on filenames with exotic characters (like newlines). diff --git a/contrib/git-jump/git-jump b/contrib/git-jump/git-jump new file mode 100755 index 0000000000..a33674e47a --- /dev/null +++ b/contrib/git-jump/git-jump @@ -0,0 +1,69 @@ +#!/bin/sh + +usage() { + cat <<\EOF +usage: git jump <mode> [<args>] + +Jump to interesting elements in an editor. +The <mode> parameter is one of: + +diff: elements are diff hunks. Arguments are given to diff. + +merge: elements are merge conflicts. Arguments are ignored. + +grep: elements are grep hits. Arguments are given to grep. +EOF +} + +open_editor() { + editor=`git var GIT_EDITOR` + eval "$editor -q \$1" +} + +mode_diff() { + git diff --relative "$@" | + perl -ne ' + if (m{^\+\+\+ b/(.*)}) { $file = $1; next } + defined($file) or next; + if (m/^@@ .*\+(\d+)/) { $line = $1; next } + defined($line) or next; + if (/^ /) { $line++; next } + if (/^[-+]\s*(.*)/) { + print "$file:$line: $1\n"; + $line = undef; + } + ' +} + +mode_merge() { + git ls-files -u | + perl -pe 's/^.*?\t//' | + sort -u | + while IFS= read fn; do + grep -Hn '^<<<<<<<' "$fn" + done +} + +# Grep -n generates nice quickfix-looking lines by itself, +# but let's clean up extra whitespace, so they look better if the +# editor shows them to us in the status bar. +mode_grep() { + git grep -n "$@" | + perl -pe ' + s/[ \t]+/ /g; + s/^ *//; + ' +} + +if test $# -lt 1; then + usage >&2 + exit 1 +fi +mode=$1; shift + +trap 'rm -f "$tmp"' 0 1 2 3 15 +tmp=`mktemp -t git-jump.XXXXXX` || exit 1 +type "mode_$mode" >/dev/null 2>&1 || { usage >&2; exit 1; } +"mode_$mode" "$@" >"$tmp" +test -s "$tmp" || exit 0 +open_editor "$tmp" diff --git a/contrib/mw-to-git/git-remote-mediawiki b/contrib/mw-to-git/git-remote-mediawiki new file mode 100755 index 0000000000..c18bfa1f15 --- /dev/null +++ b/contrib/mw-to-git/git-remote-mediawiki @@ -0,0 +1,827 @@ +#! /usr/bin/perl + +# Copyright (C) 2011 +# Jérémie Nikaes <jeremie.nikaes@ensimag.imag.fr> +# Arnaud Lacurie <arnaud.lacurie@ensimag.imag.fr> +# Claire Fousse <claire.fousse@ensimag.imag.fr> +# David Amouyal <david.amouyal@ensimag.imag.fr> +# Matthieu Moy <matthieu.moy@grenoble-inp.fr> +# License: GPL v2 or later + +# Gateway between Git and MediaWiki. +# https://github.com/Bibzball/Git-Mediawiki/wiki +# +# Known limitations: +# +# - Only wiki pages are managed, no support for [[File:...]] +# attachments. +# +# - Poor performance in the best case: it takes forever to check +# whether we're up-to-date (on fetch or push) or to fetch a few +# revisions from a large wiki, because we use exclusively a +# page-based synchronization. We could switch to a wiki-wide +# synchronization when the synchronization involves few revisions +# but the wiki is large. +# +# - Git renames could be turned into MediaWiki renames (see TODO +# below) +# +# - login/password support requires the user to write the password +# cleartext in a file (see TODO below). +# +# - No way to import "one page, and all pages included in it" +# +# - Multiple remote MediaWikis have not been very well tested. + +use strict; +use MediaWiki::API; +use DateTime::Format::ISO8601; +use encoding 'utf8'; + +# use encoding 'utf8' doesn't change STDERROR +# but we're going to output UTF-8 filenames to STDERR +binmode STDERR, ":utf8"; + +use URI::Escape; +use warnings; + +# Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced +use constant SLASH_REPLACEMENT => "%2F"; + +# It's not always possible to delete pages (may require some +# priviledges). Deleted pages are replaced with this content. +use constant DELETED_CONTENT => "[[Category:Deleted]]\n"; + +# It's not possible to create empty pages. New empty files in Git are +# sent with this content instead. +use constant EMPTY_CONTENT => "<!-- empty page -->\n"; + +# used to reflect file creation or deletion in diff. +use constant NULL_SHA1 => "0000000000000000000000000000000000000000"; + +my $remotename = $ARGV[0]; +my $url = $ARGV[1]; + +# Accept both space-separated and multiple keys in config file. +# Spaces should be written as _ anyway because we'll use chomp. +my @tracked_pages = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".pages")); +chomp(@tracked_pages); + +# Just like @tracked_pages, but for MediaWiki categories. +my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories")); +chomp(@tracked_categories); + +my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin"); +# TODO: ideally, this should be able to read from keyboard, but we're +# inside a remote helper, so our stdin is connect to git, not to a +# terminal. +my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword"); +my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain"); +chomp($wiki_login); +chomp($wiki_passwd); +chomp($wiki_domain); + +# Import only last revisions (both for clone and fetch) +my $shallow_import = run_git("config --get --bool remote.". $remotename .".shallow"); +chomp($shallow_import); +$shallow_import = ($shallow_import eq "true"); + +# Dumb push: don't update notes and mediawiki ref to reflect the last push. +# +# Configurable with mediawiki.dumbPush, or per-remote with +# remote.<remotename>.dumbPush. +# +# This means the user will have to re-import the just-pushed +# revisions. On the other hand, this means that the Git revisions +# corresponding to MediaWiki revisions are all imported from the wiki, +# regardless of whether they were initially created in Git or from the +# web interface, hence all users will get the same history (i.e. if +# the push from Git to MediaWiki loses some information, everybody +# will get the history with information lost). If the import is +# deterministic, this means everybody gets the same sha1 for each +# MediaWiki revision. +my $dumb_push = run_git("config --get --bool remote.$remotename.dumbPush"); +unless ($dumb_push) { + $dumb_push = run_git("config --get --bool mediawiki.dumbPush"); +} +chomp($dumb_push); +$dumb_push = ($dumb_push eq "true"); + +my $wiki_name = $url; +$wiki_name =~ s/[^\/]*:\/\///; +# If URL is like http://user:password@example.com/, we clearly don't +# want the password in $wiki_name. While we're there, also remove user +# and '@' sign, to avoid author like MWUser@HTTPUser@host.com +$wiki_name =~ s/^.*@//; + +# Commands parser +my $entry; +my @cmd; +while (<STDIN>) { + chomp; + @cmd = split(/ /); + if (defined($cmd[0])) { + # Line not blank + if ($cmd[0] eq "capabilities") { + die("Too many arguments for capabilities") unless (!defined($cmd[1])); + mw_capabilities(); + } elsif ($cmd[0] eq "list") { + die("Too many arguments for list") unless (!defined($cmd[2])); + mw_list($cmd[1]); + } elsif ($cmd[0] eq "import") { + die("Invalid arguments for import") unless ($cmd[1] ne "" && !defined($cmd[2])); + mw_import($cmd[1]); + } elsif ($cmd[0] eq "option") { + die("Too many arguments for option") unless ($cmd[1] ne "" && $cmd[2] ne "" && !defined($cmd[3])); + mw_option($cmd[1],$cmd[2]); + } elsif ($cmd[0] eq "push") { + mw_push($cmd[1]); + } else { + print STDERR "Unknown command. Aborting...\n"; + last; + } + } else { + # blank line: we should terminate + last; + } + + BEGIN { $| = 1 } # flush STDOUT, to make sure the previous + # command is fully processed. +} + +########################## Functions ############################## + +# MediaWiki API instance, created lazily. +my $mediawiki; + +sub mw_connect_maybe { + if ($mediawiki) { + return; + } + $mediawiki = MediaWiki::API->new; + $mediawiki->{config}->{api_url} = "$url/api.php"; + if ($wiki_login) { + if (!$mediawiki->login({ + lgname => $wiki_login, + lgpassword => $wiki_passwd, + lgdomain => $wiki_domain, + })) { + print STDERR "Failed to log in mediawiki user \"$wiki_login\" on $url\n"; + print STDERR "(error " . + $mediawiki->{error}->{code} . ': ' . + $mediawiki->{error}->{details} . ")\n"; + exit 1; + } else { + print STDERR "Logged in with user \"$wiki_login\".\n"; + } + } +} + +sub get_mw_first_pages { + my $some_pages = shift; + my @some_pages = @{$some_pages}; + + my $pages = shift; + + # pattern 'page1|page2|...' required by the API + my $titles = join('|', @some_pages); + + my $mw_pages = $mediawiki->api({ + action => 'query', + titles => $titles, + }); + if (!defined($mw_pages)) { + print STDERR "fatal: could not query the list of wiki pages.\n"; + print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; + print STDERR "fatal: make sure '$url/api.php' is a valid page.\n"; + exit 1; + } + while (my ($id, $page) = each(%{$mw_pages->{query}->{pages}})) { + if ($id < 0) { + print STDERR "Warning: page $page->{title} not found on wiki\n"; + } else { + $pages->{$page->{title}} = $page; + } + } +} + +sub get_mw_pages { + mw_connect_maybe(); + + my %pages; # hash on page titles to avoid duplicates + my $user_defined; + if (@tracked_pages) { + $user_defined = 1; + # The user provided a list of pages titles, but we + # still need to query the API to get the page IDs. + + my @some_pages = @tracked_pages; + while (@some_pages) { + my $last = 50; + if ($#some_pages < $last) { + $last = $#some_pages; + } + my @slice = @some_pages[0..$last]; + get_mw_first_pages(\@slice, \%pages); + @some_pages = @some_pages[51..$#some_pages]; + } + } + if (@tracked_categories) { + $user_defined = 1; + foreach my $category (@tracked_categories) { + if (index($category, ':') < 0) { + # Mediawiki requires the Category + # prefix, but let's not force the user + # to specify it. + $category = "Category:" . $category; + } + my $mw_pages = $mediawiki->list( { + action => 'query', + list => 'categorymembers', + cmtitle => $category, + cmlimit => 'max' } ) + || die $mediawiki->{error}->{code} . ': ' . $mediawiki->{error}->{details}; + foreach my $page (@{$mw_pages}) { + $pages{$page->{title}} = $page; + } + } + } + if (!$user_defined) { + # No user-provided list, get the list of pages from + # the API. + my $mw_pages = $mediawiki->list({ + action => 'query', + list => 'allpages', + aplimit => 500, + }); + if (!defined($mw_pages)) { + print STDERR "fatal: could not get the list of wiki pages.\n"; + print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; + print STDERR "fatal: make sure '$url/api.php' is a valid page.\n"; + exit 1; + } + foreach my $page (@{$mw_pages}) { + $pages{$page->{title}} = $page; + } + } + return values(%pages); +} + +sub run_git { + open(my $git, "-|:encoding(UTF-8)", "git " . $_[0]); + my $res = do { local $/; <$git> }; + close($git); + + return $res; +} + + +sub get_last_local_revision { + # Get note regarding last mediawiki revision + my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null"); + my @note_info = split(/ /, $note); + + my $lastrevision_number; + if (!(defined($note_info[0]) && $note_info[0] eq "mediawiki_revision:")) { + print STDERR "No previous mediawiki revision found"; + $lastrevision_number = 0; + } else { + # Notes are formatted : mediawiki_revision: #number + $lastrevision_number = $note_info[1]; + chomp($lastrevision_number); + print STDERR "Last local mediawiki revision found is $lastrevision_number"; + } + return $lastrevision_number; +} + +# Remember the timestamp corresponding to a revision id. +my %basetimestamps; + +sub get_last_remote_revision { + mw_connect_maybe(); + + my @pages = get_mw_pages(); + + my $max_rev_num = 0; + + foreach my $page (@pages) { + my $id = $page->{pageid}; + + my $query = { + action => 'query', + prop => 'revisions', + rvprop => 'ids|timestamp', + pageids => $id, + }; + + my $result = $mediawiki->api($query); + + my $lastrev = pop(@{$result->{query}->{pages}->{$id}->{revisions}}); + + $basetimestamps{$lastrev->{revid}} = $lastrev->{timestamp}; + + $max_rev_num = ($lastrev->{revid} > $max_rev_num ? $lastrev->{revid} : $max_rev_num); + } + + print STDERR "Last remote revision found is $max_rev_num.\n"; + return $max_rev_num; +} + +# Clean content before sending it to MediaWiki +sub mediawiki_clean { + my $string = shift; + my $page_created = shift; + # Mediawiki does not allow blank space at the end of a page and ends with a single \n. + # This function right trims a string and adds a \n at the end to follow this rule + $string =~ s/\s+$//; + if ($string eq "" && $page_created) { + # Creating empty pages is forbidden. + $string = EMPTY_CONTENT; + } + return $string."\n"; +} + +# Filter applied on MediaWiki data before adding them to Git +sub mediawiki_smudge { + my $string = shift; + if ($string eq EMPTY_CONTENT) { + $string = ""; + } + # This \n is important. This is due to mediawiki's way to handle end of files. + return $string."\n"; +} + +sub mediawiki_clean_filename { + my $filename = shift; + $filename =~ s/@{[SLASH_REPLACEMENT]}/\//g; + # [, ], |, {, and } are forbidden by MediaWiki, even URL-encoded. + # Do a variant of URL-encoding, i.e. looks like URL-encoding, + # but with _ added to prevent MediaWiki from thinking this is + # an actual special character. + $filename =~ s/[\[\]\{\}\|]/sprintf("_%%_%x", ord($&))/ge; + # If we use the uri escape before + # we should unescape here, before anything + + return $filename; +} + +sub mediawiki_smudge_filename { + my $filename = shift; + $filename =~ s/\//@{[SLASH_REPLACEMENT]}/g; + $filename =~ s/ /_/g; + # Decode forbidden characters encoded in mediawiki_clean_filename + $filename =~ s/_%_([0-9a-fA-F][0-9a-fA-F])/sprintf("%c", hex($1))/ge; + return $filename; +} + +sub literal_data { + my ($content) = @_; + print STDOUT "data ", bytes::length($content), "\n", $content; +} + +sub mw_capabilities { + # Revisions are imported to the private namespace + # refs/mediawiki/$remotename/ by the helper and fetched into + # refs/remotes/$remotename later by fetch. + print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n"; + print STDOUT "import\n"; + print STDOUT "list\n"; + print STDOUT "push\n"; + print STDOUT "\n"; +} + +sub mw_list { + # MediaWiki do not have branches, we consider one branch arbitrarily + # called master, and HEAD pointing to it. + print STDOUT "? refs/heads/master\n"; + print STDOUT "\@refs/heads/master HEAD\n"; + print STDOUT "\n"; +} + +sub mw_option { + print STDERR "remote-helper command 'option $_[0]' not yet implemented\n"; + print STDOUT "unsupported\n"; +} + +sub fetch_mw_revisions_for_page { + my $page = shift; + my $id = shift; + my $fetch_from = shift; + my @page_revs = (); + my $query = { + action => 'query', + prop => 'revisions', + rvprop => 'ids', + rvdir => 'newer', + rvstartid => $fetch_from, + rvlimit => 500, + pageids => $id, + }; + + my $revnum = 0; + # Get 500 revisions at a time due to the mediawiki api limit + while (1) { + my $result = $mediawiki->api($query); + + # Parse each of those 500 revisions + foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) { + my $page_rev_ids; + $page_rev_ids->{pageid} = $page->{pageid}; + $page_rev_ids->{revid} = $revision->{revid}; + push(@page_revs, $page_rev_ids); + $revnum++; + } + last unless $result->{'query-continue'}; + $query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid}; + } + if ($shallow_import && @page_revs) { + print STDERR " Found 1 revision (shallow import).\n"; + @page_revs = sort {$b->{revid} <=> $a->{revid}} (@page_revs); + return $page_revs[0]; + } + print STDERR " Found ", $revnum, " revision(s).\n"; + return @page_revs; +} + +sub fetch_mw_revisions { + my $pages = shift; my @pages = @{$pages}; + my $fetch_from = shift; + + my @revisions = (); + my $n = 1; + foreach my $page (@pages) { + my $id = $page->{pageid}; + + print STDERR "page $n/", scalar(@pages), ": ". $page->{title} ."\n"; + $n++; + my @page_revs = fetch_mw_revisions_for_page($page, $id, $fetch_from); + @revisions = (@page_revs, @revisions); + } + + return ($n, @revisions); +} + +sub import_file_revision { + my $commit = shift; + my %commit = %{$commit}; + my $full_import = shift; + my $n = shift; + + my $title = $commit{title}; + my $comment = $commit{comment}; + my $content = $commit{content}; + my $author = $commit{author}; + my $date = $commit{date}; + + print STDOUT "commit refs/mediawiki/$remotename/master\n"; + print STDOUT "mark :$n\n"; + print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n"; + literal_data($comment); + + # If it's not a clone, we need to know where to start from + if (!$full_import && $n == 1) { + print STDOUT "from refs/mediawiki/$remotename/master^0\n"; + } + if ($content ne DELETED_CONTENT) { + print STDOUT "M 644 inline $title.mw\n"; + literal_data($content); + print STDOUT "\n\n"; + } else { + print STDOUT "D $title.mw\n"; + } + + # mediawiki revision number in the git note + if ($full_import && $n == 1) { + print STDOUT "reset refs/notes/$remotename/mediawiki\n"; + } + print STDOUT "commit refs/notes/$remotename/mediawiki\n"; + print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n"; + literal_data("Note added by git-mediawiki during import"); + if (!$full_import && $n == 1) { + print STDOUT "from refs/notes/$remotename/mediawiki^0\n"; + } + print STDOUT "N inline :$n\n"; + literal_data("mediawiki_revision: " . $commit{mw_revision}); + print STDOUT "\n\n"; +} + +# parse a sequence of +# <cmd> <arg1> +# <cmd> <arg2> +# \n +# (like batch sequence of import and sequence of push statements) +sub get_more_refs { + my $cmd = shift; + my @refs; + while (1) { + my $line = <STDIN>; + if ($line =~ m/^$cmd (.*)$/) { + push(@refs, $1); + } elsif ($line eq "\n") { + return @refs; + } else { + die("Invalid command in a '$cmd' batch: ". $_); + } + } +} + +sub mw_import { + # multiple import commands can follow each other. + my @refs = (shift, get_more_refs("import")); + foreach my $ref (@refs) { + mw_import_ref($ref); + } + print STDOUT "done\n"; +} + +sub mw_import_ref { + my $ref = shift; + # The remote helper will call "import HEAD" and + # "import refs/heads/master". + # Since HEAD is a symbolic ref to master (by convention, + # followed by the output of the command "list" that we gave), + # we don't need to do anything in this case. + if ($ref eq "HEAD") { + return; + } + + mw_connect_maybe(); + + my @pages = get_mw_pages(); + + print STDERR "Searching revisions...\n"; + my $last_local = get_last_local_revision(); + my $fetch_from = $last_local + 1; + if ($fetch_from == 1) { + print STDERR ", fetching from beginning.\n"; + } else { + print STDERR ", fetching from here.\n"; + } + my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from); + + # Creation of the fast-import stream + print STDERR "Fetching & writing export data...\n"; + + $n = 0; + my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined + + foreach my $pagerevid (sort {$a->{revid} <=> $b->{revid}} @revisions) { + # fetch the content of the pages + my $query = { + action => 'query', + prop => 'revisions', + rvprop => 'content|timestamp|comment|user|ids', + revids => $pagerevid->{revid}, + }; + + my $result = $mediawiki->api($query); + + my $rev = pop(@{$result->{query}->{pages}->{$pagerevid->{pageid}}->{revisions}}); + + $n++; + + my %commit; + $commit{author} = $rev->{user} || 'Anonymous'; + $commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*'; + $commit{title} = mediawiki_smudge_filename( + $result->{query}->{pages}->{$pagerevid->{pageid}}->{title} + ); + $commit{mw_revision} = $pagerevid->{revid}; + $commit{content} = mediawiki_smudge($rev->{'*'}); + + if (!defined($rev->{timestamp})) { + $last_timestamp++; + } else { + $last_timestamp = $rev->{timestamp}; + } + $commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp); + + print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n"; + + import_file_revision(\%commit, ($fetch_from == 1), $n); + } + + if ($fetch_from == 1 && $n == 0) { + print STDERR "You appear to have cloned an empty MediaWiki.\n"; + # Something has to be done remote-helper side. If nothing is done, an error is + # thrown saying that HEAD is refering to unknown object 0000000000000000000 + # and the clone fails. + } +} + +sub error_non_fast_forward { + my $advice = run_git("config --bool advice.pushNonFastForward"); + chomp($advice); + if ($advice ne "false") { + # Native git-push would show this after the summary. + # We can't ask it to display it cleanly, so print it + # ourselves before. + print STDERR "To prevent you from losing history, non-fast-forward updates were rejected\n"; + print STDERR "Merge the remote changes (e.g. 'git pull') before pushing again. See the\n"; + print STDERR "'Note about fast-forwards' section of 'git push --help' for details.\n"; + } + print STDOUT "error $_[0] \"non-fast-forward\"\n"; + return 0; +} + +sub mw_push_file { + my $diff_info = shift; + # $diff_info contains a string in this format: + # 100644 100644 <sha1_of_blob_before_commit> <sha1_of_blob_now> <status> + my @diff_info_split = split(/[ \t]/, $diff_info); + + # Filename, including .mw extension + my $complete_file_name = shift; + # Commit message + my $summary = shift; + # MediaWiki revision number. Keep the previous one by default, + # in case there's no edit to perform. + my $newrevid = shift; + + my $new_sha1 = $diff_info_split[3]; + my $old_sha1 = $diff_info_split[2]; + my $page_created = ($old_sha1 eq NULL_SHA1); + my $page_deleted = ($new_sha1 eq NULL_SHA1); + $complete_file_name = mediawiki_clean_filename($complete_file_name); + + if (substr($complete_file_name,-3) eq ".mw") { + my $title = substr($complete_file_name,0,-3); + + my $file_content; + if ($page_deleted) { + # Deleting a page usually requires + # special priviledges. A common + # convention is to replace the page + # with this content instead: + $file_content = DELETED_CONTENT; + } else { + $file_content = run_git("cat-file blob $new_sha1"); + } + + mw_connect_maybe(); + + my $result = $mediawiki->edit( { + action => 'edit', + summary => $summary, + title => $title, + basetimestamp => $basetimestamps{$newrevid}, + text => mediawiki_clean($file_content, $page_created), + }, { + skip_encoding => 1 # Helps with names with accentuated characters + }); + if (!$result) { + if ($mediawiki->{error}->{code} == 3) { + # edit conflicts, considered as non-fast-forward + print STDERR 'Warning: Error ' . + $mediawiki->{error}->{code} . + ' from mediwiki: ' . $mediawiki->{error}->{details} . + ".\n"; + return ($newrevid, "non-fast-forward"); + } else { + # Other errors. Shouldn't happen => just die() + die 'Fatal: Error ' . + $mediawiki->{error}->{code} . + ' from mediwiki: ' . $mediawiki->{error}->{details}; + } + } + $newrevid = $result->{edit}->{newrevid}; + print STDERR "Pushed file: $new_sha1 - $title\n"; + } else { + print STDERR "$complete_file_name not a mediawiki file (Not pushable on this version of git-remote-mediawiki).\n" + } + return ($newrevid, "ok"); +} + +sub mw_push { + # multiple push statements can follow each other + my @refsspecs = (shift, get_more_refs("push")); + my $pushed; + for my $refspec (@refsspecs) { + my ($force, $local, $remote) = $refspec =~ /^(\+)?([^:]*):([^:]*)$/ + or die("Invalid refspec for push. Expected <src>:<dst> or +<src>:<dst>"); + if ($force) { + print STDERR "Warning: forced push not allowed on a MediaWiki.\n"; + } + if ($local eq "") { + print STDERR "Cannot delete remote branch on a MediaWiki\n"; + print STDOUT "error $remote cannot delete\n"; + next; + } + if ($remote ne "refs/heads/master") { + print STDERR "Only push to the branch 'master' is supported on a MediaWiki\n"; + print STDOUT "error $remote only master allowed\n"; + next; + } + if (mw_push_revision($local, $remote)) { + $pushed = 1; + } + } + + # Notify Git that the push is done + print STDOUT "\n"; + + if ($pushed && $dumb_push) { + print STDERR "Just pushed some revisions to MediaWiki.\n"; + print STDERR "The pushed revisions now have to be re-imported, and your current branch\n"; + print STDERR "needs to be updated with these re-imported commits. You can do this with\n"; + print STDERR "\n"; + print STDERR " git pull --rebase\n"; + print STDERR "\n"; + } +} + +sub mw_push_revision { + my $local = shift; + my $remote = shift; # actually, this has to be "refs/heads/master" at this point. + my $last_local_revid = get_last_local_revision(); + print STDERR ".\n"; # Finish sentence started by get_last_local_revision() + my $last_remote_revid = get_last_remote_revision(); + my $mw_revision = $last_remote_revid; + + # Get sha1 of commit pointed by local HEAD + my $HEAD_sha1 = run_git("rev-parse $local 2>/dev/null"); chomp($HEAD_sha1); + # Get sha1 of commit pointed by remotes/$remotename/master + my $remoteorigin_sha1 = run_git("rev-parse refs/remotes/$remotename/master 2>/dev/null"); + chomp($remoteorigin_sha1); + + if ($last_local_revid > 0 && + $last_local_revid < $last_remote_revid) { + return error_non_fast_forward($remote); + } + + if ($HEAD_sha1 eq $remoteorigin_sha1) { + # nothing to push + return 0; + } + + # Get every commit in between HEAD and refs/remotes/origin/master, + # including HEAD and refs/remotes/origin/master + my @commit_pairs = (); + if ($last_local_revid > 0) { + my $parsed_sha1 = $remoteorigin_sha1; + # Find a path from last MediaWiki commit to pushed commit + while ($parsed_sha1 ne $HEAD_sha1) { + my @commit_info = grep(/^$parsed_sha1/, split(/\n/, run_git("rev-list --children $local"))); + if (!@commit_info) { + return error_non_fast_forward($remote); + } + my @commit_info_split = split(/ |\n/, $commit_info[0]); + # $commit_info_split[1] is the sha1 of the commit to export + # $commit_info_split[0] is the sha1 of its direct child + push(@commit_pairs, \@commit_info_split); + $parsed_sha1 = $commit_info_split[1]; + } + } else { + # No remote mediawiki revision. Export the whole + # history (linearized with --first-parent) + print STDERR "Warning: no common ancestor, pushing complete history\n"; + my $history = run_git("rev-list --first-parent --children $local"); + my @history = split('\n', $history); + @history = @history[1..$#history]; + foreach my $line (reverse @history) { + my @commit_info_split = split(/ |\n/, $line); + push(@commit_pairs, \@commit_info_split); + } + } + + foreach my $commit_info_split (@commit_pairs) { + my $sha1_child = @{$commit_info_split}[0]; + my $sha1_commit = @{$commit_info_split}[1]; + my $diff_infos = run_git("diff-tree -r --raw -z $sha1_child $sha1_commit"); + # TODO: we could detect rename, and encode them with a #redirect on the wiki. + # TODO: for now, it's just a delete+add + my @diff_info_list = split(/\0/, $diff_infos); + # Keep the first line of the commit message as mediawiki comment for the revision + my $commit_msg = (split(/\n/, run_git("show --pretty=format:\"%s\" $sha1_commit")))[0]; + chomp($commit_msg); + # Push every blob + while (@diff_info_list) { + my $status; + # git diff-tree -z gives an output like + # <metadata>\0<filename1>\0 + # <metadata>\0<filename2>\0 + # and we've split on \0. + my $info = shift(@diff_info_list); + my $file = shift(@diff_info_list); + ($mw_revision, $status) = mw_push_file($info, $file, $commit_msg, $mw_revision); + if ($status eq "non-fast-forward") { + # we may already have sent part of the + # commit to MediaWiki, but it's too + # late to cancel it. Stop the push in + # the middle, but still give an + # accurate error message. + return error_non_fast_forward($remote); + } + if ($status ne "ok") { + die("Unknown error from mw_push_file()"); + } + } + unless ($dumb_push) { + run_git("notes --ref=$remotename/mediawiki add -m \"mediawiki_revision: $mw_revision\" $sha1_commit"); + run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child"); + } + } + + print STDOUT "ok $remote\n"; + return 1; +} diff --git a/contrib/mw-to-git/git-remote-mediawiki.txt b/contrib/mw-to-git/git-remote-mediawiki.txt new file mode 100644 index 0000000000..4d211f5b81 --- /dev/null +++ b/contrib/mw-to-git/git-remote-mediawiki.txt @@ -0,0 +1,7 @@ +Git-Mediawiki is a project which aims the creation of a gate +between git and mediawiki, allowing git users to push and pull +objects from mediawiki just as one would do with a classic git +repository thanks to remote-helpers. + +For more information, visit the wiki at +https://github.com/Bibzball/Git-Mediawiki/wiki |