diff options
Diffstat (limited to 'contrib')
40 files changed, 5150 insertions, 595 deletions
diff --git a/contrib/ciabot/INSTALL b/contrib/ciabot/INSTALL new file mode 100644 index 0000000000..7222961d35 --- /dev/null +++ b/contrib/ciabot/INSTALL @@ -0,0 +1,54 @@ += Installation instructions = + +Two scripts are included. The Python one (ciabot.py) is faster and +more capable; the shell one (ciabot.sh) is a fallback in case Python +gives your git hosting site indigestion. (I know of no such sites.) + +It is no longer necessary to modify the script in order to put it +in place; in fact, this is now discouraged. It is entirely +configurable with the following git config variables: + +ciabot.project = name of the project +ciabot.repo = name of the project repo for gitweb/cgit purposes +ciabot.xmlrpc = if true, ship notifications via XML-RPC +ciabot.revformat = format in which the revision is shown + +The revformat variable may have the following values +raw -> full hex ID of commit +short -> first 12 chars of hex ID +describe -> describe relative to last tag, falling back to short + +ciabot.project defaults to the directory name of the repository toplevel. +ciabot.repo defaults to ciabot.project lowercased. +ciabot.xmlrpc defaults to True +ciabot.revformat defaults to 'describe'. + +This means that in the normal case you need not do any configuration at all, +however setting ciabot.project will allow the hook to run slightly faster. + +Once you've set these variables, try your script with -n to see the +notification message dumped to stdout and verify that it looks sane. + +To live-test these scripts, your project needs to have been registered with +the CIA site. Here are the steps: + +1. Open an IRC window on irc://freenode/commits or your registered + project IRC channel. + +2. Run ciabot.py and/or ciabot.sh from any directory under git + control. + +You should see a notification on the channel for your most recent commit. + +After verifying correct function, install one of these scripts either +in a post-commit hook or in an update hook. + +In post-commit, run it without arguments. It will query for +current HEAD and the latest commit ID to get the information it +needs. + +In update, call it with a refname followed by a list of commits: +You want to reverse the order git rev-list emits because it lists +from most recent to oldest. + +/path/to/ciabot.py ${refname} $(git rev-list ${oldhead}..${newhead} | tac) diff --git a/contrib/ciabot/README b/contrib/ciabot/README index 3b916acece..2dfe1f91f5 100644 --- a/contrib/ciabot/README +++ b/contrib/ciabot/README @@ -8,5 +8,4 @@ You probably want the Python version; it's faster, more capable, and better documented. The shell version is maintained only as a fallback for use on hosting sites that don't permit Python hook scripts. -You will find installation instructions for each script in its comment -header. +See the file INSTALL for installation instructions. diff --git a/contrib/ciabot/ciabot.py b/contrib/ciabot/ciabot.py index 9775dffb5d..bd24395d4c 100755 --- a/contrib/ciabot/ciabot.py +++ b/contrib/ciabot/ciabot.py @@ -10,44 +10,45 @@ # usage: ciabot.py [-V] [-n] [-p projectname] [refname [commits...]] # # This script is meant to be run either in a post-commit hook or in an -# update hook. If there's nothing unusual about your hosting setup, -# you can specify the project name with a -p option and avoid having -# to modify this script. Try it with -n to see the notification mail -# dumped to stdout and verify that it looks sane. With -V it dumps its -# version and exits. +# update hook. Try it with -n to see the notification mail dumped to +# stdout and verify that it looks sane. With -V it dumps its version +# and exits. # -# In post-commit, run it without arguments (other than possibly a -p -# option). It will query for current HEAD and the latest commit ID to -# get the information it needs. +# In post-commit, run it without arguments. It will query for +# current HEAD and the latest commit ID to get the information it +# needs. # # In update, call it with a refname followed by a list of commits: -# You want to reverse the order git rev-list emits becxause it lists +# You want to reverse the order git rev-list emits because it lists # from most recent to oldest. # # /path/to/ciabot.py ${refname} $(git rev-list ${oldhead}..${newhead} | tac) # -# Note: this script uses mail, not XML-RPC, in order to avoid stalling -# until timeout when the CIA XML-RPC server is down. +# Configuration variables affecting this script: # - +# ciabot.project = name of the project +# ciabot.repo = name of the project repo for gitweb/cgit purposes +# ciabot.xmlrpc = if true (default), ship notifications via XML-RPC +# ciabot.revformat = format in which the revision is shown # -# The project as known to CIA. You will either want to change this -# or invoke the script with a -p option to set it. +# ciabot.project defaults to the directory name of the repository toplevel. +# ciabot.repo defaults to ciabot.project lowercased. # -project=None - +# This means that in the normal case you need not do any configuration at all, +# but setting the project name will speed it up slightly. # -# You may not need to change these: +# The revformat variable may have the following values +# raw -> full hex ID of commit +# short -> first 12 chars of hex ID +# describe = -> describe relative to last tag, falling back to short +# The default is 'describe'. +# +# Note: the CIA project now says only XML-RPC is reliable, so +# we default to that. # -import os, sys, commands, socket, urllib - -# Name of the repository. -# You can hardwire this to make the script faster. -repo = os.path.basename(os.getcwd()) -# Fully-qualified domain name of this host. -# You can hardwire this to make the script faster. -host = socket.getfqdn() +import os, sys, commands, socket, urllib +from xml.sax.saxutils import escape # Changeset URL prefix for your repo: when the commit ID is appended # to this, it should point at a CGI that will display the commit @@ -72,7 +73,7 @@ xml = '''\ <message> <generator> <name>CIA Python client for Git</name> - <version>%(gitver)s</version> + <version>%(version)s</version> <url>%(generator)s</url> </generator> <source> @@ -98,19 +99,18 @@ xml = '''\ # No user-serviceable parts below this line: # -# Addresses for the e-mail. The from address is a dummy, since CIA -# will never reply to this mail. -fromaddr = "CIABOT-NOREPLY@" + host -toaddr = "cia@cia.navi.cx" +# Where to ship e-mail notifications. +toaddr = "cia@cia.vc" # Identify the generator script. # Should only change when the script itself gets a new home and maintainer. -generator="http://www.catb.org/~esr/ciabot.py" +generator = "http://www.catb.org/~esr/ciabot.py" +version = "3.6" def do(command): return commands.getstatusoutput(command)[1] -def report(refname, merged): +def report(refname, merged, xmlrpc=True): "Generate a commit notification to be reported to CIA" # Try to tinyfy a reference to a web view for this commit. @@ -121,32 +121,27 @@ def report(refname, merged): branch = os.path.basename(refname) - # Compute a shortnane for the revision - rev = do("git describe '"+ merged +"' 2>/dev/null") or merged[:12] - - # Extract the neta-information for the commit - rawcommit = do("git cat-file commit " + merged) + # Compute a description for the revision + if revformat == 'raw': + rev = merged + elif revformat == 'short': + rev = '' + else: # revformat == 'describe' + rev = do("git describe %s 2>/dev/null" % merged) + if not rev: + rev = merged[:12] + + # Extract the meta-information for the commit files=do("git diff-tree -r --name-only '"+ merged +"' | sed -e '1d' -e 's-.*-<file>&</file>-'") - inheader = True - headers = {} - logmsg = "" - for line in rawcommit.split("\n"): - if inheader: - if line: - fields = line.split() - headers[fields[0]] = " ".join(fields[1:]) - else: - inheader = False - else: - logmsg = line - break - (author, ts) = headers["author"].split(">") + metainfo = do("git log -1 '--pretty=format:%an <%ae>%n%at%n%s' " + merged) + (author, ts, logmsg) = metainfo.split("\n") + logmsg = escape(logmsg) - # This discards the part of the authors addrsss after @. - # Might be bnicece to ship the full email address, if not + # This discards the part of the author's address after @. + # Might be be nice to ship the full email address, if not # for spammers' address harvesters - getting this wrong # would make the freenode #commits channel into harvester heaven. - author = author.replace("<", "").split("@")[0].split()[-1] + author = escape(author.replace("<", "").split("@")[0].split()[-1]) # This ignores the timezone. Not clear what to do with it... ts = ts.strip().split()[0] @@ -155,8 +150,7 @@ def report(refname, merged): context.update(globals()) out = xml % context - - message = '''\ + mail = '''\ Message-ID: <%(merged)s.%(author)s@%(project)s> From: %(fromaddr)s To: %(toaddr)s @@ -165,34 +159,56 @@ Subject: DeliverXML %(out)s''' % locals() - return message + if xmlrpc: + return out + else: + return mail if __name__ == "__main__": import getopt + # Get all config variables + revformat = do("git config --get ciabot.revformat") + project = do("git config --get ciabot.project") + repo = do("git config --get ciabot.repo") + xmlrpc = do("git config --get ciabot.xmlrpc") + xmlrpc = not (xmlrpc and xmlrpc == "false") + + host = socket.getfqdn() + fromaddr = "CIABOT-NOREPLY@" + host + try: - (options, arguments) = getopt.getopt(sys.argv[1:], "np:V") + (options, arguments) = getopt.getopt(sys.argv[1:], "np:xV") except getopt.GetoptError, msg: print "ciabot.py: " + str(msg) raise SystemExit, 1 - mailit = True + notify = True for (switch, val) in options: if switch == '-p': project = val elif switch == '-n': - mailit = False + notify = False + elif switch == '-x': + xmlrpc = True elif switch == '-V': - print "ciabot.py: version 3.2" + print "ciabot.py: version", version sys.exit(0) - # Cough and die if user has not specified a project + # The project variable defaults to the name of the repository toplevel. if not project: - sys.stderr.write("ciabot.py: no project specified, bailing out.\n") - sys.exit(1) - - # We'll need the git version number. - gitver = do("git --version").split()[0] + here = os.getcwd() + while True: + if os.path.exists(os.path.join(here, ".git")): + project = os.path.basename(here) + break + elif here == '/': + sys.stderr.write("ciabot.py: no .git below root!\n") + sys.exit(1) + here = os.path.dirname(here) + + if not repo: + repo = project.lower() urlprefix = urlprefix % globals() @@ -205,18 +221,29 @@ if __name__ == "__main__": refname = arguments[0] merges = arguments[1:] - if mailit: - import smtplib - server = smtplib.SMTP('localhost') + if notify: + if xmlrpc: + import xmlrpclib + server = xmlrpclib.Server('http://cia.vc/RPC2'); + else: + import smtplib + server = smtplib.SMTP('localhost') for merged in merges: - message = report(refname, merged) - if mailit: - server.sendmail(fromaddr, [toaddr], message) - else: + message = report(refname, merged, xmlrpc) + if not notify: print message + elif xmlrpc: + try: + # RPC server is flaky, this can fail due to timeout. + server.hub.deliver(message) + except socket.error, e: + sys.stderr.write("%s\n" % e) + else: + server.sendmail(fromaddr, [toaddr], message) - if mailit: - server.quit() + if notify: + if not xmlrpc: + server.quit() #End diff --git a/contrib/ciabot/ciabot.sh b/contrib/ciabot/ciabot.sh index eb87bba38e..3fbbc534ae 100755 --- a/contrib/ciabot/ciabot.sh +++ b/contrib/ciabot/ciabot.sh @@ -3,6 +3,8 @@ # Copyright (c) 2006 Fernando J. Pereda <ferdy@gentoo.org> # Copyright (c) 2008 Natanael Copa <natanael.copa@gmail.com> # Copyright (c) 2010 Eric S. Raymond <esr@thyrsus.com> +# Assistance and review by Petr Baudis, author of ciabot.pl, +# is gratefully acknowledged. # # This is a version 3.x of ciabot.sh; use -V to find the exact # version. Versions 1 and 2 were shipped in 2006 and 2008 and are not @@ -11,6 +13,7 @@ # Note: This script should be considered obsolete. # There is a faster, better-documented rewrite in Python: find it as ciabot.py # Use this only if your hosting site forbids Python hooks. +# It requires: git(1), hostname(1), cut(1), sendmail(1), and wget(1). # # Originally based on Git ciabot.pl by Petr Baudis. # This script contains porcelain and porcelain byproducts. @@ -18,15 +21,13 @@ # usage: ciabot.sh [-V] [-n] [-p projectname] [refname commit] # # This script is meant to be run either in a post-commit hook or in an -# update hook. If there's nothing unusual about your hosting setup, -# you can specify the project name with a -p option and avoid having -# to modify this script. Try it with -n first to see the notification -# mail dumped to stdout and verify that it looks sane. Use -V to dump -# the version and exit. +# update hook. Try it with -n to see the notification mail dumped to +# stdout and verify that it looks sane. With -V it dumps its version +# and exits. # -# In post-commit, run it without arguments (other than possibly a -p -# option). It will query for current HEAD and the latest commit ID to -# get the information it needs. +# In post-commit, run it without arguments. It will query for +# current HEAD and the latest commit ID to get the information it +# needs. # # In update, you have to call it once per merged commit: # @@ -34,33 +35,76 @@ # oldhead=$2 # newhead=$3 # for merged in $(git rev-list ${oldhead}..${newhead} | tac) ; do -# /path/to/ciabot.bash ${refname} ${merged} +# /path/to/ciabot.sh ${refname} ${merged} # done # -# The reason for the tac call ids that git rev-list emits commits from +# The reason for the tac call is that git rev-list emits commits from # most recent to least - better to ship notifactions from oldest to newest. # -# Note: this script uses mail, not XML-RPC, in order to avoid stalling -# until timeout when the CIA XML-RPC server is down. +# Configuration variables affecting this script: # - +# ciabot.project = name of the project +# ciabot.repo = name of the project repo for gitweb/cgit purposes +# ciabot.revformat = format in which the revision is shown # -# The project as known to CIA. You will either want to change this -# or set the project name with a -p option. +# ciabot.project defaults to the directory name of the repository toplevel. +# ciabot.repo defaults to ciabot.project lowercased. # -project= - +# This means that in the normal case you need not do any configuration at all, +# but setting the project name will speed it up slightly. # -# You may not need to change these: +# The revformat variable may have the following values +# raw -> full hex ID of commit +# short -> first 12 chars of hex ID +# describe = -> describe relative to last tag, falling back to short +# The default is 'describe'. # +# Note: the shell ancestors of this script used mail, not XML-RPC, in +# order to avoid stalling until timeout when the CIA XML-RPC server is +# down. It is unknown whether this is still an issue in 2010, but +# XML-RPC would be annoying to do from sh in any case. (XML-RPC does +# have the advantage that it guarantees notification of multiple commits +# shpped from an update in their actual order.) +# + +# The project as known to CIA. You can set this with a -p option, +# or let it default to the directory name of the repo toplevel. +project=$(git config --get ciabot.project) + +if [ -z $project ] +then + here=`pwd`; + while :; do + if [ -d $here/.git ] + then + project=`basename $here` + break + elif [ $here = '/' ] + then + echo "ciabot.sh: no .git below root!" + exit 1 + fi + here=`dirname $here` + done +fi -# Name of the repository. -# You can hardwire this to make the script faster. -repo="`basename ${PWD}`" +# Name of the repo for gitweb/cgit purposes +repo=$(git config --get ciabot.repo) +[ -z $repo] && repo=$(echo "${project}" | tr '[A-Z]' '[a-z]') -# Fully qualified domain name of the repo host. -# You can hardwire this to make the script faster. -host=`hostname --fqdn` +# What revision format do we want in the summary? +revformat=$(git config --get ciabot.revformat) + +# Fully qualified domain name of the repo host. You can hardwire this +# to make the script faster. The -f option works under Linux and FreeBSD, +# but not OpenBSD and NetBSD. But under OpenBSD and NetBSD, +# hostname without options gives the FQDN. +if hostname -f >/dev/null 2>&1 +then + hostname=`hostname -f` +else + hostname=`hostname` +fi # Changeset URL prefix for your repo: when the commit ID is appended # to this, it should point at a CGI that will display the commit @@ -73,13 +117,14 @@ urlprefix="http://${host}/cgi-bin/cgit.cgi/${repo}/commit/?id=" # You probably will not need to change the following: # -# Identify the script. Should change only when the script itself -# gets a new home and maintainer. +# Identify the script. The 'generator' variable should change only +# when the script itself gets a new home and maintainer. generator="http://www.catb.org/~esr/ciabot/ciabot.sh" +version=3.5 # Addresses for the e-mail -from="CIABOT-NOREPLY@${host}" -to="cia@cia.navi.cx" +from="CIABOT-NOREPLY@${hostname}" +to="cia@cia.vc" # SMTP client to use - may need to edit the absolute pathname for your system sendmail="sendmail -t -f ${from}" @@ -97,7 +142,7 @@ do case $opt in p) project=$2; shift ; shift ;; n) mode=dumpit; shift ;; - V) echo "ciabot.sh: version 3.2"; exit 0; shift ;; + V) echo "ciabot.sh: version $version"; exit 0; shift ;; esac done @@ -128,33 +173,29 @@ fi refname=${refname##refs/heads/} -gitver=$(git --version) -gitver=${gitver##* } - -rev=$(git describe ${merged} 2>/dev/null) -# ${merged:0:12} was the only bashism left in the 2008 version of this -# script, according to checkbashisms. Replace it with ${merged} here -# because it was just a fallback anyway, and it's worth accepting a -# longer fallback for faster execution and removing the bash -# dependency. -[ -z ${rev} ] && rev=${merged} +case $revformat in +raw) rev=$merged ;; +short) rev='' ;; +*) rev=$(git describe ${merged} 2>/dev/null) ;; +esac +[ -z ${rev} ] && rev=$(echo "$merged" | cut -c 1-12) -# This discards the part of the author's address after @. +# We discard the part of the author's address after @. # Might be nice to ship the full email address, if not # for spammers' address harvesters - getting this wrong # would make the freenode #commits channel into harvester heaven. -rawcommit=$(git cat-file commit ${merged}) -author=$(echo "$rawcommit" | sed -n -e '/^author .*<\([^@]*\).*$/s--\1-p') -logmessage=$(echo "$rawcommit" | sed -e '1,/^$/d' | head -n 1) -logmessage=$(echo "$logmessage" | sed 's/\&/&\;/g; s/</<\;/g; s/>/>\;/g') -ts=$(echo "$rawcommit" | sed -n -e '/^author .*> \([0-9]\+\).*$/s--\1-p') +author=$(git log -1 '--pretty=format:%an <%ae>' $merged) +author=$(echo "$author" | sed -n -e '/^.*<\([^@]*\).*$/s--\1-p') + +logmessage=$(git log -1 '--pretty=format:%s' $merged) +ts=$(git log -1 '--pretty=format:%at' $merged) files=$(git diff-tree -r --name-only ${merged} | sed -e '1d' -e 's-.*-<file>&</file>-') out=" <message> <generator> <name>CIA Shell client for Git</name> - <version>${gitver}</version> + <version>${version}</version> <url>${generator}</url> </generator> <source> diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 9f56ec7a6b..0492db924b 100755..100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -20,46 +20,8 @@ # 1) Copy this file to somewhere (e.g. ~/.git-completion.sh). # 2) Add the following line to your .bashrc/.zshrc: # source ~/.git-completion.sh -# -# 3) Consider changing your PS1 to also show the current branch: -# Bash: PS1='[\u@\h \W$(__git_ps1 " (%s)")]\$ ' -# ZSH: PS1='[%n@%m %c$(__git_ps1 " (%s)")]\$ ' -# -# The argument to __git_ps1 will be displayed only if you -# are currently in a git repository. The %s token will be -# the name of the current branch. -# -# In addition, if you set GIT_PS1_SHOWDIRTYSTATE to a nonempty -# value, unstaged (*) and staged (+) changes will be shown next -# to the branch name. You can configure this per-repository -# with the bash.showDirtyState variable, which defaults to true -# once GIT_PS1_SHOWDIRTYSTATE is enabled. -# -# You can also see if currently something is stashed, by setting -# GIT_PS1_SHOWSTASHSTATE to a nonempty value. If something is stashed, -# then a '$' will be shown next to the branch name. -# -# If you would like to see if there're untracked files, then you can -# set GIT_PS1_SHOWUNTRACKEDFILES to a nonempty value. If there're -# untracked files, then a '%' will be shown next to the branch name. -# -# If you would like to see the difference between HEAD and its -# upstream, set GIT_PS1_SHOWUPSTREAM="auto". A "<" indicates -# you are behind, ">" indicates you are ahead, and "<>" -# indicates you have diverged. You can further control -# behaviour by setting GIT_PS1_SHOWUPSTREAM to a space-separated -# list of values: -# verbose show number of commits ahead/behind (+/-) upstream -# legacy don't use the '--count' option available in recent -# versions of git-rev-list -# git always compare HEAD to @{upstream} -# svn always compare HEAD to your SVN upstream -# By default, __git_ps1 will compare HEAD to your SVN upstream -# if it can find one, or @{upstream} otherwise. Once you have -# set GIT_PS1_SHOWUPSTREAM, you can override it on a -# per-repository basis by setting the bash.showUpstream config -# variable. -# +# 3) Consider changing your PS1 to also show the current branch, +# see git-prompt.sh for details. if [[ -n ${ZSH_VERSION-} ]]; then autoload -U +X bashcompinit && bashcompinit @@ -74,9 +36,14 @@ esac # returns location of .git repo __gitdir () { + # Note: this function is duplicated in git-prompt.sh + # When updating it, make sure you update the other one to match. if [ -z "${1-}" ]; then if [ -n "${__git_dir-}" ]; then echo "$__git_dir" + elif [ -n "${GIT_DIR-}" ]; then + test -d "${GIT_DIR-}" || return 1 + echo "$GIT_DIR" elif [ -d .git ]; then echo .git else @@ -89,221 +56,6 @@ __gitdir () fi } -# stores the divergence from upstream in $p -# used by GIT_PS1_SHOWUPSTREAM -__git_ps1_show_upstream () -{ - local key value - local svn_remote svn_url_pattern count n - local upstream=git legacy="" verbose="" - - svn_remote=() - # get some config options from git-config - local output="$(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ')" - while read -r key value; do - case "$key" in - bash.showupstream) - GIT_PS1_SHOWUPSTREAM="$value" - if [[ -z "${GIT_PS1_SHOWUPSTREAM}" ]]; then - p="" - return - fi - ;; - svn-remote.*.url) - svn_remote[ $((${#svn_remote[@]} + 1)) ]="$value" - svn_url_pattern+="\\|$value" - upstream=svn+git # default upstream is SVN if available, else git - ;; - esac - done <<< "$output" - - # parse configuration values - for option in ${GIT_PS1_SHOWUPSTREAM}; do - case "$option" in - git|svn) upstream="$option" ;; - verbose) verbose=1 ;; - legacy) legacy=1 ;; - esac - done - - # Find our upstream - case "$upstream" in - git) upstream="@{upstream}" ;; - svn*) - # get the upstream from the "git-svn-id: ..." in a commit message - # (git-svn uses essentially the same procedure internally) - local svn_upstream=($(git log --first-parent -1 \ - --grep="^git-svn-id: \(${svn_url_pattern#??}\)" 2>/dev/null)) - if [[ 0 -ne ${#svn_upstream[@]} ]]; then - svn_upstream=${svn_upstream[ ${#svn_upstream[@]} - 2 ]} - svn_upstream=${svn_upstream%@*} - local n_stop="${#svn_remote[@]}" - for ((n=1; n <= n_stop; n++)); do - svn_upstream=${svn_upstream#${svn_remote[$n]}} - done - - if [[ -z "$svn_upstream" ]]; then - # default branch name for checkouts with no layout: - upstream=${GIT_SVN_ID:-git-svn} - else - upstream=${svn_upstream#/} - fi - elif [[ "svn+git" = "$upstream" ]]; then - upstream="@{upstream}" - fi - ;; - esac - - # Find how many commits we are ahead/behind our upstream - if [[ -z "$legacy" ]]; then - count="$(git rev-list --count --left-right \ - "$upstream"...HEAD 2>/dev/null)" - else - # produce equivalent output to --count for older versions of git - local commits - if commits="$(git rev-list --left-right "$upstream"...HEAD 2>/dev/null)" - then - local commit behind=0 ahead=0 - for commit in $commits - do - case "$commit" in - "<"*) ((behind++)) ;; - *) ((ahead++)) ;; - esac - done - count="$behind $ahead" - else - count="" - fi - fi - - # calculate the result - if [[ -z "$verbose" ]]; then - case "$count" in - "") # no upstream - p="" ;; - "0 0") # equal to upstream - p="=" ;; - "0 "*) # ahead of upstream - p=">" ;; - *" 0") # behind upstream - p="<" ;; - *) # diverged from upstream - p="<>" ;; - esac - else - case "$count" in - "") # no upstream - p="" ;; - "0 0") # equal to upstream - p=" u=" ;; - "0 "*) # ahead of upstream - p=" u+${count#0 }" ;; - *" 0") # behind upstream - p=" u-${count% 0}" ;; - *) # diverged from upstream - p=" u+${count#* }-${count% *}" ;; - esac - fi - -} - - -# __git_ps1 accepts 0 or 1 arguments (i.e., format string) -# returns text to add to bash PS1 prompt (includes branch name) -__git_ps1 () -{ - local g="$(__gitdir)" - if [ -n "$g" ]; then - local r="" - local b="" - if [ -f "$g/rebase-merge/interactive" ]; then - r="|REBASE-i" - b="$(cat "$g/rebase-merge/head-name")" - elif [ -d "$g/rebase-merge" ]; then - r="|REBASE-m" - b="$(cat "$g/rebase-merge/head-name")" - else - if [ -d "$g/rebase-apply" ]; then - if [ -f "$g/rebase-apply/rebasing" ]; then - r="|REBASE" - elif [ -f "$g/rebase-apply/applying" ]; then - r="|AM" - else - r="|AM/REBASE" - fi - elif [ -f "$g/MERGE_HEAD" ]; then - r="|MERGING" - elif [ -f "$g/CHERRY_PICK_HEAD" ]; then - r="|CHERRY-PICKING" - elif [ -f "$g/BISECT_LOG" ]; then - r="|BISECTING" - fi - - b="$(git symbolic-ref HEAD 2>/dev/null)" || { - - b="$( - case "${GIT_PS1_DESCRIBE_STYLE-}" in - (contains) - git describe --contains HEAD ;; - (branch) - git describe --contains --all HEAD ;; - (describe) - git describe HEAD ;; - (* | default) - git describe --tags --exact-match HEAD ;; - esac 2>/dev/null)" || - - b="$(cut -c1-7 "$g/HEAD" 2>/dev/null)..." || - b="unknown" - b="($b)" - } - fi - - local w="" - local i="" - local s="" - local u="" - local c="" - local p="" - - if [ "true" = "$(git rev-parse --is-inside-git-dir 2>/dev/null)" ]; then - if [ "true" = "$(git rev-parse --is-bare-repository 2>/dev/null)" ]; then - c="BARE:" - else - b="GIT_DIR!" - fi - elif [ "true" = "$(git rev-parse --is-inside-work-tree 2>/dev/null)" ]; then - if [ -n "${GIT_PS1_SHOWDIRTYSTATE-}" ]; then - if [ "$(git config --bool bash.showDirtyState)" != "false" ]; then - git diff --no-ext-diff --quiet --exit-code || w="*" - if git rev-parse --quiet --verify HEAD >/dev/null; then - git diff-index --cached --quiet HEAD -- || i="+" - else - i="#" - fi - fi - fi - if [ -n "${GIT_PS1_SHOWSTASHSTATE-}" ]; then - git rev-parse --verify refs/stash >/dev/null 2>&1 && s="$" - fi - - if [ -n "${GIT_PS1_SHOWUNTRACKEDFILES-}" ]; then - if [ -n "$(git ls-files --others --exclude-standard)" ]; then - u="%" - fi - fi - - if [ -n "${GIT_PS1_SHOWUPSTREAM-}" ]; then - __git_ps1_show_upstream - fi - fi - - local f="$w$i$s$u" - printf -- "${1:- (%s)}" "$c${b##refs/heads/}${f:+ $f}$r$p" - fi -} - __gitcomp_1 () { local c IFS=$' \t\n' @@ -676,9 +428,7 @@ __git_complete_revlist_file () *) pfx="$ref:$pfx" ;; esac - local IFS=$'\n' - COMPREPLY=($(compgen -P "$pfx" \ - -W "$(git --git-dir="$(__gitdir)" ls-tree "$ls" \ + __gitcomp_nl "$(git --git-dir="$(__gitdir)" ls-tree "$ls" \ | sed '/^100... blob /{ s,^.* ,, s,$, , @@ -692,7 +442,7 @@ __git_complete_revlist_file () s,$,/, } s/^.* //')" \ - -- "$cur_")) + "$pfx" "$cur_" "" ;; *...*) pfx="${cur_%...*}..." @@ -848,6 +598,8 @@ __git_list_porcelain_commands () checkout-index) : plumbing;; commit-tree) : plumbing;; count-objects) : infrequent;; + credential-cache) : credentials helper;; + credential-store) : credentials helper;; cvsexportcommit) : export;; cvsimport) : import;; cvsserver) : daemon;; @@ -1319,7 +1071,7 @@ _git_diff () } __git_mergetools_common="diffuse ecmerge emerge kdiff3 meld opendiff - tkdiff vimdiff gvimdiff xxdiff araxis p4merge bc3 + tkdiff vimdiff gvimdiff xxdiff araxis p4merge bc3 codecompare " _git_difftool () @@ -2280,7 +2032,7 @@ _git_config () _git_remote () { - local subcommands="add rename rm set-head set-branches set-url show prune update" + local subcommands="add rename remove set-head set-branches set-url show prune update" local subcommand="$(__git_find_on_cmdline "$subcommands")" if [ -z "$subcommand" ]; then __gitcomp "$subcommands" @@ -2288,7 +2040,7 @@ _git_remote () fi case "$subcommand" in - rename|rm|set-url|show|prune) + rename|remove|set-url|show|prune) __gitcomp_nl "$(__git_remotes)" ;; set-head|set-branches) @@ -2599,25 +2351,10 @@ _git_whatchanged () _git_log } -_git () +__git_main () { local i c=1 command __git_dir - if [[ -n ${ZSH_VERSION-} ]]; then - emulate -L bash - setopt KSH_TYPESET - - # workaround zsh's bug that leaves 'words' as a special - # variable in versions < 4.3.12 - typeset -h words - - # workaround zsh's bug that quotes spaces in the COMPREPLY - # array if IFS doesn't contain spaces. - typeset -h IFS - fi - - local cur words cword prev - _get_comp_words_by_ref -n =: cur words cword prev while [ $c -lt $cword ]; do i="${words[c]}" case "$i" in @@ -2665,24 +2402,8 @@ _git () fi } -_gitk () +__gitk_main () { - if [[ -n ${ZSH_VERSION-} ]]; then - emulate -L bash - setopt KSH_TYPESET - - # workaround zsh's bug that leaves 'words' as a special - # variable in versions < 4.3.12 - typeset -h words - - # workaround zsh's bug that quotes spaces in the COMPREPLY - # array if IFS doesn't contain spaces. - typeset -h IFS - fi - - local cur words cword prev - _get_comp_words_by_ref -n =: cur words cword prev - __git_has_doubledash && return local g="$(__gitdir)" @@ -2703,16 +2424,55 @@ _gitk () __git_complete_revlist } -complete -o bashdefault -o default -o nospace -F _git git 2>/dev/null \ - || complete -o default -o nospace -F _git git -complete -o bashdefault -o default -o nospace -F _gitk gitk 2>/dev/null \ - || complete -o default -o nospace -F _gitk gitk +__git_func_wrap () +{ + if [[ -n ${ZSH_VERSION-} ]]; then + emulate -L bash + setopt KSH_TYPESET + + # workaround zsh's bug that leaves 'words' as a special + # variable in versions < 4.3.12 + typeset -h words + + # workaround zsh's bug that quotes spaces in the COMPREPLY + # array if IFS doesn't contain spaces. + typeset -h IFS + fi + local cur words cword prev + _get_comp_words_by_ref -n =: cur words cword prev + $1 +} + +# Setup completion for certain functions defined above by setting common +# variables and workarounds. +# This is NOT a public function; use at your own risk. +__git_complete () +{ + local wrapper="__git_wrap${2}" + eval "$wrapper () { __git_func_wrap $2 ; }" + complete -o bashdefault -o default -o nospace -F $wrapper $1 2>/dev/null \ + || complete -o default -o nospace -F $wrapper $1 +} + +# wrapper for backwards compatibility +_git () +{ + __git_wrap__git_main +} + +# wrapper for backwards compatibility +_gitk () +{ + __git_wrap__gitk_main +} + +__git_complete git __git_main +__git_complete gitk __gitk_main # The following are necessary only for Cygwin, and only are needed # when the user has tab-completed the executable name and consequently # included the '.exe' suffix. # if [ Cygwin = "$(uname -o 2>/dev/null)" ]; then -complete -o bashdefault -o default -o nospace -F _git git.exe 2>/dev/null \ - || complete -o default -o nospace -F _git git.exe +__git_complete git.exe __git_main fi diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh new file mode 100644 index 0000000000..29b1ec9eb1 --- /dev/null +++ b/contrib/completion/git-prompt.sh @@ -0,0 +1,289 @@ +# bash/zsh git prompt support +# +# Copyright (C) 2006,2007 Shawn O. Pearce <spearce@spearce.org> +# Distributed under the GNU General Public License, version 2.0. +# +# This script allows you to see the current branch in your prompt. +# +# To enable: +# +# 1) Copy this file to somewhere (e.g. ~/.git-prompt.sh). +# 2) Add the following line to your .bashrc/.zshrc: +# source ~/.git-prompt.sh +# 3) Change your PS1 to also show the current branch: +# Bash: PS1='[\u@\h \W$(__git_ps1 " (%s)")]\$ ' +# ZSH: PS1='[%n@%m %c$(__git_ps1 " (%s)")]\$ ' +# +# The argument to __git_ps1 will be displayed only if you are currently +# in a git repository. The %s token will be the name of the current +# branch. +# +# In addition, if you set GIT_PS1_SHOWDIRTYSTATE to a nonempty value, +# unstaged (*) and staged (+) changes will be shown next to the branch +# name. You can configure this per-repository with the +# bash.showDirtyState variable, which defaults to true once +# GIT_PS1_SHOWDIRTYSTATE is enabled. +# +# You can also see if currently something is stashed, by setting +# GIT_PS1_SHOWSTASHSTATE to a nonempty value. If something is stashed, +# then a '$' will be shown next to the branch name. +# +# If you would like to see if there're untracked files, then you can set +# GIT_PS1_SHOWUNTRACKEDFILES to a nonempty value. If there're untracked +# files, then a '%' will be shown next to the branch name. +# +# If you would like to see the difference between HEAD and its upstream, +# set GIT_PS1_SHOWUPSTREAM="auto". A "<" indicates you are behind, ">" +# indicates you are ahead, and "<>" indicates you have diverged. You +# can further control behaviour by setting GIT_PS1_SHOWUPSTREAM to a +# space-separated list of values: +# +# verbose show number of commits ahead/behind (+/-) upstream +# legacy don't use the '--count' option available in recent +# versions of git-rev-list +# git always compare HEAD to @{upstream} +# svn always compare HEAD to your SVN upstream +# +# By default, __git_ps1 will compare HEAD to your SVN upstream if it can +# find one, or @{upstream} otherwise. Once you have set +# GIT_PS1_SHOWUPSTREAM, you can override it on a per-repository basis by +# setting the bash.showUpstream config variable. + +# __gitdir accepts 0 or 1 arguments (i.e., location) +# returns location of .git repo +__gitdir () +{ + # Note: this function is duplicated in git-completion.bash + # When updating it, make sure you update the other one to match. + if [ -z "${1-}" ]; then + if [ -n "${__git_dir-}" ]; then + echo "$__git_dir" + elif [ -n "${GIT_DIR-}" ]; then + test -d "${GIT_DIR-}" || return 1 + echo "$GIT_DIR" + elif [ -d .git ]; then + echo .git + else + git rev-parse --git-dir 2>/dev/null + fi + elif [ -d "$1/.git" ]; then + echo "$1/.git" + else + echo "$1" + fi +} + +# stores the divergence from upstream in $p +# used by GIT_PS1_SHOWUPSTREAM +__git_ps1_show_upstream () +{ + local key value + local svn_remote svn_url_pattern count n + local upstream=git legacy="" verbose="" + + svn_remote=() + # get some config options from git-config + local output="$(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ')" + while read -r key value; do + case "$key" in + bash.showupstream) + GIT_PS1_SHOWUPSTREAM="$value" + if [[ -z "${GIT_PS1_SHOWUPSTREAM}" ]]; then + p="" + return + fi + ;; + svn-remote.*.url) + svn_remote[ $((${#svn_remote[@]} + 1)) ]="$value" + svn_url_pattern+="\\|$value" + upstream=svn+git # default upstream is SVN if available, else git + ;; + esac + done <<< "$output" + + # parse configuration values + for option in ${GIT_PS1_SHOWUPSTREAM}; do + case "$option" in + git|svn) upstream="$option" ;; + verbose) verbose=1 ;; + legacy) legacy=1 ;; + esac + done + + # Find our upstream + case "$upstream" in + git) upstream="@{upstream}" ;; + svn*) + # get the upstream from the "git-svn-id: ..." in a commit message + # (git-svn uses essentially the same procedure internally) + local svn_upstream=($(git log --first-parent -1 \ + --grep="^git-svn-id: \(${svn_url_pattern#??}\)" 2>/dev/null)) + if [[ 0 -ne ${#svn_upstream[@]} ]]; then + svn_upstream=${svn_upstream[ ${#svn_upstream[@]} - 2 ]} + svn_upstream=${svn_upstream%@*} + local n_stop="${#svn_remote[@]}" + for ((n=1; n <= n_stop; n++)); do + svn_upstream=${svn_upstream#${svn_remote[$n]}} + done + + if [[ -z "$svn_upstream" ]]; then + # default branch name for checkouts with no layout: + upstream=${GIT_SVN_ID:-git-svn} + else + upstream=${svn_upstream#/} + fi + elif [[ "svn+git" = "$upstream" ]]; then + upstream="@{upstream}" + fi + ;; + esac + + # Find how many commits we are ahead/behind our upstream + if [[ -z "$legacy" ]]; then + count="$(git rev-list --count --left-right \ + "$upstream"...HEAD 2>/dev/null)" + else + # produce equivalent output to --count for older versions of git + local commits + if commits="$(git rev-list --left-right "$upstream"...HEAD 2>/dev/null)" + then + local commit behind=0 ahead=0 + for commit in $commits + do + case "$commit" in + "<"*) ((behind++)) ;; + *) ((ahead++)) ;; + esac + done + count="$behind $ahead" + else + count="" + fi + fi + + # calculate the result + if [[ -z "$verbose" ]]; then + case "$count" in + "") # no upstream + p="" ;; + "0 0") # equal to upstream + p="=" ;; + "0 "*) # ahead of upstream + p=">" ;; + *" 0") # behind upstream + p="<" ;; + *) # diverged from upstream + p="<>" ;; + esac + else + case "$count" in + "") # no upstream + p="" ;; + "0 0") # equal to upstream + p=" u=" ;; + "0 "*) # ahead of upstream + p=" u+${count#0 }" ;; + *" 0") # behind upstream + p=" u-${count% 0}" ;; + *) # diverged from upstream + p=" u+${count#* }-${count% *}" ;; + esac + fi + +} + + +# __git_ps1 accepts 0 or 1 arguments (i.e., format string) +# returns text to add to bash PS1 prompt (includes branch name) +__git_ps1 () +{ + local g="$(__gitdir)" + if [ -n "$g" ]; then + local r="" + local b="" + if [ -f "$g/rebase-merge/interactive" ]; then + r="|REBASE-i" + b="$(cat "$g/rebase-merge/head-name")" + elif [ -d "$g/rebase-merge" ]; then + r="|REBASE-m" + b="$(cat "$g/rebase-merge/head-name")" + else + if [ -d "$g/rebase-apply" ]; then + if [ -f "$g/rebase-apply/rebasing" ]; then + r="|REBASE" + elif [ -f "$g/rebase-apply/applying" ]; then + r="|AM" + else + r="|AM/REBASE" + fi + elif [ -f "$g/MERGE_HEAD" ]; then + r="|MERGING" + elif [ -f "$g/CHERRY_PICK_HEAD" ]; then + r="|CHERRY-PICKING" + elif [ -f "$g/BISECT_LOG" ]; then + r="|BISECTING" + fi + + b="$(git symbolic-ref HEAD 2>/dev/null)" || { + + b="$( + case "${GIT_PS1_DESCRIBE_STYLE-}" in + (contains) + git describe --contains HEAD ;; + (branch) + git describe --contains --all HEAD ;; + (describe) + git describe HEAD ;; + (* | default) + git describe --tags --exact-match HEAD ;; + esac 2>/dev/null)" || + + b="$(cut -c1-7 "$g/HEAD" 2>/dev/null)..." || + b="unknown" + b="($b)" + } + fi + + local w="" + local i="" + local s="" + local u="" + local c="" + local p="" + + if [ "true" = "$(git rev-parse --is-inside-git-dir 2>/dev/null)" ]; then + if [ "true" = "$(git rev-parse --is-bare-repository 2>/dev/null)" ]; then + c="BARE:" + else + b="GIT_DIR!" + fi + elif [ "true" = "$(git rev-parse --is-inside-work-tree 2>/dev/null)" ]; then + if [ -n "${GIT_PS1_SHOWDIRTYSTATE-}" ]; then + if [ "$(git config --bool bash.showDirtyState)" != "false" ]; then + git diff --no-ext-diff --quiet --exit-code || w="*" + if git rev-parse --quiet --verify HEAD >/dev/null; then + git diff-index --cached --quiet HEAD -- || i="+" + else + i="#" + fi + fi + fi + if [ -n "${GIT_PS1_SHOWSTASHSTATE-}" ]; then + git rev-parse --verify refs/stash >/dev/null 2>&1 && s="$" + fi + + if [ -n "${GIT_PS1_SHOWUNTRACKEDFILES-}" ]; then + if [ -n "$(git ls-files --others --exclude-standard)" ]; then + u="%" + fi + fi + + if [ -n "${GIT_PS1_SHOWUPSTREAM-}" ]; then + __git_ps1_show_upstream + fi + fi + + local f="$w$i$s$u" + printf -- "${1:- (%s)}" "$c${b##refs/heads/}${f:+ $f}$r$p" + fi +} diff --git a/contrib/credential/gnome-keyring/.gitignore b/contrib/credential/gnome-keyring/.gitignore new file mode 100644 index 0000000000..88d8fcdbce --- /dev/null +++ b/contrib/credential/gnome-keyring/.gitignore @@ -0,0 +1 @@ +git-credential-gnome-keyring diff --git a/contrib/credential/gnome-keyring/Makefile b/contrib/credential/gnome-keyring/Makefile new file mode 100644 index 0000000000..e6561d8db6 --- /dev/null +++ b/contrib/credential/gnome-keyring/Makefile @@ -0,0 +1,24 @@ +MAIN:=git-credential-gnome-keyring +all:: $(MAIN) + +CC = gcc +RM = rm -f +CFLAGS = -g -O2 -Wall + +-include ../../../config.mak.autogen +-include ../../../config.mak + +INCS:=$(shell pkg-config --cflags gnome-keyring-1) +LIBS:=$(shell pkg-config --libs gnome-keyring-1) + +SRCS:=$(MAIN).c +OBJS:=$(SRCS:.c=.o) + +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(INCS) -o $@ -c $< + +$(MAIN): $(OBJS) + $(CC) -o $@ $(LDFLAGS) $^ $(LIBS) + +clean: + @$(RM) $(MAIN) $(OBJS) diff --git a/contrib/credential/gnome-keyring/git-credential-gnome-keyring.c b/contrib/credential/gnome-keyring/git-credential-gnome-keyring.c new file mode 100644 index 0000000000..41f61c5db3 --- /dev/null +++ b/contrib/credential/gnome-keyring/git-credential-gnome-keyring.c @@ -0,0 +1,445 @@ +/* + * Copyright (C) 2011 John Szakmeister <john@szakmeister.net> + * 2012 Philipp A. Hartmann <pah@qo.cx> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * Credits: + * - GNOME Keyring API handling originally written by John Szakmeister + * - ported to credential helper API by Philipp A. Hartmann + */ + +#include <stdio.h> +#include <string.h> +#include <stdarg.h> +#include <stdlib.h> +#include <errno.h> +#include <gnome-keyring.h> + +/* + * This credential struct and API is simplified from git's credential.{h,c} + */ +struct credential +{ + char *protocol; + char *host; + unsigned short port; + char *path; + char *username; + char *password; +}; + +#define CREDENTIAL_INIT \ + { NULL,NULL,0,NULL,NULL,NULL } + +void credential_init(struct credential *c); +void credential_clear(struct credential *c); +int credential_read(struct credential *c); +void credential_write(const struct credential *c); + +typedef int (*credential_op_cb)(struct credential*); + +struct credential_operation +{ + char *name; + credential_op_cb op; +}; + +#define CREDENTIAL_OP_END \ + { NULL,NULL } + +/* + * Table with operation callbacks is defined in concrete + * credential helper implementation and contains entries + * like { "get", function_to_get_credential } terminated + * by CREDENTIAL_OP_END. + */ +struct credential_operation const credential_helper_ops[]; + +/* ---------------- common helper functions ----------------- */ + +static inline void free_password(char *password) +{ + char *c = password; + if (!password) + return; + + while (*c) *c++ = '\0'; + free(password); +} + +static inline void warning(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fprintf(stderr, "warning: "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n" ); + va_end(ap); +} + +static inline void error(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fprintf(stderr, "error: "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n" ); + va_end(ap); +} + +static inline void die(const char *fmt, ...) +{ + va_list ap; + + va_start(ap,fmt); + error(fmt, ap); + va_end(ap); + exit(EXIT_FAILURE); +} + +static inline void die_errno(int err) +{ + error("%s", strerror(err)); + exit(EXIT_FAILURE); +} + +static inline char *xstrdup(const char *str) +{ + char *ret = strdup(str); + if (!ret) + die_errno(errno); + + return ret; +} + +/* ----------------- GNOME Keyring functions ----------------- */ + +/* create a special keyring option string, if path is given */ +static char* keyring_object(struct credential *c) +{ + char* object = NULL; + + if (!c->path) + return object; + + object = (char*) malloc(strlen(c->host)+strlen(c->path)+8); + if(!object) + die_errno(errno); + + if(c->port) + sprintf(object,"%s:%hd/%s",c->host,c->port,c->path); + else + sprintf(object,"%s/%s",c->host,c->path); + + return object; +} + +int keyring_get(struct credential *c) +{ + char* object = NULL; + GList *entries; + GnomeKeyringNetworkPasswordData *password_data; + GnomeKeyringResult result; + + if (!c->protocol || !(c->host || c->path)) + return EXIT_FAILURE; + + object = keyring_object(c); + + result = gnome_keyring_find_network_password_sync( + c->username, + NULL /* domain */, + c->host, + object, + c->protocol, + NULL /* authtype */, + c->port, + &entries); + + free(object); + + if (result == GNOME_KEYRING_RESULT_NO_MATCH) + return EXIT_SUCCESS; + + if (result == GNOME_KEYRING_RESULT_CANCELLED) + return EXIT_SUCCESS; + + if (result != GNOME_KEYRING_RESULT_OK) { + error("%s",gnome_keyring_result_to_message(result)); + return EXIT_FAILURE; + } + + /* pick the first one from the list */ + password_data = (GnomeKeyringNetworkPasswordData *) entries->data; + + free_password(c->password); + c->password = xstrdup(password_data->password); + + if (!c->username) + c->username = xstrdup(password_data->user); + + gnome_keyring_network_password_list_free(entries); + + return EXIT_SUCCESS; +} + + +int keyring_store(struct credential *c) +{ + guint32 item_id; + char *object = NULL; + + /* + * Sanity check that what we are storing is actually sensible. + * In particular, we can't make a URL without a protocol field. + * Without either a host or pathname (depending on the scheme), + * we have no primary key. And without a username and password, + * we are not actually storing a credential. + */ + if (!c->protocol || !(c->host || c->path) || + !c->username || !c->password) + return EXIT_FAILURE; + + object = keyring_object(c); + + gnome_keyring_set_network_password_sync( + GNOME_KEYRING_DEFAULT, + c->username, + NULL /* domain */, + c->host, + object, + c->protocol, + NULL /* authtype */, + c->port, + c->password, + &item_id); + + free(object); + return EXIT_SUCCESS; +} + +int keyring_erase(struct credential *c) +{ + char *object = NULL; + GList *entries; + GnomeKeyringNetworkPasswordData *password_data; + GnomeKeyringResult result; + + /* + * Sanity check that we actually have something to match + * against. The input we get is a restrictive pattern, + * so technically a blank credential means "erase everything". + * But it is too easy to accidentally send this, since it is equivalent + * to empty input. So explicitly disallow it, and require that the + * pattern have some actual content to match. + */ + if (!c->protocol && !c->host && !c->path && !c->username) + return EXIT_FAILURE; + + object = keyring_object(c); + + result = gnome_keyring_find_network_password_sync( + c->username, + NULL /* domain */, + c->host, + object, + c->protocol, + NULL /* authtype */, + c->port, + &entries); + + free(object); + + if (result == GNOME_KEYRING_RESULT_NO_MATCH) + return EXIT_SUCCESS; + + if (result == GNOME_KEYRING_RESULT_CANCELLED) + return EXIT_SUCCESS; + + if (result != GNOME_KEYRING_RESULT_OK) + { + error("%s",gnome_keyring_result_to_message(result)); + return EXIT_FAILURE; + } + + /* pick the first one from the list (delete all matches?) */ + password_data = (GnomeKeyringNetworkPasswordData *) entries->data; + + result = gnome_keyring_item_delete_sync( + password_data->keyring, password_data->item_id); + + gnome_keyring_network_password_list_free(entries); + + if (result != GNOME_KEYRING_RESULT_OK) + { + error("%s",gnome_keyring_result_to_message(result)); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +/* + * Table with helper operation callbacks, used by generic + * credential helper main function. + */ +struct credential_operation const credential_helper_ops[] = +{ + { "get", keyring_get }, + { "store", keyring_store }, + { "erase", keyring_erase }, + CREDENTIAL_OP_END +}; + +/* ------------------ credential functions ------------------ */ + +void credential_init(struct credential *c) +{ + memset(c, 0, sizeof(*c)); +} + +void credential_clear(struct credential *c) +{ + free(c->protocol); + free(c->host); + free(c->path); + free(c->username); + free_password(c->password); + + credential_init(c); +} + +int credential_read(struct credential *c) +{ + char buf[1024]; + ssize_t line_len = 0; + char *key = buf; + char *value; + + while (fgets(buf, sizeof(buf), stdin)) + { + line_len = strlen(buf); + + if(buf[line_len-1]=='\n') + buf[--line_len]='\0'; + + if(!line_len) + break; + + value = strchr(buf,'='); + if(!value) { + warning("invalid credential line: %s", key); + return -1; + } + *value++ = '\0'; + + if (!strcmp(key, "protocol")) { + free(c->protocol); + c->protocol = xstrdup(value); + } else if (!strcmp(key, "host")) { + free(c->host); + c->host = xstrdup(value); + value = strrchr(c->host,':'); + if (value) { + *value++ = '\0'; + c->port = atoi(value); + } + } else if (!strcmp(key, "path")) { + free(c->path); + c->path = xstrdup(value); + } else if (!strcmp(key, "username")) { + free(c->username); + c->username = xstrdup(value); + } else if (!strcmp(key, "password")) { + free_password(c->password); + c->password = xstrdup(value); + while (*value) *value++ = '\0'; + } + /* + * Ignore other lines; we don't know what they mean, but + * this future-proofs us when later versions of git do + * learn new lines, and the helpers are updated to match. + */ + } + return 0; +} + +void credential_write_item(FILE *fp, const char *key, const char *value) +{ + if (!value) + return; + fprintf(fp, "%s=%s\n", key, value); +} + +void credential_write(const struct credential *c) +{ + /* only write username/password, if set */ + credential_write_item(stdout, "username", c->username); + credential_write_item(stdout, "password", c->password); +} + +static void usage(const char *name) +{ + struct credential_operation const *try_op = credential_helper_ops; + const char *basename = strrchr(name,'/'); + + basename = (basename) ? basename + 1 : name; + fprintf(stderr, "Usage: %s <", basename); + while(try_op->name) { + fprintf(stderr,"%s",(try_op++)->name); + if(try_op->name) + fprintf(stderr,"%s","|"); + } + fprintf(stderr,"%s",">\n"); +} + +int main(int argc, char *argv[]) +{ + int ret = EXIT_SUCCESS; + + struct credential_operation const *try_op = credential_helper_ops; + struct credential cred = CREDENTIAL_INIT; + + if (!argv[1]) { + usage(argv[0]); + goto out; + } + + /* lookup operation callback */ + while(try_op->name && strcmp(argv[1], try_op->name)) + try_op++; + + /* unsupported operation given -- ignore silently */ + if(!try_op->name || !try_op->op) + goto out; + + ret = credential_read(&cred); + if(ret) + goto out; + + /* perform credential operation */ + ret = (*try_op->op)(&cred); + + credential_write(&cred); + +out: + credential_clear(&cred); + return ret; +} diff --git a/contrib/credential/osxkeychain/Makefile b/contrib/credential/osxkeychain/Makefile index 75c07f8be4..4b3a08a2ba 100644 --- a/contrib/credential/osxkeychain/Makefile +++ b/contrib/credential/osxkeychain/Makefile @@ -2,10 +2,13 @@ all:: git-credential-osxkeychain CC = gcc RM = rm -f -CFLAGS = -g -Wall +CFLAGS = -g -O2 -Wall + +-include ../../../config.mak.autogen +-include ../../../config.mak git-credential-osxkeychain: git-credential-osxkeychain.o - $(CC) -o $@ $< -Wl,-framework -Wl,Security + $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) -Wl,-framework -Wl,Security git-credential-osxkeychain.o: git-credential-osxkeychain.c $(CC) -c $(CFLAGS) $< diff --git a/contrib/credential/wincred/Makefile b/contrib/credential/wincred/Makefile new file mode 100644 index 0000000000..bad45ca47a --- /dev/null +++ b/contrib/credential/wincred/Makefile @@ -0,0 +1,14 @@ +all: git-credential-wincred.exe + +CC = gcc +RM = rm -f +CFLAGS = -O2 -Wall + +-include ../../../config.mak.autogen +-include ../../../config.mak + +git-credential-wincred.exe : git-credential-wincred.c + $(LINK.c) $^ $(LOADLIBES) $(LDLIBS) -o $@ + +clean: + $(RM) git-credential-wincred.exe diff --git a/contrib/credential/wincred/git-credential-wincred.c b/contrib/credential/wincred/git-credential-wincred.c new file mode 100644 index 0000000000..cbaec5f24b --- /dev/null +++ b/contrib/credential/wincred/git-credential-wincred.c @@ -0,0 +1,357 @@ +/* + * A git credential helper that interface with Windows' Credential Manager + * + */ +#include <windows.h> +#include <stdio.h> +#include <io.h> +#include <fcntl.h> + +/* common helpers */ + +static void die(const char *err, ...) +{ + char msg[4096]; + va_list params; + va_start(params, err); + vsnprintf(msg, sizeof(msg), err, params); + fprintf(stderr, "%s\n", msg); + va_end(params); + exit(1); +} + +static void *xmalloc(size_t size) +{ + void *ret = malloc(size); + if (!ret && !size) + ret = malloc(1); + if (!ret) + die("Out of memory"); + return ret; +} + +static char *xstrdup(const char *str) +{ + char *ret = strdup(str); + if (!ret) + die("Out of memory"); + return ret; +} + +/* MinGW doesn't have wincred.h, so we need to define stuff */ + +typedef struct _CREDENTIAL_ATTRIBUTEW { + LPWSTR Keyword; + DWORD Flags; + DWORD ValueSize; + LPBYTE Value; +} CREDENTIAL_ATTRIBUTEW, *PCREDENTIAL_ATTRIBUTEW; + +typedef struct _CREDENTIALW { + DWORD Flags; + DWORD Type; + LPWSTR TargetName; + LPWSTR Comment; + FILETIME LastWritten; + DWORD CredentialBlobSize; + LPBYTE CredentialBlob; + DWORD Persist; + DWORD AttributeCount; + PCREDENTIAL_ATTRIBUTEW Attributes; + LPWSTR TargetAlias; + LPWSTR UserName; +} CREDENTIALW, *PCREDENTIALW; + +#define CRED_TYPE_GENERIC 1 +#define CRED_PERSIST_LOCAL_MACHINE 2 +#define CRED_MAX_ATTRIBUTES 64 + +typedef BOOL (WINAPI *CredWriteWT)(PCREDENTIALW, DWORD); +typedef BOOL (WINAPI *CredUnPackAuthenticationBufferWT)(DWORD, PVOID, DWORD, + LPWSTR, DWORD *, LPWSTR, DWORD *, LPWSTR, DWORD *); +typedef BOOL (WINAPI *CredEnumerateWT)(LPCWSTR, DWORD, DWORD *, + PCREDENTIALW **); +typedef BOOL (WINAPI *CredPackAuthenticationBufferWT)(DWORD, LPWSTR, LPWSTR, + PBYTE, DWORD *); +typedef VOID (WINAPI *CredFreeT)(PVOID); +typedef BOOL (WINAPI *CredDeleteWT)(LPCWSTR, DWORD, DWORD); + +static HMODULE advapi, credui; +static CredWriteWT CredWriteW; +static CredUnPackAuthenticationBufferWT CredUnPackAuthenticationBufferW; +static CredEnumerateWT CredEnumerateW; +static CredPackAuthenticationBufferWT CredPackAuthenticationBufferW; +static CredFreeT CredFree; +static CredDeleteWT CredDeleteW; + +static void load_cred_funcs(void) +{ + /* load DLLs */ + advapi = LoadLibrary("advapi32.dll"); + credui = LoadLibrary("credui.dll"); + if (!advapi || !credui) + die("failed to load DLLs"); + + /* get function pointers */ + CredWriteW = (CredWriteWT)GetProcAddress(advapi, "CredWriteW"); + CredUnPackAuthenticationBufferW = (CredUnPackAuthenticationBufferWT) + GetProcAddress(credui, "CredUnPackAuthenticationBufferW"); + CredEnumerateW = (CredEnumerateWT)GetProcAddress(advapi, + "CredEnumerateW"); + CredPackAuthenticationBufferW = (CredPackAuthenticationBufferWT) + GetProcAddress(credui, "CredPackAuthenticationBufferW"); + CredFree = (CredFreeT)GetProcAddress(advapi, "CredFree"); + CredDeleteW = (CredDeleteWT)GetProcAddress(advapi, "CredDeleteW"); + if (!CredWriteW || !CredUnPackAuthenticationBufferW || + !CredEnumerateW || !CredPackAuthenticationBufferW || !CredFree || + !CredDeleteW) + die("failed to load functions"); +} + +static char target_buf[1024]; +static char *protocol, *host, *path, *username; +static WCHAR *wusername, *password, *target; + +static void write_item(const char *what, WCHAR *wbuf) +{ + char *buf; + int len = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, NULL, 0, NULL, + FALSE); + buf = xmalloc(len); + + if (!WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, buf, len, NULL, FALSE)) + die("WideCharToMultiByte failed!"); + + printf("%s=", what); + fwrite(buf, 1, len - 1, stdout); + putchar('\n'); + free(buf); +} + +static int match_attr(const CREDENTIALW *cred, const WCHAR *keyword, + const char *want) +{ + int i; + if (!want) + return 1; + + for (i = 0; i < cred->AttributeCount; ++i) + if (!wcscmp(cred->Attributes[i].Keyword, keyword)) + return !strcmp((const char *)cred->Attributes[i].Value, + want); + + return 0; /* not found */ +} + +static int match_cred(const CREDENTIALW *cred) +{ + return (!wusername || !wcscmp(wusername, cred->UserName)) && + match_attr(cred, L"git_protocol", protocol) && + match_attr(cred, L"git_host", host) && + match_attr(cred, L"git_path", path); +} + +static void get_credential(void) +{ + WCHAR *user_buf, *pass_buf; + DWORD user_buf_size = 0, pass_buf_size = 0; + CREDENTIALW **creds, *cred = NULL; + DWORD num_creds; + int i; + + if (!CredEnumerateW(L"git:*", 0, &num_creds, &creds)) + return; + + /* search for the first credential that matches username */ + for (i = 0; i < num_creds; ++i) + if (match_cred(creds[i])) { + cred = creds[i]; + break; + } + if (!cred) + return; + + CredUnPackAuthenticationBufferW(0, cred->CredentialBlob, + cred->CredentialBlobSize, NULL, &user_buf_size, NULL, NULL, + NULL, &pass_buf_size); + + user_buf = xmalloc(user_buf_size * sizeof(WCHAR)); + pass_buf = xmalloc(pass_buf_size * sizeof(WCHAR)); + + if (!CredUnPackAuthenticationBufferW(0, cred->CredentialBlob, + cred->CredentialBlobSize, user_buf, &user_buf_size, NULL, NULL, + pass_buf, &pass_buf_size)) + die("CredUnPackAuthenticationBuffer failed"); + + CredFree(creds); + + /* zero-terminate (sizes include zero-termination) */ + user_buf[user_buf_size - 1] = L'\0'; + pass_buf[pass_buf_size - 1] = L'\0'; + + write_item("username", user_buf); + write_item("password", pass_buf); + + free(user_buf); + free(pass_buf); +} + +static void write_attr(CREDENTIAL_ATTRIBUTEW *attr, const WCHAR *keyword, + const char *value) +{ + attr->Keyword = (LPWSTR)keyword; + attr->Flags = 0; + attr->ValueSize = strlen(value) + 1; /* store zero-termination */ + attr->Value = (LPBYTE)value; +} + +static void store_credential(void) +{ + CREDENTIALW cred; + BYTE *auth_buf; + DWORD auth_buf_size = 0; + CREDENTIAL_ATTRIBUTEW attrs[CRED_MAX_ATTRIBUTES]; + + if (!wusername || !password) + return; + + /* query buffer size */ + CredPackAuthenticationBufferW(0, wusername, password, + NULL, &auth_buf_size); + + auth_buf = xmalloc(auth_buf_size); + + if (!CredPackAuthenticationBufferW(0, wusername, password, + auth_buf, &auth_buf_size)) + die("CredPackAuthenticationBuffer failed"); + + cred.Flags = 0; + cred.Type = CRED_TYPE_GENERIC; + cred.TargetName = target; + cred.Comment = L"saved by git-credential-wincred"; + cred.CredentialBlobSize = auth_buf_size; + cred.CredentialBlob = auth_buf; + cred.Persist = CRED_PERSIST_LOCAL_MACHINE; + cred.AttributeCount = 1; + cred.Attributes = attrs; + cred.TargetAlias = NULL; + cred.UserName = wusername; + + write_attr(attrs, L"git_protocol", protocol); + + if (host) { + write_attr(attrs + cred.AttributeCount, L"git_host", host); + cred.AttributeCount++; + } + + if (path) { + write_attr(attrs + cred.AttributeCount, L"git_path", path); + cred.AttributeCount++; + } + + if (!CredWriteW(&cred, 0)) + die("CredWrite failed"); +} + +static void erase_credential(void) +{ + CREDENTIALW **creds; + DWORD num_creds; + int i; + + if (!CredEnumerateW(L"git:*", 0, &num_creds, &creds)) + return; + + for (i = 0; i < num_creds; ++i) { + if (match_cred(creds[i])) + CredDeleteW(creds[i]->TargetName, creds[i]->Type, 0); + } + + CredFree(creds); +} + +static WCHAR *utf8_to_utf16_dup(const char *str) +{ + int wlen = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); + WCHAR *wstr = xmalloc(sizeof(WCHAR) * wlen); + MultiByteToWideChar(CP_UTF8, 0, str, -1, wstr, wlen); + return wstr; +} + +static void read_credential(void) +{ + char buf[1024]; + + while (fgets(buf, sizeof(buf), stdin)) { + char *v; + + if (!strcmp(buf, "\n")) + break; + buf[strlen(buf)-1] = '\0'; + + v = strchr(buf, '='); + if (!v) + die("bad input: %s", buf); + *v++ = '\0'; + + if (!strcmp(buf, "protocol")) + protocol = xstrdup(v); + else if (!strcmp(buf, "host")) + host = xstrdup(v); + else if (!strcmp(buf, "path")) + path = xstrdup(v); + else if (!strcmp(buf, "username")) { + username = xstrdup(v); + wusername = utf8_to_utf16_dup(v); + } else if (!strcmp(buf, "password")) + password = utf8_to_utf16_dup(v); + else + die("unrecognized input"); + } +} + +int main(int argc, char *argv[]) +{ + const char *usage = + "Usage: git credential-wincred <get|store|erase>\n"; + + if (!argv[1]) + die(usage); + + /* git use binary pipes to avoid CRLF-issues */ + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + + read_credential(); + + load_cred_funcs(); + + if (!protocol || !(host || path)) + return 0; + + /* prepare 'target', the unique key for the credential */ + strncat(target_buf, "git:", sizeof(target_buf)); + strncat(target_buf, protocol, sizeof(target_buf)); + strncat(target_buf, "://", sizeof(target_buf)); + if (username) { + strncat(target_buf, username, sizeof(target_buf)); + strncat(target_buf, "@", sizeof(target_buf)); + } + if (host) + strncat(target_buf, host, sizeof(target_buf)); + if (path) { + strncat(target_buf, "/", sizeof(target_buf)); + strncat(target_buf, path, sizeof(target_buf)); + } + + target = utf8_to_utf16_dup(target_buf); + + if (!strcmp(argv[1], "get")) + get_credential(); + else if (!strcmp(argv[1], "store")) + store_credential(); + else if (!strcmp(argv[1], "erase")) + erase_credential(); + /* otherwise, ignore unknown action */ + return 0; +} diff --git a/contrib/emacs/git-blame.el b/contrib/emacs/git-blame.el index d351cfb6e7..e671f6c1c6 100644 --- a/contrib/emacs/git-blame.el +++ b/contrib/emacs/git-blame.el @@ -304,7 +304,7 @@ See also function `git-blame-mode'." (defun git-blame-cleanup () "Remove all blame properties" - (mapcar 'delete-overlay git-blame-overlays) + (mapc 'delete-overlay git-blame-overlays) (setq git-blame-overlays nil) (remove-git-blame-text-properties (point-min) (point-max))) @@ -337,16 +337,16 @@ See also function `git-blame-mode'." (defvar in-blame-filter nil) (defun git-blame-filter (proc str) - (save-excursion - (set-buffer (process-buffer proc)) - (goto-char (process-mark proc)) - (insert-before-markers str) - (goto-char 0) - (unless in-blame-filter - (let ((more t) - (in-blame-filter t)) - (while more - (setq more (git-blame-parse))))))) + (with-current-buffer (process-buffer proc) + (save-excursion + (goto-char (process-mark proc)) + (insert-before-markers str) + (goto-char (point-min)) + (unless in-blame-filter + (let ((more t) + (in-blame-filter t)) + (while more + (setq more (git-blame-parse)))))))) (defun git-blame-parse () (cond ((looking-at "\\([0-9a-f]\\{40\\}\\) \\([0-9]+\\) \\([0-9]+\\) \\([0-9]+\\)\n") @@ -385,32 +385,33 @@ See also function `git-blame-mode'." info)))) (defun git-blame-create-overlay (info start-line num-lines) - (save-excursion - (set-buffer git-blame-file) - (let ((inhibit-point-motion-hooks t) - (inhibit-modification-hooks t)) - (goto-line start-line) - (let* ((start (point)) - (end (progn (forward-line num-lines) (point))) - (ovl (make-overlay start end)) - (hash (car info)) - (spec `((?h . ,(substring hash 0 6)) - (?H . ,hash) - (?a . ,(git-blame-get-info info 'author)) - (?A . ,(git-blame-get-info info 'author-mail)) - (?c . ,(git-blame-get-info info 'committer)) - (?C . ,(git-blame-get-info info 'committer-mail)) - (?s . ,(git-blame-get-info info 'summary))))) - (push ovl git-blame-overlays) - (overlay-put ovl 'git-blame info) - (overlay-put ovl 'help-echo - (format-spec git-blame-mouseover-format spec)) - (if git-blame-use-colors - (overlay-put ovl 'face (list :background - (cdr (assq 'color (cdr info)))))) - (overlay-put ovl 'line-prefix - (propertize (format-spec git-blame-prefix-format spec) - 'face 'git-blame-prefix-face)))))) + (with-current-buffer git-blame-file + (save-excursion + (let ((inhibit-point-motion-hooks t) + (inhibit-modification-hooks t)) + (goto-char (point-min)) + (forward-line (1- start-line)) + (let* ((start (point)) + (end (progn (forward-line num-lines) (point))) + (ovl (make-overlay start end)) + (hash (car info)) + (spec `((?h . ,(substring hash 0 6)) + (?H . ,hash) + (?a . ,(git-blame-get-info info 'author)) + (?A . ,(git-blame-get-info info 'author-mail)) + (?c . ,(git-blame-get-info info 'committer)) + (?C . ,(git-blame-get-info info 'committer-mail)) + (?s . ,(git-blame-get-info info 'summary))))) + (push ovl git-blame-overlays) + (overlay-put ovl 'git-blame info) + (overlay-put ovl 'help-echo + (format-spec git-blame-mouseover-format spec)) + (if git-blame-use-colors + (overlay-put ovl 'face (list :background + (cdr (assq 'color (cdr info)))))) + (overlay-put ovl 'line-prefix + (propertize (format-spec git-blame-prefix-format spec) + 'face 'git-blame-prefix-face))))))) (defun git-blame-add-info (info key value) (nconc info (list (cons (intern key) value)))) diff --git a/contrib/examples/builtin-fetch--tool.c b/contrib/examples/builtin-fetch--tool.c index 3140e405fa..8bc8c7533a 100644 --- a/contrib/examples/builtin-fetch--tool.c +++ b/contrib/examples/builtin-fetch--tool.c @@ -96,7 +96,7 @@ static int update_local_ref(const char *name, strcpy(oldh, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV)); strcpy(newh, find_unique_abbrev(sha1_new, DEFAULT_ABBREV)); - if (in_merge_bases(current, &updated, 1)) { + if (in_merge_bases(current, updated)) { fprintf(stderr, "* %s: fast-forward to %s\n", name, note); fprintf(stderr, " old..new: %s..%s\n", oldh, newh); @@ -518,7 +518,7 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) filename = git_path("FETCH_HEAD"); fp = fopen(filename, "a"); if (!fp) - return error("cannot open %s: %s\n", filename, strerror(errno)); + return error("cannot open %s: %s", filename, strerror(errno)); result = append_fetch_head(fp, argv[2], argv[3], argv[4], argv[5], argv[6], !!argv[7][0], @@ -536,7 +536,7 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) filename = git_path("FETCH_HEAD"); fp = fopen(filename, "a"); if (!fp) - return error("cannot open %s: %s\n", filename, strerror(errno)); + return error("cannot open %s: %s", filename, strerror(errno)); result = fetch_native_store(fp, argv[2], argv[3], argv[4], verbose, force); fclose(fp); diff --git a/contrib/mw-to-git/Makefile b/contrib/mw-to-git/Makefile new file mode 100644 index 0000000000..3ed728b0ef --- /dev/null +++ b/contrib/mw-to-git/Makefile @@ -0,0 +1,47 @@ +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +## Build git-remote-mediawiki + +-include ../../config.mak.autogen +-include ../../config.mak + +ifndef PERL_PATH + PERL_PATH = /usr/bin/perl +endif +ifndef gitexecdir + gitexecdir = $(shell git --exec-path) +endif + +PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH)) +gitexecdir_SQ = $(subst ','\'',$(gitexecdir)) +SCRIPT = git-remote-mediawiki + +.PHONY: install help doc test clean + +help: + @echo 'This is the help target of the Makefile. Current configuration:' + @echo ' gitexecdir = $(gitexecdir_SQ)' + @echo ' PERL_PATH = $(PERL_PATH_SQ)' + @echo 'Run "$(MAKE) install" to install $(SCRIPT) in gitexecdir' + @echo 'Run "$(MAKE) test" to run the testsuite' + +install: + sed -e '1s|#!.*/perl|#!$(PERL_PATH_SQ)|' $(SCRIPT) \ + > '$(gitexecdir_SQ)/$(SCRIPT)' + chmod +x '$(gitexecdir)/$(SCRIPT)' + +doc: + @echo 'Sorry, "make doc" is not implemented yet for $(SCRIPT)' + +test: + $(MAKE) -C t/ test + +clean: + $(RM) '$(gitexecdir)/$(SCRIPT)' + $(MAKE) -C t/ clean diff --git a/contrib/mw-to-git/git-remote-mediawiki b/contrib/mw-to-git/git-remote-mediawiki index c18bfa1f15..68555d4265 100755 --- a/contrib/mw-to-git/git-remote-mediawiki +++ b/contrib/mw-to-git/git-remote-mediawiki @@ -9,40 +9,19 @@ # License: GPL v2 or later # Gateway between Git and MediaWiki. -# https://github.com/Bibzball/Git-Mediawiki/wiki -# -# Known limitations: -# -# - Only wiki pages are managed, no support for [[File:...]] -# attachments. -# -# - Poor performance in the best case: it takes forever to check -# whether we're up-to-date (on fetch or push) or to fetch a few -# revisions from a large wiki, because we use exclusively a -# page-based synchronization. We could switch to a wiki-wide -# synchronization when the synchronization involves few revisions -# but the wiki is large. -# -# - Git renames could be turned into MediaWiki renames (see TODO -# below) -# -# - login/password support requires the user to write the password -# cleartext in a file (see TODO below). -# -# - No way to import "one page, and all pages included in it" -# -# - Multiple remote MediaWikis have not been very well tested. +# Documentation & bugtracker: https://github.com/moy/Git-Mediawiki/ use strict; use MediaWiki::API; use DateTime::Format::ISO8601; -use encoding 'utf8'; -# use encoding 'utf8' doesn't change STDERROR -# but we're going to output UTF-8 filenames to STDERR +# By default, use UTF-8 to communicate with Git and the user binmode STDERR, ":utf8"; +binmode STDOUT, ":utf8"; use URI::Escape; +use IPC::Open2; + use warnings; # Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced @@ -59,6 +38,9 @@ use constant EMPTY_CONTENT => "<!-- empty page -->\n"; # used to reflect file creation or deletion in diff. use constant NULL_SHA1 => "0000000000000000000000000000000000000000"; +# Used on Git's side to reflect empty edit messages on the wiki +use constant EMPTY_MESSAGE => '*Empty MediaWiki Message*'; + my $remotename = $ARGV[0]; my $url = $ARGV[1]; @@ -71,10 +53,18 @@ chomp(@tracked_pages); my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories")); chomp(@tracked_categories); +# Import media files on pull +my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport"); +chomp($import_media); +$import_media = ($import_media eq "true"); + +# Export media files on push +my $export_media = run_git("config --get --bool remote.". $remotename .".mediaexport"); +chomp($export_media); +$export_media = !($export_media eq "false"); + my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin"); -# TODO: ideally, this should be able to read from keyboard, but we're -# inside a remote helper, so our stdin is connect to git, not to a -# terminal. +# Note: mwPassword is discourraged. Use the credential system instead. my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword"); my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain"); chomp($wiki_login); @@ -86,6 +76,21 @@ my $shallow_import = run_git("config --get --bool remote.". $remotename .".shall chomp($shallow_import); $shallow_import = ($shallow_import eq "true"); +# Fetch (clone and pull) by revisions instead of by pages. This behavior +# is more efficient when we have a wiki with lots of pages and we fetch +# the revisions quite often so that they concern only few pages. +# Possible values: +# - by_rev: perform one query per new revision on the remote wiki +# - by_page: query each tracked page for new revision +my $fetch_strategy = run_git("config --get remote.$remotename.fetchStrategy"); +unless ($fetch_strategy) { + $fetch_strategy = run_git("config --get mediawiki.fetchStrategy"); +} +chomp($fetch_strategy); +unless ($fetch_strategy) { + $fetch_strategy = "by_page"; +} + # Dumb push: don't update notes and mediawiki ref to reflect the last push. # # Configurable with mediawiki.dumbPush, or per-remote with @@ -151,32 +156,152 @@ while (<STDIN>) { ########################## Functions ############################## +## credential API management (generic functions) + +sub credential_read { + my %credential; + my $reader = shift; + my $op = shift; + while (<$reader>) { + my ($key, $value) = /([^=]*)=(.*)/; + if (not defined $key) { + die "ERROR receiving response from git credential $op:\n$_\n"; + } + $credential{$key} = $value; + } + return %credential; +} + +sub credential_write { + my $credential = shift; + my $writer = shift; + # url overwrites other fields, so it must come first + print $writer "url=$credential->{url}\n" if exists $credential->{url}; + while (my ($key, $value) = each(%$credential) ) { + if (length $value && $key ne 'url') { + print $writer "$key=$value\n"; + } + } +} + +sub credential_run { + my $op = shift; + my $credential = shift; + my $pid = open2(my $reader, my $writer, "git credential $op"); + credential_write($credential, $writer); + print $writer "\n"; + close($writer); + + if ($op eq "fill") { + %$credential = credential_read($reader, $op); + } else { + if (<$reader>) { + die "ERROR while running git credential $op:\n$_"; + } + } + close($reader); + waitpid($pid, 0); + my $child_exit_status = $? >> 8; + if ($child_exit_status != 0) { + die "'git credential $op' failed with code $child_exit_status."; + } +} + # MediaWiki API instance, created lazily. my $mediawiki; sub mw_connect_maybe { if ($mediawiki) { - return; + return; } $mediawiki = MediaWiki::API->new; $mediawiki->{config}->{api_url} = "$url/api.php"; if ($wiki_login) { - if (!$mediawiki->login({ - lgname => $wiki_login, - lgpassword => $wiki_passwd, - lgdomain => $wiki_domain, - })) { - print STDERR "Failed to log in mediawiki user \"$wiki_login\" on $url\n"; - print STDERR "(error " . - $mediawiki->{error}->{code} . ': ' . - $mediawiki->{error}->{details} . ")\n"; - exit 1; + my %credential = (url => $url); + $credential{username} = $wiki_login; + $credential{password} = $wiki_passwd; + credential_run("fill", \%credential); + my $request = {lgname => $credential{username}, + lgpassword => $credential{password}, + lgdomain => $wiki_domain}; + if ($mediawiki->login($request)) { + credential_run("approve", \%credential); + print STDERR "Logged in mediawiki user \"$credential{username}\".\n"; } else { - print STDERR "Logged in with user \"$wiki_login\".\n"; + print STDERR "Failed to log in mediawiki user \"$credential{username}\" on $url\n"; + print STDERR " (error " . + $mediawiki->{error}->{code} . ': ' . + $mediawiki->{error}->{details} . ")\n"; + credential_run("reject", \%credential); + exit 1; + } + } +} + +## Functions for listing pages on the remote wiki +sub get_mw_tracked_pages { + my $pages = shift; + get_mw_page_list(\@tracked_pages, $pages); +} + +sub get_mw_page_list { + my $page_list = shift; + my $pages = shift; + my @some_pages = @$page_list; + while (@some_pages) { + my $last = 50; + if ($#some_pages < $last) { + $last = $#some_pages; + } + my @slice = @some_pages[0..$last]; + get_mw_first_pages(\@slice, $pages); + @some_pages = @some_pages[51..$#some_pages]; + } +} + +sub get_mw_tracked_categories { + my $pages = shift; + foreach my $category (@tracked_categories) { + if (index($category, ':') < 0) { + # Mediawiki requires the Category + # prefix, but let's not force the user + # to specify it. + $category = "Category:" . $category; + } + my $mw_pages = $mediawiki->list( { + action => 'query', + list => 'categorymembers', + cmtitle => $category, + cmlimit => 'max' } ) + || die $mediawiki->{error}->{code} . ': ' + . $mediawiki->{error}->{details}; + foreach my $page (@{$mw_pages}) { + $pages->{$page->{title}} = $page; } } } +sub get_mw_all_pages { + my $pages = shift; + # No user-provided list, get the list of pages from the API. + my $mw_pages = $mediawiki->list({ + action => 'query', + list => 'allpages', + aplimit => 'max' + }); + if (!defined($mw_pages)) { + print STDERR "fatal: could not get the list of wiki pages.\n"; + print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; + print STDERR "fatal: make sure '$url/api.php' is a valid page.\n"; + exit 1; + } + foreach my $page (@{$mw_pages}) { + $pages->{$page->{title}} = $page; + } +} + +# queries the wiki for a set of pages. Meant to be used within a loop +# querying the wiki for slices of page list. sub get_mw_first_pages { my $some_pages = shift; my @some_pages = @{$some_pages}; @@ -205,70 +330,45 @@ sub get_mw_first_pages { } } +# Get the list of pages to be fetched according to configuration. sub get_mw_pages { mw_connect_maybe(); + print STDERR "Listing pages on remote wiki...\n"; + my %pages; # hash on page titles to avoid duplicates my $user_defined; if (@tracked_pages) { $user_defined = 1; # The user provided a list of pages titles, but we # still need to query the API to get the page IDs. - - my @some_pages = @tracked_pages; - while (@some_pages) { - my $last = 50; - if ($#some_pages < $last) { - $last = $#some_pages; - } - my @slice = @some_pages[0..$last]; - get_mw_first_pages(\@slice, \%pages); - @some_pages = @some_pages[51..$#some_pages]; - } + get_mw_tracked_pages(\%pages); } if (@tracked_categories) { $user_defined = 1; - foreach my $category (@tracked_categories) { - if (index($category, ':') < 0) { - # Mediawiki requires the Category - # prefix, but let's not force the user - # to specify it. - $category = "Category:" . $category; - } - my $mw_pages = $mediawiki->list( { - action => 'query', - list => 'categorymembers', - cmtitle => $category, - cmlimit => 'max' } ) - || die $mediawiki->{error}->{code} . ': ' . $mediawiki->{error}->{details}; - foreach my $page (@{$mw_pages}) { - $pages{$page->{title}} = $page; - } - } + get_mw_tracked_categories(\%pages); } if (!$user_defined) { - # No user-provided list, get the list of pages from - # the API. - my $mw_pages = $mediawiki->list({ - action => 'query', - list => 'allpages', - aplimit => 500, - }); - if (!defined($mw_pages)) { - print STDERR "fatal: could not get the list of wiki pages.\n"; - print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; - print STDERR "fatal: make sure '$url/api.php' is a valid page.\n"; - exit 1; - } - foreach my $page (@{$mw_pages}) { - $pages{$page->{title}} = $page; + get_mw_all_pages(\%pages); + } + if ($import_media) { + print STDERR "Getting media files for selected pages...\n"; + if ($user_defined) { + get_linked_mediafiles(\%pages); + } else { + get_all_mediafiles(\%pages); } } - return values(%pages); + print STDERR (scalar keys %pages) . " pages found.\n"; + return %pages; } +# usage: $out = run_git("command args"); +# $out = run_git("command args", "raw"); # don't interpret output as UTF-8. sub run_git { - open(my $git, "-|:encoding(UTF-8)", "git " . $_[0]); + my $args = shift; + my $encoding = (shift || "encoding(UTF-8)"); + open(my $git, "-|:$encoding", "git " . $args); my $res = do { local $/; <$git> }; close($git); @@ -276,6 +376,123 @@ sub run_git { } +sub get_all_mediafiles { + my $pages = shift; + # Attach list of all pages for media files from the API, + # they are in a different namespace, only one namespace + # can be queried at the same moment + my $mw_pages = $mediawiki->list({ + action => 'query', + list => 'allpages', + apnamespace => get_mw_namespace_id("File"), + aplimit => 'max' + }); + if (!defined($mw_pages)) { + print STDERR "fatal: could not get the list of pages for media files.\n"; + print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; + print STDERR "fatal: make sure '$url/api.php' is a valid page.\n"; + exit 1; + } + foreach my $page (@{$mw_pages}) { + $pages->{$page->{title}} = $page; + } +} + +sub get_linked_mediafiles { + my $pages = shift; + my @titles = map $_->{title}, values(%{$pages}); + + # The query is split in small batches because of the MW API limit of + # the number of links to be returned (500 links max). + my $batch = 10; + while (@titles) { + if ($#titles < $batch) { + $batch = $#titles; + } + my @slice = @titles[0..$batch]; + + # pattern 'page1|page2|...' required by the API + my $mw_titles = join('|', @slice); + + # Media files could be included or linked from + # a page, get all related + my $query = { + action => 'query', + prop => 'links|images', + titles => $mw_titles, + plnamespace => get_mw_namespace_id("File"), + pllimit => 'max' + }; + my $result = $mediawiki->api($query); + + while (my ($id, $page) = each(%{$result->{query}->{pages}})) { + my @media_titles; + if (defined($page->{links})) { + my @link_titles = map $_->{title}, @{$page->{links}}; + push(@media_titles, @link_titles); + } + if (defined($page->{images})) { + my @image_titles = map $_->{title}, @{$page->{images}}; + push(@media_titles, @image_titles); + } + if (@media_titles) { + get_mw_page_list(\@media_titles, $pages); + } + } + + @titles = @titles[($batch+1)..$#titles]; + } +} + +sub get_mw_mediafile_for_page_revision { + # Name of the file on Wiki, with the prefix. + my $filename = shift; + my $timestamp = shift; + my %mediafile; + + # Search if on a media file with given timestamp exists on + # MediaWiki. In that case download the file. + my $query = { + action => 'query', + prop => 'imageinfo', + titles => "File:" . $filename, + iistart => $timestamp, + iiend => $timestamp, + iiprop => 'timestamp|archivename|url', + iilimit => 1 + }; + my $result = $mediawiki->api($query); + + my ($fileid, $file) = each( %{$result->{query}->{pages}} ); + # If not defined it means there is no revision of the file for + # given timestamp. + if (defined($file->{imageinfo})) { + $mediafile{title} = $filename; + + my $fileinfo = pop(@{$file->{imageinfo}}); + $mediafile{timestamp} = $fileinfo->{timestamp}; + # Mediawiki::API's download function doesn't support https URLs + # and can't download old versions of files. + print STDERR "\tDownloading file $mediafile{title}, version $mediafile{timestamp}\n"; + $mediafile{content} = download_mw_mediafile($fileinfo->{url}); + } + return %mediafile; +} + +sub download_mw_mediafile { + my $url = shift; + + my $response = $mediawiki->{ua}->get($url); + if ($response->code == 200) { + return $response->decoded_content; + } else { + print STDERR "Error downloading mediafile from :\n"; + print STDERR "URL: $url\n"; + print STDERR "Server response: " . $response->code . " " . $response->message . "\n"; + exit 1; + } +} + sub get_last_local_revision { # Get note regarding last mediawiki revision my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null"); @@ -297,13 +514,36 @@ sub get_last_local_revision { # Remember the timestamp corresponding to a revision id. my %basetimestamps; +# Get the last remote revision without taking in account which pages are +# tracked or not. This function makes a single request to the wiki thus +# avoid a loop onto all tracked pages. This is useful for the fetch-by-rev +# option. +sub get_last_global_remote_rev { + mw_connect_maybe(); + + my $query = { + action => 'query', + list => 'recentchanges', + prop => 'revisions', + rclimit => '1', + rcdir => 'older', + }; + my $result = $mediawiki->api($query); + return $result->{query}->{recentchanges}[0]->{revid}; +} + +# Get the last remote revision concerning the tracked pages and the tracked +# categories. sub get_last_remote_revision { mw_connect_maybe(); - my @pages = get_mw_pages(); + my %pages_hash = get_mw_pages(); + my @pages = values(%pages_hash); my $max_rev_num = 0; + print STDERR "Getting last revision id on tracked pages...\n"; + foreach my $page (@pages) { my $id = $page->{pageid}; @@ -379,6 +619,16 @@ sub literal_data { print STDOUT "data ", bytes::length($content), "\n", $content; } +sub literal_data_raw { + # Output possibly binary content. + my ($content) = @_; + # Avoid confusion between size in bytes and in characters + utf8::downgrade($content); + binmode STDOUT, ":raw"; + print STDOUT "data ", bytes::length($content), "\n", $content; + binmode STDOUT, ":utf8"; +} + sub mw_capabilities { # Revisions are imported to the private namespace # refs/mediawiki/$remotename/ by the helper and fetched into @@ -466,6 +716,11 @@ sub import_file_revision { my %commit = %{$commit}; my $full_import = shift; my $n = shift; + my $mediafile = shift; + my %mediafile; + if ($mediafile) { + %mediafile = %{$mediafile}; + } my $title = $commit{title}; my $comment = $commit{comment}; @@ -485,6 +740,10 @@ sub import_file_revision { if ($content ne DELETED_CONTENT) { print STDOUT "M 644 inline $title.mw\n"; literal_data($content); + if (%mediafile) { + print STDOUT "M 644 inline $mediafile{title}\n"; + literal_data_raw($mediafile{content}); + } print STDOUT "\n\n"; } else { print STDOUT "D $title.mw\n"; @@ -547,8 +806,6 @@ sub mw_import_ref { mw_connect_maybe(); - my @pages = get_mw_pages(); - print STDERR "Searching revisions...\n"; my $last_local = get_last_local_revision(); my $fetch_from = $last_local + 1; @@ -557,36 +814,111 @@ sub mw_import_ref { } else { print STDERR ", fetching from here.\n"; } + + my $n = 0; + if ($fetch_strategy eq "by_rev") { + print STDERR "Fetching & writing export data by revs...\n"; + $n = mw_import_ref_by_revs($fetch_from); + } elsif ($fetch_strategy eq "by_page") { + print STDERR "Fetching & writing export data by pages...\n"; + $n = mw_import_ref_by_pages($fetch_from); + } else { + print STDERR "fatal: invalid fetch strategy \"$fetch_strategy\".\n"; + print STDERR "Check your configuration variables remote.$remotename.fetchStrategy and mediawiki.fetchStrategy\n"; + exit 1; + } + + if ($fetch_from == 1 && $n == 0) { + print STDERR "You appear to have cloned an empty MediaWiki.\n"; + # Something has to be done remote-helper side. If nothing is done, an error is + # thrown saying that HEAD is refering to unknown object 0000000000000000000 + # and the clone fails. + } +} + +sub mw_import_ref_by_pages { + + my $fetch_from = shift; + my %pages_hash = get_mw_pages(); + my @pages = values(%pages_hash); + my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from); - # Creation of the fast-import stream - print STDERR "Fetching & writing export data...\n"; + @revisions = sort {$a->{revid} <=> $b->{revid}} @revisions; + my @revision_ids = map $_->{revid}, @revisions; - $n = 0; + return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash); +} + +sub mw_import_ref_by_revs { + + my $fetch_from = shift; + my %pages_hash = get_mw_pages(); + + my $last_remote = get_last_global_remote_rev(); + my @revision_ids = $fetch_from..$last_remote; + return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash); +} + +# Import revisions given in second argument (array of integers). +# Only pages appearing in the third argument (hash indexed by page titles) +# will be imported. +sub mw_import_revids { + my $fetch_from = shift; + my $revision_ids = shift; + my $pages = shift; + + my $n = 0; + my $n_actual = 0; my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined - foreach my $pagerevid (sort {$a->{revid} <=> $b->{revid}} @revisions) { + foreach my $pagerevid (@$revision_ids) { + # Count page even if we skip it, since we display + # $n/$total and $total includes skipped pages. + $n++; + # fetch the content of the pages my $query = { action => 'query', prop => 'revisions', rvprop => 'content|timestamp|comment|user|ids', - revids => $pagerevid->{revid}, + revids => $pagerevid, }; my $result = $mediawiki->api($query); - my $rev = pop(@{$result->{query}->{pages}->{$pagerevid->{pageid}}->{revisions}}); + if (!$result) { + die "Failed to retrieve modified page for revision $pagerevid"; + } - $n++; + if (defined($result->{query}->{badrevids}->{$pagerevid})) { + # The revision id does not exist on the remote wiki. + next; + } + + if (!defined($result->{query}->{pages})) { + die "Invalid revision $pagerevid."; + } + + my @result_pages = values(%{$result->{query}->{pages}}); + my $result_page = $result_pages[0]; + my $rev = $result_pages[0]->{revisions}->[0]; + + my $page_title = $result_page->{title}; + + if (!exists($pages->{$page_title})) { + print STDERR "$n/", scalar(@$revision_ids), + ": Skipping revision #$rev->{revid} of $page_title\n"; + next; + } + + $n_actual++; my %commit; $commit{author} = $rev->{user} || 'Anonymous'; - $commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*'; - $commit{title} = mediawiki_smudge_filename( - $result->{query}->{pages}->{$pagerevid->{pageid}}->{title} - ); - $commit{mw_revision} = $pagerevid->{revid}; + $commit{comment} = $rev->{comment} || EMPTY_MESSAGE; + $commit{title} = mediawiki_smudge_filename($page_title); + $commit{mw_revision} = $rev->{revid}; $commit{content} = mediawiki_smudge($rev->{'*'}); if (!defined($rev->{timestamp})) { @@ -596,17 +928,23 @@ sub mw_import_ref { } $commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp); - print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n"; - - import_file_revision(\%commit, ($fetch_from == 1), $n); + # Differentiates classic pages and media files. + my ($namespace, $filename) = $page_title =~ /^([^:]*):(.*)$/; + my %mediafile; + if ($namespace) { + my $id = get_mw_namespace_id($namespace); + if ($id && $id == get_mw_namespace_id("File")) { + %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp}); + } + } + # If this is a revision of the media page for new version + # of a file do one common commit for both file and media page. + # Else do commit only for that page. + print STDERR "$n/", scalar(@$revision_ids), ": Revision #$rev->{revid} of $commit{title}\n"; + import_file_revision(\%commit, ($fetch_from == 1), $n_actual, \%mediafile); } - if ($fetch_from == 1 && $n == 0) { - print STDERR "You appear to have cloned an empty MediaWiki.\n"; - # Something has to be done remote-helper side. If nothing is done, an error is - # thrown saying that HEAD is refering to unknown object 0000000000000000000 - # and the clone fails. - } + return $n_actual; } sub error_non_fast_forward { @@ -624,6 +962,63 @@ sub error_non_fast_forward { return 0; } +sub mw_upload_file { + my $complete_file_name = shift; + my $new_sha1 = shift; + my $extension = shift; + my $file_deleted = shift; + my $summary = shift; + my $newrevid; + my $path = "File:" . $complete_file_name; + my %hashFiles = get_allowed_file_extensions(); + if (!exists($hashFiles{$extension})) { + print STDERR "$complete_file_name is not a permitted file on this wiki.\n"; + print STDERR "Check the configuration of file uploads in your mediawiki.\n"; + return $newrevid; + } + # Deleting and uploading a file requires a priviledged user + if ($file_deleted) { + mw_connect_maybe(); + my $query = { + action => 'delete', + title => $path, + reason => $summary + }; + if (!$mediawiki->edit($query)) { + print STDERR "Failed to delete file on remote wiki\n"; + print STDERR "Check your permissions on the remote site. Error code:\n"; + print STDERR $mediawiki->{error}->{code} . ':' . $mediawiki->{error}->{details}; + exit 1; + } + } else { + # Don't let perl try to interpret file content as UTF-8 => use "raw" + my $content = run_git("cat-file blob $new_sha1", "raw"); + if ($content ne "") { + mw_connect_maybe(); + $mediawiki->{config}->{upload_url} = + "$url/index.php/Special:Upload"; + $mediawiki->edit({ + action => 'upload', + filename => $complete_file_name, + comment => $summary, + file => [undef, + $complete_file_name, + Content => $content], + ignorewarnings => 1, + }, { + skip_encoding => 1 + } ) || die $mediawiki->{error}->{code} . ':' + . $mediawiki->{error}->{details}; + my $last_file_page = $mediawiki->get_page({title => $path}); + $newrevid = $last_file_page->{revid}; + print STDERR "Pushed file: $new_sha1 - $complete_file_name.\n"; + } else { + print STDERR "Empty file $complete_file_name not pushed.\n"; + } + } + return $newrevid; +} + sub mw_push_file { my $diff_info = shift; # $diff_info contains a string in this format: @@ -636,7 +1031,12 @@ sub mw_push_file { my $summary = shift; # MediaWiki revision number. Keep the previous one by default, # in case there's no edit to perform. - my $newrevid = shift; + my $oldrevid = shift; + my $newrevid; + + if ($summary eq EMPTY_MESSAGE) { + $summary = ''; + } my $new_sha1 = $diff_info_split[3]; my $old_sha1 = $diff_info_split[2]; @@ -644,9 +1044,16 @@ sub mw_push_file { my $page_deleted = ($new_sha1 eq NULL_SHA1); $complete_file_name = mediawiki_clean_filename($complete_file_name); - if (substr($complete_file_name,-3) eq ".mw") { - my $title = substr($complete_file_name,0,-3); - + my ($title, $extension) = $complete_file_name =~ /^(.*)\.([^\.]*)$/; + if (!defined($extension)) { + $extension = ""; + } + if ($extension eq "mw") { + my $ns = get_mw_namespace_id_for_page($complete_file_name); + if ($ns && $ns == get_mw_namespace_id("File") && (!$export_media)) { + print STDERR "Ignoring media file related page: $complete_file_name\n"; + return ($oldrevid, "ok"); + } my $file_content; if ($page_deleted) { # Deleting a page usually requires @@ -664,7 +1071,7 @@ sub mw_push_file { action => 'edit', summary => $summary, title => $title, - basetimestamp => $basetimestamps{$newrevid}, + basetimestamp => $basetimestamps{$oldrevid}, text => mediawiki_clean($file_content, $page_created), }, { skip_encoding => 1 # Helps with names with accentuated characters @@ -676,7 +1083,7 @@ sub mw_push_file { $mediawiki->{error}->{code} . ' from mediwiki: ' . $mediawiki->{error}->{details} . ".\n"; - return ($newrevid, "non-fast-forward"); + return ($oldrevid, "non-fast-forward"); } else { # Other errors. Shouldn't happen => just die() die 'Fatal: Error ' . @@ -686,9 +1093,14 @@ sub mw_push_file { } $newrevid = $result->{edit}->{newrevid}; print STDERR "Pushed file: $new_sha1 - $title\n"; + } elsif ($export_media) { + $newrevid = mw_upload_file($complete_file_name, $new_sha1, + $extension, $page_deleted, + $summary); } else { - print STDERR "$complete_file_name not a mediawiki file (Not pushable on this version of git-remote-mediawiki).\n" + print STDERR "Ignoring media file $title\n"; } + $newrevid = ($newrevid or $oldrevid); return ($newrevid, "ok"); } @@ -760,16 +1172,26 @@ sub mw_push_revision { if ($last_local_revid > 0) { my $parsed_sha1 = $remoteorigin_sha1; # Find a path from last MediaWiki commit to pushed commit + print STDERR "Computing path from local to remote ...\n"; + my @local_ancestry = split(/\n/, run_git("rev-list --boundary --parents $local ^$parsed_sha1")); + my %local_ancestry; + foreach my $line (@local_ancestry) { + if (my ($child, $parents) = $line =~ m/^-?([a-f0-9]+) ([a-f0-9 ]+)/) { + foreach my $parent (split(' ', $parents)) { + $local_ancestry{$parent} = $child; + } + } elsif (!$line =~ m/^([a-f0-9]+)/) { + die "Unexpected output from git rev-list: $line"; + } + } while ($parsed_sha1 ne $HEAD_sha1) { - my @commit_info = grep(/^$parsed_sha1/, split(/\n/, run_git("rev-list --children $local"))); - if (!@commit_info) { + my $child = $local_ancestry{$parsed_sha1}; + if (!$child) { + printf STDERR "Cannot find a path in history from remote commit to last commit\n"; return error_non_fast_forward($remote); } - my @commit_info_split = split(/ |\n/, $commit_info[0]); - # $commit_info_split[1] is the sha1 of the commit to export - # $commit_info_split[0] is the sha1 of its direct child - push(@commit_pairs, \@commit_info_split); - $parsed_sha1 = $commit_info_split[1]; + push(@commit_pairs, [$parsed_sha1, $child]); + $parsed_sha1 = $child; } } else { # No remote mediawiki revision. Export the whole @@ -791,8 +1213,8 @@ sub mw_push_revision { # TODO: we could detect rename, and encode them with a #redirect on the wiki. # TODO: for now, it's just a delete+add my @diff_info_list = split(/\0/, $diff_infos); - # Keep the first line of the commit message as mediawiki comment for the revision - my $commit_msg = (split(/\n/, run_git("show --pretty=format:\"%s\" $sha1_commit")))[0]; + # Keep the subject line of the commit message as mediawiki comment for the revision + my $commit_msg = run_git("log --no-walk --format=\"%s\" $sha1_commit"); chomp($commit_msg); # Push every blob while (@diff_info_list) { @@ -817,7 +1239,7 @@ sub mw_push_revision { } } unless ($dumb_push) { - run_git("notes --ref=$remotename/mediawiki add -m \"mediawiki_revision: $mw_revision\" $sha1_commit"); + run_git("notes --ref=$remotename/mediawiki add -f -m \"mediawiki_revision: $mw_revision\" $sha1_commit"); run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child"); } } @@ -825,3 +1247,104 @@ sub mw_push_revision { print STDOUT "ok $remote\n"; return 1; } + +sub get_allowed_file_extensions { + mw_connect_maybe(); + + my $query = { + action => 'query', + meta => 'siteinfo', + siprop => 'fileextensions' + }; + my $result = $mediawiki->api($query); + my @file_extensions= map $_->{ext},@{$result->{query}->{fileextensions}}; + my %hashFile = map {$_ => 1}@file_extensions; + + return %hashFile; +} + +# In memory cache for MediaWiki namespace ids. +my %namespace_id; + +# Namespaces whose id is cached in the configuration file +# (to avoid duplicates) +my %cached_mw_namespace_id; + +# Return MediaWiki id for a canonical namespace name. +# Ex.: "File", "Project". +sub get_mw_namespace_id { + mw_connect_maybe(); + my $name = shift; + + if (!exists $namespace_id{$name}) { + # Look at configuration file, if the record for that namespace is + # already cached. Namespaces are stored in form: + # "Name_of_namespace:Id_namespace", ex.: "File:6". + my @temp = split(/[\n]/, run_git("config --get-all remote." + . $remotename .".namespaceCache")); + chomp(@temp); + foreach my $ns (@temp) { + my ($n, $id) = split(/:/, $ns); + if ($id eq 'notANameSpace') { + $namespace_id{$n} = {is_namespace => 0}; + } else { + $namespace_id{$n} = {is_namespace => 1, id => $id}; + } + $cached_mw_namespace_id{$n} = 1; + } + } + + if (!exists $namespace_id{$name}) { + print STDERR "Namespace $name not found in cache, querying the wiki ...\n"; + # NS not found => get namespace id from MW and store it in + # configuration file. + my $query = { + action => 'query', + meta => 'siteinfo', + siprop => 'namespaces' + }; + my $result = $mediawiki->api($query); + + while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) { + if (defined($ns->{id}) && defined($ns->{canonical})) { + $namespace_id{$ns->{canonical}} = {is_namespace => 1, id => $ns->{id}}; + if ($ns->{'*'}) { + # alias (e.g. french Fichier: as alias for canonical File:) + $namespace_id{$ns->{'*'}} = {is_namespace => 1, id => $ns->{id}}; + } + } + } + } + + my $ns = $namespace_id{$name}; + my $id; + + unless (defined $ns) { + print STDERR "No such namespace $name on MediaWiki.\n"; + $ns = {is_namespace => 0}; + $namespace_id{$name} = $ns; + } + + if ($ns->{is_namespace}) { + $id = $ns->{id}; + } + + # Store "notANameSpace" as special value for inexisting namespaces + my $store_id = ($id || 'notANameSpace'); + + # Store explicitely requested namespaces on disk + if (!exists $cached_mw_namespace_id{$name}) { + run_git("config --add remote.". $remotename + .".namespaceCache \"". $name .":". $store_id ."\""); + $cached_mw_namespace_id{$name} = 1; + } + return $id; +} + +sub get_mw_namespace_id_for_page { + if (my ($namespace) = $_[0] =~ /^([^:]*):/) { + return get_mw_namespace_id($namespace); + } else { + return; + } +} diff --git a/contrib/mw-to-git/t/.gitignore b/contrib/mw-to-git/t/.gitignore new file mode 100644 index 0000000000..a7a40b4964 --- /dev/null +++ b/contrib/mw-to-git/t/.gitignore @@ -0,0 +1,4 @@ +WEB/ +wiki/ +trash directory.t*/ +test-results/ diff --git a/contrib/mw-to-git/t/Makefile b/contrib/mw-to-git/t/Makefile new file mode 100644 index 0000000000..f422203fa0 --- /dev/null +++ b/contrib/mw-to-git/t/Makefile @@ -0,0 +1,31 @@ +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +## Test git-remote-mediawiki + +all: test + +-include ../../../config.mak.autogen +-include ../../../config.mak + +T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) + +.PHONY: help test clean all + +help: + @echo 'Run "$(MAKE) test" to launch test scripts' + @echo 'Run "$(MAKE) clean" to remove trash folders' + +test: + @for t in $(T); do \ + echo "$$t"; \ + "./$$t" || exit 1; \ + done + +clean: + $(RM) -r 'trash directory'.* diff --git a/contrib/mw-to-git/t/README b/contrib/mw-to-git/t/README new file mode 100644 index 0000000000..96e97390cf --- /dev/null +++ b/contrib/mw-to-git/t/README @@ -0,0 +1,124 @@ +Tests for Mediawiki-to-Git +========================== + +Introduction +------------ +This manual describes how to install the git-remote-mediawiki test +environment on a machine with git installed on it. + +Prerequisite +------------ + +In order to run this test environment correctly, you will need to +install the following packages (Debian/Ubuntu names, may need to be +adapted for another distribution): + +* lighttpd +* php5 +* php5-cgi +* php5-cli +* php5-curl +* php5-sqlite + +Principles and Technical Choices +-------------------------------- + +The test environment makes it easy to install and manipulate one or +several MediaWiki instances. To allow developers to run the testsuite +easily, the environment does not require root priviledge (except to +install the required packages if needed). It starts a webserver +instance on the user's account (using lighttpd greatly helps for +that), and does not need a separate database daemon (thanks to the use +of sqlite). + +Run the test environment +------------------------ + +Install a new wiki +~~~~~~~~~~~~~~~~~~ + +Once you have all the prerequisite, you need to install a MediaWiki +instance on your machine. If you already have one, it is still +strongly recommended to install one with the script provided. Here's +how to work it: + +a. change directory to contrib/mw-to-git/t/ +b. if needed, edit test.config to choose your installation parameters +c. run `./install-wiki.sh install` +d. check on your favourite web browser if your wiki is correctly + installed. + +Remove an existing wiki +~~~~~~~~~~~~~~~~~~~~~~~ + +Edit the file test.config to fit the wiki you want to delete, and then +execute the command `./install-wiki.sh delete` from the +contrib/mw-to-git/t directory. + +Run the existing tests +~~~~~~~~~~~~~~~~~~~~~~ + +The provided tests are currently in the `contrib/mw-to-git/t` directory. +The files are all the t936[0-9]-*.sh shell scripts. + +a. Run all tests: +To do so, run "make test" from the contrib/mw-to-git/ directory. + +b. Run a specific test: +To run a given test <test_name>, run ./<test_name> from the +contrib/mw-to-git/t directory. + +How to create new tests +----------------------- + +Available functions +~~~~~~~~~~~~~~~~~~~ + +The test environment of git-remote-mediawiki provides some functions +useful to test its behaviour. for more details about the functions' +parameters, please refer to the `test-gitmw-lib.sh` and +`test-gitmw.pl` files. + +** `test_check_wiki_precond`: +Check if the tests must be skipped or not. Please use this function +at the beggining of each new test file. + +** `wiki_getpage`: +Fetch a given page from the wiki and puts its content in the +directory in parameter. + +** `wiki_delete_page`: +Delete a given page from the wiki. + +** `wiki_edit_page`: +Create or modify a given page in the wiki. You can specify several +parameters like a summary for the page edition, or add the page to a +given category. +See test-gitmw.pl for more details. + +** `wiki_getallpage`: +Fetch all pages from the wiki into a given directory. The directory +is created if it does not exists. + +** `test_diff_directories`: +Compare the content of two directories. The content must be the same. +Use this function to compare the content of a git directory and a wiki +one created by wiki_getallpage. + +** `test_contains_N_files`: +Check if the given directory contains a given number of file. + +** `wiki_page_exists`: +Tests if a given page exists on the wiki. + +** `wiki_reset`: +Reset the wiki, i.e. flush the database. Use this function at the +begining of each new test, except if the test re-uses the same wiki +(and history) as the previous test. + +How to write a new test +~~~~~~~~~~~~~~~~~~~~~~~ + +Please, follow the standards given by git. See git/t/README. +New file should be named as t936[0-9]-*.sh. +Be sure to reset your wiki regulary with the function `wiki_reset`. diff --git a/contrib/mw-to-git/t/install-wiki.sh b/contrib/mw-to-git/t/install-wiki.sh new file mode 100755 index 0000000000..c6d6fa3aef --- /dev/null +++ b/contrib/mw-to-git/t/install-wiki.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# This script installs or deletes a MediaWiki on your computer. +# It requires a web server with PHP and SQLite running. In addition, if you +# do not have MediaWiki sources on your computer, the option 'install' +# downloads them for you. +# Please set the CONFIGURATION VARIABLES in ./test-gitmw-lib.sh + +WIKI_TEST_DIR=$(cd "$(dirname "$0")" && pwd) + +if test -z "$WIKI_TEST_DIR" +then + WIKI_TEST_DIR=. +fi + +. "$WIKI_TEST_DIR"/test-gitmw-lib.sh +usage () { + echo "Usage: " + echo " ./install-wiki.sh <install | delete | --help>" + echo " install | -i : Install a wiki on your computer." + echo " delete | -d : Delete the wiki and all its pages and " + echo " content." +} + + +# Argument: install, delete, --help | -h +case "$1" in + "install" | "-i") + wiki_install + exit 0 + ;; + "delete" | "-d") + wiki_delete + exit 0 + ;; + "--help" | "-h") + usage + exit 0 + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; +esac diff --git a/contrib/mw-to-git/t/install-wiki/.gitignore b/contrib/mw-to-git/t/install-wiki/.gitignore new file mode 100644 index 0000000000..b5a2a4408c --- /dev/null +++ b/contrib/mw-to-git/t/install-wiki/.gitignore @@ -0,0 +1 @@ +wikidb.sqlite diff --git a/contrib/mw-to-git/t/install-wiki/LocalSettings.php b/contrib/mw-to-git/t/install-wiki/LocalSettings.php new file mode 100644 index 0000000000..29f125116b --- /dev/null +++ b/contrib/mw-to-git/t/install-wiki/LocalSettings.php @@ -0,0 +1,129 @@ +<?php +# This file was automatically generated by the MediaWiki 1.19.0 +# installer. If you make manual changes, please keep track in case you +# need to recreate them later. +# +# See includes/DefaultSettings.php for all configurable settings +# and their default values, but don't forget to make changes in _this_ +# file, not there. +# +# Further documentation for configuration settings may be found at: +# http://www.mediawiki.org/wiki/Manual:Configuration_settings + +# Protect against web entry +if ( !defined( 'MEDIAWIKI' ) ) { + exit; +} + +## Uncomment this to disable output compression +# $wgDisableOutputCompression = true; + +$wgSitename = "Git-MediaWiki-Test"; +$wgMetaNamespace = "Git-MediaWiki-Test"; + +## The URL base path to the directory containing the wiki; +## defaults for all runtime URL paths are based off of this. +## For more information on customizing the URLs please see: +## http://www.mediawiki.org/wiki/Manual:Short_URL +$wgScriptPath = "@WG_SCRIPT_PATH@"; +$wgScriptExtension = ".php"; + +## The protocol and server name to use in fully-qualified URLs +$wgServer = "@WG_SERVER@"; + +## The relative URL path to the skins directory +$wgStylePath = "$wgScriptPath/skins"; + +## The relative URL path to the logo. Make sure you change this from the default, +## or else you'll overwrite your logo when you upgrade! +$wgLogo = "$wgStylePath/common/images/wiki.png"; + +## UPO means: this is also a user preference option + +$wgEnableEmail = true; +$wgEnableUserEmail = true; # UPO + +$wgEmergencyContact = "apache@localhost"; +$wgPasswordSender = "apache@localhost"; + +$wgEnotifUserTalk = false; # UPO +$wgEnotifWatchlist = false; # UPO +$wgEmailAuthentication = true; + +## Database settings +$wgDBtype = "sqlite"; +$wgDBserver = ""; +$wgDBname = "@WG_SQLITE_DATAFILE@"; +$wgDBuser = ""; +$wgDBpassword = ""; + +# SQLite-specific settings +$wgSQLiteDataDir = "@WG_SQLITE_DATADIR@"; + + +## Shared memory settings +$wgMainCacheType = CACHE_NONE; +$wgMemCachedServers = array(); + +## To enable image uploads, make sure the 'images' directory +## is writable, then set this to true: +$wgEnableUploads = true; +$wgUseImageMagick = true; +$wgImageMagickConvertCommand ="@CONVERT@"; +$wgFileExtensions[] = 'txt'; + +# InstantCommons allows wiki to use images from http://commons.wikimedia.org +$wgUseInstantCommons = false; + +## If you use ImageMagick (or any other shell command) on a +## Linux server, this will need to be set to the name of an +## available UTF-8 locale +$wgShellLocale = "en_US.utf8"; + +## If you want to use image uploads under safe mode, +## create the directories images/archive, images/thumb and +## images/temp, and make them all writable. Then uncomment +## this, if it's not already uncommented: +#$wgHashedUploadDirectory = false; + +## Set $wgCacheDirectory to a writable directory on the web server +## to make your wiki go slightly faster. The directory should not +## be publically accessible from the web. +#$wgCacheDirectory = "$IP/cache"; + +# Site language code, should be one of the list in ./languages/Names.php +$wgLanguageCode = "en"; + +$wgSecretKey = "1c912bfe3519fb70f5dc523ecc698111cd43d81a11c585b3eefb28f29c2699b7"; +#$wgSecretKey = "@SECRETKEY@"; + + +# Site upgrade key. Must be set to a string (default provided) to turn on the +# web installer while LocalSettings.php is in place +$wgUpgradeKey = "ddae7dc87cd0a645"; + +## Default skin: you can change the default skin. Use the internal symbolic +## names, ie 'standard', 'nostalgia', 'cologneblue', 'monobook', 'vector': +$wgDefaultSkin = "vector"; + +## For attaching licensing metadata to pages, and displaying an +## appropriate copyright notice / icon. GNU Free Documentation +## License and Creative Commons licenses are supported so far. +$wgRightsPage = ""; # Set to the title of a wiki page that describes your license/copyright +$wgRightsUrl = ""; +$wgRightsText = ""; +$wgRightsIcon = ""; + +# Path to the GNU diff3 utility. Used for conflict resolution. +$wgDiff3 = "/usr/bin/diff3"; + +# Query string length limit for ResourceLoader. You should only set this if +# your web server has a query string length limit (then set it to that limit), +# or if you have suhosin.get.max_value_length set in php.ini (then set it to +# that value) +$wgResourceLoaderMaxQueryLength = -1; + + + +# End of automatically generated settings. +# Add more configuration options below. diff --git a/contrib/mw-to-git/t/install-wiki/db_install.php b/contrib/mw-to-git/t/install-wiki/db_install.php new file mode 100644 index 0000000000..0f3f4e018a --- /dev/null +++ b/contrib/mw-to-git/t/install-wiki/db_install.php @@ -0,0 +1,120 @@ +<?php +/** + * This script generates a SQLite database for a MediaWiki version 1.19.0 + * You must specify the login of the admin (argument 1) and its + * password (argument 2) and the folder where the database file + * is located (absolute path in argument 3). + * It is used by the script install-wiki.sh in order to make easy the + * installation of a MediaWiki. + * + * In order to generate a SQLite database file, MediaWiki ask the user + * to submit some forms in its web browser. This script simulates this + * behavior though the functions <get> and <submit> + * + */ +$argc = $_SERVER['argc']; +$argv = $_SERVER['argv']; + +$login = $argv[2]; +$pass = $argv[3]; +$tmp = $argv[4]; +$port = $argv[5]; + +$url = 'http://localhost:'.$port.'/wiki/mw-config/index.php'; +$db_dir = urlencode($tmp); +$tmp_cookie = tempnam($tmp, "COOKIE_"); +/* + * Fetchs a page with cURL. + */ +function get($page_name = "") { + $curl = curl_init(); + $page_name_add = ""; + if ($page_name != "") { + $page_name_add = '?page='.$page_name; + } + $url = $GLOBALS['url'].$page_name_add; + $tmp_cookie = $GLOBALS['tmp_cookie']; + curl_setopt($curl, CURLOPT_COOKIEJAR, $tmp_cookie); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($curl, CURLOPT_COOKIEFILE, $tmp_cookie); + curl_setopt($curl, CURLOPT_HEADER, true); + curl_setopt($curl, CURLOPT_URL, $url); + + $page = curl_exec($curl); + if (!$page) { + die("Could not get page: $url\n"); + } + curl_close($curl); + return $page; +} + +/* + * Submits a form with cURL. + */ +function submit($page_name, $option = "") { + $curl = curl_init(); + $datapost = 'submit-continue=Continue+%E2%86%92'; + if ($option != "") { + $datapost = $option.'&'.$datapost; + } + $url = $GLOBALS['url'].'?page='.$page_name; + $tmp_cookie = $GLOBALS['tmp_cookie']; + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_POST, true); + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($curl, CURLOPT_POSTFIELDS, $datapost); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_COOKIEJAR, $tmp_cookie); + curl_setopt($curl, CURLOPT_COOKIEFILE, $tmp_cookie); + + $page = curl_exec($curl); + if (!$page) { + die("Could not get page: $url\n"); + } + curl_close($curl); + return "$page"; +} + +/* + * Here starts this script: simulates the behavior of the user + * submitting forms to generates the database file. + * Note this simulation was made for the MediaWiki version 1.19.0, + * we can't assume it works with other versions. + * + */ + +$page = get(); +if (!preg_match('/input type="hidden" value="([0-9]+)" name="LanguageRequestTime"/', + $page, $matches)) { + echo "Unexpected content for page downloaded:\n"; + echo "$page"; + die; +}; +$timestamp = $matches[1]; +$language = "LanguageRequestTime=$timestamp&uselang=en&ContLang=en"; +$page = submit('Language', $language); + +submit('Welcome'); + +$db_config = 'DBType=sqlite'; +$db_config = $db_config.'&sqlite_wgSQLiteDataDir='.$db_dir; +$db_config = $db_config.'&sqlite_wgDBname='.$argv[1]; +submit('DBConnect', $db_config); + +$wiki_config = 'config_wgSitename=TEST'; +$wiki_config = $wiki_config.'&config__NamespaceType=site-name'; +$wiki_config = $wiki_config.'&config_wgMetaNamespace=MyWiki'; +$wiki_config = $wiki_config.'&config__AdminName='.$login; + +$wiki_config = $wiki_config.'&config__AdminPassword='.$pass; +$wiki_config = $wiki_config.'&config__AdminPassword2='.$pass; + +$wiki_config = $wiki_config.'&wiki__configEmail=email%40email.org'; +$wiki_config = $wiki_config.'&config__SkipOptional=skip'; +submit('Name', $wiki_config); +submit('Install'); +submit('Install'); + +unlink($tmp_cookie); +?> diff --git a/contrib/mw-to-git/t/push-pull-tests.sh b/contrib/mw-to-git/t/push-pull-tests.sh new file mode 100644 index 0000000000..9da2dc5ff0 --- /dev/null +++ b/contrib/mw-to-git/t/push-pull-tests.sh @@ -0,0 +1,144 @@ +test_push_pull () { + + test_expect_success 'Git pull works after adding a new wiki page' ' + wiki_reset && + + git clone mediawiki::'"$WIKI_URL"' mw_dir_1 && + wiki_editpage Foo "page created after the git clone" false && + + ( + cd mw_dir_1 && + git pull + ) && + + wiki_getallpage ref_page_1 && + test_diff_directories mw_dir_1 ref_page_1 + ' + + test_expect_success 'Git pull works after editing a wiki page' ' + wiki_reset && + + wiki_editpage Foo "page created before the git clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_2 && + wiki_editpage Foo "new line added on the wiki" true && + + ( + cd mw_dir_2 && + git pull + ) && + + wiki_getallpage ref_page_2 && + test_diff_directories mw_dir_2 ref_page_2 + ' + + test_expect_success 'git pull works on conflict handled by auto-merge' ' + wiki_reset && + + wiki_editpage Foo "1 init +3 +5 + " false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_3 && + + wiki_editpage Foo "1 init +2 content added on wiki after clone +3 +5 + " false && + + ( + cd mw_dir_3 && + echo "1 init +3 +4 content added on git after clone +5 +" >Foo.mw && + git commit -am "conflicting change on foo" && + git pull && + git push + ) + ' + + test_expect_success 'Git push works after adding a file .mw' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_4 && + wiki_getallpage ref_page_4 && + ( + cd mw_dir_4 && + test_path_is_missing Foo.mw && + touch Foo.mw && + echo "hello world" >>Foo.mw && + git add Foo.mw && + git commit -m "Foo" && + git push + ) && + wiki_getallpage ref_page_4 && + test_diff_directories mw_dir_4 ref_page_4 + ' + + test_expect_success 'Git push works after editing a file .mw' ' + wiki_reset && + wiki_editpage "Foo" "page created before the git clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_5 && + + ( + cd mw_dir_5 && + echo "new line added in the file Foo.mw" >>Foo.mw && + git commit -am "edit file Foo.mw" && + git push + ) && + + wiki_getallpage ref_page_5 && + test_diff_directories mw_dir_5 ref_page_5 + ' + + test_expect_failure 'Git push works after deleting a file' ' + wiki_reset && + wiki_editpage Foo "wiki page added before git clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_6 && + + ( + cd mw_dir_6 && + git rm Foo.mw && + git commit -am "page Foo.mw deleted" && + git push + ) && + + test_must_fail wiki_page_exist Foo + ' + + test_expect_success 'Merge conflict expected and solving it' ' + wiki_reset && + + git clone mediawiki::'"$WIKI_URL"' mw_dir_7 && + wiki_editpage Foo "1 conflict +3 wiki +4" false && + + ( + cd mw_dir_7 && + echo "1 conflict +2 git +4" >Foo.mw && + git add Foo.mw && + git commit -m "conflict created" && + test_must_fail git pull && + "$PERL_PATH" -pi -e "s/[<=>].*//g" Foo.mw && + git commit -am "merge conflict solved" && + git push + ) + ' + + test_expect_failure 'git pull works after deleting a wiki page' ' + wiki_reset && + wiki_editpage Foo "wiki page added before the git clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_8 && + + wiki_delete_page Foo && + ( + cd mw_dir_8 && + git pull && + test_path_is_missing Foo.mw + ) + ' +} diff --git a/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh b/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh new file mode 100755 index 0000000000..811a90c9ae --- /dev/null +++ b/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh @@ -0,0 +1,257 @@ +#!/bin/sh +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +# License: GPL v2 or later + + +test_description='Test the Git Mediawiki remote helper: git clone' + +. ./test-gitmw-lib.sh +. $TEST_DIRECTORY/test-lib.sh + + +test_check_precond + + +test_expect_success 'Git clone creates the expected git log with one file' ' + wiki_reset && + wiki_editpage foo "this is not important" false -c cat -s "this must be the same" && + git clone mediawiki::'"$WIKI_URL"' mw_dir_1 && + ( + cd mw_dir_1 && + git log --format=%s HEAD^..HEAD >log.tmp + ) && + echo "this must be the same" >msg.tmp && + diff -b mw_dir_1/log.tmp msg.tmp +' + + +test_expect_success 'Git clone creates the expected git log with multiple files' ' + wiki_reset && + wiki_editpage daddy "this is not important" false -s="this must be the same" && + wiki_editpage daddy "neither is this" true -s="this must also be the same" && + wiki_editpage daddy "neither is this" true -s="same same same" && + wiki_editpage dj "dont care" false -s="identical" && + wiki_editpage dj "dont care either" true -s="identical too" && + git clone mediawiki::'"$WIKI_URL"' mw_dir_2 && + ( + cd mw_dir_2 && + git log --format=%s Daddy.mw >logDaddy.tmp && + git log --format=%s Dj.mw >logDj.tmp + ) && + echo "same same same" >msgDaddy.tmp && + echo "this must also be the same" >>msgDaddy.tmp && + echo "this must be the same" >>msgDaddy.tmp && + echo "identical too" >msgDj.tmp && + echo "identical" >>msgDj.tmp && + diff -b mw_dir_2/logDaddy.tmp msgDaddy.tmp && + diff -b mw_dir_2/logDj.tmp msgDj.tmp +' + + +test_expect_success 'Git clone creates only Main_Page.mw with an empty wiki' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_3 && + test_contains_N_files mw_dir_3 1 && + test_path_is_file mw_dir_3/Main_Page.mw +' + +test_expect_success 'Git clone does not fetch a deleted page' ' + wiki_reset && + wiki_editpage foo "this page must be deleted before the clone" false && + wiki_delete_page foo && + git clone mediawiki::'"$WIKI_URL"' mw_dir_4 && + test_contains_N_files mw_dir_4 1 && + test_path_is_file mw_dir_4/Main_Page.mw && + test_path_is_missing mw_dir_4/Foo.mw +' + +test_expect_success 'Git clone works with page added' ' + wiki_reset && + wiki_editpage foo " I will be cloned" false && + wiki_editpage bar "I will be cloned" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_5 && + wiki_getallpage ref_page_5 && + test_diff_directories mw_dir_5 ref_page_5 && + wiki_delete_page foo && + wiki_delete_page bar +' + +test_expect_success 'Git clone works with an edited page ' ' + wiki_reset && + wiki_editpage foo "this page will be edited" \ + false -s "first edition of page foo"&& + wiki_editpage foo "this page has been edited and must be on the clone " true && + git clone mediawiki::'"$WIKI_URL"' mw_dir_6 && + test_path_is_file mw_dir_6/Foo.mw && + test_path_is_file mw_dir_6/Main_Page.mw && + wiki_getallpage mw_dir_6/page_ref_6 && + test_diff_directories mw_dir_6 mw_dir_6/page_ref_6 && + ( + cd mw_dir_6 && + git log --format=%s HEAD^ Foo.mw > ../Foo.log + ) && + echo "first edition of page foo" > FooExpect.log && + diff FooExpect.log Foo.log +' + + +test_expect_success 'Git clone works with several pages and some deleted ' ' + wiki_reset && + wiki_editpage foo "this page will not be deleted" false && + wiki_editpage bar "I must not be erased" false && + wiki_editpage namnam "I will not be there at the end" false && + wiki_editpage nyancat "nyan nyan nyan delete me" false && + wiki_delete_page namnam && + wiki_delete_page nyancat && + git clone mediawiki::'"$WIKI_URL"' mw_dir_7 && + test_path_is_file mw_dir_7/Foo.mw && + test_path_is_file mw_dir_7/Bar.mw && + test_path_is_missing mw_dir_7/Namnam.mw && + test_path_is_missing mw_dir_7/Nyancat.mw && + wiki_getallpage mw_dir_7/page_ref_7 && + test_diff_directories mw_dir_7 mw_dir_7/page_ref_7 +' + + +test_expect_success 'Git clone works with one specific page cloned ' ' + wiki_reset && + wiki_editpage foo "I will not be cloned" false && + wiki_editpage bar "Do not clone me" false && + wiki_editpage namnam "I will be cloned :)" false -s="this log must stay" && + wiki_editpage nyancat "nyan nyan nyan you cant clone me" false && + git clone -c remote.origin.pages=namnam \ + mediawiki::'"$WIKI_URL"' mw_dir_8 && + test_contains_N_files mw_dir_8 1 && + test_path_is_file mw_dir_8/Namnam.mw && + test_path_is_missing mw_dir_8/Main_Page.mw && + ( + cd mw_dir_8 && + echo "this log must stay" >msg.tmp && + git log --format=%s >log.tmp && + diff -b msg.tmp log.tmp + ) && + wiki_check_content mw_dir_8/Namnam.mw Namnam +' + +test_expect_success 'Git clone works with multiple specific page cloned ' ' + wiki_reset && + wiki_editpage foo "I will be there" false && + wiki_editpage bar "I will not disapear" false && + wiki_editpage namnam "I be erased" false && + wiki_editpage nyancat "nyan nyan nyan you will not erase me" false && + wiki_delete_page namnam && + git clone -c remote.origin.pages="foo bar nyancat namnam" \ + mediawiki::'"$WIKI_URL"' mw_dir_9 && + test_contains_N_files mw_dir_9 3 && + test_path_is_missing mw_dir_9/Namnam.mw && + test_path_is_file mw_dir_9/Foo.mw && + test_path_is_file mw_dir_9/Nyancat.mw && + test_path_is_file mw_dir_9/Bar.mw && + wiki_check_content mw_dir_9/Foo.mw Foo && + wiki_check_content mw_dir_9/Bar.mw Bar && + wiki_check_content mw_dir_9/Nyancat.mw Nyancat +' + +test_expect_success 'Mediawiki-clone of several specific pages on wiki' ' + wiki_reset && + wiki_editpage foo "foo 1" false && + wiki_editpage bar "bar 1" false && + wiki_editpage dummy "dummy 1" false && + wiki_editpage cloned_1 "cloned_1 1" false && + wiki_editpage cloned_2 "cloned_2 2" false && + wiki_editpage cloned_3 "cloned_3 3" false && + mkdir -p ref_page_10 && + wiki_getpage cloned_1 ref_page_10 && + wiki_getpage cloned_2 ref_page_10 && + wiki_getpage cloned_3 ref_page_10 && + git clone -c remote.origin.pages="cloned_1 cloned_2 cloned_3" \ + mediawiki::'"$WIKI_URL"' mw_dir_10 && + test_diff_directories mw_dir_10 ref_page_10 +' + +test_expect_success 'Git clone works with the shallow option' ' + wiki_reset && + wiki_editpage foo "1st revision, should be cloned" false && + wiki_editpage bar "1st revision, should be cloned" false && + wiki_editpage nyan "1st revision, should not be cloned" false && + wiki_editpage nyan "2nd revision, should be cloned" false && + git -c remote.origin.shallow=true clone \ + mediawiki::'"$WIKI_URL"' mw_dir_11 && + test_contains_N_files mw_dir_11 4 && + test_path_is_file mw_dir_11/Nyan.mw && + test_path_is_file mw_dir_11/Foo.mw && + test_path_is_file mw_dir_11/Bar.mw && + test_path_is_file mw_dir_11/Main_Page.mw && + ( + cd mw_dir_11 && + test `git log --oneline Nyan.mw | wc -l` -eq 1 && + test `git log --oneline Foo.mw | wc -l` -eq 1 && + test `git log --oneline Bar.mw | wc -l` -eq 1 && + test `git log --oneline Main_Page.mw | wc -l ` -eq 1 + ) && + wiki_check_content mw_dir_11/Nyan.mw Nyan && + wiki_check_content mw_dir_11/Foo.mw Foo && + wiki_check_content mw_dir_11/Bar.mw Bar && + wiki_check_content mw_dir_11/Main_Page.mw Main_Page +' + +test_expect_success 'Git clone works with the shallow option with a delete page' ' + wiki_reset && + wiki_editpage foo "1st revision, will be deleted" false && + wiki_editpage bar "1st revision, should be cloned" false && + wiki_editpage nyan "1st revision, should not be cloned" false && + wiki_editpage nyan "2nd revision, should be cloned" false && + wiki_delete_page foo && + git -c remote.origin.shallow=true clone \ + mediawiki::'"$WIKI_URL"' mw_dir_12 && + test_contains_N_files mw_dir_12 3 && + test_path_is_file mw_dir_12/Nyan.mw && + test_path_is_missing mw_dir_12/Foo.mw && + test_path_is_file mw_dir_12/Bar.mw && + test_path_is_file mw_dir_12/Main_Page.mw && + ( + cd mw_dir_12 && + test `git log --oneline Nyan.mw | wc -l` -eq 1 && + test `git log --oneline Bar.mw | wc -l` -eq 1 && + test `git log --oneline Main_Page.mw | wc -l ` -eq 1 + ) && + wiki_check_content mw_dir_12/Nyan.mw Nyan && + wiki_check_content mw_dir_12/Bar.mw Bar && + wiki_check_content mw_dir_12/Main_Page.mw Main_Page +' + +test_expect_success 'Test of fetching a category' ' + wiki_reset && + wiki_editpage Foo "I will be cloned" false -c=Category && + wiki_editpage Bar "Meet me on the repository" false -c=Category && + wiki_editpage Dummy "I will not come" false && + wiki_editpage BarWrong "I will stay online only" false -c=NotCategory && + git clone -c remote.origin.categories="Category" \ + mediawiki::'"$WIKI_URL"' mw_dir_13 && + wiki_getallpage ref_page_13 Category && + test_diff_directories mw_dir_13 ref_page_13 +' + +test_expect_success 'Test of resistance to modification of category on wiki for clone' ' + wiki_reset && + wiki_editpage Tobedeleted "this page will be deleted" false -c=Catone && + wiki_editpage Tobeedited "this page will be modified" false -c=Catone && + wiki_editpage Normalone "this page wont be modified and will be on git" false -c=Catone && + wiki_editpage Notconsidered "this page will not appear on local" false && + wiki_editpage Othercategory "this page will not appear on local" false -c=Cattwo && + wiki_editpage Tobeedited "this page have been modified" true -c=Catone && + wiki_delete_page Tobedeleted + git clone -c remote.origin.categories="Catone" \ + mediawiki::'"$WIKI_URL"' mw_dir_14 && + wiki_getallpage ref_page_14 Catone && + test_diff_directories mw_dir_14 ref_page_14 +' + +test_done diff --git a/contrib/mw-to-git/t/t9361-mw-to-git-push-pull.sh b/contrib/mw-to-git/t/t9361-mw-to-git-push-pull.sh new file mode 100755 index 0000000000..9ea201459b --- /dev/null +++ b/contrib/mw-to-git/t/t9361-mw-to-git-push-pull.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +# License: GPL v2 or later + +# tests for git-remote-mediawiki + +test_description='Test the Git Mediawiki remote helper: git push and git pull simple test cases' + +. ./test-gitmw-lib.sh +. ./push-pull-tests.sh +. $TEST_DIRECTORY/test-lib.sh + +test_check_precond + +test_push_pull + +test_done diff --git a/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh b/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh new file mode 100755 index 0000000000..246d47d8fb --- /dev/null +++ b/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh @@ -0,0 +1,321 @@ +#!/bin/sh +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +# License: GPL v2 or later + +# tests for git-remote-mediawiki + +test_description='Test git-mediawiki with special characters in filenames' + +. ./test-gitmw-lib.sh +. $TEST_DIRECTORY/test-lib.sh + + +test_check_precond + + +test_expect_success 'Git clone works for a wiki with accents in the page names' ' + wiki_reset && + wiki_editpage féé "This page must be délétéd before clone" false && + wiki_editpage kèè "This page must be deleted before clone" false && + wiki_editpage hà à "This page must be deleted before clone" false && + wiki_editpage kîî "This page must be deleted before clone" false && + wiki_editpage foo "This page must be deleted before clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_1 && + wiki_getallpage ref_page_1 && + test_diff_directories mw_dir_1 ref_page_1 +' + + +test_expect_success 'Git pull works with a wiki with accents in the pages names' ' + wiki_reset && + wiki_editpage kîî "this page must be cloned" false && + wiki_editpage foo "this page must be cloned" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_2 && + wiki_editpage éà îôû "This page must be pulled" false && + ( + cd mw_dir_2 && + git pull + ) && + wiki_getallpage ref_page_2 && + test_diff_directories mw_dir_2 ref_page_2 +' + + +test_expect_success 'Cloning a chosen page works with accents' ' + wiki_reset && + wiki_editpage kîî "this page must be cloned" false && + git clone -c remote.origin.pages=kîî \ + mediawiki::'"$WIKI_URL"' mw_dir_3 && + wiki_check_content mw_dir_3/Kîî.mw Kîî && + test_path_is_file mw_dir_3/Kîî.mw && + rm -rf mw_dir_3 +' + + +test_expect_success 'The shallow option works with accents' ' + wiki_reset && + wiki_editpage néoà "1st revision, should not be cloned" false && + wiki_editpage néoà "2nd revision, should be cloned" false && + git -c remote.origin.shallow=true clone \ + mediawiki::'"$WIKI_URL"' mw_dir_4 && + test_contains_N_files mw_dir_4 2 && + test_path_is_file mw_dir_4/Néoà .mw && + test_path_is_file mw_dir_4/Main_Page.mw && + ( + cd mw_dir_4 && + test `git log --oneline Néoà .mw | wc -l` -eq 1 && + test `git log --oneline Main_Page.mw | wc -l ` -eq 1 + ) && + wiki_check_content mw_dir_4/Néoà .mw Néoà && + wiki_check_content mw_dir_4/Main_Page.mw Main_Page +' + + +test_expect_success 'Cloning works when page name first letter has an accent' ' + wiki_reset && + wiki_editpage îî "this page must be cloned" false && + git clone -c remote.origin.pages=îî \ + mediawiki::'"$WIKI_URL"' mw_dir_5 && + test_path_is_file mw_dir_5/Îî.mw && + wiki_check_content mw_dir_5/Îî.mw Îî +' + + +test_expect_success 'Git push works with a wiki with accents' ' + wiki_reset && + wiki_editpage féé "lots of accents : éèà Ö" false && + wiki_editpage foo "this page must be cloned" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_6 && + ( + cd mw_dir_6 && + echo "A wild Pîkächû appears on the wiki" >Pîkächû.mw && + git add Pîkächû.mw && + git commit -m "A new page appears" && + git push + ) && + wiki_getallpage ref_page_6 && + test_diff_directories mw_dir_6 ref_page_6 +' + +test_expect_success 'Git clone works with accentsand spaces' ' + wiki_reset && + wiki_editpage "é à î" "this page must be délété before the clone" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_7 && + wiki_getallpage ref_page_7 && + test_diff_directories mw_dir_7 ref_page_7 +' + +test_expect_success 'character $ in page name (mw -> git)' ' + wiki_reset && + wiki_editpage file_\$_foo "expect to be called file_$_foo" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_8 && + test_path_is_file mw_dir_8/File_\$_foo.mw && + wiki_getallpage ref_page_8 && + test_diff_directories mw_dir_8 ref_page_8 +' + + + +test_expect_success 'character $ in file name (git -> mw) ' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_9 && + ( + cd mw_dir_9 && + echo "this file is called File_\$_foo.mw" >File_\$_foo.mw && + git add . && + git commit -am "file File_\$_foo.mw" && + git pull && + git push + ) && + wiki_getallpage ref_page_9 && + test_diff_directories mw_dir_9 ref_page_9 +' + + +test_expect_failure 'capital at the begining of file names' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_10 && + ( + cd mw_dir_10 && + echo "my new file foo" >foo.mw && + echo "my new file Foo... Finger crossed" >Foo.mw && + git add . && + git commit -am "file foo.mw" && + git pull && + git push + ) && + wiki_getallpage ref_page_10 && + test_diff_directories mw_dir_10 ref_page_10 +' + + +test_expect_failure 'special character at the begining of file name from mw to git' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_11 && + wiki_editpage {char_1 "expect to be renamed {char_1" false && + wiki_editpage [char_2 "expect to be renamed [char_2" false && + ( + cd mw_dir_11 && + git pull + ) && + test_path_is_file mw_dir_11/{char_1 && + test_path_is_file mw_dir_11/[char_2 +' + +test_expect_success 'Pull page with title containing ":" other than namespace separator' ' + wiki_editpage Foo:Bar content false && + ( + cd mw_dir_11 && + git pull + ) && + test_path_is_file mw_dir_11/Foo:Bar.mw +' + +test_expect_success 'Push page with title containing ":" other than namespace separator' ' + ( + cd mw_dir_11 && + echo content >NotANameSpace:Page.mw && + git add NotANameSpace:Page.mw && + git commit -m "add page with colon" && + git push + ) && + wiki_page_exist NotANameSpace:Page +' + +test_expect_success 'test of correct formating for file name from mw to git' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_12 && + wiki_editpage char_%_7b_1 "expect to be renamed char{_1" false && + wiki_editpage char_%_5b_2 "expect to be renamed char{_2" false && + ( + cd mw_dir_12 && + git pull + ) && + test_path_is_file mw_dir_12/Char\{_1.mw && + test_path_is_file mw_dir_12/Char\[_2.mw && + wiki_getallpage ref_page_12 && + mv ref_page_12/Char_%_7b_1.mw ref_page_12/Char\{_1.mw && + mv ref_page_12/Char_%_5b_2.mw ref_page_12/Char\[_2.mw && + test_diff_directories mw_dir_12 ref_page_12 +' + + +test_expect_failure 'test of correct formating for file name begining with special character' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_13 && + ( + cd mw_dir_13 && + echo "my new file {char_1" >\{char_1.mw && + echo "my new file [char_2" >\[char_2.mw && + git add . && + git commit -am "commiting some exotic file name..." && + git push && + git pull + ) && + wiki_getallpage ref_page_13 && + test_path_is_file ref_page_13/{char_1.mw && + test_path_is_file ref_page_13/[char_2.mw && + test_diff_directories mw_dir_13 ref_page_13 +' + + +test_expect_success 'test of correct formating for file name from git to mw' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_14 && + ( + cd mw_dir_14 && + echo "my new file char{_1" >Char\{_1.mw && + echo "my new file char[_2" >Char\[_2.mw && + git add . && + git commit -m "commiting some exotic file name..." && + git push + ) && + wiki_getallpage ref_page_14 && + mv mw_dir_14/Char\{_1.mw mw_dir_14/Char_%_7b_1.mw && + mv mw_dir_14/Char\[_2.mw mw_dir_14/Char_%_5b_2.mw && + test_diff_directories mw_dir_14 ref_page_14 +' + + +test_expect_success 'git clone with /' ' + wiki_reset && + wiki_editpage \/fo\/o "this is not important" false -c=Deleted && + git clone mediawiki::'"$WIKI_URL"' mw_dir_15 && + test_path_is_file mw_dir_15/%2Ffo%2Fo.mw && + wiki_check_content mw_dir_15/%2Ffo%2Fo.mw \/fo\/o +' + + +test_expect_success 'git push with /' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_16 && + echo "I will be on the wiki" >mw_dir_16/%2Ffo%2Fo.mw && + ( + cd mw_dir_16 && + git add %2Ffo%2Fo.mw && + git commit -m " %2Ffo%2Fo added" && + git push + ) && + wiki_page_exist \/fo\/o && + wiki_check_content mw_dir_16/%2Ffo%2Fo.mw \/fo\/o + +' + + +test_expect_success 'git clone with \' ' + wiki_reset && + wiki_editpage \\ko\\o "this is not important" false -c=Deleted && + git clone mediawiki::'"$WIKI_URL"' mw_dir_17 && + test_path_is_file mw_dir_17/\\ko\\o.mw && + wiki_check_content mw_dir_17/\\ko\\o.mw \\ko\\o +' + + +test_expect_success 'git push with \' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_18 && + echo "I will be on the wiki" >mw_dir_18/\\ko\\o.mw && + ( + cd mw_dir_18 && + git add \\ko\\o.mw && + git commit -m " \\ko\\o added" && + git push + )&& + wiki_page_exist \\ko\\o && + wiki_check_content mw_dir_18/\\ko\\o.mw \\ko\\o + +' + +test_expect_success 'git clone with \ in format control' ' + wiki_reset && + wiki_editpage \\no\\o "this is not important" false && + git clone mediawiki::'"$WIKI_URL"' mw_dir_19 && + test_path_is_file mw_dir_19/\\no\\o.mw && + wiki_check_content mw_dir_19/\\no\\o.mw \\no\\o +' + + +test_expect_success 'git push with \ in format control' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir_20 && + echo "I will be on the wiki" >mw_dir_20/\\fo\\o.mw && + ( + cd mw_dir_20 && + git add \\fo\\o.mw && + git commit -m " \\fo\\o added" && + git push + )&& + wiki_page_exist \\fo\\o && + wiki_check_content mw_dir_20/\\fo\\o.mw \\fo\\o + +' + + +test_done diff --git a/contrib/mw-to-git/t/t9363-mw-to-git-export-import.sh b/contrib/mw-to-git/t/t9363-mw-to-git-export-import.sh new file mode 100755 index 0000000000..5a0373935f --- /dev/null +++ b/contrib/mw-to-git/t/t9363-mw-to-git-export-import.sh @@ -0,0 +1,198 @@ +#!/bin/sh +# +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# +# License: GPL v2 or later + +# tests for git-remote-mediawiki + +test_description='Test the Git Mediawiki remote helper: git push and git pull simple test cases' + +. ./test-gitmw-lib.sh +. $TEST_DIRECTORY/test-lib.sh + + +test_check_precond + + +test_git_reimport () { + git -c remote.origin.dumbPush=true push && + git -c remote.origin.mediaImport=true pull --rebase +} + +# Don't bother with permissions, be administrator by default +test_expect_success 'setup config' ' + git config --global remote.origin.mwLogin WikiAdmin && + git config --global remote.origin.mwPassword AdminPass && + test_might_fail git config --global --unset remote.origin.mediaImport +' + +test_expect_success 'git push can upload media (File:) files' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + ( + cd mw_dir && + echo "hello world" >Foo.txt && + git add Foo.txt && + git commit -m "add a text file" && + git push && + "$PERL_PATH" -e "print STDOUT \"binary content: \".chr(255);" >Foo.txt && + git add Foo.txt && + git commit -m "add a text file with binary content" && + git push + ) +' + +test_expect_success 'git clone works on previously created wiki with media files' ' + test_when_finished "rm -rf mw_dir mw_dir_clone" && + git clone -c remote.origin.mediaimport=true \ + mediawiki::'"$WIKI_URL"' mw_dir_clone && + test_cmp mw_dir_clone/Foo.txt mw_dir/Foo.txt && + (cd mw_dir_clone && git checkout HEAD^) && + (cd mw_dir && git checkout HEAD^) && + test_cmp mw_dir_clone/Foo.txt mw_dir/Foo.txt +' + +test_expect_success 'git push & pull work with locally renamed media files' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + test_when_finished "rm -fr mw_dir" && + ( + cd mw_dir && + echo "A File" >Foo.txt && + git add Foo.txt && + git commit -m "add a file" && + git mv Foo.txt Bar.txt && + git commit -m "Rename a file" && + test_git_reimport && + echo "A File" >expect && + test_cmp expect Bar.txt && + test_path_is_missing Foo.txt + ) +' + +test_expect_success 'git push can propagate local page deletion' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + test_when_finished "rm -fr mw_dir" && + ( + cd mw_dir && + test_path_is_missing Foo.mw && + echo "hello world" >Foo.mw && + git add Foo.mw && + git commit -m "Add the page Foo" && + git push && + rm -f Foo.mw && + git commit -am "Delete the page Foo" && + test_git_reimport && + test_path_is_missing Foo.mw + ) +' + +test_expect_success 'git push can propagate local media file deletion' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + test_when_finished "rm -fr mw_dir" && + ( + cd mw_dir && + echo "hello world" >Foo.txt && + git add Foo.txt && + git commit -m "Add the text file Foo" && + git rm Foo.txt && + git commit -m "Delete the file Foo" && + test_git_reimport && + test_path_is_missing Foo.txt + ) +' + +# test failure: the file is correctly uploaded, and then deleted but +# as no page link to it, the import (which looks at page revisions) +# doesn't notice the file deletion on the wiki. We fetch the list of +# files from the wiki, but as the file is deleted, it doesn't appear. +test_expect_failure 'git pull correctly imports media file deletion when no page link to it' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + test_when_finished "rm -fr mw_dir" && + ( + cd mw_dir && + echo "hello world" >Foo.txt && + git add Foo.txt && + git commit -m "Add the text file Foo" && + git push && + git rm Foo.txt && + git commit -m "Delete the file Foo" && + test_git_reimport && + test_path_is_missing Foo.txt + ) +' + +test_expect_success 'git push properly warns about insufficient permissions' ' + wiki_reset && + git clone mediawiki::'"$WIKI_URL"' mw_dir && + test_when_finished "rm -fr mw_dir" && + ( + cd mw_dir && + echo "A File" >foo.forbidden && + git add foo.forbidden && + git commit -m "add a file" && + git push 2>actual && + test_i18ngrep "foo.forbidden is not a permitted file" actual + ) +' + +test_expect_success 'setup a repository with media files' ' + wiki_reset && + wiki_editpage testpage "I am linking a file [[File:File.txt]]" false && + echo "File content" >File.txt && + wiki_upload_file File.txt && + echo "Another file content" >AnotherFile.txt && + wiki_upload_file AnotherFile.txt +' + +test_expect_success 'git clone works with one specific page cloned and mediaimport=true' ' + git clone -c remote.origin.pages=testpage \ + -c remote.origin.mediaimport=true \ + mediawiki::'"$WIKI_URL"' mw_dir_15 && + test_when_finished "rm -rf mw_dir_15" && + test_contains_N_files mw_dir_15 3 && + test_path_is_file mw_dir_15/Testpage.mw && + test_path_is_file mw_dir_15/File:File.txt.mw && + test_path_is_file mw_dir_15/File.txt && + test_path_is_missing mw_dir_15/Main_Page.mw && + test_path_is_missing mw_dir_15/File:AnotherFile.txt.mw && + test_path_is_missing mw_dir_15/AnothetFile.txt && + wiki_check_content mw_dir_15/Testpage.mw Testpage && + test_cmp mw_dir_15/File.txt File.txt +' + +test_expect_success 'git clone works with one specific page cloned and mediaimport=false' ' + test_when_finished "rm -rf mw_dir_16" && + git clone -c remote.origin.pages=testpage \ + mediawiki::'"$WIKI_URL"' mw_dir_16 && + test_contains_N_files mw_dir_16 1 && + test_path_is_file mw_dir_16/Testpage.mw && + test_path_is_missing mw_dir_16/File:File.txt.mw && + test_path_is_missing mw_dir_16/File.txt && + test_path_is_missing mw_dir_16/Main_Page.mw && + wiki_check_content mw_dir_16/Testpage.mw Testpage +' + +# should behave like mediaimport=false +test_expect_success 'git clone works with one specific page cloned and mediaimport unset' ' + test_when_finished "rm -fr mw_dir_17" && + git clone -c remote.origin.pages=testpage \ + mediawiki::'"$WIKI_URL"' mw_dir_17 && + test_contains_N_files mw_dir_17 1 && + test_path_is_file mw_dir_17/Testpage.mw && + test_path_is_missing mw_dir_17/File:File.txt.mw && + test_path_is_missing mw_dir_17/File.txt && + test_path_is_missing mw_dir_17/Main_Page.mw && + wiki_check_content mw_dir_17/Testpage.mw Testpage +' + +test_done diff --git a/contrib/mw-to-git/t/t9364-pull-by-rev.sh b/contrib/mw-to-git/t/t9364-pull-by-rev.sh new file mode 100755 index 0000000000..5c22457a0b --- /dev/null +++ b/contrib/mw-to-git/t/t9364-pull-by-rev.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +test_description='Test the Git Mediawiki remote helper: git pull by revision' + +. ./test-gitmw-lib.sh +. ./push-pull-tests.sh +. $TEST_DIRECTORY/test-lib.sh + +test_check_precond + +test_expect_success 'configuration' ' + git config --global mediawiki.fetchStrategy by_rev +' + +test_push_pull + +test_done diff --git a/contrib/mw-to-git/t/test-gitmw-lib.sh b/contrib/mw-to-git/t/test-gitmw-lib.sh new file mode 100755 index 0000000000..3b2cfacf51 --- /dev/null +++ b/contrib/mw-to-git/t/test-gitmw-lib.sh @@ -0,0 +1,435 @@ +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# License: GPL v2 or later + +# +# CONFIGURATION VARIABLES +# You might want to change these ones +# + +. ./test.config + +WIKI_URL=http://"$SERVER_ADDR:$PORT/$WIKI_DIR_NAME" +CURR_DIR=$(pwd) +TEST_OUTPUT_DIRECTORY=$(pwd) +TEST_DIRECTORY="$CURR_DIR"/../../../t + +export TEST_OUTPUT_DIRECTORY TEST_DIRECTORY CURR_DIR + +if test "$LIGHTTPD" = "false" ; then + PORT=80 +else + WIKI_DIR_INST="$CURR_DIR/$WEB_WWW" +fi + +wiki_upload_file () { + "$CURR_DIR"/test-gitmw.pl upload_file "$@" +} + +wiki_getpage () { + "$CURR_DIR"/test-gitmw.pl get_page "$@" +} + +wiki_delete_page () { + "$CURR_DIR"/test-gitmw.pl delete_page "$@" +} + +wiki_editpage () { + "$CURR_DIR"/test-gitmw.pl edit_page "$@" +} + +die () { + die_with_status 1 "$@" +} + +die_with_status () { + status=$1 + shift + echo >&2 "$*" + exit "$status" +} + + +# Check the preconditions to run git-remote-mediawiki's tests +test_check_precond () { + if ! test_have_prereq PERL + then + skip_all='skipping gateway git-mw tests, perl not available' + test_done + fi + + if [ ! -f "$GIT_BUILD_DIR"/git-remote-mediawiki ]; + then + echo "No remote mediawiki for git found. Copying it in git" + echo "cp $GIT_BUILD_DIR/contrib/mw-to-git/git-remote-mediawiki $GIT_BUILD_DIR/" + ln -s "$GIT_BUILD_DIR"/contrib/mw-to-git/git-remote-mediawiki "$GIT_BUILD_DIR" + fi + + if [ ! -d "$WIKI_DIR_INST/$WIKI_DIR_NAME" ]; + then + skip_all='skipping gateway git-mw tests, no mediawiki found' + test_done + fi +} + +# test_diff_directories <dir_git> <dir_wiki> +# +# Compare the contents of directories <dir_git> and <dir_wiki> with diff +# and errors if they do not match. The program will +# not look into .git in the process. +# Warning: the first argument MUST be the directory containing the git data +test_diff_directories () { + rm -rf "$1_tmp" + mkdir -p "$1_tmp" + cp "$1"/*.mw "$1_tmp" + diff -r -b "$1_tmp" "$2" +} + +# $1=<dir> +# $2=<N> +# +# Check that <dir> contains exactly <N> files +test_contains_N_files () { + if test `ls -- "$1" | wc -l` -ne "$2"; then + echo "directory $1 sould contain $2 files" + echo "it contains these files:" + ls "$1" + false + fi +} + + +# wiki_check_content <file_name> <page_name> +# +# Compares the contents of the file <file_name> and the wiki page +# <page_name> and exits with error 1 if they do not match. +wiki_check_content () { + mkdir -p wiki_tmp + wiki_getpage "$2" wiki_tmp + # replacement of forbidden character in file name + page_name=$(printf "%s\n" "$2" | sed -e "s/\//%2F/g") + + diff -b "$1" wiki_tmp/"$page_name".mw + if test $? -ne 0 + then + rm -rf wiki_tmp + error "ERROR: file $2 not found on wiki" + fi + rm -rf wiki_tmp +} + +# wiki_page_exist <page_name> +# +# Check the existence of the page <page_name> on the wiki and exits +# with error if it is absent from it. +wiki_page_exist () { + mkdir -p wiki_tmp + wiki_getpage "$1" wiki_tmp + page_name=$(printf "%s\n" "$1" | sed "s/\//%2F/g") + if test -f wiki_tmp/"$page_name".mw ; then + rm -rf wiki_tmp + else + rm -rf wiki_tmp + error "test failed: file $1 not found on wiki" + fi +} + +# wiki_getallpagename +# +# Fetch the name of each page on the wiki. +wiki_getallpagename () { + "$CURR_DIR"/test-gitmw.pl getallpagename +} + +# wiki_getallpagecategory <category> +# +# Fetch the name of each page belonging to <category> on the wiki. +wiki_getallpagecategory () { + "$CURR_DIR"/test-gitmw.pl getallpagename "$@" +} + +# wiki_getallpage <dest_dir> [<category>] +# +# Fetch all the pages from the wiki and place them in the directory +# <dest_dir>. +# If <category> is define, then wiki_getallpage fetch the pages included +# in <category>. +wiki_getallpage () { + if test -z "$2"; + then + wiki_getallpagename + else + wiki_getallpagecategory "$2" + fi + mkdir -p "$1" + while read -r line; do + wiki_getpage "$line" $1; + done < all.txt +} + +# ================= Install part ================= + +error () { + echo "$@" >&2 + exit 1 +} + +# config_lighttpd +# +# Create the configuration files and the folders necessary to start lighttpd. +# Overwrite any existing file. +config_lighttpd () { + mkdir -p $WEB + mkdir -p $WEB_TMP + mkdir -p $WEB_WWW + cat > $WEB/lighttpd.conf <<EOF + server.document-root = "$CURR_DIR/$WEB_WWW" + server.port = $PORT + server.pid-file = "$CURR_DIR/$WEB_TMP/pid" + + server.modules = ( + "mod_rewrite", + "mod_redirect", + "mod_access", + "mod_accesslog", + "mod_fastcgi" + ) + + index-file.names = ("index.php" , "index.html") + + mimetype.assign = ( + ".pdf" => "application/pdf", + ".sig" => "application/pgp-signature", + ".spl" => "application/futuresplash", + ".class" => "application/octet-stream", + ".ps" => "application/postscript", + ".torrent" => "application/x-bittorrent", + ".dvi" => "application/x-dvi", + ".gz" => "application/x-gzip", + ".pac" => "application/x-ns-proxy-autoconfig", + ".swf" => "application/x-shockwave-flash", + ".tar.gz" => "application/x-tgz", + ".tgz" => "application/x-tgz", + ".tar" => "application/x-tar", + ".zip" => "application/zip", + ".mp3" => "audio/mpeg", + ".m3u" => "audio/x-mpegurl", + ".wma" => "audio/x-ms-wma", + ".wax" => "audio/x-ms-wax", + ".ogg" => "application/ogg", + ".wav" => "audio/x-wav", + ".gif" => "image/gif", + ".jpg" => "image/jpeg", + ".jpeg" => "image/jpeg", + ".png" => "image/png", + ".xbm" => "image/x-xbitmap", + ".xpm" => "image/x-xpixmap", + ".xwd" => "image/x-xwindowdump", + ".css" => "text/css", + ".html" => "text/html", + ".htm" => "text/html", + ".js" => "text/javascript", + ".asc" => "text/plain", + ".c" => "text/plain", + ".cpp" => "text/plain", + ".log" => "text/plain", + ".conf" => "text/plain", + ".text" => "text/plain", + ".txt" => "text/plain", + ".dtd" => "text/xml", + ".xml" => "text/xml", + ".mpeg" => "video/mpeg", + ".mpg" => "video/mpeg", + ".mov" => "video/quicktime", + ".qt" => "video/quicktime", + ".avi" => "video/x-msvideo", + ".asf" => "video/x-ms-asf", + ".asx" => "video/x-ms-asf", + ".wmv" => "video/x-ms-wmv", + ".bz2" => "application/x-bzip", + ".tbz" => "application/x-bzip-compressed-tar", + ".tar.bz2" => "application/x-bzip-compressed-tar", + "" => "text/plain" + ) + + fastcgi.server = ( ".php" => + ("localhost" => + ( "socket" => "$CURR_DIR/$WEB_TMP/php.socket", + "bin-path" => "$PHP_DIR/php-cgi -c $CURR_DIR/$WEB/php.ini" + + ) + ) + ) +EOF + + cat > $WEB/php.ini <<EOF + session.save_path ='$CURR_DIR/$WEB_TMP' +EOF +} + +# start_lighttpd +# +# Start or restart daemon lighttpd. If restart, rewrite configuration files. +start_lighttpd () { + if test -f "$WEB_TMP/pid"; then + echo "Instance already running. Restarting..." + stop_lighttpd + fi + config_lighttpd + "$LIGHTTPD_DIR"/lighttpd -f "$WEB"/lighttpd.conf + + if test $? -ne 0 ; then + echo "Could not execute http deamon lighttpd" + exit 1 + fi +} + +# stop_lighttpd +# +# Kill daemon lighttpd and removes files and folders associated. +stop_lighttpd () { + test -f "$WEB_TMP/pid" && kill $(cat "$WEB_TMP/pid") + rm -rf "$WEB" +} + +# Create the SQLite database of the MediaWiki. If the database file already +# exists, it will be deleted. +# This script should be runned from the directory where $FILES_FOLDER is +# located. +create_db () { + rm -f "$TMP/$DB_FILE" + + echo "Generating the SQLite database file. It can take some time ..." + # Run the php script to generate the SQLite database file + # with cURL calls. + php "$FILES_FOLDER/$DB_INSTALL_SCRIPT" $(basename "$DB_FILE" .sqlite) \ + "$WIKI_ADMIN" "$WIKI_PASSW" "$TMP" "$PORT" + + if [ ! -f "$TMP/$DB_FILE" ] ; then + error "Can't create database file $TMP/$DB_FILE. Try to run ./install-wiki.sh delete first." + fi + + # Copy the generated database file into the directory the + # user indicated. + cp "$TMP/$DB_FILE" "$FILES_FOLDER" || + error "Unable to copy $TMP/$DB_FILE to $FILES_FOLDER" +} + +# Install a wiki in your web server directory. +wiki_install () { + if test $LIGHTTPD = "true" ; then + start_lighttpd + fi + + SERVER_ADDR=$SERVER_ADDR:$PORT + # In this part, we change directory to $TMP in order to download, + # unpack and copy the files of MediaWiki + ( + mkdir -p "$WIKI_DIR_INST/$WIKI_DIR_NAME" + if [ ! -d "$WIKI_DIR_INST/$WIKI_DIR_NAME" ] ; then + error "Folder $WIKI_DIR_INST/$WIKI_DIR_NAME doesn't exist. + Please create it and launch the script again." + fi + + # Fetch MediaWiki's archive if not already present in the TMP directory + cd "$TMP" + if [ ! -f "$MW_VERSION.tar.gz" ] ; then + echo "Downloading $MW_VERSION sources ..." + wget "http://download.wikimedia.org/mediawiki/1.19/mediawiki-1.19.0.tar.gz" || + error "Unable to download "\ + "http://download.wikimedia.org/mediawiki/1.19/"\ + "mediawiki-1.19.0.tar.gz. "\ + "Please fix your connection and launch the script again." + echo "$MW_VERSION.tar.gz downloaded in `pwd`. "\ + "You can delete it later if you want." + else + echo "Reusing existing $MW_VERSION.tar.gz downloaded in `pwd`." + fi + archive_abs_path=$(pwd)/"$MW_VERSION.tar.gz" + cd "$WIKI_DIR_INST/$WIKI_DIR_NAME/" || + error "can't cd to $WIKI_DIR_INST/$WIKI_DIR_NAME/" + tar xzf "$archive_abs_path" --strip-components=1 || + error "Unable to extract WikiMedia's files from $archive_abs_path to "\ + "$WIKI_DIR_INST/$WIKI_DIR_NAME" + ) || exit 1 + + create_db + + # Copy the generic LocalSettings.php in the web server's directory + # And modify parameters according to the ones set at the top + # of this script. + # Note that LocalSettings.php is never modified. + if [ ! -f "$FILES_FOLDER/LocalSettings.php" ] ; then + error "Can't find $FILES_FOLDER/LocalSettings.php " \ + "in the current folder. "\ + "Please run the script inside its folder." + fi + cp "$FILES_FOLDER/LocalSettings.php" \ + "$FILES_FOLDER/LocalSettings-tmp.php" || + error "Unable to copy $FILES_FOLDER/LocalSettings.php " \ + "to $FILES_FOLDER/LocalSettings-tmp.php" + + # Parse and set the LocalSettings file of the user according to the + # CONFIGURATION VARIABLES section at the beginning of this script + file_swap="$FILES_FOLDER/LocalSettings-swap.php" + sed "s,@WG_SCRIPT_PATH@,/$WIKI_DIR_NAME," \ + "$FILES_FOLDER/LocalSettings-tmp.php" > "$file_swap" + mv "$file_swap" "$FILES_FOLDER/LocalSettings-tmp.php" + sed "s,@WG_SERVER@,http://$SERVER_ADDR," \ + "$FILES_FOLDER/LocalSettings-tmp.php" > "$file_swap" + mv "$file_swap" "$FILES_FOLDER/LocalSettings-tmp.php" + sed "s,@WG_SQLITE_DATADIR@,$TMP," \ + "$FILES_FOLDER/LocalSettings-tmp.php" > "$file_swap" + mv "$file_swap" "$FILES_FOLDER/LocalSettings-tmp.php" + sed "s,@WG_SQLITE_DATAFILE@,$( basename $DB_FILE .sqlite)," \ + "$FILES_FOLDER/LocalSettings-tmp.php" > "$file_swap" + mv "$file_swap" "$FILES_FOLDER/LocalSettings-tmp.php" + + mv "$FILES_FOLDER/LocalSettings-tmp.php" \ + "$WIKI_DIR_INST/$WIKI_DIR_NAME/LocalSettings.php" || + error "Unable to move $FILES_FOLDER/LocalSettings-tmp.php" \ + "in $WIKI_DIR_INST/$WIKI_DIR_NAME" + echo "File $FILES_FOLDER/LocalSettings.php is set in" \ + " $WIKI_DIR_INST/$WIKI_DIR_NAME" + + echo "Your wiki has been installed. You can check it at + http://$SERVER_ADDR/$WIKI_DIR_NAME" +} + +# Reset the database of the wiki and the password of the admin +# +# Warning: This function must be called only in a subdirectory of t/ directory +wiki_reset () { + # Copy initial database of the wiki + if [ ! -f "../$FILES_FOLDER/$DB_FILE" ] ; then + error "Can't find ../$FILES_FOLDER/$DB_FILE in the current folder." + fi + cp "../$FILES_FOLDER/$DB_FILE" "$TMP" || + error "Can't copy ../$FILES_FOLDER/$DB_FILE in $TMP" + echo "File $FILES_FOLDER/$DB_FILE is set in $TMP" +} + +# Delete the wiki created in the web server's directory and all its content +# saved in the database. +wiki_delete () { + if test $LIGHTTPD = "true"; then + stop_lighttpd + else + # Delete the wiki's directory. + rm -rf "$WIKI_DIR_INST/$WIKI_DIR_NAME" || + error "Wiki's directory $WIKI_DIR_INST/" \ + "$WIKI_DIR_NAME could not be deleted" + # Delete the wiki's SQLite database. + rm -f "$TMP/$DB_FILE" || + error "Database $TMP/$DB_FILE could not be deleted." + fi + + # Delete the wiki's SQLite database + rm -f "$TMP/$DB_FILE" || error "Database $TMP/$DB_FILE could not be deleted." + rm -f "$FILES_FOLDER/$DB_FILE" + rm -rf "$TMP/$MW_VERSION" +} diff --git a/contrib/mw-to-git/t/test-gitmw.pl b/contrib/mw-to-git/t/test-gitmw.pl new file mode 100755 index 0000000000..0ff76259fa --- /dev/null +++ b/contrib/mw-to-git/t/test-gitmw.pl @@ -0,0 +1,225 @@ +#!/usr/bin/perl -w -s +# Copyright (C) 2012 +# Charles Roussel <charles.roussel@ensimag.imag.fr> +# Simon Cathebras <simon.cathebras@ensimag.imag.fr> +# Julien Khayat <julien.khayat@ensimag.imag.fr> +# Guillaume Sasdy <guillaume.sasdy@ensimag.imag.fr> +# Simon Perrat <simon.perrat@ensimag.imag.fr> +# License: GPL v2 or later + +# Usage: +# ./test-gitmw.pl <command> [argument]* +# Execute in terminal using the name of the function to call as first +# parameter, and the function's arguments as following parameters +# +# Example: +# ./test-gitmw.pl "get_page" foo . +# will call <wiki_getpage> with arguments <foo> and <.> +# +# Available functions are: +# "get_page" +# "delete_page" +# "edit_page" +# "getallpagename" + +use MediaWiki::API; +use Getopt::Long; +use encoding 'utf8'; +use DateTime::Format::ISO8601; +use open ':encoding(utf8)'; +use constant SLASH_REPLACEMENT => "%2F"; + +#Parsing of the config file + +my $configfile = "$ENV{'CURR_DIR'}/test.config"; +my %config; +open my $CONFIG, "<", $configfile or die "can't open $configfile: $!"; +while (<$CONFIG>) +{ + chomp; + s/#.*//; + s/^\s+//; + s/\s+$//; + next unless length; + my ($key, $value) = split (/\s*=\s*/,$_, 2); + $config{$key} = $value; + last if ($key eq 'LIGHTTPD' and $value eq 'false'); + last if ($key eq 'PORT'); +} +close $CONFIG or die "can't close $configfile: $!"; + +my $wiki_address = "http://$config{'SERVER_ADDR'}".":"."$config{'PORT'}"; +my $wiki_url = "$wiki_address/$config{'WIKI_DIR_NAME'}/api.php"; +my $wiki_admin = "$config{'WIKI_ADMIN'}"; +my $wiki_admin_pass = "$config{'WIKI_PASSW'}"; +my $mw = MediaWiki::API->new; +$mw->{config}->{api_url} = $wiki_url; + + +# wiki_login <name> <password> +# +# Logs the user with <name> and <password> in the global variable +# of the mediawiki $mw +sub wiki_login { + $mw->login( { lgname => "$_[0]",lgpassword => "$_[1]" } ) + || die "getpage: login failed"; +} + +# wiki_getpage <wiki_page> <dest_path> +# +# fetch a page <wiki_page> from the wiki referenced in the global variable +# $mw and copies its content in directory dest_path +sub wiki_getpage { + my $pagename = $_[0]; + my $destdir = $_[1]; + + my $page = $mw->get_page( { title => $pagename } ); + if (!defined($page)) { + die "getpage: wiki does not exist"; + } + + my $content = $page->{'*'}; + if (!defined($content)) { + die "getpage: page does not exist"; + } + + $pagename=$page->{'title'}; + # Replace spaces by underscore in the page name + $pagename =~ s/ /_/g; + $pagename =~ s/\//%2F/g; + open(my $file, ">$destdir/$pagename.mw"); + print $file "$content"; + close ($file); + +} + +# wiki_delete_page <page_name> +# +# delete the page with name <page_name> from the wiki referenced +# in the global variable $mw +sub wiki_delete_page { + my $pagename = $_[0]; + + my $exist=$mw->get_page({title => $pagename}); + + if (defined($exist->{'*'})){ + $mw->edit({ action => 'delete', + title => $pagename}) + || die $mw->{error}->{code} . ": " . $mw->{error}->{details}; + } else { + die "no page with such name found: $pagename\n"; + } +} + +# wiki_editpage <wiki_page> <wiki_content> <wiki_append> [-c=<category>] [-s=<summary>] +# +# Edit a page named <wiki_page> with content <wiki_content> on the wiki +# referenced with the global variable $mw +# If <wiki_append> == true : append <wiki_content> at the end of the actual +# content of the page <wiki_page> +# If <wik_page> doesn't exist, that page is created with the <wiki_content> +sub wiki_editpage { + my $wiki_page = $_[0]; + my $wiki_content = $_[1]; + my $wiki_append = $_[2]; + my $summary = ""; + my ($summ, $cat) = (); + GetOptions('s=s' => \$summ, 'c=s' => \$cat); + + my $append = 0; + if (defined($wiki_append) && $wiki_append eq 'true') { + $append=1; + } + + my $previous_text =""; + + if ($append) { + my $ref = $mw->get_page( { title => $wiki_page } ); + $previous_text = $ref->{'*'}; + } + + my $text = $wiki_content; + if (defined($previous_text)) { + $text="$previous_text$text"; + } + + # Eventually, add this page to a category. + if (defined($cat)) { + my $category_name="[[Category:$cat]]"; + $text="$text\n $category_name"; + } + if(defined($summ)){ + $summary=$summ; + } + + $mw->edit( { action => 'edit', title => $wiki_page, summary => $summary, text => "$text"} ); +} + +# wiki_getallpagename [<category>] +# +# Fetch all pages of the wiki referenced by the global variable $mw +# and print the names of each one in the file all.txt with a new line +# ("\n") between these. +# If the argument <category> is defined, then this function get only the pages +# belonging to <category>. +sub wiki_getallpagename { + # fetch the pages of the wiki + if (defined($_[0])) { + my $mw_pages = $mw->list ( { action => 'query', + list => 'categorymembers', + cmtitle => "Category:$_[0]", + cmnamespace => 0, + cmlimit => 500 }, + ) + || die $mw->{error}->{code}.": ".$mw->{error}->{details}; + open(my $file, ">all.txt"); + foreach my $page (@{$mw_pages}) { + print $file "$page->{title}\n"; + } + close ($file); + + } else { + my $mw_pages = $mw->list({ + action => 'query', + list => 'allpages', + aplimit => 500, + }) + || die $mw->{error}->{code}.": ".$mw->{error}->{details}; + open(my $file, ">all.txt"); + foreach my $page (@{$mw_pages}) { + print $file "$page->{title}\n"; + } + close ($file); + } +} + +sub wiki_upload_file { + my $file_name = $_[0]; + my $resultat = $mw->edit ( { + action => 'upload', + filename => $file_name, + comment => 'upload a file', + file => [ $file_name ], + ignorewarnings=>1, + }, { + skip_encoding => 1 + } ) || die $mw->{error}->{code} . ' : ' . $mw->{error}->{details}; +} + + + +# Main part of this script: parse the command line arguments +# and select which function to execute +my $fct_to_call = shift; + +wiki_login($wiki_admin, $wiki_admin_pass); + +my %functions_to_call = qw( + upload_file wiki_upload_file + get_page wiki_getpage + delete_page wiki_delete_page + edit_page wiki_editpage + getallpagename wiki_getallpagename +); +die "$0 ERROR: wrong argument" unless exists $functions_to_call{$fct_to_call}; +&{$functions_to_call{$fct_to_call}}(@ARGV); diff --git a/contrib/mw-to-git/t/test.config b/contrib/mw-to-git/t/test.config new file mode 100644 index 0000000000..958b37b4a7 --- /dev/null +++ b/contrib/mw-to-git/t/test.config @@ -0,0 +1,35 @@ +# Name of the web server's directory dedicated to the wiki is WIKI_DIR_NAME +WIKI_DIR_NAME=wiki + +# Login and password of the wiki's admin +WIKI_ADMIN=WikiAdmin +WIKI_PASSW=AdminPass + +# Address of the web server +SERVER_ADDR=localhost + +# SQLite database of the wiki, named DB_FILE, is located in TMP +TMP=/tmp +DB_FILE=wikidb.sqlite + +# If LIGHTTPD is not set to true, the script will use the defaut +# web server running in WIKI_DIR_INST. +WIKI_DIR_INST=/var/www + +# If LIGHTTPD is set to true, the script will use Lighttpd to run +# the wiki. +LIGHTTPD=true + +# The variables below are useful only if LIGHTTPD is set to true. +PORT=1234 +PHP_DIR=/usr/bin +LIGHTTPD_DIR=/usr/sbin +WEB=WEB +WEB_TMP=$WEB/tmp +WEB_WWW=$WEB/www + +# The variables below are used by the script to install a wiki. +# You should not modify these unless you are modifying the script itself. +MW_VERSION=mediawiki-1.19.0 +FILES_FOLDER=install-wiki +DB_INSTALL_SCRIPT=db_install.php diff --git a/contrib/persistent-https/LICENSE b/contrib/persistent-https/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/contrib/persistent-https/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/contrib/persistent-https/Makefile b/contrib/persistent-https/Makefile new file mode 100644 index 0000000000..92baa3beee --- /dev/null +++ b/contrib/persistent-https/Makefile @@ -0,0 +1,38 @@ +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BUILD_LABEL=$(shell date +"%s") +TAR_OUT=$(shell go env GOOS)_$(shell go env GOARCH).tar.gz + +all: git-remote-persistent-https git-remote-persistent-https--proxy \ + git-remote-persistent-http + +git-remote-persistent-https--proxy: git-remote-persistent-https + ln -f -s git-remote-persistent-https git-remote-persistent-https--proxy + +git-remote-persistent-http: git-remote-persistent-https + ln -f -s git-remote-persistent-https git-remote-persistent-http + +git-remote-persistent-https: + go build -o git-remote-persistent-https \ + -ldflags "-X main._BUILD_EMBED_LABEL $(BUILD_LABEL)" + +clean: + rm -f git-remote-persistent-http* *.tar.gz + +tar: clean all + @chmod 555 git-remote-persistent-https + @tar -czf $(TAR_OUT) git-remote-persistent-http* README LICENSE + @echo + @echo "Created $(TAR_OUT)" diff --git a/contrib/persistent-https/README b/contrib/persistent-https/README new file mode 100644 index 0000000000..f784dd2e66 --- /dev/null +++ b/contrib/persistent-https/README @@ -0,0 +1,62 @@ +git-remote-persistent-https + +The git-remote-persistent-https binary speeds up SSL operations +by running a daemon job (git-remote-persistent-https--proxy) that +keeps a connection open to a server. + + +PRE-BUILT BINARIES + +Darwin amd64: +https://commondatastorage.googleapis.com/git-remote-persistent-https/darwin_amd64.tar.gz + +Linux amd64: +https://commondatastorage.googleapis.com/git-remote-persistent-https/linux_amd64.tar.gz + + +INSTALLING + +Move all of the git-remote-persistent-http* binaries to a directory +in PATH. + + +USAGE + +HTTPS requests can be delegated to the proxy by using the +"persistent-https" scheme, e.g. + +git clone persistent-https://kernel.googlesource.com/pub/scm/git/git + +Likewise, .gitconfig can be updated as follows to rewrite https urls +to use persistent-https: + +[url "persistent-https"] + insteadof = https +[url "persistent-http"] + insteadof = http + + +##################################################################### +# BUILDING FROM SOURCE +##################################################################### + +LOCATION + +The source is available in the contrib/persistent-https directory of +the Git source repository. The Git source repository is available at +git://git.kernel.org/pub/scm/git/git.git/ +https://kernel.googlesource.com/pub/scm/git/git + + +PREREQUISITES + +The code is written in Go (http://golang.org/) and the Go compiler is +required. Currently, the compiler must be built and installed from tip +of source, in order to include a fix in the reverse http proxy: +http://code.google.com/p/go/source/detail?r=a615b796570a2cd8591884767a7d67ede74f6648 + + +BUILDING + +Run "make" to build the binaries. See the section on +INSTALLING above. diff --git a/contrib/persistent-https/client.go b/contrib/persistent-https/client.go new file mode 100644 index 0000000000..71125b5832 --- /dev/null +++ b/contrib/persistent-https/client.go @@ -0,0 +1,189 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +type Client struct { + ProxyBin string + Args []string + + insecure bool +} + +func (c *Client) Run() error { + if err := c.resolveArgs(); err != nil { + return fmt.Errorf("resolveArgs() got error: %v", err) + } + + // Connect to the proxy. + uconn, hconn, addr, err := c.connect() + if err != nil { + return fmt.Errorf("connect() got error: %v", err) + } + // Keep the unix socket connection open for the duration of the request. + defer uconn.Close() + // Keep a connection to the HTTP server open, so no other user can + // bind on the same address so long as the process is running. + defer hconn.Close() + + // Start the git-remote-http subprocess. + cargs := []string{"-c", fmt.Sprintf("http.proxy=%v", addr), "remote-http"} + cargs = append(cargs, c.Args...) + cmd := exec.Command("git", cargs...) + + for _, v := range os.Environ() { + if !strings.HasPrefix(v, "GIT_PERSISTENT_HTTPS_SECURE=") { + cmd.Env = append(cmd.Env, v) + } + } + // Set the GIT_PERSISTENT_HTTPS_SECURE environment variable when + // the proxy is using a SSL connection. This allows credential helpers + // to identify secure proxy connections, despite being passed an HTTP + // scheme. + if !c.insecure { + cmd.Env = append(cmd.Env, "GIT_PERSISTENT_HTTPS_SECURE=1") + } + + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if eerr, ok := err.(*exec.ExitError); ok { + if stat, ok := eerr.ProcessState.Sys().(syscall.WaitStatus); ok && stat.ExitStatus() != 0 { + os.Exit(stat.ExitStatus()) + } + } + return fmt.Errorf("git-remote-http subprocess got error: %v", err) + } + return nil +} + +func (c *Client) connect() (uconn net.Conn, hconn net.Conn, addr string, err error) { + uconn, err = DefaultSocket.Dial() + if err != nil { + if e, ok := err.(*net.OpError); ok && (os.IsNotExist(e.Err) || e.Err == syscall.ECONNREFUSED) { + if err = c.startProxy(); err == nil { + uconn, err = DefaultSocket.Dial() + } + } + if err != nil { + return + } + } + + if addr, err = c.readAddr(uconn); err != nil { + return + } + + // Open a tcp connection to the proxy. + if hconn, err = net.Dial("tcp", addr); err != nil { + return + } + + // Verify the address hasn't changed ownership. + var addr2 string + if addr2, err = c.readAddr(uconn); err != nil { + return + } else if addr != addr2 { + err = fmt.Errorf("address changed after connect. got %q, want %q", addr2, addr) + return + } + return +} + +func (c *Client) readAddr(conn net.Conn) (string, error) { + conn.SetDeadline(time.Now().Add(5 * time.Second)) + data := make([]byte, 100) + n, err := conn.Read(data) + if err != nil { + return "", fmt.Errorf("error reading unix socket: %v", err) + } else if n == 0 { + return "", errors.New("empty data response") + } + conn.Write([]byte{1}) // Ack + + var addr string + if addrs := strings.Split(string(data[:n]), "\n"); len(addrs) != 2 { + return "", fmt.Errorf("got %q, wanted 2 addresses", data[:n]) + } else if c.insecure { + addr = addrs[1] + } else { + addr = addrs[0] + } + return addr, nil +} + +func (c *Client) startProxy() error { + cmd := exec.Command(c.ProxyBin) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + defer stdout.Close() + if err := cmd.Start(); err != nil { + return err + } + result := make(chan error) + go func() { + bytes, _, err := bufio.NewReader(stdout).ReadLine() + if line := string(bytes); err == nil && line != "OK" { + err = fmt.Errorf("proxy returned %q, want \"OK\"", line) + } + result <- err + }() + select { + case err := <-result: + return err + case <-time.After(5 * time.Second): + return errors.New("timeout waiting for proxy to start") + } + panic("not reachable") +} + +func (c *Client) resolveArgs() error { + if nargs := len(c.Args); nargs == 0 { + return errors.New("remote needed") + } else if nargs > 2 { + return fmt.Errorf("want at most 2 args, got %v", c.Args) + } + + // Rewrite the url scheme to be http. + idx := len(c.Args) - 1 + rawurl := c.Args[idx] + rurl, err := url.Parse(rawurl) + if err != nil { + return fmt.Errorf("invalid remote: %v", err) + } + c.insecure = rurl.Scheme == "persistent-http" + rurl.Scheme = "http" + c.Args[idx] = rurl.String() + if idx != 0 && c.Args[0] == rawurl { + c.Args[0] = c.Args[idx] + } + return nil +} diff --git a/contrib/persistent-https/main.go b/contrib/persistent-https/main.go new file mode 100644 index 0000000000..fd1b107743 --- /dev/null +++ b/contrib/persistent-https/main.go @@ -0,0 +1,82 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The git-remote-persistent-https binary speeds up SSL operations by running +// a daemon job that keeps a connection open to a Git server. This ensures the +// git-remote-persistent-https--proxy is running and delegating execution +// to the git-remote-http binary with the http_proxy set to the daemon job. +// A unix socket is used to authenticate the proxy and discover the +// HTTP address. Note, both the client and proxy are included in the same +// binary. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + "time" +) + +var ( + forceProxy = flag.Bool("proxy", false, "Whether to start the binary in proxy mode") + proxyBin = flag.String("proxy_bin", "git-remote-persistent-https--proxy", "Path to the proxy binary") + printLabel = flag.Bool("print_label", false, "Prints the build label for the binary") + + // Variable that should be defined through the -X linker flag. + _BUILD_EMBED_LABEL string +) + +const ( + defaultMaxIdleDuration = 24 * time.Hour + defaultPollUpdateInterval = 15 * time.Minute +) + +func main() { + flag.Parse() + if *printLabel { + // Short circuit execution to print the build label + fmt.Println(buildLabel()) + return + } + + var err error + if *forceProxy || strings.HasSuffix(os.Args[0], "--proxy") { + log.SetPrefix("git-remote-persistent-https--proxy: ") + proxy := &Proxy{ + BuildLabel: buildLabel(), + MaxIdleDuration: defaultMaxIdleDuration, + PollUpdateInterval: defaultPollUpdateInterval, + } + err = proxy.Run() + } else { + log.SetPrefix("git-remote-persistent-https: ") + client := &Client{ + ProxyBin: *proxyBin, + Args: flag.Args(), + } + err = client.Run() + } + if err != nil { + log.Fatalln(err) + } +} + +func buildLabel() string { + if _BUILD_EMBED_LABEL == "" { + log.Println(`unlabeled build; build with "make" to label`) + } + return _BUILD_EMBED_LABEL +} diff --git a/contrib/persistent-https/proxy.go b/contrib/persistent-https/proxy.go new file mode 100644 index 0000000000..bb0cdba386 --- /dev/null +++ b/contrib/persistent-https/proxy.go @@ -0,0 +1,190 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "net" + "net/http" + "net/http/httputil" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" +) + +type Proxy struct { + BuildLabel string + MaxIdleDuration time.Duration + PollUpdateInterval time.Duration + + ul net.Listener + httpAddr string + httpsAddr string +} + +func (p *Proxy) Run() error { + hl, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("http listen failed: %v", err) + } + defer hl.Close() + + hsl, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("https listen failed: %v", err) + } + defer hsl.Close() + + p.ul, err = DefaultSocket.Listen() + if err != nil { + c, derr := DefaultSocket.Dial() + if derr == nil { + c.Close() + fmt.Println("OK\nA proxy is already running... exiting") + return nil + } else if e, ok := derr.(*net.OpError); ok && e.Err == syscall.ECONNREFUSED { + // Nothing is listening on the socket, unlink it and try again. + syscall.Unlink(DefaultSocket.Path()) + p.ul, err = DefaultSocket.Listen() + } + if err != nil { + return fmt.Errorf("unix listen failed on %v: %v", DefaultSocket.Path(), err) + } + } + defer p.ul.Close() + go p.closeOnSignal() + go p.closeOnUpdate() + + p.httpAddr = hl.Addr().String() + p.httpsAddr = hsl.Addr().String() + fmt.Printf("OK\nListening on unix socket=%v http=%v https=%v\n", + p.ul.Addr(), p.httpAddr, p.httpsAddr) + + result := make(chan error, 2) + go p.serveUnix(result) + go func() { + result <- http.Serve(hl, &httputil.ReverseProxy{ + FlushInterval: 500 * time.Millisecond, + Director: func(r *http.Request) {}, + }) + }() + go func() { + result <- http.Serve(hsl, &httputil.ReverseProxy{ + FlushInterval: 500 * time.Millisecond, + Director: func(r *http.Request) { + r.URL.Scheme = "https" + }, + }) + }() + return <-result +} + +type socketContext struct { + sync.WaitGroup + mutex sync.Mutex + last time.Time +} + +func (sc *socketContext) Done() { + sc.mutex.Lock() + defer sc.mutex.Unlock() + sc.last = time.Now() + sc.WaitGroup.Done() +} + +func (p *Proxy) serveUnix(result chan<- error) { + sockCtx := &socketContext{} + go p.closeOnIdle(sockCtx) + + var err error + for { + var uconn net.Conn + uconn, err = p.ul.Accept() + if err != nil { + err = fmt.Errorf("accept failed: %v", err) + break + } + sockCtx.Add(1) + go p.handleUnixConn(sockCtx, uconn) + } + sockCtx.Wait() + result <- err +} + +func (p *Proxy) handleUnixConn(sockCtx *socketContext, uconn net.Conn) { + defer sockCtx.Done() + defer uconn.Close() + data := []byte(fmt.Sprintf("%v\n%v", p.httpsAddr, p.httpAddr)) + uconn.SetDeadline(time.Now().Add(5 * time.Second)) + for i := 0; i < 2; i++ { + if n, err := uconn.Write(data); err != nil { + log.Printf("error sending http addresses: %+v\n", err) + return + } else if n != len(data) { + log.Printf("sent %d data bytes, wanted %d\n", n, len(data)) + return + } + if _, err := uconn.Read([]byte{0, 0, 0, 0}); err != nil { + log.Printf("error waiting for Ack: %+v\n", err) + return + } + } + // Wait without a deadline for the client to finish via EOF + uconn.SetDeadline(time.Time{}) + uconn.Read([]byte{0, 0, 0, 0}) +} + +func (p *Proxy) closeOnIdle(sockCtx *socketContext) { + for d := p.MaxIdleDuration; d > 0; { + time.Sleep(d) + sockCtx.Wait() + sockCtx.mutex.Lock() + if d = sockCtx.last.Add(p.MaxIdleDuration).Sub(time.Now()); d <= 0 { + log.Println("graceful shutdown from idle timeout") + p.ul.Close() + } + sockCtx.mutex.Unlock() + } +} + +func (p *Proxy) closeOnUpdate() { + for { + time.Sleep(p.PollUpdateInterval) + if out, err := exec.Command(os.Args[0], "--print_label").Output(); err != nil { + log.Printf("error polling for updated binary: %v\n", err) + } else if s := string(out[:len(out)-1]); p.BuildLabel != s { + log.Printf("graceful shutdown from updated binary: %q --> %q\n", p.BuildLabel, s) + p.ul.Close() + break + } + } +} + +func (p *Proxy) closeOnSignal() { + ch := make(chan os.Signal, 10) + signal.Notify(ch, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM), os.Signal(syscall.SIGHUP)) + sig := <-ch + p.ul.Close() + switch sig { + case os.Signal(syscall.SIGHUP): + log.Printf("graceful shutdown from signal: %v\n", sig) + default: + log.Fatalf("exiting from signal: %v\n", sig) + } +} diff --git a/contrib/persistent-https/socket.go b/contrib/persistent-https/socket.go new file mode 100644 index 0000000000..193b911dd1 --- /dev/null +++ b/contrib/persistent-https/socket.go @@ -0,0 +1,97 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "net" + "os" + "path/filepath" + "syscall" +) + +// A Socket is a wrapper around a Unix socket that verifies directory +// permissions. +type Socket struct { + Dir string +} + +func defaultDir() string { + sockPath := ".git-credential-cache" + if home := os.Getenv("HOME"); home != "" { + return filepath.Join(home, sockPath) + } + log.Printf("socket: cannot find HOME path. using relative directory %q for socket", sockPath) + return sockPath +} + +// DefaultSocket is a Socket in the $HOME/.git-credential-cache directory. +var DefaultSocket = Socket{Dir: defaultDir()} + +// Listen announces the local network address of the unix socket. The +// permissions on the socket directory are verified before attempting +// the actual listen. +func (s Socket) Listen() (net.Listener, error) { + network, addr := "unix", s.Path() + if err := s.mkdir(); err != nil { + return nil, &net.OpError{Op: "listen", Net: network, Addr: &net.UnixAddr{Name: addr, Net: network}, Err: err} + } + return net.Listen(network, addr) +} + +// Dial connects to the unix socket. The permissions on the socket directory +// are verified before attempting the actual dial. +func (s Socket) Dial() (net.Conn, error) { + network, addr := "unix", s.Path() + if err := s.checkPermissions(); err != nil { + return nil, &net.OpError{Op: "dial", Net: network, Addr: &net.UnixAddr{Name: addr, Net: network}, Err: err} + } + return net.Dial(network, addr) +} + +// Path returns the fully specified file name of the unix socket. +func (s Socket) Path() string { + return filepath.Join(s.Dir, "persistent-https-proxy-socket") +} + +func (s Socket) mkdir() error { + if err := s.checkPermissions(); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(s.Dir, 0700); err != nil { + return err + } + return s.checkPermissions() +} + +func (s Socket) checkPermissions() error { + fi, err := os.Stat(s.Dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("socket: got file, want directory for %q", s.Dir) + } + if fi.Mode().Perm() != 0700 { + return fmt.Errorf("socket: got perm %o, want 700 for %q", fi.Mode().Perm(), s.Dir) + } + if st := fi.Sys().(*syscall.Stat_t); int(st.Uid) != os.Getuid() { + return fmt.Errorf("socket: got uid %d, want %d for %q", st.Uid, os.Getuid(), s.Dir) + } + return nil +} diff --git a/contrib/rerere-train.sh b/contrib/rerere-train.sh index 2cfe1b936b..36b6feebe0 100755 --- a/contrib/rerere-train.sh +++ b/contrib/rerere-train.sh @@ -7,7 +7,7 @@ USAGE="$me rev-list-args" SUBDIRECTORY_OK=Yes OPTIONS_SPEC= -. git-sh-setup +. $(git --exec-path)/git-sh-setup require_work_tree cd_to_toplevel |