Merge branch 'mh/multimail'

* mh/multimail:
  git-multimail: update to version 1.0.0

5 files changed, 249 insertions, 56 deletions

diff --git a/contrib/hooks/multimail/CHANGES b/contrib/hooks/multimail/CHANGES
new file mode 100644
index 0000000000..3603d56c26
--- /dev/null
+++ b/contrib/hooks/multimail/CHANGES
@@ -0,0 +1,33 @@
+Release 1.0.0
+=============
+
+* Fix encoding of non-ASCII email addresses in email headers.
+
+* Fix backwards-compatibility bugs for older Python 2.x versions.
+
+* Fix a backwards-compatibility bug for Git 1.7.1.
+
+* Add an option commitDiffOpts to customize logs for revisions.
+
+* Pass "-oi" to sendmail by default to prevent premature termination
+  on a line containing only ".".
+
+* Stagger email "Date:" values in an attempt to help mail clients
+  thread the emails in the right order.
+
+* If a mailing list setting is missing, just skip sending the
+  corresponding email (with a warning) instead of failing.
+
+* Add a X-Git-Host header that can be used for email filtering.
+
+* Allow the sender's fully-qualified domain name to be configured.
+
+* Minor documentation improvements.
+
+* Add this CHANGES file.
+
+
+Release 0.9.0
+=============
+
+* Initial release.
diff --git a/contrib/hooks/multimail/README b/contrib/hooks/multimail/README
index 9904396710..477d65fed3 100644
--- a/contrib/hooks/multimail/README
+++ b/contrib/hooks/multimail/README
@@ -91,9 +91,10 @@ Requirements
   been tested; if you do so, please report your results.)
 
 * To send emails using the default configuration, a standard sendmail
-  program must be located at '/usr/sbin/sendmail' and configured
-  correctly to send emails.  If this is not the case, see the
-  multimailhook.mailer configuration variable below for how to
+  program must be located at '/usr/sbin/sendmail' or
+  '/usr/lib/sendmail' and must be configured correctly to send emails.
+  If this is not the case, set multimailhook.sendmailCommand, or see
+  the multimailhook.mailer configuration variable below for how to
   configure git-multimail to send emails via an SMTP server.
 
 
@@ -169,7 +170,7 @@ multimailhook.repoName
     for gitolite repositories, or otherwise to derive this value from
     the repository path name.
 
-multimailhook.mailinglist
+multimailhook.mailingList
 
     The list of email addresses to which notification emails should be
     sent, as RFC 2822 email addresses separated by commas.  This
@@ -184,26 +185,29 @@ multimailhook.refchangeList
     reference changes should be sent, as RFC 2822 email addresses
     separated by commas.  This configuration option can be
     multivalued.  The default is the value in
-    multimailhook.mailinglist.  Set this value to the empty string to
-    prevent reference change emails from being sent.
+    multimailhook.mailingList.  Set this value to the empty string to
+    prevent reference change emails from being sent even if
+    multimailhook.mailingList is set.
 
 multimailhook.announceList
 
     The list of email addresses to which emails about new annotated
     tags should be sent, as RFC 2822 email addresses separated by
     commas.  This configuration option can be multivalued.  The
-    default is the value in multimailhook.refchangelist or
-    multimailhook.mailinglist.  Set this value to the empty string to
-    prevent annotated tag announcement emails from being sent.
+    default is the value in multimailhook.refchangeList or
+    multimailhook.mailingList.  Set this value to the empty string to
+    prevent annotated tag announcement emails from being sent even if
+    one of the other values is set.
 
 multimailhook.commitList
 
     The list of email addresses to which emails about individual new
     commits should be sent, as RFC 2822 email addresses separated by
     commas.  This configuration option can be multivalued.  The
-    default is the value in multimailhook.mailinglist.  Set this value
+    default is the value in multimailhook.mailingList.  Set this value
     to the empty string to prevent notification emails about
-    individual commits from being sent.
+    individual commits from being sent even if
+    multimailhook.mailingList is set.
 
 multimailhook.announceShortlog
 
@@ -237,10 +241,11 @@ multimailhook.mailer
            quoting is allowed in the value of this setting, but remember that
            Git requires double-quotes to be escaped; e.g.,
 
-             git config multimailhook.sendmailcommand '/usr/sbin/sendmail -t -F \"Git Repo\"'
+             git config multimailhook.sendmailcommand '/usr/sbin/sendmail -oi -t -F \"Git Repo\"'
 
-           Default is '/usr/sbin/sendmail -t' or '/usr/lib/sendmail
-           -t' (depending on which file is present and executable).
+           Default is '/usr/sbin/sendmail -oi -t' or
+           '/usr/lib/sendmail -oi -t' (depending on which file is
+           present and executable).
 
        multimailhook.envelopeSender
 
@@ -344,6 +349,14 @@ multimailhook.logOpts
       [multimailhook]
               logopts = --pretty=format:\"%h %aN <%aE>%n%s%n%n%b%n\"
 
+multimailhook.commitLogOpts
+
+    Options passed to "git log" to generate additional info for
+    revision change emails.  For example, adding --ignore-all-spaces
+    will suppress whitespace changes.  The default options are "-C
+    --stat -p --cc".  Shell quoting is allowed; see
+    multimailhook.logOpts for details.
+
 multimailhook.emailDomain
 
     Domain name appended to the username of the person doing the push
@@ -381,8 +394,8 @@ Email filtering aids
 
 All emails include extra headers to enable fine tuned filtering and
 give information for debugging.  All emails include the headers
-"X-Git-Repo", "X-Git-Refname", and "X-Git-Reftype".  ReferenceChange
-emails also include headers "X-Git-Oldrev" and "X-Git-Newrev";
+"X-Git-Host", "X-Git-Repo", "X-Git-Refname", and "X-Git-Reftype".
+ReferenceChange emails also include headers "X-Git-Oldrev" and "X-Git-Newrev";
 Revision emails also include header "X-Git-Rev".
 
 
@@ -463,6 +476,7 @@ The git-multimail project itself is currently hosted on GitHub:
 We use the GitHub issue tracker to keep track of bugs and feature
 requests, and GitHub pull requests to exchange patches (though, if you
 prefer, you can send patches via the Git mailing list with cc to me).
+Please sign off your patches as per the Git project practice.
 
 Please note that although a copy of git-multimail will probably be
 distributed in the "contrib" section of the main Git project,
diff --git a/contrib/hooks/multimail/README.Git b/contrib/hooks/multimail/README.Git
index 9c2e66a69a..129b771410 100644
--- a/contrib/hooks/multimail/README.Git
+++ b/contrib/hooks/multimail/README.Git
@@ -6,10 +6,10 @@ website:
     https://github.com/mhagger/git-multimail
 
 The version in this directory was obtained from the upstream project
-on 2013-07-14 and consists of the "git-multimail" subdirectory from
+on 2014-04-07 and consists of the "git-multimail" subdirectory from
 revision
 
-    1a5cb09c698a74d15a715a86b09ead5f56bf4b06
+    1b32653bafc4f902535b9fc1cd9cae911325b870
 
 Please see the README file in this directory for information about how
 to report bugs or contribute to git-multimail.
diff --git a/contrib/hooks/multimail/git_multimail.py b/contrib/hooks/multimail/git_multimail.py
index 81c6a51706..8b58ed6444 100755
--- a/contrib/hooks/multimail/git_multimail.py
+++ b/contrib/hooks/multimail/git_multimail.py
@@ -1,6 +1,6 @@
 #! /usr/bin/env python2
 
-# Copyright (c) 2012,2013 Michael Haggerty
+# Copyright (c) 2012-2014 Michael Haggerty and others
 # Derived from contrib/hooks/post-receive-email, which is
 # Copyright (c) 2007 Andy Parkins
 # and also includes contributions by other authors.
@@ -49,21 +49,25 @@ import sys
 import os
 import re
 import bisect
+import socket
 import subprocess
 import shlex
 import optparse
 import smtplib
+import time
 
 try:
     from email.utils import make_msgid
     from email.utils import getaddresses
     from email.utils import formataddr
+    from email.utils import formatdate
     from email.header import Header
 except ImportError:
     # Prior to Python 2.5, the email module used different names:
     from email.Utils import make_msgid
     from email.Utils import getaddresses
     from email.Utils import formataddr
+    from email.Utils import formatdate
     from email.Header import Header
 
 
@@ -73,6 +77,7 @@ ZEROS = '0' * 40
 LOGBEGIN = '- Log -----------------------------------------------------------------\n'
 LOGEND = '-----------------------------------------------------------------------\n'
 
+ADDR_HEADERS = set(['from', 'to', 'cc', 'bcc', 'reply-to', 'sender'])
 
 # It is assumed in many places that the encoding is uniformly UTF-8,
 # so changing these constants is unsupported.  But define them here
@@ -95,6 +100,7 @@ REF_DELETED_SUBJECT_TEMPLATE = (
     )
 
 REFCHANGE_HEADER_TEMPLATE = """\
+Date: %(send_date)s
 To: %(recipients)s
 Subject: %(subject)s
 MIME-Version: 1.0
@@ -103,6 +109,7 @@ Content-Transfer-Encoding: 8bit
 Message-ID: %(msgid)s
 From: %(fromaddr)s
 Reply-To: %(reply_to)s
+X-Git-Host: %(fqdn)s
 X-Git-Repo: %(repo_shortname)s
 X-Git-Refname: %(refname)s
 X-Git-Reftype: %(refname_type)s
@@ -221,6 +228,7 @@ how to provide full information about this reference change.
 
 
 REVISION_HEADER_TEMPLATE = """\
+Date: %(send_date)s
 To: %(recipients)s
 Subject: %(emailprefix)s%(num)02d/%(tot)02d: %(oneline)s
 MIME-Version: 1.0
@@ -230,6 +238,7 @@ From: %(fromaddr)s
 Reply-To: %(reply_to)s
 In-Reply-To: %(reply_to_msgid)s
 References: %(reply_to_msgid)s
+X-Git-Host: %(fqdn)s
 X-Git-Repo: %(repo_shortname)s
 X-Git-Refname: %(refname)s
 X-Git-Reftype: %(refname_type)s
@@ -263,13 +272,43 @@ class ConfigurationException(Exception):
     pass
 
 
+# The "git" program (this could be changed to include a full path):
+GIT_EXECUTABLE = 'git'
+
+
+# How "git" should be invoked (including global arguments), as a list
+# of words.  This variable is usually initialized automatically by
+# read_git_output() via choose_git_command(), but if a value is set
+# here then it will be used unconditionally.
+GIT_CMD = None
+
+
+def choose_git_command():
+    """Decide how to invoke git, and record the choice in GIT_CMD."""
+
+    global GIT_CMD
+
+    if GIT_CMD is None:
+        try:
+            # Check to see whether the "-c" option is accepted (it was
+            # only added in Git 1.7.2).  We don't actually use the
+            # output of "git --version", though if we needed more
+            # specific version information this would be the place to
+            # do it.
+            cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version']
+            read_output(cmd)
+            GIT_CMD = [GIT_EXECUTABLE, '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)]
+        except CommandError:
+            GIT_CMD = [GIT_EXECUTABLE]
+
+
 def read_git_output(args, input=None, keepends=False, **kw):
     """Read the output of a Git command."""
 
-    return read_output(
-        ['git', '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)] + args,
-        input=input, keepends=keepends, **kw
-        )
+    if GIT_CMD is None:
+        choose_git_command()
+
+    return read_output(GIT_CMD + args, input=input, keepends=keepends, **kw)
 
 
 def read_output(cmd, input=None, keepends=False, **kw):
@@ -297,6 +336,31 @@ def read_git_lines(args, keepends=False, **kw):
     return read_git_output(args, keepends=True, **kw).splitlines(keepends)
 
 
+def header_encode(text, header_name=None):
+    """Encode and line-wrap the value of an email header field."""
+
+    try:
+        if isinstance(text, str):
+            text = text.decode(ENCODING, 'replace')
+        return Header(text, header_name=header_name).encode()
+    except UnicodeEncodeError:
+        return Header(text, header_name=header_name, charset=CHARSET,
+                      errors='replace').encode()
+
+
+def addr_header_encode(text, header_name=None):
+    """Encode and line-wrap the value of an email header field containing
+    email addresses."""
+
+    return Header(
+        ', '.join(
+            formataddr((header_encode(name), emailaddr))
+            for name, emailaddr in getaddresses([text])
+            ),
+        header_name=header_name
+        ).encode()
+
+
 class Config(object):
     def __init__(self, section, git_config=None):
         """Represent a section of the git configuration.
@@ -578,11 +642,11 @@ class Change(object):
                         % (e.args[0], line,)
                         )
             else:
-                try:
-                    h = Header(value, header_name=name)
-                except UnicodeDecodeError:
-                    h = Header(value, header_name=name, charset=CHARSET, errors='replace')
-                for splitline in ('%s: %s\n' % (name, h.encode(),)).splitlines(True):
+                if name.lower() in ADDR_HEADERS:
+                    value = addr_header_encode(value, name)
+                else:
+                    value = header_encode(value, name)
+                for splitline in ('%s: %s\n' % (name, value)).splitlines(True):
                     yield splitline
 
     def generate_email_header(self):
@@ -616,15 +680,19 @@ class Change(object):
 
         raise NotImplementedError()
 
-    def generate_email(self, push, body_filter=None):
+    def generate_email(self, push, body_filter=None, extra_header_values={}):
         """Generate an email describing this change.
 
         Iterate over the lines (including the header lines) of an
         email describing this change.  If body_filter is not None,
         then use it to filter the lines that are intended for the
-        email body."""
+        email body.
+
+        The extra_header_values field is received as a dict and not as
+        **kwargs, to allow passing other keyword arguments in the
+        future (e.g. passing extra values to generate_email_intro()"""
 
-        for line in self.generate_email_header():
+        for line in self.generate_email_header(**extra_header_values):
             yield line
         yield '\n'
         for line in self.generate_email_intro():
@@ -680,8 +748,10 @@ class Revision(Change):
 
         return values
 
-    def generate_email_header(self):
-        for line in self.expand_header_lines(REVISION_HEADER_TEMPLATE):
+    def generate_email_header(self, **extra_values):
+        for line in self.expand_header_lines(
+            REVISION_HEADER_TEMPLATE, **extra_values
+            ):
             yield line
 
     def generate_email_intro(self):
@@ -692,11 +762,7 @@ class Revision(Change):
         """Show this revision."""
 
         return read_git_lines(
-            [
-                'log', '-C',
-                 '--stat', '-p', '--cc',
-                '-1', self.rev.sha1,
-                ],
+            ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1],
             keepends=True,
             )
 
@@ -800,6 +866,7 @@ class ReferenceChange(Change):
         self.msgid = make_msgid()
         self.diffopts = environment.diffopts
         self.logopts = environment.logopts
+        self.commitlogopts = environment.commitlogopts
         self.showlog = environment.refchange_showlog
 
     def _compute_values(self):
@@ -835,9 +902,12 @@ class ReferenceChange(Change):
             }[self.change_type]
         return self.expand(template)
 
-    def generate_email_header(self):
+    def generate_email_header(self, **extra_values):
+        if 'subject' not in extra_values:
+            extra_values['subject'] = self.get_subject()
+
         for line in self.expand_header_lines(
-            REFCHANGE_HEADER_TEMPLATE, subject=self.get_subject(),
+            REFCHANGE_HEADER_TEMPLATE, **extra_values
             ):
             yield line
 
@@ -1273,7 +1343,7 @@ class Mailer(object):
 
 
 class SendMailer(Mailer):
-    """Send emails using 'sendmail -t'."""
+    """Send emails using 'sendmail -oi -t'."""
 
     SENDMAIL_CANDIDATES = [
         '/usr/sbin/sendmail',
@@ -1302,7 +1372,7 @@ class SendMailer(Mailer):
         if command:
             self.command = command[:]
         else:
-            self.command = [self.find_sendmail(), '-t']
+            self.command = [self.find_sendmail(), '-oi', '-t']
 
         if envelopesender:
             self.command.extend(['-f', envelopesender])
@@ -1495,6 +1565,12 @@ class Environment(object):
             'git log' when generating the detailed log for a set of
             commits (see refchange_showlog)
 
+        commitlogopts (list of strings)
+
+            The options that should be passed to 'git log' for each
+            commit mail.  The value should be a list of strings
+            representing words to be passed to the command.
+
     """
 
     REPO_NAME_RE = re.compile(r'^(?P<name>.+?)(?:\.git)$')
@@ -1506,6 +1582,7 @@ class Environment(object):
         self.diffopts = ['--stat', '--summary', '--find-copies-harder']
         self.logopts = []
         self.refchange_showlog = False
+        self.commitlogopts = ['-C', '--stat', '-p', '--cc']
 
         self.COMPUTED_KEYS = [
             'administrator',
@@ -1672,6 +1749,10 @@ class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin):
         if logopts is not None:
             self.logopts = shlex.split(logopts)
 
+        commitlogopts = config.get('commitlogopts')
+        if commitlogopts is not None:
+            self.commitlogopts = shlex.split(commitlogopts)
+
         reply_to = config.get('replyTo')
         self.__reply_to_refchange = config.get('replyToRefchange', default=reply_to)
         if (
@@ -1829,6 +1910,47 @@ class ConfigMaxlinesEnvironmentMixin(
             )
 
 
+class FQDNEnvironmentMixin(Environment):
+    """A mixin that sets the host's FQDN to its constructor argument."""
+
+    def __init__(self, fqdn, **kw):
+        super(FQDNEnvironmentMixin, self).__init__(**kw)
+        self.COMPUTED_KEYS += ['fqdn']
+        self.__fqdn = fqdn
+
+    def get_fqdn(self):
+        """Return the fully-qualified domain name for this host.
+
+        Return None if it is unavailable or unwanted."""
+
+        return self.__fqdn
+
+
+class ConfigFQDNEnvironmentMixin(
+    ConfigEnvironmentMixin,
+    FQDNEnvironmentMixin,
+    ):
+    """Read the FQDN from the config."""
+
+    def __init__(self, config, **kw):
+        fqdn = config.get('fqdn')
+        super(ConfigFQDNEnvironmentMixin, self).__init__(
+            config=config,
+            fqdn=fqdn,
+            **kw
+            )
+
+
+class ComputeFQDNEnvironmentMixin(FQDNEnvironmentMixin):
+    """Get the FQDN by calling socket.getfqdn()."""
+
+    def __init__(self, **kw):
+        super(ComputeFQDNEnvironmentMixin, self).__init__(
+            fqdn=socket.getfqdn(),
+            **kw
+            )
+
+
 class PusherDomainEnvironmentMixin(ConfigEnvironmentMixin):
     """Deduce pusher_email from pusher by appending an emaildomain."""
 
@@ -1861,6 +1983,10 @@ class StaticRecipientsEnvironmentMixin(Environment):
         # actual *contents* of the change being reported, we only
         # choose based on the *type* of the change.  Therefore we can
         # compute them once and for all:
+        if not (refchange_recipients
+                or announce_recipients
+                or revision_recipients):
+            raise ConfigurationException('No email recipients configured!')
         self.__refchange_recipients = refchange_recipients
         self.__announce_recipients = announce_recipients
         self.__revision_recipients = revision_recipients
@@ -1911,17 +2037,8 @@ class ConfigRecipientsEnvironmentMixin(
             retval = config.get_recipients(name)
             if retval is not None:
                 return retval
-        if len(names) == 1:
-            hint = 'Please set "%s.%s"' % (config.section, name)
         else:
-            hint = (
-                'Please set one of the following:\n    "%s"'
-                % ('"\n    "'.join('%s.%s' % (config.section, name) for name in names))
-                )
-
-        raise ConfigurationException(
-            'The list of recipients for %s is not configured.\n%s' % (names[0], hint)
-            )
+            return ''
 
 
 class ProjectdescEnvironmentMixin(Environment):
@@ -1956,6 +2073,7 @@ class GenericEnvironmentMixin(Environment):
 class GenericEnvironment(
     ProjectdescEnvironmentMixin,
     ConfigMaxlinesEnvironmentMixin,
+    ComputeFQDNEnvironmentMixin,
     ConfigFilterLinesEnvironmentMixin,
     ConfigRecipientsEnvironmentMixin,
     PusherDomainEnvironmentMixin,
@@ -1980,9 +2098,27 @@ class GitoliteEnvironmentMixin(Environment):
         return self.osenv.get('GL_USER', 'unknown user')
 
 
+class IncrementalDateTime(object):
+    """Simple wrapper to give incremental date/times.
+
+    Each call will result in a date/time a second later than the
+    previous call.  This can be used to falsify email headers, to
+    increase the likelihood that email clients sort the emails
+    correctly."""
+
+    def __init__(self):
+        self.time = time.time()
+
+    def next(self):
+        formatted = formatdate(self.time, True)
+        self.time += 1
+        return formatted
+
+
 class GitoliteEnvironment(
     ProjectdescEnvironmentMixin,
     ConfigMaxlinesEnvironmentMixin,
+    ComputeFQDNEnvironmentMixin,
     ConfigFilterLinesEnvironmentMixin,
     ConfigRecipientsEnvironmentMixin,
     PusherDomainEnvironmentMixin,
@@ -2187,6 +2323,7 @@ class Push(object):
         # guarantee that one (and only one) email is generated for
         # each new commit.
         unhandled_sha1s = set(self.get_new_commits())
+        send_date = IncrementalDateTime()
         for change in self.changes:
             # Check if we've got anyone to send to
             if not change.recipients:
@@ -2197,7 +2334,11 @@ class Push(object):
                     )
             else:
                 sys.stderr.write('Sending notification emails to: %s\n' % (change.recipients,))
-                mailer.send(change.generate_email(self, body_filter), change.recipients)
+                extra_values = {'send_date' : send_date.next()}
+                mailer.send(
+                    change.generate_email(self, body_filter, extra_values),
+                    change.recipients,
+                    )
 
             sha1s = []
             for sha1 in reversed(list(self.get_new_commits(change))):
@@ -2217,7 +2358,11 @@ class Push(object):
             for (num, sha1) in enumerate(sha1s):
                 rev = Revision(change, GitObject(sha1), num=num+1, tot=len(sha1s))
                 if rev.recipients:
-                    mailer.send(rev.generate_email(self, body_filter), rev.recipients)
+                    extra_values = {'send_date' : send_date.next()}
+                    mailer.send(
+                        rev.generate_email(self, body_filter, extra_values),
+                        rev.recipients,
+                        )
 
         # Consistency check:
         if unhandled_sha1s:
@@ -2288,6 +2433,7 @@ def choose_environment(config, osenv=None, env=None, recipients=None):
     environment_mixins = [
         ProjectdescEnvironmentMixin,
         ConfigMaxlinesEnvironmentMixin,
+        ComputeFQDNEnvironmentMixin,
         ConfigFilterLinesEnvironmentMixin,
         PusherDomainEnvironmentMixin,
         ConfigOptionsEnvironmentMixin,
diff --git a/contrib/hooks/multimail/post-receive b/contrib/hooks/multimail/post-receive
index 93ebb437d1..4d46828ba5 100755
--- a/contrib/hooks/multimail/post-receive
+++ b/contrib/hooks/multimail/post-receive
@@ -66,10 +66,10 @@ mailer = git_multimail.choose_mailer(config, environment)
 # Alternatively, you may hardcode the mailer using code like one of
 # the following:
 
-# Use "/usr/sbin/sendmail -t" to send emails.  The envelopesender
+# Use "/usr/sbin/sendmail -oi -t" to send emails.  The envelopesender
 # argument is optional:
 #mailer = git_multimail.SendMailer(
-#    command=['/usr/sbin/sendmail', '-t'],
+#    command=['/usr/sbin/sendmail', '-oi', '-t'],
 #    envelopesender='git-repo@example.com',
 #    )