Merge branch 'mh/multimail'

* mh/multimail:
  git-multimail: update to version 1.0.0

1 files changed, 182 insertions, 36 deletions

diff --git a/contrib/hooks/multimail/git_multimail.py b/contrib/hooks/multimail/git_multimail.py
index 81c6a51706..8b58ed6444 100755
--- a/contrib/hooks/multimail/git_multimail.py
+++ b/contrib/hooks/multimail/git_multimail.py
@@ -1,6 +1,6 @@
 #! /usr/bin/env python2
 
-# Copyright (c) 2012,2013 Michael Haggerty
+# Copyright (c) 2012-2014 Michael Haggerty and others
 # Derived from contrib/hooks/post-receive-email, which is
 # Copyright (c) 2007 Andy Parkins
 # and also includes contributions by other authors.
@@ -49,21 +49,25 @@ import sys
 import os
 import re
 import bisect
+import socket
 import subprocess
 import shlex
 import optparse
 import smtplib
+import time
 
 try:
     from email.utils import make_msgid
     from email.utils import getaddresses
     from email.utils import formataddr
+    from email.utils import formatdate
     from email.header import Header
 except ImportError:
     # Prior to Python 2.5, the email module used different names:
     from email.Utils import make_msgid
     from email.Utils import getaddresses
     from email.Utils import formataddr
+    from email.Utils import formatdate
     from email.Header import Header
 
 
@@ -73,6 +77,7 @@ ZEROS = '0' * 40
 LOGBEGIN = '- Log -----------------------------------------------------------------\n'
 LOGEND = '-----------------------------------------------------------------------\n'
 
+ADDR_HEADERS = set(['from', 'to', 'cc', 'bcc', 'reply-to', 'sender'])
 
 # It is assumed in many places that the encoding is uniformly UTF-8,
 # so changing these constants is unsupported.  But define them here
@@ -95,6 +100,7 @@ REF_DELETED_SUBJECT_TEMPLATE = (
     )
 
 REFCHANGE_HEADER_TEMPLATE = """\
+Date: %(send_date)s
 To: %(recipients)s
 Subject: %(subject)s
 MIME-Version: 1.0
@@ -103,6 +109,7 @@ Content-Transfer-Encoding: 8bit
 Message-ID: %(msgid)s
 From: %(fromaddr)s
 Reply-To: %(reply_to)s
+X-Git-Host: %(fqdn)s
 X-Git-Repo: %(repo_shortname)s
 X-Git-Refname: %(refname)s
 X-Git-Reftype: %(refname_type)s
@@ -221,6 +228,7 @@ how to provide full information about this reference change.
 
 
 REVISION_HEADER_TEMPLATE = """\
+Date: %(send_date)s
 To: %(recipients)s
 Subject: %(emailprefix)s%(num)02d/%(tot)02d: %(oneline)s
 MIME-Version: 1.0
@@ -230,6 +238,7 @@ From: %(fromaddr)s
 Reply-To: %(reply_to)s
 In-Reply-To: %(reply_to_msgid)s
 References: %(reply_to_msgid)s
+X-Git-Host: %(fqdn)s
 X-Git-Repo: %(repo_shortname)s
 X-Git-Refname: %(refname)s
 X-Git-Reftype: %(refname_type)s
@@ -263,13 +272,43 @@ class ConfigurationException(Exception):
     pass
 
 
+# The "git" program (this could be changed to include a full path):
+GIT_EXECUTABLE = 'git'
+
+
+# How "git" should be invoked (including global arguments), as a list
+# of words.  This variable is usually initialized automatically by
+# read_git_output() via choose_git_command(), but if a value is set
+# here then it will be used unconditionally.
+GIT_CMD = None
+
+
+def choose_git_command():
+    """Decide how to invoke git, and record the choice in GIT_CMD."""
+
+    global GIT_CMD
+
+    if GIT_CMD is None:
+        try:
+            # Check to see whether the "-c" option is accepted (it was
+            # only added in Git 1.7.2).  We don't actually use the
+            # output of "git --version", though if we needed more
+            # specific version information this would be the place to
+            # do it.
+            cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version']
+            read_output(cmd)
+            GIT_CMD = [GIT_EXECUTABLE, '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)]
+        except CommandError:
+            GIT_CMD = [GIT_EXECUTABLE]
+
+
 def read_git_output(args, input=None, keepends=False, **kw):
     """Read the output of a Git command."""
 
-    return read_output(
-        ['git', '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)] + args,
-        input=input, keepends=keepends, **kw
-        )
+    if GIT_CMD is None:
+        choose_git_command()
+
+    return read_output(GIT_CMD + args, input=input, keepends=keepends, **kw)
 
 
 def read_output(cmd, input=None, keepends=False, **kw):
@@ -297,6 +336,31 @@ def read_git_lines(args, keepends=False, **kw):
     return read_git_output(args, keepends=True, **kw).splitlines(keepends)
 
 
+def header_encode(text, header_name=None):
+    """Encode and line-wrap the value of an email header field."""
+
+    try:
+        if isinstance(text, str):
+            text = text.decode(ENCODING, 'replace')
+        return Header(text, header_name=header_name).encode()
+    except UnicodeEncodeError:
+        return Header(text, header_name=header_name, charset=CHARSET,
+                      errors='replace').encode()
+
+
+def addr_header_encode(text, header_name=None):
+    """Encode and line-wrap the value of an email header field containing
+    email addresses."""
+
+    return Header(
+        ', '.join(
+            formataddr((header_encode(name), emailaddr))
+            for name, emailaddr in getaddresses([text])
+            ),
+        header_name=header_name
+        ).encode()
+
+
 class Config(object):
     def __init__(self, section, git_config=None):
         """Represent a section of the git configuration.
@@ -578,11 +642,11 @@ class Change(object):
                         % (e.args[0], line,)
                         )
             else:
-                try:
-                    h = Header(value, header_name=name)
-                except UnicodeDecodeError:
-                    h = Header(value, header_name=name, charset=CHARSET, errors='replace')
-                for splitline in ('%s: %s\n' % (name, h.encode(),)).splitlines(True):
+                if name.lower() in ADDR_HEADERS:
+                    value = addr_header_encode(value, name)
+                else:
+                    value = header_encode(value, name)
+                for splitline in ('%s: %s\n' % (name, value)).splitlines(True):
                     yield splitline
 
     def generate_email_header(self):
@@ -616,15 +680,19 @@ class Change(object):
 
         raise NotImplementedError()
 
-    def generate_email(self, push, body_filter=None):
+    def generate_email(self, push, body_filter=None, extra_header_values={}):
         """Generate an email describing this change.
 
         Iterate over the lines (including the header lines) of an
         email describing this change.  If body_filter is not None,
         then use it to filter the lines that are intended for the
-        email body."""
+        email body.
+
+        The extra_header_values field is received as a dict and not as
+        **kwargs, to allow passing other keyword arguments in the
+        future (e.g. passing extra values to generate_email_intro()"""
 
-        for line in self.generate_email_header():
+        for line in self.generate_email_header(**extra_header_values):
             yield line
         yield '\n'
         for line in self.generate_email_intro():
@@ -680,8 +748,10 @@ class Revision(Change):
 
         return values
 
-    def generate_email_header(self):
-        for line in self.expand_header_lines(REVISION_HEADER_TEMPLATE):
+    def generate_email_header(self, **extra_values):
+        for line in self.expand_header_lines(
+            REVISION_HEADER_TEMPLATE, **extra_values
+            ):
             yield line
 
     def generate_email_intro(self):
@@ -692,11 +762,7 @@ class Revision(Change):
         """Show this revision."""
 
         return read_git_lines(
-            [
-                'log', '-C',
-                 '--stat', '-p', '--cc',
-                '-1', self.rev.sha1,
-                ],
+            ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1],
             keepends=True,
             )
 
@@ -800,6 +866,7 @@ class ReferenceChange(Change):
         self.msgid = make_msgid()
         self.diffopts = environment.diffopts
         self.logopts = environment.logopts
+        self.commitlogopts = environment.commitlogopts
         self.showlog = environment.refchange_showlog
 
     def _compute_values(self):
@@ -835,9 +902,12 @@ class ReferenceChange(Change):
             }[self.change_type]
         return self.expand(template)
 
-    def generate_email_header(self):
+    def generate_email_header(self, **extra_values):
+        if 'subject' not in extra_values:
+            extra_values['subject'] = self.get_subject()
+
         for line in self.expand_header_lines(
-            REFCHANGE_HEADER_TEMPLATE, subject=self.get_subject(),
+            REFCHANGE_HEADER_TEMPLATE, **extra_values
             ):
             yield line
 
@@ -1273,7 +1343,7 @@ class Mailer(object):
 
 
 class SendMailer(Mailer):
-    """Send emails using 'sendmail -t'."""
+    """Send emails using 'sendmail -oi -t'."""
 
     SENDMAIL_CANDIDATES = [
         '/usr/sbin/sendmail',
@@ -1302,7 +1372,7 @@ class SendMailer(Mailer):
         if command:
             self.command = command[:]
         else:
-            self.command = [self.find_sendmail(), '-t']
+            self.command = [self.find_sendmail(), '-oi', '-t']
 
         if envelopesender:
             self.command.extend(['-f', envelopesender])
@@ -1495,6 +1565,12 @@ class Environment(object):
             'git log' when generating the detailed log for a set of
             commits (see refchange_showlog)
 
+        commitlogopts (list of strings)
+
+            The options that should be passed to 'git log' for each
+            commit mail.  The value should be a list of strings
+            representing words to be passed to the command.
+
     """
 
     REPO_NAME_RE = re.compile(r'^(?P<name>.+?)(?:\.git)$')
@@ -1506,6 +1582,7 @@ class Environment(object):
         self.diffopts = ['--stat', '--summary', '--find-copies-harder']
         self.logopts = []
         self.refchange_showlog = False
+        self.commitlogopts = ['-C', '--stat', '-p', '--cc']
 
         self.COMPUTED_KEYS = [
             'administrator',
@@ -1672,6 +1749,10 @@ class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin):
         if logopts is not None:
             self.logopts = shlex.split(logopts)
 
+        commitlogopts = config.get('commitlogopts')
+        if commitlogopts is not None:
+            self.commitlogopts = shlex.split(commitlogopts)
+
         reply_to = config.get('replyTo')
         self.__reply_to_refchange = config.get('replyToRefchange', default=reply_to)
         if (
@@ -1829,6 +1910,47 @@ class ConfigMaxlinesEnvironmentMixin(
             )
 
 
+class FQDNEnvironmentMixin(Environment):
+    """A mixin that sets the host's FQDN to its constructor argument."""
+
+    def __init__(self, fqdn, **kw):
+        super(FQDNEnvironmentMixin, self).__init__(**kw)
+        self.COMPUTED_KEYS += ['fqdn']
+        self.__fqdn = fqdn
+
+    def get_fqdn(self):
+        """Return the fully-qualified domain name for this host.
+
+        Return None if it is unavailable or unwanted."""
+
+        return self.__fqdn
+
+
+class ConfigFQDNEnvironmentMixin(
+    ConfigEnvironmentMixin,
+    FQDNEnvironmentMixin,
+    ):
+    """Read the FQDN from the config."""
+
+    def __init__(self, config, **kw):
+        fqdn = config.get('fqdn')
+        super(ConfigFQDNEnvironmentMixin, self).__init__(
+            config=config,
+            fqdn=fqdn,
+            **kw
+            )
+
+
+class ComputeFQDNEnvironmentMixin(FQDNEnvironmentMixin):
+    """Get the FQDN by calling socket.getfqdn()."""
+
+    def __init__(self, **kw):
+        super(ComputeFQDNEnvironmentMixin, self).__init__(
+            fqdn=socket.getfqdn(),
+            **kw
+            )
+
+
 class PusherDomainEnvironmentMixin(ConfigEnvironmentMixin):
     """Deduce pusher_email from pusher by appending an emaildomain."""
 
@@ -1861,6 +1983,10 @@ class StaticRecipientsEnvironmentMixin(Environment):
         # actual *contents* of the change being reported, we only
         # choose based on the *type* of the change.  Therefore we can
         # compute them once and for all:
+        if not (refchange_recipients
+                or announce_recipients
+                or revision_recipients):
+            raise ConfigurationException('No email recipients configured!')
         self.__refchange_recipients = refchange_recipients
         self.__announce_recipients = announce_recipients
         self.__revision_recipients = revision_recipients
@@ -1911,17 +2037,8 @@ class ConfigRecipientsEnvironmentMixin(
             retval = config.get_recipients(name)
             if retval is not None:
                 return retval
-        if len(names) == 1:
-            hint = 'Please set "%s.%s"' % (config.section, name)
         else:
-            hint = (
-                'Please set one of the following:\n    "%s"'
-                % ('"\n    "'.join('%s.%s' % (config.section, name) for name in names))
-                )
-
-        raise ConfigurationException(
-            'The list of recipients for %s is not configured.\n%s' % (names[0], hint)
-            )
+            return ''
 
 
 class ProjectdescEnvironmentMixin(Environment):
@@ -1956,6 +2073,7 @@ class GenericEnvironmentMixin(Environment):
 class GenericEnvironment(
     ProjectdescEnvironmentMixin,
     ConfigMaxlinesEnvironmentMixin,
+    ComputeFQDNEnvironmentMixin,
     ConfigFilterLinesEnvironmentMixin,
     ConfigRecipientsEnvironmentMixin,
     PusherDomainEnvironmentMixin,
@@ -1980,9 +2098,27 @@ class GitoliteEnvironmentMixin(Environment):
         return self.osenv.get('GL_USER', 'unknown user')
 
 
+class IncrementalDateTime(object):
+    """Simple wrapper to give incremental date/times.
+
+    Each call will result in a date/time a second later than the
+    previous call.  This can be used to falsify email headers, to
+    increase the likelihood that email clients sort the emails
+    correctly."""
+
+    def __init__(self):
+        self.time = time.time()
+
+    def next(self):
+        formatted = formatdate(self.time, True)
+        self.time += 1
+        return formatted
+
+
 class GitoliteEnvironment(
     ProjectdescEnvironmentMixin,
     ConfigMaxlinesEnvironmentMixin,
+    ComputeFQDNEnvironmentMixin,
     ConfigFilterLinesEnvironmentMixin,
     ConfigRecipientsEnvironmentMixin,
     PusherDomainEnvironmentMixin,
@@ -2187,6 +2323,7 @@ class Push(object):
         # guarantee that one (and only one) email is generated for
         # each new commit.
         unhandled_sha1s = set(self.get_new_commits())
+        send_date = IncrementalDateTime()
         for change in self.changes:
             # Check if we've got anyone to send to
             if not change.recipients:
@@ -2197,7 +2334,11 @@ class Push(object):
                     )
             else:
                 sys.stderr.write('Sending notification emails to: %s\n' % (change.recipients,))
-                mailer.send(change.generate_email(self, body_filter), change.recipients)
+                extra_values = {'send_date' : send_date.next()}
+                mailer.send(
+                    change.generate_email(self, body_filter, extra_values),
+                    change.recipients,
+                    )
 
             sha1s = []
             for sha1 in reversed(list(self.get_new_commits(change))):
@@ -2217,7 +2358,11 @@ class Push(object):
             for (num, sha1) in enumerate(sha1s):
                 rev = Revision(change, GitObject(sha1), num=num+1, tot=len(sha1s))
                 if rev.recipients:
-                    mailer.send(rev.generate_email(self, body_filter), rev.recipients)
+                    extra_values = {'send_date' : send_date.next()}
+                    mailer.send(
+                        rev.generate_email(self, body_filter, extra_values),
+                        rev.recipients,
+                        )
 
         # Consistency check:
         if unhandled_sha1s:
@@ -2288,6 +2433,7 @@ def choose_environment(config, osenv=None, env=None, recipients=None):
     environment_mixins = [
         ProjectdescEnvironmentMixin,
         ConfigMaxlinesEnvironmentMixin,
+        ComputeFQDNEnvironmentMixin,
         ConfigFilterLinesEnvironmentMixin,
         PusherDomainEnvironmentMixin,
         ConfigOptionsEnvironmentMixin,