diff options
Diffstat (limited to 'contrib/hooks/multimail/git_multimail.py')
| -rwxr-xr-x | contrib/hooks/multimail/git_multimail.py | 2517 |
1 files changed, 2108 insertions, 409 deletions
diff --git a/contrib/hooks/multimail/git_multimail.py b/contrib/hooks/multimail/git_multimail.py index 8b58ed6444..c7f86403cf 100755 --- a/contrib/hooks/multimail/git_multimail.py +++ b/contrib/hooks/multimail/git_multimail.py @@ -1,5 +1,8 @@ -#! /usr/bin/env python2 +#! /usr/bin/env python +__version__ = '1.4.0' + +# Copyright (c) 2015-2016 Matthieu Moy and others # Copyright (c) 2012-2014 Michael Haggerty and others # Derived from contrib/hooks/post-receive-email, which is # Copyright (c) 2007 Andy Parkins @@ -53,10 +56,83 @@ import socket import subprocess import shlex import optparse +import logging import smtplib +try: + import ssl +except ImportError: + # Python < 2.6 do not have ssl, but that's OK if we don't use it. + pass import time +import cgi + +PYTHON3 = sys.version_info >= (3, 0) + +if sys.version_info <= (2, 5): + def all(iterable): + for element in iterable: + if not element: + return False + return True + + +def is_ascii(s): + return all(ord(c) < 128 and ord(c) > 0 for c in s) + + +if PYTHON3: + def is_string(s): + return isinstance(s, str) + + def str_to_bytes(s): + return s.encode(ENCODING) + + def bytes_to_str(s, errors='strict'): + return s.decode(ENCODING, errors) + + unicode = str + + def write_str(f, msg): + # Try outputing with the default encoding. If it fails, + # try UTF-8. + try: + f.buffer.write(msg.encode(sys.getdefaultencoding())) + except UnicodeEncodeError: + f.buffer.write(msg.encode(ENCODING)) + + def read_line(f): + # Try reading with the default encoding. If it fails, + # try UTF-8. + out = f.buffer.readline() + try: + return out.decode(sys.getdefaultencoding()) + except UnicodeEncodeError: + return out.decode(ENCODING) +else: + def is_string(s): + try: + return isinstance(s, basestring) + except NameError: # Silence Pyflakes warning + raise + + def str_to_bytes(s): + return s + + def bytes_to_str(s, errors='strict'): + return s + + def write_str(f, msg): + f.write(msg) + + def read_line(f): + return f.readline() + + def next(it): + return it.next() + try: + from email.charset import Charset from email.utils import make_msgid from email.utils import getaddresses from email.utils import formataddr @@ -64,6 +140,7 @@ try: from email.header import Header except ImportError: # Prior to Python 2.5, the email module used different names: + from email.Charset import Charset from email.Utils import make_msgid from email.Utils import getaddresses from email.Utils import formataddr @@ -99,12 +176,16 @@ REF_DELETED_SUBJECT_TEMPLATE = ( ' (was %(oldrev_short)s)' ) +COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE = ( + '%(emailprefix)s%(refname_type)s %(short_refname)s updated: %(oneline)s' + ) + REFCHANGE_HEADER_TEMPLATE = """\ Date: %(send_date)s To: %(recipients)s Subject: %(subject)s MIME-Version: 1.0 -Content-Type: text/plain; charset=%(charset)s +Content-Type: text/%(contenttype)s; charset=%(charset)s Content-Transfer-Encoding: 8bit Message-ID: %(msgid)s From: %(fromaddr)s @@ -115,6 +196,8 @@ X-Git-Refname: %(refname)s X-Git-Reftype: %(refname_type)s X-Git-Oldrev: %(oldrev)s X-Git-Newrev: %(newrev)s +X-Git-NotificationType: ref_changed +X-Git-Multimail-Version: %(multimail_version)s Auto-Submitted: auto-generated """ @@ -143,8 +226,8 @@ reference pointing at a previous point in the repository history. \\ O -- O -- O (%(oldrev_short)s) -Any revisions marked "omits" are not gone; other references still -refer to them. Any revisions marked "discards" are gone forever. +Any revisions marked "omit" are not gone; other references still +refer to them. Any revisions marked "discard" are gone forever. """ @@ -163,8 +246,8 @@ You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. -Any revisions marked "omits" are not gone; other references still -refer to them. Any revisions marked "discards" are gone forever. +Any revisions marked "omit" are not gone; other references still +refer to them. Any revisions marked "discard" are gone forever. """ @@ -188,22 +271,22 @@ from the repository. NEW_REVISIONS_TEMPLATE = """\ The %(tot)s revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions -listed as "adds" were already present in the repository and have only +listed as "add" were already present in the repository and have only been added to this reference. """ TAG_CREATED_TEMPLATE = """\ - at %(newrev_short)-9s (%(newrev_type)s) + at %(newrev_short)-8s (%(newrev_type)s) """ TAG_UPDATED_TEMPLATE = """\ *** WARNING: tag %(short_refname)s was modified! *** - from %(oldrev_short)-9s (%(oldrev_type)s) - to %(newrev_short)-9s (%(newrev_type)s) + from %(oldrev_short)-8s (%(oldrev_type)s) + to %(newrev_short)-8s (%(newrev_type)s) """ @@ -216,7 +299,7 @@ TAG_DELETED_TEMPLATE = """\ # The template used in summary tables. It looks best if this uses the # same alignment as TAG_CREATED_TEMPLATE and TAG_UPDATED_TEMPLATE. BRIEF_SUMMARY_TEMPLATE = """\ -%(action)10s %(rev_short)-9s %(text)s +%(action)8s %(rev_short)-8s %(text)s """ @@ -230,9 +313,10 @@ how to provide full information about this reference change. REVISION_HEADER_TEMPLATE = """\ Date: %(send_date)s To: %(recipients)s +Cc: %(cc_recipients)s Subject: %(emailprefix)s%(num)02d/%(tot)02d: %(oneline)s MIME-Version: 1.0 -Content-Type: text/plain; charset=%(charset)s +Content-Type: text/%(contenttype)s; charset=%(charset)s Content-Transfer-Encoding: 8bit From: %(fromaddr)s Reply-To: %(reply_to)s @@ -243,6 +327,8 @@ X-Git-Repo: %(repo_shortname)s X-Git-Refname: %(refname)s X-Git-Reftype: %(refname_type)s X-Git-Rev: %(rev)s +X-Git-NotificationType: diff +X-Git-Multimail-Version: %(multimail_version)s Auto-Submitted: auto-generated """ @@ -254,10 +340,54 @@ in repository %(repo_shortname)s. """ +LINK_TEXT_TEMPLATE = """\ +View the commit online: +%(browse_url)s + +""" + +LINK_HTML_TEMPLATE = """\ +<p><a href="%(browse_url)s">View the commit online</a>.</p> +""" + REVISION_FOOTER_TEMPLATE = FOOTER_TEMPLATE +# Combined, meaning refchange+revision email (for single-commit additions) +COMBINED_HEADER_TEMPLATE = """\ +Date: %(send_date)s +To: %(recipients)s +Subject: %(subject)s +MIME-Version: 1.0 +Content-Type: text/%(contenttype)s; charset=%(charset)s +Content-Transfer-Encoding: 8bit +Message-ID: %(msgid)s +From: %(fromaddr)s +Reply-To: %(reply_to)s +X-Git-Host: %(fqdn)s +X-Git-Repo: %(repo_shortname)s +X-Git-Refname: %(refname)s +X-Git-Reftype: %(refname_type)s +X-Git-Oldrev: %(oldrev)s +X-Git-Newrev: %(newrev)s +X-Git-Rev: %(rev)s +X-Git-NotificationType: ref_changed_plus_diff +X-Git-Multimail-Version: %(multimail_version)s +Auto-Submitted: auto-generated +""" + +COMBINED_INTRO_TEMPLATE = """\ +This is an automated email from the git hooks/post-receive script. + +%(pusher)s pushed a commit to %(refname_type)s %(short_refname)s +in repository %(repo_shortname)s. + +""" + +COMBINED_FOOTER_TEMPLATE = FOOTER_TEMPLATE + + class CommandError(Exception): def __init__(self, cmd, retcode): self.cmd = cmd @@ -314,12 +444,19 @@ def read_git_output(args, input=None, keepends=False, **kw): def read_output(cmd, input=None, keepends=False, **kw): if input: stdin = subprocess.PIPE + input = str_to_bytes(input) else: stdin = None + errors = 'strict' + if 'errors' in kw: + errors = kw['errors'] + del kw['errors'] p = subprocess.Popen( - cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw + tuple(str_to_bytes(w) for w in cmd), + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw ) (out, err) = p.communicate(input) + out = bytes_to_str(out, errors=errors) retcode = p.wait() if retcode: raise CommandError(cmd, retcode) @@ -336,29 +473,81 @@ def read_git_lines(args, keepends=False, **kw): return read_git_output(args, keepends=True, **kw).splitlines(keepends) +def git_rev_list_ish(cmd, spec, args=None, **kw): + """Common functionality for invoking a 'git rev-list'-like command. + + Parameters: + * cmd is the Git command to run, e.g., 'rev-list' or 'log'. + * spec is a list of revision arguments to pass to the named + command. If None, this function returns an empty list. + * args is a list of extra arguments passed to the named command. + * All other keyword arguments (if any) are passed to the + underlying read_git_lines() function. + + Return the output of the Git command in the form of a list, one + entry per output line. + """ + if spec is None: + return [] + if args is None: + args = [] + args = [cmd, '--stdin'] + args + spec_stdin = ''.join(s + '\n' for s in spec) + return read_git_lines(args, input=spec_stdin, **kw) + + +def git_rev_list(spec, **kw): + """Run 'git rev-list' with the given list of revision arguments. + + See git_rev_list_ish() for parameter and return value + documentation. + """ + return git_rev_list_ish('rev-list', spec, **kw) + + +def git_log(spec, **kw): + """Run 'git log' with the given list of revision arguments. + + See git_rev_list_ish() for parameter and return value + documentation. + """ + return git_rev_list_ish('log', spec, **kw) + + def header_encode(text, header_name=None): """Encode and line-wrap the value of an email header field.""" - try: - if isinstance(text, str): - text = text.decode(ENCODING, 'replace') - return Header(text, header_name=header_name).encode() - except UnicodeEncodeError: - return Header(text, header_name=header_name, charset=CHARSET, - errors='replace').encode() + # Convert to unicode, if required. + if not isinstance(text, unicode): + text = unicode(text, 'utf-8') + + if is_ascii(text): + charset = 'ascii' + else: + charset = 'utf-8' + + return Header(text, header_name=header_name, charset=Charset(charset)).encode() def addr_header_encode(text, header_name=None): """Encode and line-wrap the value of an email header field containing email addresses.""" - return Header( - ', '.join( - formataddr((header_encode(name), emailaddr)) - for name, emailaddr in getaddresses([text]) - ), - header_name=header_name - ).encode() + # Convert to unicode, if required. + if not isinstance(text, unicode): + text = unicode(text, 'utf-8') + + text = ', '.join( + formataddr((header_encode(name), emailaddr)) + for name, emailaddr in getaddresses([text]) + ) + + if is_ascii(text): + charset = 'ascii' + else: + charset = 'utf-8' + + return Header(text, header_name=header_name, charset=Charset(charset)).encode() class Config(object): @@ -385,12 +574,34 @@ class Config(object): assert words[-1] == '' return words[:-1] + @staticmethod + def add_config_parameters(c): + """Add configuration parameters to Git. + + c is either an str or a list of str, each element being of the + form 'var=val' or 'var', with the same syntax and meaning as + the argument of 'git -c var=val'. + """ + if isinstance(c, str): + c = (c,) + parameters = os.environ.get('GIT_CONFIG_PARAMETERS', '') + if parameters: + parameters += ' ' + # git expects GIT_CONFIG_PARAMETERS to be of the form + # "'name1=value1' 'name2=value2' 'name3=value3'" + # including everything inside the double quotes (but not the double + # quotes themselves). Spacing is critical. Also, if a value contains + # a literal single quote that quote must be represented using the + # four character sequence: '\'' + parameters += ' '.join("'" + x.replace("'", "'\\''") + "'" for x in c) + os.environ['GIT_CONFIG_PARAMETERS'] = parameters + def get(self, name, default=None): try: values = self._split(read_git_output( - ['config', '--get', '--null', '%s.%s' % (self.section, name)], - env=self.env, keepends=True, - )) + ['config', '--get', '--null', '%s.%s' % (self.section, name)], + env=self.env, keepends=True, + )) assert len(values) == 1 return values[0] except CommandError: @@ -417,7 +628,8 @@ class Config(object): ['config', '--get-all', '--null', '%s.%s' % (self.section, name)], env=self.env, keepends=True, )) - except CommandError, e: + except CommandError: + t, e, traceback = sys.exc_info() if e.retcode == 1: # "the section or key is invalid"; i.e., there is no # value for the specified key. @@ -425,18 +637,6 @@ class Config(object): else: raise - def get_recipients(self, name, default=None): - """Read a recipients list from the configuration. - - Return the result as a comma-separated list of email - addresses, or default if the option is unset. If the setting - has multiple values, concatenate them with comma separators.""" - - lines = self.get_all(name, default=None) - if lines is None: - return default - return ', '.join(line.strip() for line in lines) - def set(self, name, value): read_git_output( ['config', '%s.%s' % (self.section, name), value], @@ -449,16 +649,22 @@ class Config(object): env=self.env, ) - def has_key(self, name): + def __contains__(self, name): return self.get_all(name, default=None) is not None + # We don't use this method anymore internally, but keep it here in + # case somebody is calling it from their own code: + def has_key(self, name): + return name in self + def unset_all(self, name): try: read_git_output( ['config', '--unset-all', '%s.%s' % (self.section, name)], env=self.env, ) - except CommandError, e: + except CommandError: + t, e, traceback = sys.exc_info() if e.retcode == 5: # The name doesn't exist, which is what we wanted anyway... pass @@ -552,7 +758,7 @@ class GitObject(object): if not self.sha1: raise ValueError('Empty commit has no summary') - return iter(generate_summaries('--no-walk', self.sha1)).next() + return next(iter(generate_summaries('--no-walk', self.sha1))) def __eq__(self, other): return isinstance(other, GitObject) and self.sha1 == other.sha1 @@ -563,6 +769,10 @@ class GitObject(object): def __nonzero__(self): return bool(self.sha1) + def __bool__(self): + """Python 2 backward compatibility""" + return self.__nonzero__() + def __str__(self): return self.sha1 or ZEROS @@ -577,19 +787,36 @@ class Change(object): def __init__(self, environment): self.environment = environment self._values = None + self._contains_html_diff = False + + def _contains_diff(self): + # We do contain a diff, should it be rendered in HTML? + if self.environment.commit_email_format == "html": + self._contains_html_diff = True def _compute_values(self): - """Return a dictionary {keyword : expansion} for this Change. + """Return a dictionary {keyword: expansion} for this Change. Derived classes overload this method to add more entries to the return value. This method is used internally by get_values(). The return value should always be a new dictionary.""" - return self.environment.get_values() + values = self.environment.get_values() + fromaddr = self.environment.get_fromaddr(change=self) + if fromaddr is not None: + values['fromaddr'] = fromaddr + values['multimail_version'] = get_version() + return values + + # Aliases usable in template strings. Tuple of pairs (destination, + # source). + VALUES_ALIAS = ( + ("id", "newrev"), + ) def get_values(self, **extra_values): - """Return a dictionary {keyword : expansion} for this Change. + """Return a dictionary {keyword: expansion} for this Change. Return a dictionary mapping keywords to the values that they should be expanded to for this Change (used when interpolating @@ -603,6 +830,9 @@ class Change(object): values = self._values.copy() if extra_values: values.update(extra_values) + + for alias, val in self.VALUES_ALIAS: + values[alias] = values[val] return values def expand(self, template, **extra_values): @@ -615,10 +845,14 @@ class Change(object): return template % self.get_values(**extra_values) - def expand_lines(self, template, **extra_values): + def expand_lines(self, template, html_escape_val=False, **extra_values): """Break template into lines and expand each line.""" values = self.get_values(**extra_values) + if html_escape_val: + for k in values: + if is_string(values[k]): + values[k] = cgi.escape(values[k], True) for line in template.splitlines(True): yield line % values @@ -629,14 +863,21 @@ class Change(object): skip lines that contain references to unknown variables.""" values = self.get_values(**extra_values) + if self._contains_html_diff: + self._content_type = 'html' + else: + self._content_type = 'plain' + values['contenttype'] = self._content_type + for line in template.splitlines(): - (name, value) = line.split(':', 1) + (name, value) = line.split(': ', 1) try: value = value % values - except KeyError, e: + except KeyError: + t, e, traceback = sys.exc_info() if DEBUG: - sys.stderr.write( + self.environment.log_warning( 'Warning: unknown variable %r in the following line; line skipped:\n' ' %s\n' % (e.args[0], line,) @@ -656,7 +897,11 @@ class Change(object): raise NotImplementedError() - def generate_email_intro(self): + def generate_browse_link(self, base_url): + """Generate a link to an online repository browser.""" + return iter(()) + + def generate_email_intro(self, html_escape_val=False): """Generate the email intro for this Change, a line at a time. The output will be used as the standard boilerplate at the top @@ -672,7 +917,7 @@ class Change(object): raise NotImplementedError() - def generate_email_footer(self): + def generate_email_footer(self, html_escape_val): """Generate the footer of the email, a line at a time. The footer is always included, irrespective of @@ -680,6 +925,24 @@ class Change(object): raise NotImplementedError() + def _wrap_for_html(self, lines): + """Wrap the lines in HTML <pre> tag when using HTML format. + + Escape special HTML characters and add <pre> and </pre> tags around + the given lines if we should be generating HTML as indicated by + self._contains_html_diff being set to true. + """ + if self._contains_html_diff: + yield "<pre style='margin:0'>\n" + + for line in lines: + yield cgi.escape(line) + + yield '</pre>\n' + else: + for line in lines: + yield line + def generate_email(self, push, body_filter=None, extra_header_values={}): """Generate an email describing this change. @@ -695,22 +958,97 @@ class Change(object): for line in self.generate_email_header(**extra_header_values): yield line yield '\n' - for line in self.generate_email_intro(): + html_escape_val = (self.environment.html_in_intro and + self._contains_html_diff) + intro = self.generate_email_intro(html_escape_val) + if not self.environment.html_in_intro: + intro = self._wrap_for_html(intro) + for line in intro: yield line + if self.environment.commitBrowseURL: + for line in self.generate_browse_link(self.environment.commitBrowseURL): + yield line + body = self.generate_email_body(push) if body_filter is not None: body = body_filter(body) + + diff_started = False + if self._contains_html_diff: + # "white-space: pre" is the default, but we need to + # specify it again in case the message is viewed in a + # webmail which wraps it in an element setting white-space + # to something else (Zimbra does this and sets + # white-space: pre-line). + yield '<pre style="white-space: pre; background: #F8F8F8">' for line in body: - yield line + if self._contains_html_diff: + # This is very, very naive. It would be much better to really + # parse the diff, i.e. look at how many lines do we have in + # the hunk headers instead of blindly highlighting everything + # that looks like it might be part of a diff. + bgcolor = '' + fgcolor = '' + if line.startswith('--- a/'): + diff_started = True + bgcolor = 'e0e0ff' + elif line.startswith('diff ') or line.startswith('index '): + diff_started = True + fgcolor = '808080' + elif diff_started: + if line.startswith('+++ '): + bgcolor = 'e0e0ff' + elif line.startswith('@@'): + bgcolor = 'e0e0e0' + elif line.startswith('+'): + bgcolor = 'e0ffe0' + elif line.startswith('-'): + bgcolor = 'ffe0e0' + elif line.startswith('commit '): + fgcolor = '808000' + elif line.startswith(' '): + fgcolor = '404040' + + # Chop the trailing LF, we don't want it inside <pre>. + line = cgi.escape(line[:-1]) + + if bgcolor or fgcolor: + style = 'display:block; white-space:pre;' + if bgcolor: + style += 'background:#' + bgcolor + ';' + if fgcolor: + style += 'color:#' + fgcolor + ';' + # Use a <span style='display:block> to color the + # whole line. The newline must be inside the span + # to display properly both in Firefox and in + # text-based browser. + line = "<span style='%s'>%s\n</span>" % (style, line) + else: + line = line + '\n' - for line in self.generate_email_footer(): yield line + if self._contains_html_diff: + yield '</pre>' + html_escape_val = (self.environment.html_in_footer and + self._contains_html_diff) + footer = self.generate_email_footer(html_escape_val) + if not self.environment.html_in_footer: + footer = self._wrap_for_html(footer) + for line in footer: + yield line + + def get_specific_fromaddr(self): + """For kinds of Changes which specify it, return the kind-specific + From address to use.""" + return None class Revision(Change): """A Change consisting of a single git commit.""" + CC_RE = re.compile(r'^\s*C[Cc]:\s*(?P<to>[^#]+@[^\s#]*)\s*(#.*)?$') + def __init__(self, reference_change, rev, num, tot): Change.__init__(self, reference_change.environment) self.reference_change = reference_change @@ -722,6 +1060,24 @@ class Revision(Change): self.author = read_git_output(['log', '--no-walk', '--format=%aN <%aE>', self.rev.sha1]) self.recipients = self.environment.get_revision_recipients(self) + self.cc_recipients = '' + if self.environment.get_scancommitforcc(): + self.cc_recipients = ', '.join(to.strip() for to in self._cc_recipients()) + if self.cc_recipients: + self.environment.log_msg( + 'Add %s to CC for %s' % (self.cc_recipients, self.rev.sha1)) + + def _cc_recipients(self): + cc_recipients = [] + message = read_git_output(['log', '--no-walk', '--format=%b', self.rev.sha1]) + lines = message.strip().split('\n') + for line in lines: + m = re.match(self.CC_RE, line) + if m: + cc_recipients.append(m.group('to')) + + return cc_recipients + def _compute_values(self): values = Change._compute_values(self) @@ -729,16 +1085,23 @@ class Revision(Change): ['log', '--format=%s', '--no-walk', self.rev.sha1] ) + max_subject_length = self.environment.get_max_subject_length() + if max_subject_length > 0 and len(oneline) > max_subject_length: + oneline = oneline[:max_subject_length - 6] + ' [...]' + values['rev'] = self.rev.sha1 values['rev_short'] = self.rev.short values['change_type'] = self.change_type values['refname'] = self.refname + values['newrev'] = self.rev.sha1 values['short_refname'] = self.reference_change.short_refname values['refname_type'] = self.reference_change.refname_type values['reply_to_msgid'] = self.reference_change.msgid values['num'] = self.num values['tot'] = self.tot values['recipients'] = self.recipients + if self.cc_recipients: + values['cc_recipients'] = self.cc_recipients values['oneline'] = oneline values['author'] = self.author @@ -750,24 +1113,54 @@ class Revision(Change): def generate_email_header(self, **extra_values): for line in self.expand_header_lines( - REVISION_HEADER_TEMPLATE, **extra_values - ): + REVISION_HEADER_TEMPLATE, **extra_values + ): yield line - def generate_email_intro(self): - for line in self.expand_lines(REVISION_INTRO_TEMPLATE): + def generate_browse_link(self, base_url): + if '%(' not in base_url: + base_url += '%(id)s' + url = "".join(self.expand_lines(base_url)) + if self._content_type == 'html': + for line in self.expand_lines(LINK_HTML_TEMPLATE, + html_escape_val=True, + browse_url=url): + yield line + elif self._content_type == 'plain': + for line in self.expand_lines(LINK_TEXT_TEMPLATE, + html_escape_val=False, + browse_url=url): + yield line + else: + raise NotImplementedError("Content-type %s unsupported. Please report it as a bug.") + + def generate_email_intro(self, html_escape_val=False): + for line in self.expand_lines(REVISION_INTRO_TEMPLATE, + html_escape_val=html_escape_val): yield line def generate_email_body(self, push): """Show this revision.""" - return read_git_lines( - ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1], - keepends=True, - ) + for line in read_git_lines( + ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1], + keepends=True, + errors='replace'): + if line.startswith('Date: ') and self.environment.date_substitute: + yield self.environment.date_substitute + line[len('Date: '):] + else: + yield line - def generate_email_footer(self): - return self.expand_lines(REVISION_FOOTER_TEMPLATE) + def generate_email_footer(self, html_escape_val): + return self.expand_lines(REVISION_FOOTER_TEMPLATE, + html_escape_val=html_escape_val) + + def generate_email(self, push, body_filter=None, extra_header_values={}): + self._contains_diff() + return Change.generate_email(self, push, body_filter, extra_header_values) + + def get_specific_fromaddr(self): + return self.environment.from_commit class ReferenceChange(Change): @@ -822,26 +1215,26 @@ class ReferenceChange(Change): klass = BranchChange elif area == 'remotes': # Tracking branch: - sys.stderr.write( + environment.log_warning( '*** Push-update of tracking branch %r\n' - '*** - incomplete email generated.\n' - % (refname,) + '*** - incomplete email generated.' + % (refname,) ) klass = OtherReferenceChange else: # Some other reference namespace: - sys.stderr.write( + environment.log_warning( '*** Push-update of strange reference %r\n' - '*** - incomplete email generated.\n' - % (refname,) + '*** - incomplete email generated.' + % (refname,) ) klass = OtherReferenceChange else: # Anything else (is there anything else?) - sys.stderr.write( + environment.log_warning( '*** Unknown type of update to %r (%s)\n' - '*** - incomplete email generated.\n' - % (refname, rev.type,) + '*** - incomplete email generated.' + % (refname, rev.type,) ) klass = OtherReferenceChange @@ -854,9 +1247,9 @@ class ReferenceChange(Change): def __init__(self, environment, refname, short_refname, old, new, rev): Change.__init__(self, environment) self.change_type = { - (False, True) : 'create', - (True, True) : 'update', - (True, False) : 'delete', + (False, True): 'create', + (True, True): 'update', + (True, False): 'delete', }[bool(old), bool(new)] self.refname = refname self.short_refname = short_refname @@ -865,10 +1258,16 @@ class ReferenceChange(Change): self.rev = rev self.msgid = make_msgid() self.diffopts = environment.diffopts + self.graphopts = environment.graphopts self.logopts = environment.logopts self.commitlogopts = environment.commitlogopts + self.showgraph = environment.refchange_showgraph self.showlog = environment.refchange_showlog + self.header_template = REFCHANGE_HEADER_TEMPLATE + self.intro_template = REFCHANGE_INTRO_TEMPLATE + self.footer_template = FOOTER_TEMPLATE + def _compute_values(self): values = Change._compute_values(self) @@ -894,11 +1293,39 @@ class ReferenceChange(Change): return values + def send_single_combined_email(self, known_added_sha1s): + """Determine if a combined refchange/revision email should be sent + + If there is only a single new (non-merge) commit added by a + change, it is useful to combine the ReferenceChange and + Revision emails into one. In such a case, return the single + revision; otherwise, return None. + + This method is overridden in BranchChange.""" + + return None + + def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}): + """Generate an email describing this change AND specified revision. + + Iterate over the lines (including the header lines) of an + email describing this change. If body_filter is not None, + then use it to filter the lines that are intended for the + email body. + + The extra_header_values field is received as a dict and not as + **kwargs, to allow passing other keyword arguments in the + future (e.g. passing extra values to generate_email_intro() + + This method is overridden in BranchChange.""" + + raise NotImplementedError + def get_subject(self): template = { - 'create' : REF_CREATED_SUBJECT_TEMPLATE, - 'update' : REF_UPDATED_SUBJECT_TEMPLATE, - 'delete' : REF_DELETED_SUBJECT_TEMPLATE, + 'create': REF_CREATED_SUBJECT_TEMPLATE, + 'update': REF_UPDATED_SUBJECT_TEMPLATE, + 'delete': REF_DELETED_SUBJECT_TEMPLATE, }[self.change_type] return self.expand(template) @@ -907,12 +1334,13 @@ class ReferenceChange(Change): extra_values['subject'] = self.get_subject() for line in self.expand_header_lines( - REFCHANGE_HEADER_TEMPLATE, **extra_values - ): + self.header_template, **extra_values + ): yield line - def generate_email_intro(self): - for line in self.expand_lines(REFCHANGE_INTRO_TEMPLATE): + def generate_email_intro(self, html_escape_val=False): + for line in self.expand_lines(self.intro_template, + html_escape_val=html_escape_val): yield line def generate_email_body(self, push): @@ -922,9 +1350,9 @@ class ReferenceChange(Change): generate_update_summary() / generate_delete_summary().""" change_summary = { - 'create' : self.generate_create_summary, - 'delete' : self.generate_delete_summary, - 'update' : self.generate_update_summary, + 'create': self.generate_create_summary, + 'delete': self.generate_delete_summary, + 'update': self.generate_update_summary, }[self.change_type](push) for line in change_summary: yield line @@ -932,22 +1360,47 @@ class ReferenceChange(Change): for line in self.generate_revision_change_summary(push): yield line - def generate_email_footer(self): - return self.expand_lines(FOOTER_TEMPLATE) + def generate_email_footer(self, html_escape_val): + return self.expand_lines(self.footer_template, + html_escape_val=html_escape_val) + + def generate_revision_change_graph(self, push): + if self.showgraph: + args = ['--graph'] + self.graphopts + for newold in ('new', 'old'): + has_newold = False + spec = push.get_commits_spec(newold, self) + for line in git_log(spec, args=args, keepends=True): + if not has_newold: + has_newold = True + yield '\n' + yield 'Graph of %s commits:\n\n' % ( + {'new': 'new', 'old': 'discarded'}[newold],) + yield ' ' + line + if has_newold: + yield '\n' def generate_revision_change_log(self, new_commits_list): if self.showlog: yield '\n' yield 'Detailed log of new commits:\n\n' for line in read_git_lines( - ['log', '--no-walk'] - + self.logopts - + new_commits_list - + ['--'], + ['log', '--no-walk'] + + self.logopts + + new_commits_list + + ['--'], keepends=True, - ): + ): yield line + def generate_new_revision_summary(self, tot, new_commits_list, push): + for line in self.expand_lines(NEW_REVISIONS_TEMPLATE, tot=tot): + yield line + for line in self.generate_revision_change_graph(push): + yield line + for line in self.generate_revision_change_log(new_commits_list): + yield line + def generate_revision_change_summary(self, push): """Generate a summary of the revisions added/removed by this change.""" @@ -960,7 +1413,7 @@ class ReferenceChange(Change): sha1s.reverse() tot = len(sha1s) new_revisions = [ - Revision(self, GitObject(sha1), num=i+1, tot=tot) + Revision(self, GitObject(sha1), num=i + 1, tot=tot) for (i, sha1) in enumerate(sha1s) ] @@ -973,9 +1426,8 @@ class ReferenceChange(Change): BRIEF_SUMMARY_TEMPLATE, action='new', text=subject, ) yield '\n' - for line in self.expand_lines(NEW_REVISIONS_TEMPLATE, tot=tot): - yield line - for line in self.generate_revision_change_log([r.rev.sha1 for r in new_revisions]): + for line in self.generate_new_revision_summary( + tot, [r.rev.sha1 for r in new_revisions], push): yield line else: for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE): @@ -993,16 +1445,16 @@ class ReferenceChange(Change): # revisions in the summary even though we will not send # new notification emails for them. adds = list(generate_summaries( - '--topo-order', '--reverse', '%s..%s' - % (self.old.commit_sha1, self.new.commit_sha1,) - )) + '--topo-order', '--reverse', '%s..%s' + % (self.old.commit_sha1, self.new.commit_sha1,) + )) # List of the revisions that were removed from the branch # by this update. This will be empty except for # non-fast-forward updates. discards = list(generate_summaries( - '%s..%s' % (self.new.commit_sha1, self.old.commit_sha1,) - )) + '%s..%s' % (self.new.commit_sha1, self.old.commit_sha1,) + )) if adds: new_commits_list = push.get_new_commits(self) @@ -1018,9 +1470,9 @@ class ReferenceChange(Change): if discards and adds: for (sha1, subject) in discards: if sha1 in discarded_commits: - action = 'discards' + action = 'discard' else: - action = 'omits' + action = 'omit' yield self.expand( BRIEF_SUMMARY_TEMPLATE, action=action, rev_short=sha1, text=subject, @@ -1029,7 +1481,7 @@ class ReferenceChange(Change): if sha1 in new_commits: action = 'new' else: - action = 'adds' + action = 'add' yield self.expand( BRIEF_SUMMARY_TEMPLATE, action=action, rev_short=sha1, text=subject, @@ -1041,9 +1493,9 @@ class ReferenceChange(Change): elif discards: for (sha1, subject) in discards: if sha1 in discarded_commits: - action = 'discards' + action = 'discard' else: - action = 'omits' + action = 'omit' yield self.expand( BRIEF_SUMMARY_TEMPLATE, action=action, rev_short=sha1, text=subject, @@ -1062,7 +1514,7 @@ class ReferenceChange(Change): if sha1 in new_commits: action = 'new' else: - action = 'adds' + action = 'add' yield self.expand( BRIEF_SUMMARY_TEMPLATE, action=action, rev_short=sha1, text=subject, @@ -1071,13 +1523,14 @@ class ReferenceChange(Change): yield '\n' if new_commits: - for line in self.expand_lines(NEW_REVISIONS_TEMPLATE, tot=len(new_commits)): - yield line - for line in self.generate_revision_change_log(new_commits_list): + for line in self.generate_new_revision_summary( + len(new_commits), new_commits_list, push): yield line else: for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE): yield line + for line in self.generate_revision_change_graph(push): + yield line # The diffstat is shown from the old revision to the new # revision. This is to show the truth of what happened in @@ -1089,11 +1542,11 @@ class ReferenceChange(Change): yield '\n' yield 'Summary of changes:\n' for line in read_git_lines( - ['diff-tree'] - + self.diffopts - + ['%s..%s' % (self.old.commit_sha1, self.new.commit_sha1,)], - keepends=True, - ): + ['diff-tree'] + + self.diffopts + + ['%s..%s' % (self.old.commit_sha1, self.new.commit_sha1,)], + keepends=True, + ): yield line elif self.old.commit_sha1 and not self.new.commit_sha1: @@ -1103,7 +1556,7 @@ class ReferenceChange(Change): sha1s = list(push.get_discarded_commits(self)) tot = len(sha1s) discarded_revisions = [ - Revision(self, GitObject(sha1), num=i+1, tot=tot) + Revision(self, GitObject(sha1), num=i + 1, tot=tot) for (i, sha1) in enumerate(sha1s) ] @@ -1114,8 +1567,10 @@ class ReferenceChange(Change): for r in discarded_revisions: (sha1, subject) = r.rev.get_summary() yield r.expand( - BRIEF_SUMMARY_TEMPLATE, action='discards', text=subject, + BRIEF_SUMMARY_TEMPLATE, action='discard', text=subject, ) + for line in self.generate_revision_change_graph(push): + yield line else: for line in self.expand_lines(NO_DISCARDED_REVISIONS_TEMPLATE): yield line @@ -1150,6 +1605,9 @@ class ReferenceChange(Change): ) yield '\n' + def get_specific_fromaddr(self): + return self.environment.from_refchange + class BranchChange(ReferenceChange): refname_type = 'branch' @@ -1161,6 +1619,159 @@ class BranchChange(ReferenceChange): old=old, new=new, rev=rev, ) self.recipients = environment.get_refchange_recipients(self) + self._single_revision = None + + def send_single_combined_email(self, known_added_sha1s): + if not self.environment.combine_when_single_commit: + return None + + # In the sadly-all-too-frequent usecase of people pushing only + # one of their commits at a time to a repository, users feel + # the reference change summary emails are noise rather than + # important signal. This is because, in this particular + # usecase, there is a reference change summary email for each + # new commit, and all these summaries do is point out that + # there is one new commit (which can readily be inferred by + # the existence of the individual revision email that is also + # sent). In such cases, our users prefer there to be a combined + # reference change summary/new revision email. + # + # So, if the change is an update and it doesn't discard any + # commits, and it adds exactly one non-merge commit (gerrit + # forces a workflow where every commit is individually merged + # and the git-multimail hook fired off for just this one + # change), then we send a combined refchange/revision email. + try: + # If this change is a reference update that doesn't discard + # any commits... + if self.change_type != 'update': + return None + + if read_git_lines( + ['merge-base', self.old.sha1, self.new.sha1] + ) != [self.old.sha1]: + return None + + # Check if this update introduced exactly one non-merge + # commit: + + def split_line(line): + """Split line into (sha1, [parent,...]).""" + + words = line.split() + return (words[0], words[1:]) + + # Get the new commits introduced by the push as a list of + # (sha1, [parent,...]) + new_commits = [ + split_line(line) + for line in read_git_lines( + [ + 'log', '-3', '--format=%H %P', + '%s..%s' % (self.old.sha1, self.new.sha1), + ] + ) + ] + + if not new_commits: + return None + + # If the newest commit is a merge, save it for a later check + # but otherwise ignore it + merge = None + tot = len(new_commits) + if len(new_commits[0][1]) > 1: + merge = new_commits[0][0] + del new_commits[0] + + # Our primary check: we can't combine if more than one commit + # is introduced. We also currently only combine if the new + # commit is a non-merge commit, though it may make sense to + # combine if it is a merge as well. + if not ( + len(new_commits) == 1 and + len(new_commits[0][1]) == 1 and + new_commits[0][0] in known_added_sha1s + ): + return None + + # We do not want to combine revision and refchange emails if + # those go to separate locations. + rev = Revision(self, GitObject(new_commits[0][0]), 1, tot) + if rev.recipients != self.recipients: + return None + + # We ignored the newest commit if it was just a merge of the one + # commit being introduced. But we don't want to ignore that + # merge commit it it involved conflict resolutions. Check that. + if merge and merge != read_git_output(['diff-tree', '--cc', merge]): + return None + + # We can combine the refchange and one new revision emails + # into one. Return the Revision that a combined email should + # be sent about. + return rev + except CommandError: + # Cannot determine number of commits in old..new or new..old; + # don't combine reference/revision emails: + return None + + def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}): + values = revision.get_values() + if extra_header_values: + values.update(extra_header_values) + if 'subject' not in extra_header_values: + values['subject'] = self.expand(COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE, **values) + + self._single_revision = revision + self._contains_diff() + self.header_template = COMBINED_HEADER_TEMPLATE + self.intro_template = COMBINED_INTRO_TEMPLATE + self.footer_template = COMBINED_FOOTER_TEMPLATE + + def revision_gen_link(base_url): + # revision is used only to generate the body, and + # _content_type is set while generating headers. Get it + # from the BranchChange object. + revision._content_type = self._content_type + return revision.generate_browse_link(base_url) + self.generate_browse_link = revision_gen_link + for line in self.generate_email(push, body_filter, values): + yield line + + def generate_email_body(self, push): + '''Call the appropriate body generation routine. + + If this is a combined refchange/revision email, the special logic + for handling this combined email comes from this function. For + other cases, we just use the normal handling.''' + + # If self._single_revision isn't set; don't override + if not self._single_revision: + for line in super(BranchChange, self).generate_email_body(push): + yield line + return + + # This is a combined refchange/revision email; we first provide + # some info from the refchange portion, and then call the revision + # generate_email_body function to handle the revision portion. + adds = list(generate_summaries( + '--topo-order', '--reverse', '%s..%s' + % (self.old.commit_sha1, self.new.commit_sha1,) + )) + + yield self.expand("The following commit(s) were added to %(refname)s by this push:\n") + for (sha1, subject) in adds: + yield self.expand( + BRIEF_SUMMARY_TEMPLATE, action='new', + rev_short=sha1, text=subject, + ) + + yield self._single_revision.rev.short + " is described below\n" + yield '\n' + + for line in self._single_revision.generate_email_body(push): + yield line class AnnotatedTagChange(ReferenceChange): @@ -1204,13 +1815,13 @@ class AnnotatedTagChange(ReferenceChange): except CommandError: prevtag = None if prevtag: - yield ' replaces %s\n' % (prevtag,) + yield ' replaces %s\n' % (prevtag,) else: prevtag = None - yield ' length %s bytes\n' % (read_git_output(['cat-file', '-s', tagobject]),) + yield ' length %s bytes\n' % (read_git_output(['cat-file', '-s', tagobject]),) - yield ' tagged by %s\n' % (tagger,) - yield ' on %s\n' % (tagged,) + yield ' by %s\n' % (tagger,) + yield ' on %s\n' % (tagged,) yield '\n' # Show the content of the tag message; this might contain a @@ -1327,6 +1938,9 @@ class OtherReferenceChange(ReferenceChange): class Mailer(object): """An object that can send emails.""" + def __init__(self, environment): + self.environment = environment + def send(self, lines, to_addrs): """Send an email consisting of lines. @@ -1361,14 +1975,14 @@ class SendMailer(Mailer): 'Try setting multimailhook.sendmailCommand.' ) - def __init__(self, command=None, envelopesender=None): + def __init__(self, environment, command=None, envelopesender=None): """Construct a SendMailer instance. command should be the command and arguments used to invoke sendmail, as a list of strings. If an envelopesender is provided, it will also be passed to the command, via '-f envelopesender'.""" - + super(SendMailer, self).__init__(environment) if command: self.command = command[:] else: @@ -1380,22 +1994,28 @@ class SendMailer(Mailer): def send(self, lines, to_addrs): try: p = subprocess.Popen(self.command, stdin=subprocess.PIPE) - except OSError, e: - sys.stderr.write( - '*** Cannot execute command: %s\n' % ' '.join(self.command) - + '*** %s\n' % str(e) - + '*** Try setting multimailhook.mailer to "smtp"\n' + except OSError: + self.environment.get_logger().error( + '*** Cannot execute command: %s\n' % ' '.join(self.command) + + '*** %s\n' % sys.exc_info()[1] + + '*** Try setting multimailhook.mailer to "smtp"\n' + '*** to send emails without using the sendmail command.\n' ) sys.exit(1) try: + lines = (str_to_bytes(line) for line in lines) p.stdin.writelines(lines) - except: - sys.stderr.write( + except Exception: + self.environment.get_logger().error( '*** Error while generating commit email\n' '*** - mail sending aborted.\n' ) - p.terminate() + if hasattr(p, 'terminate'): + # subprocess.terminate() is not available in Python 2.4 + p.terminate() + else: + import signal + os.kill(p.pid, signal.SIGTERM) raise else: p.stdin.close() @@ -1407,36 +2027,140 @@ class SendMailer(Mailer): class SMTPMailer(Mailer): """Send emails using Python's smtplib.""" - def __init__(self, envelopesender, smtpserver): + def __init__(self, environment, + envelopesender, smtpserver, + smtpservertimeout=10.0, smtpserverdebuglevel=0, + smtpencryption='none', + smtpuser='', smtppass='', + smtpcacerts='' + ): + super(SMTPMailer, self).__init__(environment) if not envelopesender: - sys.stderr.write( + self.environment.get_logger().error( 'fatal: git_multimail: cannot use SMTPMailer without a sender address.\n' 'please set either multimailhook.envelopeSender or user.email\n' ) sys.exit(1) + if smtpencryption == 'ssl' and not (smtpuser and smtppass): + raise ConfigurationException( + 'Cannot use SMTPMailer with security option ssl ' + 'without options username and password.' + ) self.envelopesender = envelopesender self.smtpserver = smtpserver + self.smtpservertimeout = smtpservertimeout + self.smtpserverdebuglevel = smtpserverdebuglevel + self.security = smtpencryption + self.username = smtpuser + self.password = smtppass + self.smtpcacerts = smtpcacerts try: - self.smtp = smtplib.SMTP(self.smtpserver) - except Exception, e: - sys.stderr.write('*** Error establishing SMTP connection to %s***\n' % self.smtpserver) - sys.stderr.write('*** %s\n' % str(e)) + def call(klass, server, timeout): + try: + return klass(server, timeout=timeout) + except TypeError: + # Old Python versions do not have timeout= argument. + return klass(server) + if self.security == 'none': + self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout) + elif self.security == 'ssl': + if self.smtpcacerts: + raise smtplib.SMTPException( + "Checking certificate is not supported for ssl, prefer starttls" + ) + self.smtp = call(smtplib.SMTP_SSL, self.smtpserver, timeout=self.smtpservertimeout) + elif self.security == 'tls': + if 'ssl' not in sys.modules: + self.environment.get_logger().error( + '*** Your Python version does not have the ssl library installed\n' + '*** smtpEncryption=tls is not available.\n' + '*** Either upgrade Python to 2.6 or later\n' + ' or use git_multimail.py version 1.2.\n') + if ':' not in self.smtpserver: + self.smtpserver += ':587' # default port for TLS + self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout) + # start: ehlo + starttls + # equivalent to + # self.smtp.ehlo() + # self.smtp.starttls() + # with acces to the ssl layer + self.smtp.ehlo() + if not self.smtp.has_extn("starttls"): + raise smtplib.SMTPException("STARTTLS extension not supported by server") + resp, reply = self.smtp.docmd("STARTTLS") + if resp != 220: + raise smtplib.SMTPException("Wrong answer to the STARTTLS command") + if self.smtpcacerts: + self.smtp.sock = ssl.wrap_socket( + self.smtp.sock, + ca_certs=self.smtpcacerts, + cert_reqs=ssl.CERT_REQUIRED + ) + else: + self.smtp.sock = ssl.wrap_socket( + self.smtp.sock, + cert_reqs=ssl.CERT_NONE + ) + self.environment.get_logger().error( + '*** Warning, the server certificat is not verified (smtp) ***\n' + '*** set the option smtpCACerts ***\n' + ) + if not hasattr(self.smtp.sock, "read"): + # using httplib.FakeSocket with Python 2.5.x or earlier + self.smtp.sock.read = self.smtp.sock.recv + self.smtp.file = smtplib.SSLFakeFile(self.smtp.sock) + self.smtp.helo_resp = None + self.smtp.ehlo_resp = None + self.smtp.esmtp_features = {} + self.smtp.does_esmtp = 0 + # end: ehlo + starttls + self.smtp.ehlo() + else: + sys.stdout.write('*** Error: Control reached an invalid option. ***') + sys.exit(1) + if self.smtpserverdebuglevel > 0: + sys.stdout.write( + "*** Setting debug on for SMTP server connection (%s) ***\n" + % self.smtpserverdebuglevel) + self.smtp.set_debuglevel(self.smtpserverdebuglevel) + except Exception: + self.environment.get_logger().error( + '*** Error establishing SMTP connection to %s ***\n' + '*** %s\n' + % (self.smtpserver, sys.exc_info()[1])) sys.exit(1) def __del__(self): - self.smtp.quit() + if hasattr(self, 'smtp'): + self.smtp.quit() + del self.smtp def send(self, lines, to_addrs): try: + if self.username or self.password: + self.smtp.login(self.username, self.password) msg = ''.join(lines) # turn comma-separated list into Python list if needed. - if isinstance(to_addrs, basestring): + if is_string(to_addrs): to_addrs = [email for (name, email) in getaddresses([to_addrs])] self.smtp.sendmail(self.envelopesender, to_addrs, msg) - except Exception, e: - sys.stderr.write('*** Error sending email***\n') - sys.stderr.write('*** %s\n' % str(e)) - self.smtp.quit() + except smtplib.SMTPResponseException: + err = sys.exc_info()[1] + self.environment.get_logger().error( + '*** Error sending email ***\n' + '*** Error %d: %s\n' + % (err.smtp_code, bytes_to_str(err.smtp_error))) + try: + smtp = self.smtp + # delete the field before quit() so that in case of + # error, self.smtp is deleted anyway. + del self.smtp + smtp.quit() + except: + self.environment.get_logger().error( + '*** Error closing the SMTP connection ***\n' + '*** Exiting anyway ... ***\n' + '*** %s\n' % sys.exc_info()[1]) sys.exit(1) @@ -1451,9 +2175,10 @@ class OutputMailer(Mailer): self.f = f def send(self, lines, to_addrs): - self.f.write(self.SEPARATOR) - self.f.writelines(lines) - self.f.write(self.SEPARATOR) + write_str(self.f, self.SEPARATOR) + for line in lines: + write_str(self.f, line) + write_str(self.f, self.SEPARATOR) def get_git_dir(): @@ -1519,11 +2244,13 @@ class Environment(object): Return the address to be used as the 'From' email address in the email envelope. - get_fromaddr() + get_fromaddr(change=None) Return the 'From' email address used in the email 'From:' - headers. (May be a full RFC 2822 email address like 'Joe - User <user@example.com>'.) + headers. If the change is known when this function is + called, it is passed in as the 'change' parameter. (May + be a full RFC 2822 email address like 'Joe User + <user@example.com>'.) get_administrator() @@ -1543,12 +2270,46 @@ class Environment(object): get_reply_to_commit() is used for individual commit emails. + get_ref_filter_regex() + + Return a tuple -- a compiled regex, and a boolean indicating + whether the regex picks refs to include (if False, the regex + matches on refs to exclude). + + get_default_ref_ignore_regex() + + Return a regex that should be ignored for both what emails + to send and when computing what commits are considered new + to the repository. Default is "^refs/notes/". + + get_max_subject_length() + + Return an int giving the maximal length for the subject + (git log --oneline). + They should also define the following attributes: announce_show_shortlog (bool) True iff announce emails should include a shortlog. + commit_email_format (string) + + If "html", generate commit emails in HTML instead of plain text + used by default. + + html_in_intro (bool) + html_in_footer (bool) + + When generating HTML emails, the introduction (respectively, + the footer) will be HTML-escaped iff html_in_intro (respectively, + the footer) is true. When false, only the values used to expand + the template are escaped. + + refchange_showgraph (bool) + + True iff refchanges emails should include a detailed graph. + refchange_showlog (bool) True iff refchanges emails should include a detailed log. @@ -1559,6 +2320,12 @@ class Environment(object): summary email. The value should be a list of strings representing words to be passed to the command. + graphopts (list of strings) + + Analogous to diffopts, but contains options passed to + 'git log --graph' when generating the detailed graph for + a set of commits (see refchange_showgraph) + logopts (list of strings) Analogous to diffopts, but contains options passed to @@ -1571,6 +2338,38 @@ class Environment(object): commit mail. The value should be a list of strings representing words to be passed to the command. + date_substitute (string) + + String to be used in substitution for 'Date:' at start of + line in the output of 'git log'. + + quiet (bool) + On success do not write to stderr + + stdout (bool) + Write email to stdout rather than emailing. Useful for debugging + + combine_when_single_commit (bool) + + True if a combined email should be produced when a single + new commit is pushed to a branch, False otherwise. + + from_refchange, from_commit (strings) + + Addresses to use for the From: field for refchange emails + and commit emails respectively. Set from + multimailhook.fromRefchange and multimailhook.fromCommit + by ConfigEnvironmentMixin. + + log_file, error_log_file, debug_log_file (string) + + Name of a file to which logs should be sent. + + verbose (int) + + How verbose the system should be. + - 0 (default): show info, errors, ... + - 1 : show basic debug info """ REPO_NAME_RE = re.compile(r'^(?P<name>.+?)(?:\.git)$') @@ -1578,17 +2377,27 @@ class Environment(object): def __init__(self, osenv=None): self.osenv = osenv or os.environ self.announce_show_shortlog = False + self.commit_email_format = "text" + self.html_in_intro = False + self.html_in_footer = False + self.commitBrowseURL = None self.maxcommitemails = 500 self.diffopts = ['--stat', '--summary', '--find-copies-harder'] + self.graphopts = ['--oneline', '--decorate'] self.logopts = [] + self.refchange_showgraph = False self.refchange_showlog = False self.commitlogopts = ['-C', '--stat', '-p', '--cc'] + self.date_substitute = 'AuthorDate: ' + self.quiet = False + self.stdout = False + self.combine_when_single_commit = True + self.logger = None self.COMPUTED_KEYS = [ 'administrator', 'charset', 'emailprefix', - 'fromaddr', 'pusher', 'pusher_email', 'repo_path', @@ -1598,6 +2407,12 @@ class Environment(object): self._values = None + def get_logger(self): + """Get (possibly creates) the logger associated to this environment.""" + if self.logger is None: + self.logger = Logger(self) + return self.logger + def get_repo_shortname(self): """Use the last part of the repo path, with ".git" stripped off if present.""" @@ -1614,6 +2429,14 @@ class Environment(object): def get_pusher_email(self): return None + def get_fromaddr(self, change=None): + config = Config('user') + fromname = config.get('name', default='') + fromemail = config.get('email', default='') + if fromemail: + return formataddr([fromname, fromemail]) + return self.get_sender() + def get_administrator(self): return 'the administrator of this repository' @@ -1631,7 +2454,7 @@ class Environment(object): return CHARSET def get_values(self): - """Return a dictionary {keyword : expansion} for this Environment. + """Return a dictionary {keyword: expansion} for this Environment. This method is called by Change._compute_values(). The keys in the returned dictionary are available to be used in any of @@ -1641,7 +2464,7 @@ class Environment(object): The return value is always a new dictionary.""" if self._values is None: - values = {} + values = {'': ''} # %()s expands to the empty string. for key in self.COMPUTED_KEYS: value = getattr(self, 'get_%s' % (key,))() @@ -1688,6 +2511,20 @@ class Environment(object): def get_reply_to_commit(self, revision): return revision.author + def get_default_ref_ignore_regex(self): + # The commit messages of git notes are essentially meaningless + # and "filenames" in git notes commits are an implementational + # detail that might surprise users at first. As such, we + # would need a completely different method for handling emails + # of git notes in order for them to be of benefit for users, + # which we simply do not have right now. + return "^refs/notes/" + + def get_max_subject_length(self): + """Return the maximal subject line (git log --oneline) length. + Longer subject lines will be truncated.""" + raise NotImplementedError() + def filter_body(self, lines): """Filter the lines intended for an email body. @@ -1699,6 +2536,27 @@ class Environment(object): return lines + def log_msg(self, msg): + """Write the string msg on a log file or on stderr. + + Sends the text to stderr by default, override to change the behavior.""" + self.get_logger().info(msg) + + def log_warning(self, msg): + """Write the string msg on a log file or on stderr. + + Sends the text to stderr by default, override to change the behavior.""" + self.get_logger().warning(msg) + + def log_error(self, msg): + """Write the string msg on a log file or on stderr. + + Sends the text to stderr by default, override to change the behavior.""" + self.get_logger().error(msg) + + def check(self): + pass + class ConfigEnvironmentMixin(Environment): """A mixin that sets self.config to its constructor's config argument. @@ -1718,33 +2576,70 @@ class ConfigEnvironmentMixin(Environment): class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin): """An Environment that reads most of its information from "git config".""" + @staticmethod + def forbid_field_values(name, value, forbidden): + for forbidden_val in forbidden: + if value is not None and value.lower() == forbidden: + raise ConfigurationException( + '"%s" is not an allowed setting for %s' % (value, name) + ) + def __init__(self, config, **kw): super(ConfigOptionsEnvironmentMixin, self).__init__( config=config, **kw ) - self.announce_show_shortlog = config.get_bool( - 'announceshortlog', default=self.announce_show_shortlog - ) + for var, cfg in ( + ('announce_show_shortlog', 'announceshortlog'), + ('refchange_showgraph', 'refchangeShowGraph'), + ('refchange_showlog', 'refchangeshowlog'), + ('quiet', 'quiet'), + ('stdout', 'stdout'), + ): + val = config.get_bool(cfg) + if val is not None: + setattr(self, var, val) + + commit_email_format = config.get('commitEmailFormat') + if commit_email_format is not None: + if commit_email_format != "html" and commit_email_format != "text": + self.log_warning( + '*** Unknown value for multimailhook.commitEmailFormat: %s\n' % + commit_email_format + + '*** Expected either "text" or "html". Ignoring.\n' + ) + else: + self.commit_email_format = commit_email_format - self.refchange_showlog = config.get_bool( - 'refchangeshowlog', default=self.refchange_showlog - ) + html_in_intro = config.get_bool('htmlInIntro') + if html_in_intro is not None: + self.html_in_intro = html_in_intro + + html_in_footer = config.get_bool('htmlInFooter') + if html_in_footer is not None: + self.html_in_footer = html_in_footer + + self.commitBrowseURL = config.get('commitBrowseURL') maxcommitemails = config.get('maxcommitemails') if maxcommitemails is not None: try: self.maxcommitemails = int(maxcommitemails) except ValueError: - sys.stderr.write( - '*** Malformed value for multimailhook.maxCommitEmails: %s\n' % maxcommitemails - + '*** Expected a number. Ignoring.\n' + self.log_warning( + '*** Malformed value for multimailhook.maxCommitEmails: %s\n' + % maxcommitemails + + '*** Expected a number. Ignoring.\n' ) diffopts = config.get('diffopts') if diffopts is not None: self.diffopts = shlex.split(diffopts) + graphopts = config.get('graphOpts') + if graphopts is not None: + self.graphopts = shlex.split(graphopts) + logopts = config.get('logopts') if logopts is not None: self.logopts = shlex.split(logopts) @@ -1753,80 +2648,122 @@ class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin): if commitlogopts is not None: self.commitlogopts = shlex.split(commitlogopts) + date_substitute = config.get('dateSubstitute') + if date_substitute == 'none': + self.date_substitute = None + elif date_substitute is not None: + self.date_substitute = date_substitute + reply_to = config.get('replyTo') self.__reply_to_refchange = config.get('replyToRefchange', default=reply_to) - if ( - self.__reply_to_refchange is not None - and self.__reply_to_refchange.lower() == 'author' - ): - raise ConfigurationException( - '"author" is not an allowed setting for replyToRefchange' - ) + self.forbid_field_values('replyToRefchange', + self.__reply_to_refchange, + ['author']) self.__reply_to_commit = config.get('replyToCommit', default=reply_to) + self.from_refchange = config.get('fromRefchange') + self.forbid_field_values('fromRefchange', + self.from_refchange, + ['author', 'none']) + self.from_commit = config.get('fromCommit') + self.forbid_field_values('fromCommit', + self.from_commit, + ['none']) + + combine = config.get_bool('combineWhenSingleCommit') + if combine is not None: + self.combine_when_single_commit = combine + + self.log_file = config.get('logFile', default=None) + self.error_log_file = config.get('errorLogFile', default=None) + self.debug_log_file = config.get('debugLogFile', default=None) + if config.get_bool('Verbose', default=False): + self.verbose = 1 + else: + self.verbose = 0 + def get_administrator(self): return ( - self.config.get('administrator') - or self.get_sender() - or super(ConfigOptionsEnvironmentMixin, self).get_administrator() + self.config.get('administrator') or + self.get_sender() or + super(ConfigOptionsEnvironmentMixin, self).get_administrator() ) def get_repo_shortname(self): return ( - self.config.get('reponame') - or super(ConfigOptionsEnvironmentMixin, self).get_repo_shortname() + self.config.get('reponame') or + super(ConfigOptionsEnvironmentMixin, self).get_repo_shortname() ) def get_emailprefix(self): emailprefix = self.config.get('emailprefix') - if emailprefix and emailprefix.strip(): - return emailprefix.strip() + ' ' + if emailprefix is not None: + emailprefix = emailprefix.strip() + if emailprefix: + emailprefix += ' ' else: - return '[%s] ' % (self.get_repo_shortname(),) + emailprefix = '[%(repo_shortname)s] ' + short_name = self.get_repo_shortname() + try: + return emailprefix % {'repo_shortname': short_name} + except: + self.get_logger().error( + '*** Invalid multimailhook.emailPrefix: %s\n' % emailprefix + + '*** %s\n' % sys.exc_info()[1] + + "*** Only the '%(repo_shortname)s' placeholder is allowed\n" + ) + raise ConfigurationException( + '"%s" is not an allowed setting for emailPrefix' % emailprefix + ) def get_sender(self): return self.config.get('envelopesender') - def get_fromaddr(self): + def process_addr(self, addr, change): + if addr.lower() == 'author': + if hasattr(change, 'author'): + return change.author + else: + return None + elif addr.lower() == 'pusher': + return self.get_pusher_email() + elif addr.lower() == 'none': + return None + else: + return addr + + def get_fromaddr(self, change=None): fromaddr = self.config.get('from') + if change: + specific_fromaddr = change.get_specific_fromaddr() + if specific_fromaddr: + fromaddr = specific_fromaddr + if fromaddr: + fromaddr = self.process_addr(fromaddr, change) if fromaddr: return fromaddr - else: - config = Config('user') - fromname = config.get('name', default='') - fromemail = config.get('email', default='') - if fromemail: - return formataddr([fromname, fromemail]) - else: - return self.get_sender() + return super(ConfigOptionsEnvironmentMixin, self).get_fromaddr(change) def get_reply_to_refchange(self, refchange): if self.__reply_to_refchange is None: return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_refchange(refchange) - elif self.__reply_to_refchange.lower() == 'pusher': - return self.get_pusher_email() - elif self.__reply_to_refchange.lower() == 'none': - return None else: - return self.__reply_to_refchange + return self.process_addr(self.__reply_to_refchange, refchange) def get_reply_to_commit(self, revision): if self.__reply_to_commit is None: return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_commit(revision) - elif self.__reply_to_commit.lower() == 'author': - return revision.get_author() - elif self.__reply_to_commit.lower() == 'pusher': - return self.get_pusher_email() - elif self.__reply_to_commit.lower() == 'none': - return None else: - return self.__reply_to_commit + return self.process_addr(self.__reply_to_commit, revision) + + def get_scancommitforcc(self): + return self.config.get('scancommitforcc') class FilterLinesEnvironmentMixin(Environment): """Handle encoding and maximum line length of body lines. - emailmaxlinelength (int or None) + email_max_line_length (int or None) The maximum length of any single line in the email body. Longer lines are truncated at that length with ' [...]' @@ -1841,30 +2778,38 @@ class FilterLinesEnvironmentMixin(Environment): """ - def __init__(self, strict_utf8=True, emailmaxlinelength=500, **kw): + def __init__(self, strict_utf8=True, + email_max_line_length=500, max_subject_length=500, + **kw): super(FilterLinesEnvironmentMixin, self).__init__(**kw) self.__strict_utf8 = strict_utf8 - self.__emailmaxlinelength = emailmaxlinelength + self.__email_max_line_length = email_max_line_length + self.__max_subject_length = max_subject_length def filter_body(self, lines): lines = super(FilterLinesEnvironmentMixin, self).filter_body(lines) if self.__strict_utf8: - lines = (line.decode(ENCODING, 'replace') for line in lines) + if not PYTHON3: + lines = (line.decode(ENCODING, 'replace') for line in lines) # Limit the line length in Unicode-space to avoid # splitting characters: - if self.__emailmaxlinelength: - lines = limit_linelength(lines, self.__emailmaxlinelength) - lines = (line.encode(ENCODING, 'replace') for line in lines) - elif self.__emailmaxlinelength: - lines = limit_linelength(lines, self.__emailmaxlinelength) + if self.__email_max_line_length > 0: + lines = limit_linelength(lines, self.__email_max_line_length) + if not PYTHON3: + lines = (line.encode(ENCODING, 'replace') for line in lines) + elif self.__email_max_line_length: + lines = limit_linelength(lines, self.__email_max_line_length) return lines + def get_max_subject_length(self): + return self.__max_subject_length + class ConfigFilterLinesEnvironmentMixin( - ConfigEnvironmentMixin, - FilterLinesEnvironmentMixin, - ): + ConfigEnvironmentMixin, + FilterLinesEnvironmentMixin, + ): """Handle encoding and maximum line length based on config.""" def __init__(self, config, **kw): @@ -1872,9 +2817,13 @@ class ConfigFilterLinesEnvironmentMixin( if strict_utf8 is not None: kw['strict_utf8'] = strict_utf8 - emailmaxlinelength = config.get('emailmaxlinelength') - if emailmaxlinelength is not None: - kw['emailmaxlinelength'] = int(emailmaxlinelength) + email_max_line_length = config.get('emailmaxlinelength') + if email_max_line_length is not None: + kw['email_max_line_length'] = int(email_max_line_length) + + max_subject_length = config.get('subjectMaxLength', default=email_max_line_length) + if max_subject_length is not None: + kw['max_subject_length'] = int(max_subject_length) super(ConfigFilterLinesEnvironmentMixin, self).__init__( config=config, **kw @@ -1890,15 +2839,15 @@ class MaxlinesEnvironmentMixin(Environment): def filter_body(self, lines): lines = super(MaxlinesEnvironmentMixin, self).filter_body(lines) - if self.__emailmaxlines: + if self.__emailmaxlines > 0: lines = limit_lines(lines, self.__emailmaxlines) return lines class ConfigMaxlinesEnvironmentMixin( - ConfigEnvironmentMixin, - MaxlinesEnvironmentMixin, - ): + ConfigEnvironmentMixin, + MaxlinesEnvironmentMixin, + ): """Limit the email body to the number of lines specified in config.""" def __init__(self, config, **kw): @@ -1927,9 +2876,9 @@ class FQDNEnvironmentMixin(Environment): class ConfigFQDNEnvironmentMixin( - ConfigEnvironmentMixin, - FQDNEnvironmentMixin, - ): + ConfigEnvironmentMixin, + FQDNEnvironmentMixin, + ): """Read the FQDN from the config.""" def __init__(self, config, **kw): @@ -1970,10 +2919,10 @@ class StaticRecipientsEnvironmentMixin(Environment): """Set recipients statically based on constructor parameters.""" def __init__( - self, - refchange_recipients, announce_recipients, revision_recipients, - **kw - ): + self, + refchange_recipients, announce_recipients, revision_recipients, scancommitforcc, + **kw + ): super(StaticRecipientsEnvironmentMixin, self).__init__(**kw) # The recipients for various types of notification emails, as @@ -1983,28 +2932,68 @@ class StaticRecipientsEnvironmentMixin(Environment): # actual *contents* of the change being reported, we only # choose based on the *type* of the change. Therefore we can # compute them once and for all: - if not (refchange_recipients - or announce_recipients - or revision_recipients): - raise ConfigurationException('No email recipients configured!') self.__refchange_recipients = refchange_recipients self.__announce_recipients = announce_recipients self.__revision_recipients = revision_recipients + def check(self): + if not (self.get_refchange_recipients(None) or + self.get_announce_recipients(None) or + self.get_revision_recipients(None) or + self.get_scancommitforcc()): + raise ConfigurationException('No email recipients configured!') + super(StaticRecipientsEnvironmentMixin, self).check() + def get_refchange_recipients(self, refchange): + if self.__refchange_recipients is None: + return super(StaticRecipientsEnvironmentMixin, + self).get_refchange_recipients(refchange) return self.__refchange_recipients def get_announce_recipients(self, annotated_tag_change): + if self.__announce_recipients is None: + return super(StaticRecipientsEnvironmentMixin, + self).get_refchange_recipients(annotated_tag_change) return self.__announce_recipients def get_revision_recipients(self, revision): + if self.__revision_recipients is None: + return super(StaticRecipientsEnvironmentMixin, + self).get_refchange_recipients(revision) return self.__revision_recipients +class CLIRecipientsEnvironmentMixin(Environment): + """Mixin storing recipients information comming from the + command-line.""" + + def __init__(self, cli_recipients=None, **kw): + super(CLIRecipientsEnvironmentMixin, self).__init__(**kw) + self.__cli_recipients = cli_recipients + + def get_refchange_recipients(self, refchange): + if self.__cli_recipients is None: + return super(CLIRecipientsEnvironmentMixin, + self).get_refchange_recipients(refchange) + return self.__cli_recipients + + def get_announce_recipients(self, annotated_tag_change): + if self.__cli_recipients is None: + return super(CLIRecipientsEnvironmentMixin, + self).get_announce_recipients(annotated_tag_change) + return self.__cli_recipients + + def get_revision_recipients(self, revision): + if self.__cli_recipients is None: + return super(CLIRecipientsEnvironmentMixin, + self).get_revision_recipients(revision) + return self.__cli_recipients + + class ConfigRecipientsEnvironmentMixin( - ConfigEnvironmentMixin, - StaticRecipientsEnvironmentMixin - ): + ConfigEnvironmentMixin, + StaticRecipientsEnvironmentMixin + ): """Determine recipients statically based on config.""" def __init__(self, config, **kw): @@ -2019,6 +3008,7 @@ class ConfigRecipientsEnvironmentMixin( revision_recipients=self._get_recipients( config, 'commitlist', 'mailinglist', ), + scancommitforcc=config.get('scancommitforcc'), **kw ) @@ -2034,13 +3024,100 @@ class ConfigRecipientsEnvironmentMixin( found, raise a ConfigurationException.""" for name in names: - retval = config.get_recipients(name) - if retval is not None: - return retval + lines = config.get_all(name) + if lines is not None: + lines = [line.strip() for line in lines] + # Single "none" is a special value equivalen to empty string. + if lines == ['none']: + lines = [''] + return ', '.join(lines) else: return '' +class StaticRefFilterEnvironmentMixin(Environment): + """Set branch filter statically based on constructor parameters.""" + + def __init__(self, ref_filter_incl_regex, ref_filter_excl_regex, + ref_filter_do_send_regex, ref_filter_dont_send_regex, + **kw): + super(StaticRefFilterEnvironmentMixin, self).__init__(**kw) + + if ref_filter_incl_regex and ref_filter_excl_regex: + raise ConfigurationException( + "Cannot specify both a ref inclusion and exclusion regex.") + self.__is_inclusion_filter = bool(ref_filter_incl_regex) + default_exclude = self.get_default_ref_ignore_regex() + if ref_filter_incl_regex: + ref_filter_regex = ref_filter_incl_regex + elif ref_filter_excl_regex: + ref_filter_regex = ref_filter_excl_regex + '|' + default_exclude + else: + ref_filter_regex = default_exclude + try: + self.__compiled_regex = re.compile(ref_filter_regex) + except Exception: + raise ConfigurationException( + 'Invalid Ref Filter Regex "%s": %s' % (ref_filter_regex, sys.exc_info()[1])) + + if ref_filter_do_send_regex and ref_filter_dont_send_regex: + raise ConfigurationException( + "Cannot specify both a ref doSend and dontSend regex.") + self.__is_do_send_filter = bool(ref_filter_do_send_regex) + if ref_filter_do_send_regex: + ref_filter_send_regex = ref_filter_do_send_regex + elif ref_filter_dont_send_regex: + ref_filter_send_regex = ref_filter_dont_send_regex + else: + ref_filter_send_regex = '.*' + self.__is_do_send_filter = True + try: + self.__send_compiled_regex = re.compile(ref_filter_send_regex) + except Exception: + raise ConfigurationException( + 'Invalid Ref Filter Regex "%s": %s' % + (ref_filter_send_regex, sys.exc_info()[1])) + + def get_ref_filter_regex(self, send_filter=False): + if send_filter: + return self.__send_compiled_regex, self.__is_do_send_filter + else: + return self.__compiled_regex, self.__is_inclusion_filter + + +class ConfigRefFilterEnvironmentMixin( + ConfigEnvironmentMixin, + StaticRefFilterEnvironmentMixin + ): + """Determine branch filtering statically based on config.""" + + def _get_regex(self, config, key): + """Get a list of whitespace-separated regex. The refFilter* config + variables are multivalued (hence the use of get_all), and we + allow each entry to be a whitespace-separated list (hence the + split on each line). The whole thing is glued into a single regex.""" + values = config.get_all(key) + if values is None: + return values + items = [] + for line in values: + for i in line.split(): + items.append(i) + if items == []: + return None + return '|'.join(items) + + def __init__(self, config, **kw): + super(ConfigRefFilterEnvironmentMixin, self).__init__( + config=config, + ref_filter_incl_regex=self._get_regex(config, 'refFilterInclusionRegex'), + ref_filter_excl_regex=self._get_regex(config, 'refFilterExclusionRegex'), + ref_filter_do_send_regex=self._get_regex(config, 'refFilterDoSendRegex'), + ref_filter_dont_send_regex=self._get_regex(config, 'refFilterDontSendRegex'), + **kw + ) + + class ProjectdescEnvironmentMixin(Environment): """Make a "projectdesc" value available for templates. @@ -2067,35 +3144,62 @@ class ProjectdescEnvironmentMixin(Environment): class GenericEnvironmentMixin(Environment): def get_pusher(self): - return self.osenv.get('USER', 'unknown user') + return self.osenv.get('USER', self.osenv.get('USERNAME', 'unknown user')) -class GenericEnvironment( - ProjectdescEnvironmentMixin, - ConfigMaxlinesEnvironmentMixin, - ComputeFQDNEnvironmentMixin, - ConfigFilterLinesEnvironmentMixin, - ConfigRecipientsEnvironmentMixin, - PusherDomainEnvironmentMixin, - ConfigOptionsEnvironmentMixin, - GenericEnvironmentMixin, - Environment, - ): - pass +class GitoliteEnvironmentHighPrecMixin(Environment): + def get_pusher(self): + return self.osenv.get('GL_USER', 'unknown user') -class GitoliteEnvironmentMixin(Environment): +class GitoliteEnvironmentLowPrecMixin(Environment): def get_repo_shortname(self): # The gitolite environment variable $GL_REPO is a pretty good # repo_shortname (though it's probably not as good as a value # the user might have explicitly put in his config). return ( - self.osenv.get('GL_REPO', None) - or super(GitoliteEnvironmentMixin, self).get_repo_shortname() + self.osenv.get('GL_REPO', None) or + super(GitoliteEnvironmentLowPrecMixin, self).get_repo_shortname() ) - def get_pusher(self): - return self.osenv.get('GL_USER', 'unknown user') + def get_fromaddr(self, change=None): + GL_USER = self.osenv.get('GL_USER') + if GL_USER is not None: + # Find the path to gitolite.conf. Note that gitolite v3 + # did away with the GL_ADMINDIR and GL_CONF environment + # variables (they are now hard-coded). + GL_ADMINDIR = self.osenv.get( + 'GL_ADMINDIR', + os.path.expanduser(os.path.join('~', '.gitolite'))) + GL_CONF = self.osenv.get( + 'GL_CONF', + os.path.join(GL_ADMINDIR, 'conf', 'gitolite.conf')) + if os.path.isfile(GL_CONF): + f = open(GL_CONF, 'rU') + try: + in_user_emails_section = False + re_template = r'^\s*#\s*%s\s*$' + re_begin, re_user, re_end = ( + re.compile(re_template % x) + for x in ( + r'BEGIN\s+USER\s+EMAILS', + re.escape(GL_USER) + r'\s+(.*)', + r'END\s+USER\s+EMAILS', + )) + for l in f: + l = l.rstrip('\n') + if not in_user_emails_section: + if re_begin.match(l): + in_user_emails_section = True + continue + if re_end.match(l): + break + m = re_user.match(l) + if m: + return m.group(1) + finally: + f.close() + return super(GitoliteEnvironmentLowPrecMixin, self).get_fromaddr(change) class IncrementalDateTime(object): @@ -2108,25 +3212,109 @@ class IncrementalDateTime(object): def __init__(self): self.time = time.time() + self.next = self.__next__ # Python 2 backward compatibility - def next(self): + def __next__(self): formatted = formatdate(self.time, True) self.time += 1 return formatted -class GitoliteEnvironment( - ProjectdescEnvironmentMixin, - ConfigMaxlinesEnvironmentMixin, - ComputeFQDNEnvironmentMixin, - ConfigFilterLinesEnvironmentMixin, - ConfigRecipientsEnvironmentMixin, - PusherDomainEnvironmentMixin, - ConfigOptionsEnvironmentMixin, - GitoliteEnvironmentMixin, - Environment, - ): - pass +class StashEnvironmentHighPrecMixin(Environment): + def __init__(self, user=None, repo=None, **kw): + super(StashEnvironmentHighPrecMixin, + self).__init__(user=user, repo=repo, **kw) + self.__user = user + self.__repo = repo + + def get_pusher(self): + return re.match('(.*?)\s*<', self.__user).group(1) + + def get_pusher_email(self): + return self.__user + + +class StashEnvironmentLowPrecMixin(Environment): + def __init__(self, user=None, repo=None, **kw): + super(StashEnvironmentLowPrecMixin, self).__init__(**kw) + self.__repo = repo + self.__user = user + + def get_repo_shortname(self): + return self.__repo + + def get_fromaddr(self, change=None): + return self.__user + + +class GerritEnvironmentHighPrecMixin(Environment): + def __init__(self, project=None, submitter=None, update_method=None, **kw): + super(GerritEnvironmentHighPrecMixin, + self).__init__(submitter=submitter, project=project, **kw) + self.__project = project + self.__submitter = submitter + self.__update_method = update_method + "Make an 'update_method' value available for templates." + self.COMPUTED_KEYS += ['update_method'] + + def get_pusher(self): + if self.__submitter: + if self.__submitter.find('<') != -1: + # Submitter has a configured email, we transformed + # __submitter into an RFC 2822 string already. + return re.match('(.*?)\s*<', self.__submitter).group(1) + else: + # Submitter has no configured email, it's just his name. + return self.__submitter + else: + # If we arrive here, this means someone pushed "Submit" from + # the gerrit web UI for the CR (or used one of the programmatic + # APIs to do the same, such as gerrit review) and the + # merge/push was done by the Gerrit user. It was technically + # triggered by someone else, but sadly we have no way of + # determining who that someone else is at this point. + return 'Gerrit' # 'unknown user'? + + def get_pusher_email(self): + if self.__submitter: + return self.__submitter + else: + return super(GerritEnvironmentHighPrecMixin, self).get_pusher_email() + + def get_default_ref_ignore_regex(self): + default = super(GerritEnvironmentHighPrecMixin, self).get_default_ref_ignore_regex() + return default + '|^refs/changes/|^refs/cache-automerge/|^refs/meta/' + + def get_revision_recipients(self, revision): + # Merge commits created by Gerrit when users hit "Submit this patchset" + # in the Web UI (or do equivalently with REST APIs or the gerrit review + # command) are not something users want to see an individual email for. + # Filter them out. + committer = read_git_output(['log', '--no-walk', '--format=%cN', + revision.rev.sha1]) + if committer == 'Gerrit Code Review': + return [] + else: + return super(GerritEnvironmentHighPrecMixin, self).get_revision_recipients(revision) + + def get_update_method(self): + return self.__update_method + + +class GerritEnvironmentLowPrecMixin(Environment): + def __init__(self, project=None, submitter=None, **kw): + super(GerritEnvironmentLowPrecMixin, self).__init__(**kw) + self.__project = project + self.__submitter = submitter + + def get_repo_shortname(self): + return self.__project + + def get_fromaddr(self, change=None): + if self.__submitter and self.__submitter.find('<') != -1: + return self.__submitter + else: + return super(GerritEnvironmentLowPrecMixin, self).get_fromaddr(change) class Push(object): @@ -2149,9 +3337,9 @@ class Push(object): references. The first step is to determine the "other" references--those - unaffected by the current push. They are computed by - Push._compute_other_ref_sha1s() by listing all references then - removing any affected by this push. + unaffected by the current push. They are computed by listing all + references then removing any affected by this push. The results + are stored in Push._other_ref_sha1s. The commits contained in the repository before this push were @@ -2187,7 +3375,7 @@ class Push(object): possible and working with SHA1s thereafter (because SHA1s are immutable).""" - # A map {(changeclass, changetype) : integer} specifying the order + # A map {(changeclass, changetype): integer} specifying the order # that reference changes will be processed if multiple reference # changes are included in a single push. The order is significant # mostly because new commit notifications are threaded together @@ -2211,66 +3399,139 @@ class Push(object): ]) ) - def __init__(self, changes): + def __init__(self, environment, changes, ignore_other_refs=False): self.changes = sorted(changes, key=self._sort_key) + self.__other_ref_sha1s = None + self.__cached_commits_spec = {} + self.environment = environment + + if ignore_other_refs: + self.__other_ref_sha1s = set() - # The SHA-1s of commits referred to by references unaffected - # by this push: - other_ref_sha1s = self._compute_other_ref_sha1s() + @classmethod + def _sort_key(klass, change): + return (klass.SORT_ORDER[change.__class__, change.change_type], change.refname,) - self._old_rev_exclusion_spec = self._compute_rev_exclusion_spec( - other_ref_sha1s.union( - change.old.sha1 + @property + def _other_ref_sha1s(self): + """The GitObjects referred to by references unaffected by this push. + """ + if self.__other_ref_sha1s is None: + # The refnames being changed by this push: + updated_refs = set( + change.refname for change in self.changes - if change.old.type in ['commit', 'tag'] ) - ) - self._new_rev_exclusion_spec = self._compute_rev_exclusion_spec( - other_ref_sha1s.union( - change.new.sha1 - for change in self.changes - if change.new.type in ['commit', 'tag'] + + # The SHA-1s of commits referred to by all references in this + # repository *except* updated_refs: + sha1s = set() + fmt = ( + '%(objectname) %(objecttype) %(refname)\n' + '%(*objectname) %(*objecttype) %(refname)' ) - ) + ref_filter_regex, is_inclusion_filter = \ + self.environment.get_ref_filter_regex() + for line in read_git_lines( + ['for-each-ref', '--format=%s' % (fmt,)]): + (sha1, type, name) = line.split(' ', 2) + if (sha1 and type == 'commit' and + name not in updated_refs and + include_ref(name, ref_filter_regex, is_inclusion_filter)): + sha1s.add(sha1) - @classmethod - def _sort_key(klass, change): - return (klass.SORT_ORDER[change.__class__, change.change_type], change.refname,) + self.__other_ref_sha1s = sha1s + + return self.__other_ref_sha1s + + def _get_commits_spec_incl(self, new_or_old, reference_change=None): + """Get new or old SHA-1 from one or each of the changed refs. - def _compute_other_ref_sha1s(self): - """Return the GitObjects referred to by references unaffected by this push.""" + Return a list of SHA-1 commit identifier strings suitable as + arguments to 'git rev-list' (or 'git log' or ...). The + returned identifiers are either the old or new values from one + or all of the changed references, depending on the values of + new_or_old and reference_change. - # The refnames being changed by this push: - updated_refs = set( - change.refname + new_or_old is either the string 'new' or the string 'old'. If + 'new', the returned SHA-1 identifiers are the new values from + each changed reference. If 'old', the SHA-1 identifiers are + the old values from each changed reference. + + If reference_change is specified and not None, only the new or + old reference from the specified reference is included in the + return value. + + This function returns None if there are no matching revisions + (e.g., because a branch was deleted and new_or_old is 'new'). + """ + + if not reference_change: + incl_spec = sorted( + getattr(change, new_or_old).sha1 + for change in self.changes + if getattr(change, new_or_old) + ) + if not incl_spec: + incl_spec = None + elif not getattr(reference_change, new_or_old).commit_sha1: + incl_spec = None + else: + incl_spec = [getattr(reference_change, new_or_old).commit_sha1] + return incl_spec + + def _get_commits_spec_excl(self, new_or_old): + """Get exclusion revisions for determining new or discarded commits. + + Return a list of strings suitable as arguments to 'git + rev-list' (or 'git log' or ...) that will exclude all + commits that, depending on the value of new_or_old, were + either previously in the repository (useful for determining + which commits are new to the repository) or currently in the + repository (useful for determining which commits were + discarded from the repository). + + new_or_old is either the string 'new' or the string 'old'. If + 'new', the commits to be excluded are those that were in the + repository before the push. If 'old', the commits to be + excluded are those that are currently in the repository. """ + + old_or_new = {'old': 'new', 'new': 'old'}[new_or_old] + excl_revs = self._other_ref_sha1s.union( + getattr(change, old_or_new).sha1 for change in self.changes + if getattr(change, old_or_new).type in ['commit', 'tag'] ) + return ['^' + sha1 for sha1 in sorted(excl_revs)] - # The SHA-1s of commits referred to by all references in this - # repository *except* updated_refs: - sha1s = set() - fmt = ( - '%(objectname) %(objecttype) %(refname)\n' - '%(*objectname) %(*objecttype) %(refname)' - ) - for line in read_git_lines(['for-each-ref', '--format=%s' % (fmt,)]): - (sha1, type, name) = line.split(' ', 2) - if sha1 and type == 'commit' and name not in updated_refs: - sha1s.add(sha1) + def get_commits_spec(self, new_or_old, reference_change=None): + """Get rev-list arguments for added or discarded commits. - return sha1s + Return a list of strings suitable as arguments to 'git + rev-list' (or 'git log' or ...) that select those commits + that, depending on the value of new_or_old, are either new to + the repository or were discarded from the repository. - def _compute_rev_exclusion_spec(self, sha1s): - """Return an exclusion specification for 'git rev-list'. + new_or_old is either the string 'new' or the string 'old'. If + 'new', the returned list is used to select commits that are + new to the repository. If 'old', the returned value is used + to select the commits that have been discarded from the + repository. - git_objects is an iterable over GitObject instances. Return a - string that can be passed to the standard input of 'git - rev-list --stdin' to exclude all of the commits referred to by - git_objects.""" + If reference_change is specified and not None, the new or + discarded commits are limited to those that are reachable from + the new or old value of the specified reference. - return ''.join( - ['^%s\n' % (sha1,) for sha1 in sorted(sha1s)] - ) + This function returns None if there are no added (or discarded) + revisions. + """ + key = (new_or_old, reference_change) + if key not in self.__cached_commits_spec: + ret = self._get_commits_spec_incl(new_or_old, reference_change) + if ret is not None: + ret.extend(self._get_commits_spec_excl(new_or_old)) + self.__cached_commits_spec[key] = ret + return self.__cached_commits_spec[key] def get_new_commits(self, reference_change=None): """Return a list of commits added by this push. @@ -2280,19 +3541,8 @@ class Push(object): reference_change is None, then return a list of *all* commits added by this push.""" - if not reference_change: - new_revs = sorted( - change.new.sha1 - for change in self.changes - if change.new - ) - elif not reference_change.new.commit_sha1: - return [] - else: - new_revs = [reference_change.new.commit_sha1] - - cmd = ['rev-list', '--stdin'] + new_revs - return read_git_lines(cmd, input=self._old_rev_exclusion_spec) + spec = self.get_commits_spec('new', reference_change) + return git_rev_list(spec) def get_discarded_commits(self, reference_change): """Return a list of commits discarded by this push. @@ -2301,13 +3551,8 @@ class Push(object): entirely discarded from the repository by the part of this push represented by reference_change.""" - if not reference_change.old.commit_sha1: - return [] - else: - old_revs = [reference_change.old.commit_sha1] - - cmd = ['rev-list', '--stdin'] + old_revs - return read_git_lines(cmd, input=self._new_rev_exclusion_spec) + spec = self.get_commits_spec('old', reference_change) + return git_rev_list(spec) def send_emails(self, mailer, body_filter=None): """Use send all of the notification emails needed for this push. @@ -2325,40 +3570,57 @@ class Push(object): unhandled_sha1s = set(self.get_new_commits()) send_date = IncrementalDateTime() for change in self.changes: + sha1s = [] + for sha1 in reversed(list(self.get_new_commits(change))): + if sha1 in unhandled_sha1s: + sha1s.append(sha1) + unhandled_sha1s.remove(sha1) + # Check if we've got anyone to send to if not change.recipients: - sys.stderr.write( + change.environment.log_warning( '*** no recipients configured so no email will be sent\n' - '*** for %r update %s->%s\n' + '*** for %r update %s->%s' % (change.refname, change.old.sha1, change.new.sha1,) ) else: - sys.stderr.write('Sending notification emails to: %s\n' % (change.recipients,)) - extra_values = {'send_date' : send_date.next()} - mailer.send( - change.generate_email(self, body_filter, extra_values), - change.recipients, - ) + if not change.environment.quiet: + change.environment.log_msg( + 'Sending notification emails to: %s' % (change.recipients,)) + extra_values = {'send_date': next(send_date)} - sha1s = [] - for sha1 in reversed(list(self.get_new_commits(change))): - if sha1 in unhandled_sha1s: - sha1s.append(sha1) - unhandled_sha1s.remove(sha1) + rev = change.send_single_combined_email(sha1s) + if rev: + mailer.send( + change.generate_combined_email(self, rev, body_filter, extra_values), + rev.recipients, + ) + # This change is now fully handled; no need to handle + # individual revisions any further. + continue + else: + mailer.send( + change.generate_email(self, body_filter, extra_values), + change.recipients, + ) max_emails = change.environment.maxcommitemails if max_emails and len(sha1s) > max_emails: - sys.stderr.write( - '*** Too many new commits (%d), not sending commit emails.\n' % len(sha1s) - + '*** Try setting multimailhook.maxCommitEmails to a greater value\n' - + '*** Currently, multimailhook.maxCommitEmails=%d\n' % max_emails + change.environment.log_warning( + '*** Too many new commits (%d), not sending commit emails.\n' % len(sha1s) + + '*** Try setting multimailhook.maxCommitEmails to a greater value\n' + + '*** Currently, multimailhook.maxCommitEmails=%d' % max_emails ) return for (num, sha1) in enumerate(sha1s): - rev = Revision(change, GitObject(sha1), num=num+1, tot=len(sha1s)) + rev = Revision(change, GitObject(sha1), num=num + 1, tot=len(sha1s)) + if not rev.recipients and rev.cc_recipients: + change.environment.log_msg('*** Replacing Cc: with To:') + rev.recipients = rev.cc_recipients + rev.cc_recipients = None if rev.recipients: - extra_values = {'send_date' : send_date.next()} + extra_values = {'send_date': next(send_date)} mailer.send( rev.generate_email(self, body_filter, extra_values), rev.recipients, @@ -2366,25 +3628,57 @@ class Push(object): # Consistency check: if unhandled_sha1s: - sys.stderr.write( + change.environment.log_error( 'ERROR: No emails were sent for the following new commits:\n' - ' %s\n' + ' %s' % ('\n '.join(sorted(unhandled_sha1s)),) ) +def include_ref(refname, ref_filter_regex, is_inclusion_filter): + does_match = bool(ref_filter_regex.search(refname)) + if is_inclusion_filter: + return does_match + else: # exclusion filter -- we include the ref if the regex doesn't match + return not does_match + + def run_as_post_receive_hook(environment, mailer): + environment.check() + send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True) + ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False) changes = [] - for line in sys.stdin: + while True: + line = read_line(sys.stdin) + if line == '': + break (oldrev, newrev, refname) = line.strip().split(' ', 2) + environment.get_logger().debug( + "run_as_post_receive_hook: oldrev=%s, newrev=%s, refname=%s" % + (oldrev, newrev, refname)) + + if not include_ref(refname, ref_filter_regex, is_inclusion_filter): + continue + if not include_ref(refname, send_filter_regex, send_is_inclusion_filter): + continue changes.append( ReferenceChange.create(environment, oldrev, newrev, refname) ) - push = Push(changes) - push.send_emails(mailer, body_filter=environment.filter_body) - - -def run_as_update_hook(environment, mailer, refname, oldrev, newrev): + if changes: + push = Push(environment, changes) + push.send_emails(mailer, body_filter=environment.filter_body) + if hasattr(mailer, '__del__'): + mailer.__del__() + + +def run_as_update_hook(environment, mailer, refname, oldrev, newrev, force_send=False): + environment.check() + send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True) + ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False) + if not include_ref(refname, ref_filter_regex, is_inclusion_filter): + return + if not include_ref(refname, send_filter_regex, send_is_inclusion_filter): + return changes = [ ReferenceChange.create( environment, @@ -2393,8 +3687,79 @@ def run_as_update_hook(environment, mailer, refname, oldrev, newrev): refname, ), ] - push = Push(changes) + push = Push(environment, changes, force_send) push.send_emails(mailer, body_filter=environment.filter_body) + if hasattr(mailer, '__del__'): + mailer.__del__() + + +def check_ref_filter(environment): + send_filter_regex, send_is_inclusion = environment.get_ref_filter_regex(True) + ref_filter_regex, ref_is_inclusion = environment.get_ref_filter_regex(False) + + def inc_exc_lusion(b): + if b: + return 'inclusion' + else: + return 'exclusion' + + if send_filter_regex: + sys.stdout.write("DoSend/DontSend filter regex (" + + (inc_exc_lusion(send_is_inclusion)) + + '): ' + send_filter_regex.pattern + + '\n') + if send_filter_regex: + sys.stdout.write("Include/Exclude filter regex (" + + (inc_exc_lusion(ref_is_inclusion)) + + '): ' + ref_filter_regex.pattern + + '\n') + sys.stdout.write(os.linesep) + + sys.stdout.write( + "Refs marked as EXCLUDE are excluded by either refFilterInclusionRegex\n" + "or refFilterExclusionRegex. No emails will be sent for commits included\n" + "in these refs.\n" + "Refs marked as DONT-SEND are excluded by either refFilterDoSendRegex or\n" + "refFilterDontSendRegex, but not by either refFilterInclusionRegex or\n" + "refFilterExclusionRegex. Emails will be sent for commits included in these\n" + "refs only when the commit reaches a ref which isn't excluded.\n" + "Refs marked as DO-SEND are not excluded by any filter. Emails will\n" + "be sent normally for commits included in these refs.\n") + + sys.stdout.write(os.linesep) + + for refname in read_git_lines(['for-each-ref', '--format', '%(refname)']): + sys.stdout.write(refname) + if not include_ref(refname, ref_filter_regex, ref_is_inclusion): + sys.stdout.write(' EXCLUDE') + elif not include_ref(refname, send_filter_regex, send_is_inclusion): + sys.stdout.write(' DONT-SEND') + else: + sys.stdout.write(' DO-SEND') + + sys.stdout.write(os.linesep) + + +def show_env(environment, out): + out.write('Environment values:\n') + for (k, v) in sorted(environment.get_values().items()): + if k: # Don't show the {'' : ''} pair. + out.write(' %s : %r\n' % (k, v)) + out.write('\n') + # Flush to avoid interleaving with further log output + out.flush() + + +def check_setup(environment): + environment.check() + show_env(environment, sys.stdout) + sys.stdout.write("Now, checking that git-multimail's standard input " + "is properly set ..." + os.linesep) + sys.stdout.write("Please type some text and then press Return" + os.linesep) + stdin = sys.stdin.readline() + sys.stdout.write("You have just entered:" + os.linesep) + sys.stdout.write(stdin) + sys.stdout.write("git-multimail seems properly set up." + os.linesep) def choose_mailer(config, environment): @@ -2402,47 +3767,61 @@ def choose_mailer(config, environment): if mailer == 'smtp': smtpserver = config.get('smtpserver', default='localhost') + smtpservertimeout = float(config.get('smtpservertimeout', default=10.0)) + smtpserverdebuglevel = int(config.get('smtpserverdebuglevel', default=0)) + smtpencryption = config.get('smtpencryption', default='none') + smtpuser = config.get('smtpuser', default='') + smtppass = config.get('smtppass', default='') + smtpcacerts = config.get('smtpcacerts', default='') mailer = SMTPMailer( + environment, envelopesender=(environment.get_sender() or environment.get_fromaddr()), - smtpserver=smtpserver, + smtpserver=smtpserver, smtpservertimeout=smtpservertimeout, + smtpserverdebuglevel=smtpserverdebuglevel, + smtpencryption=smtpencryption, + smtpuser=smtpuser, + smtppass=smtppass, + smtpcacerts=smtpcacerts ) elif mailer == 'sendmail': command = config.get('sendmailcommand') if command: command = shlex.split(command) - mailer = SendMailer(command=command, envelopesender=environment.get_sender()) + mailer = SendMailer(environment, + command=command, envelopesender=environment.get_sender()) else: - sys.stderr.write( - 'fatal: multimailhook.mailer is set to an incorrect value: "%s"\n' % mailer - + 'please use one of "smtp" or "sendmail".\n' + environment.log_error( + 'fatal: multimailhook.mailer is set to an incorrect value: "%s"\n' % mailer + + 'please use one of "smtp" or "sendmail".' ) sys.exit(1) return mailer KNOWN_ENVIRONMENTS = { - 'generic' : GenericEnvironmentMixin, - 'gitolite' : GitoliteEnvironmentMixin, + 'generic': {'highprec': GenericEnvironmentMixin}, + 'gitolite': {'highprec': GitoliteEnvironmentHighPrecMixin, + 'lowprec': GitoliteEnvironmentLowPrecMixin}, + 'stash': {'highprec': StashEnvironmentHighPrecMixin, + 'lowprec': StashEnvironmentLowPrecMixin}, + 'gerrit': {'highprec': GerritEnvironmentHighPrecMixin, + 'lowprec': GerritEnvironmentLowPrecMixin}, } -def choose_environment(config, osenv=None, env=None, recipients=None): +def choose_environment(config, osenv=None, env=None, recipients=None, + hook_info=None): + env_name = choose_environment_name(config, env, osenv) + environment_klass = build_environment_klass(env_name) + env = build_environment(environment_klass, env_name, config, + osenv, recipients, hook_info) + return env + + +def choose_environment_name(config, env, osenv): if not osenv: osenv = os.environ - environment_mixins = [ - ProjectdescEnvironmentMixin, - ConfigMaxlinesEnvironmentMixin, - ComputeFQDNEnvironmentMixin, - ConfigFilterLinesEnvironmentMixin, - PusherDomainEnvironmentMixin, - ConfigOptionsEnvironmentMixin, - ] - environment_kw = { - 'osenv' : osenv, - 'config' : config, - } - if not env: env = config.get('environment') @@ -2451,25 +3830,250 @@ def choose_environment(config, osenv=None, env=None, recipients=None): env = 'gitolite' else: env = 'generic' + return env - environment_mixins.append(KNOWN_ENVIRONMENTS[env]) - - if recipients: - environment_mixins.insert(0, StaticRecipientsEnvironmentMixin) - environment_kw['refchange_recipients'] = recipients - environment_kw['announce_recipients'] = recipients - environment_kw['revision_recipients'] = recipients - else: - environment_mixins.insert(0, ConfigRecipientsEnvironmentMixin) +COMMON_ENVIRONMENT_MIXINS = [ + ConfigRecipientsEnvironmentMixin, + CLIRecipientsEnvironmentMixin, + ConfigRefFilterEnvironmentMixin, + ProjectdescEnvironmentMixin, + ConfigMaxlinesEnvironmentMixin, + ComputeFQDNEnvironmentMixin, + ConfigFilterLinesEnvironmentMixin, + PusherDomainEnvironmentMixin, + ConfigOptionsEnvironmentMixin, + ] + + +def build_environment_klass(env_name): + if 'class' in KNOWN_ENVIRONMENTS[env_name]: + return KNOWN_ENVIRONMENTS[env_name]['class'] + + environment_mixins = [] + known_env = KNOWN_ENVIRONMENTS[env_name] + if 'highprec' in known_env: + high_prec_mixin = known_env['highprec'] + environment_mixins.append(high_prec_mixin) + environment_mixins = environment_mixins + COMMON_ENVIRONMENT_MIXINS + if 'lowprec' in known_env: + low_prec_mixin = known_env['lowprec'] + environment_mixins.append(low_prec_mixin) + environment_mixins.append(Environment) + klass_name = env_name.capitalize() + 'Environement' environment_klass = type( - 'EffectiveEnvironment', - tuple(environment_mixins) + (Environment,), + klass_name, + tuple(environment_mixins), {}, ) + KNOWN_ENVIRONMENTS[env_name]['class'] = environment_klass + return environment_klass + + +GerritEnvironment = build_environment_klass('gerrit') +StashEnvironment = build_environment_klass('stash') +GitoliteEnvironment = build_environment_klass('gitolite') +GenericEnvironment = build_environment_klass('generic') + + +def build_environment(environment_klass, env, config, + osenv, recipients, hook_info): + environment_kw = { + 'osenv': osenv, + 'config': config, + } + + if env == 'stash': + environment_kw['user'] = hook_info['stash_user'] + environment_kw['repo'] = hook_info['stash_repo'] + elif env == 'gerrit': + environment_kw['project'] = hook_info['project'] + environment_kw['submitter'] = hook_info['submitter'] + environment_kw['update_method'] = hook_info['update_method'] + + environment_kw['cli_recipients'] = recipients + return environment_klass(**environment_kw) +def get_version(): + oldcwd = os.getcwd() + try: + try: + os.chdir(os.path.dirname(os.path.realpath(__file__))) + git_version = read_git_output(['describe', '--tags', 'HEAD']) + if git_version == __version__: + return git_version + else: + return '%s (%s)' % (__version__, git_version) + except: + pass + finally: + os.chdir(oldcwd) + return __version__ + + +def compute_gerrit_options(options, args, required_gerrit_options, + raw_refname): + if None in required_gerrit_options: + raise SystemExit("Error: Specify all of --oldrev, --newrev, --refname, " + "and --project; or none of them.") + + if options.environment not in (None, 'gerrit'): + raise SystemExit("Non-gerrit environments incompatible with --oldrev, " + "--newrev, --refname, and --project") + options.environment = 'gerrit' + + if args: + raise SystemExit("Error: Positional parameters not allowed with " + "--oldrev, --newrev, and --refname.") + + # Gerrit oddly omits 'refs/heads/' in the refname when calling + # ref-updated hook; put it back. + git_dir = get_git_dir() + if (not os.path.exists(os.path.join(git_dir, raw_refname)) and + os.path.exists(os.path.join(git_dir, 'refs', 'heads', + raw_refname))): + options.refname = 'refs/heads/' + options.refname + + # New revisions can appear in a gerrit repository either due to someone + # pushing directly (in which case options.submitter will be set), or they + # can press "Submit this patchset" in the web UI for some CR (in which + # case options.submitter will not be set and gerrit will not have provided + # us the information about who pressed the button). + # + # Note for the nit-picky: I'm lumping in REST API calls and the ssh + # gerrit review command in with "Submit this patchset" button, since they + # have the same effect. + if options.submitter: + update_method = 'pushed' + # The submitter argument is almost an RFC 2822 email address; change it + # from 'User Name (email@domain)' to 'User Name <email@domain>' so it is + options.submitter = options.submitter.replace('(', '<').replace(')', '>') + else: + update_method = 'submitted' + # Gerrit knew who submitted this patchset, but threw that information + # away when it invoked this hook. However, *IF* Gerrit created a + # merge to bring the patchset in (project 'Submit Type' is either + # "Always Merge", or is "Merge if Necessary" and happens to be + # necessary for this particular CR), then it will have the committer + # of that merge be 'Gerrit Code Review' and the author will be the + # person who requested the submission of the CR. Since this is fairly + # likely for most gerrit installations (of a reasonable size), it's + # worth the extra effort to try to determine the actual submitter. + rev_info = read_git_lines(['log', '--no-walk', '--merges', + '--format=%cN%n%aN <%aE>', options.newrev]) + if rev_info and rev_info[0] == 'Gerrit Code Review': + options.submitter = rev_info[1] + + # We pass back refname, oldrev, newrev as args because then the + # gerrit ref-updated hook is much like the git update hook + return (options, + [options.refname, options.oldrev, options.newrev], + {'project': options.project, 'submitter': options.submitter, + 'update_method': update_method}) + + +def check_hook_specific_args(options, args): + raw_refname = options.refname + # Convert each string option unicode for Python3. + if PYTHON3: + opts = ['environment', 'recipients', 'oldrev', 'newrev', 'refname', + 'project', 'submitter', 'stash_user', 'stash_repo'] + for opt in opts: + if not hasattr(options, opt): + continue + obj = getattr(options, opt) + if obj: + enc = obj.encode('utf-8', 'surrogateescape') + dec = enc.decode('utf-8', 'replace') + setattr(options, opt, dec) + + # First check for stash arguments + if (options.stash_user is None) != (options.stash_repo is None): + raise SystemExit("Error: Specify both of --stash-user and " + "--stash-repo or neither.") + if options.stash_user: + options.environment = 'stash' + return options, args, {'stash_user': options.stash_user, + 'stash_repo': options.stash_repo} + + # Finally, check for gerrit specific arguments + required_gerrit_options = (options.oldrev, options.newrev, options.refname, + options.project) + if required_gerrit_options != (None,) * 4: + return compute_gerrit_options(options, args, required_gerrit_options, + raw_refname) + + # No special options in use, just return what we started with + return options, args, {} + + +class Logger(object): + def parse_verbose(self, verbose): + if verbose > 0: + return logging.DEBUG + else: + return logging.INFO + + def create_log_file(self, environment, name, path, verbosity): + log_file = logging.getLogger(name) + file_handler = logging.FileHandler(path) + log_fmt = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") + file_handler.setFormatter(log_fmt) + log_file.addHandler(file_handler) + log_file.setLevel(verbosity) + return log_file + + def __init__(self, environment): + self.environment = environment + self.loggers = [] + stderr_log = logging.getLogger('git_multimail.stderr') + + class EncodedStderr(object): + def write(self, x): + write_str(sys.stderr, x) + + def flush(self): + sys.stderr.flush() + + stderr_handler = logging.StreamHandler(EncodedStderr()) + stderr_log.addHandler(stderr_handler) + stderr_log.setLevel(self.parse_verbose(environment.verbose)) + self.loggers.append(stderr_log) + + if environment.debug_log_file is not None: + debug_log_file = self.create_log_file( + environment, 'git_multimail.debug', environment.debug_log_file, logging.DEBUG) + self.loggers.append(debug_log_file) + + if environment.log_file is not None: + log_file = self.create_log_file( + environment, 'git_multimail.file', environment.log_file, logging.INFO) + self.loggers.append(log_file) + + if environment.error_log_file is not None: + error_log_file = self.create_log_file( + environment, 'git_multimail.error', environment.error_log_file, logging.ERROR) + self.loggers.append(error_log_file) + + def info(self, msg): + for l in self.loggers: + l.info(msg) + + def debug(self, msg): + for l in self.loggers: + l.debug(msg) + + def warning(self, msg): + for l in self.loggers: + l.warning(msg) + + def error(self, msg): + for l in self.loggers: + l.error(msg) + + def main(args): parser = optparse.OptionParser( description=__doc__, @@ -2478,7 +4082,7 @@ def main(args): parser.add_option( '--environment', '--env', action='store', type='choice', - choices=['generic', 'gitolite'], default=None, + choices=list(KNOWN_ENVIRONMENTS.keys()), default=None, help=( 'Choose type of environment is in use. Default is taken from ' 'multimailhook.environment if set; otherwise "generic".' @@ -2496,44 +4100,139 @@ def main(args): '--show-env', action='store_true', default=False, help=( 'Write to stderr the values determined for the environment ' - '(intended for debugging purposes).' + '(intended for debugging purposes), then proceed normally.' + ), + ) + parser.add_option( + '--force-send', action='store_true', default=False, + help=( + 'Force sending refchange email when using as an update hook. ' + 'This is useful to work around the unreliable new commits ' + 'detection in this mode.' + ), + ) + parser.add_option( + '-c', metavar="<name>=<value>", action='append', + help=( + 'Pass a configuration parameter through to git. The value given ' + 'will override values from configuration files. See the -c option ' + 'of git(1) for more details. (Only works with git >= 1.7.3)' + ), + ) + parser.add_option( + '--version', '-v', action='store_true', default=False, + help=( + "Display git-multimail's version" + ), + ) + + parser.add_option( + '--python-version', action='store_true', default=False, + help=( + "Display the version of Python used by git-multimail" ), ) + parser.add_option( + '--check-ref-filter', action='store_true', default=False, + help=( + 'List refs and show information on how git-multimail ' + 'will process them.' + ) + ) + + # The following options permit this script to be run as a gerrit + # ref-updated hook. See e.g. + # code.google.com/p/gerrit/source/browse/Documentation/config-hooks.txt + # We suppress help for these items, since these are specific to gerrit, + # and we don't want users directly using them any way other than how the + # gerrit ref-updated hook is called. + parser.add_option('--oldrev', action='store', help=optparse.SUPPRESS_HELP) + parser.add_option('--newrev', action='store', help=optparse.SUPPRESS_HELP) + parser.add_option('--refname', action='store', help=optparse.SUPPRESS_HELP) + parser.add_option('--project', action='store', help=optparse.SUPPRESS_HELP) + parser.add_option('--submitter', action='store', help=optparse.SUPPRESS_HELP) + + # The following allow this to be run as a stash asynchronous post-receive + # hook (almost identical to a git post-receive hook but triggered also for + # merges of pull requests from the UI). We suppress help for these items, + # since these are specific to stash. + parser.add_option('--stash-user', action='store', help=optparse.SUPPRESS_HELP) + parser.add_option('--stash-repo', action='store', help=optparse.SUPPRESS_HELP) + (options, args) = parser.parse_args(args) + (options, args, hook_info) = check_hook_specific_args(options, args) + + if options.version: + sys.stdout.write('git-multimail version ' + get_version() + '\n') + return + + if options.python_version: + sys.stdout.write('Python version ' + sys.version + '\n') + return + + if options.c: + Config.add_config_parameters(options.c) config = Config('multimailhook') + environment = None try: environment = choose_environment( config, osenv=os.environ, env=options.environment, recipients=options.recipients, + hook_info=hook_info, ) if options.show_env: - sys.stderr.write('Environment values:\n') - for (k,v) in sorted(environment.get_values().items()): - sys.stderr.write(' %s : %r\n' % (k,v)) - sys.stderr.write('\n') + show_env(environment, sys.stderr) - if options.stdout: + if options.stdout or environment.stdout: mailer = OutputMailer(sys.stdout) else: mailer = choose_mailer(config, environment) + must_check_setup = os.environ.get('GIT_MULTIMAIL_CHECK_SETUP') + if must_check_setup == '': + must_check_setup = False + if options.check_ref_filter: + check_ref_filter(environment) + elif must_check_setup: + check_setup(environment) # Dual mode: if arguments were specified on the command line, run # like an update hook; otherwise, run as a post-receive hook. - if args: + elif args: if len(args) != 3: parser.error('Need zero or three non-option arguments') (refname, oldrev, newrev) = args - run_as_update_hook(environment, mailer, refname, oldrev, newrev) + environment.get_logger().debug( + "run_as_update_hook: refname=%s, oldrev=%s, newrev=%s, force_send=%s" % + (refname, oldrev, newrev, options.force_send)) + run_as_update_hook(environment, mailer, refname, oldrev, newrev, options.force_send) else: run_as_post_receive_hook(environment, mailer) - except ConfigurationException, e: - sys.exit(str(e)) - + except ConfigurationException: + sys.exit(sys.exc_info()[1]) + except SystemExit: + raise + except Exception: + t, e, tb = sys.exc_info() + import traceback + sys.stderr.write('\n') # Avoid mixing message with previous output + msg = ( + 'Exception \'' + t.__name__ + + '\' raised. Please report this as a bug to\n' + 'https://github.com/git-multimail/git-multimail/issues\n' + 'with the information below:\n\n' + 'git-multimail version ' + get_version() + '\n' + 'Python version ' + sys.version + '\n' + + traceback.format_exc()) + try: + environment.get_logger().error(msg) + except: + sys.stderr.write(msg) + sys.exit(1) if __name__ == '__main__': main(sys.argv[1:]) |