diff options
108 files changed, 2786 insertions, 6939 deletions
diff --git a/Documentation/Makefile b/Documentation/Makefile index 2aae4c9cbb..f5605b7767 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -139,6 +139,7 @@ ASCIIDOC_CONF = -f asciidoc.conf ASCIIDOC_COMMON = $(ASCIIDOC) $(ASCIIDOC_EXTRA) $(ASCIIDOC_CONF) \ -amanversion=$(GIT_VERSION) \ -amanmanual='Git Manual' -amansource='Git' +ASCIIDOC_DEPS = asciidoc.conf GIT-ASCIIDOCFLAGS TXT_TO_HTML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_HTML) TXT_TO_XML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_DOCBOOK) MANPAGE_XSL = manpage-normal.xsl @@ -193,6 +194,7 @@ ASCIIDOC_DOCBOOK = docbook5 ASCIIDOC_EXTRA += -acompat-mode -atabsize=8 ASCIIDOC_EXTRA += -I. -rasciidoctor-extensions ASCIIDOC_EXTRA += -alitdd='&\#x2d;&\#x2d;' +ASCIIDOC_DEPS = asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS DBLATEX_COMMON = XMLTO_EXTRA += --skip-validation XMLTO_EXTRA += -x manpage.xsl @@ -294,9 +296,7 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(DOC_DEP_TXT) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(PERL_PATH) ./build-docdep.perl >$@ $(QUIET_STDERR) ifneq ($(MAKECMDGOALS),clean) -include doc.dep @@ -316,8 +316,7 @@ cmds_txt = cmds-ancillaryinterrogators.txt \ $(cmds_txt): cmd-list.made cmd-list.made: cmd-list.perl ../command-list.txt $(MAN1_TXT) - $(QUIET_GEN)$(RM) $@ && \ - $(PERL_PATH) ./cmd-list.perl ../command-list.txt $(cmds_txt) $(QUIET_STDERR) && \ + $(QUIET_GEN)$(PERL_PATH) ./cmd-list.perl ../command-list.txt $(cmds_txt) $(QUIET_STDERR) && \ date >$@ mergetools_txt = mergetools-diff.txt mergetools-merge.txt @@ -325,7 +324,7 @@ mergetools_txt = mergetools-diff.txt mergetools-merge.txt $(mergetools_txt): mergetools-list.made mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) - $(QUIET_GEN)$(RM) $@ && \ + $(QUIET_GEN) \ $(SHELL_PATH) -c 'MERGE_TOOLS_DIR=../mergetools && \ . ../git-mergetool--lib.sh && \ show_tool_names can_diff "* " || :' >mergetools-diff.txt && \ @@ -354,32 +353,23 @@ clean: $(RM) manpage-base-url.xsl $(RM) GIT-ASCIIDOCFLAGS -$(MAN_HTML): %.html : %.txt asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ +$(MAN_HTML): %.html : %.txt $(ASCIIDOC_DEPS) + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -d manpage -o $@ $< -$(OBSOLETE_HTML): %.html : %.txto asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ +$(OBSOLETE_HTML): %.html : %.txto $(ASCIIDOC_DEPS) + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -o $@ $< manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_GEN)sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ %.1 %.5 %.7 : %.xml manpage-base-url.xsl $(wildcard manpage*.xsl) - $(QUIET_XMLTO)$(RM) $@ && \ - $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< + $(QUIET_XMLTO)$(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< -%.xml : %.txt asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ +%.xml : %.txt $(ASCIIDOC_DEPS) + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d manpage -o $@ $< user-manual.xml: user-manual.txt user-manual.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d book -o $@ $< technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -400,46 +390,35 @@ XSLTOPTS += --stringparam html.stylesheet docbook-xsl.css XSLTOPTS += --param generate.consistent.ids 1 user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)xsltproc $(XSLTOPTS) -o $@ $(XSLT) $< git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@+ && \ + $(PERL_PATH) fix-texi.perl <$@+ >$@ && \ + $(RM) $@+ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(DBLATEX) -o $@ $(DBLATEX_COMMON) $< gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI) \ ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ - rm $(xml)+ &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(RM) $(xml)+ &&) true) > $@+ && \ + $(PERL_PATH) cat-texi.perl $@ <$@+ >$@ && \ + $(RM) $@+ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@ howto-index.txt: howto-index.sh $(HOWTO_TXT) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)'$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -448,10 +427,9 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(HOWTO_TXT)): %.html : %.txt GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC) \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) @@ -492,4 +470,7 @@ doc-l10n install-l10n:: $(MAKE) -C po $@ endif +# Delete the target file on error +.DELETE_ON_ERROR: + .PHONY: FORCE diff --git a/Documentation/RelNotes/1.6.0.3.txt b/Documentation/RelNotes/1.6.0.3.txt index ae0577836a..ad36c0f0b7 100644 --- a/Documentation/RelNotes/1.6.0.3.txt +++ b/Documentation/RelNotes/1.6.0.3.txt @@ -50,7 +50,7 @@ Fixes since v1.6.0.2 if the working tree is currently dirty. * "git for-each-ref --format=%(subject)" fixed for commits with no - no newline in the message body. + newline in the message body. * "git remote" fixed to protect printf from user input. diff --git a/Documentation/RelNotes/1.8.4.txt b/Documentation/RelNotes/1.8.4.txt index 255e185af6..2e7529928b 100644 --- a/Documentation/RelNotes/1.8.4.txt +++ b/Documentation/RelNotes/1.8.4.txt @@ -365,7 +365,7 @@ details). (merge 2fbd4f9 mh/maint-lockfile-overflow later to maint). * Invocations of "git checkout" used internally by "git rebase" were - counted as "checkout", and affected later "git checkout -" to the + counted as "checkout", and affected later "git checkout -", which took the user to an unexpected place. (merge 3bed291 rr/rebase-checkout-reflog later to maint). diff --git a/Documentation/RelNotes/2.29.0.txt b/Documentation/RelNotes/2.29.0.txt index 06ba2f803f..1f41302ebb 100644 --- a/Documentation/RelNotes/2.29.0.txt +++ b/Documentation/RelNotes/2.29.0.txt @@ -184,8 +184,8 @@ Performance, Internal Implementation, Development Support etc. the ref backend in use, as its format is much richer than the normal refs, and written directly by "git fetch" as a plain file.. - * An unused binary has been discarded, and and a bunch of commands - have been turned into into built-in. + * An unused binary has been discarded, and a bunch of commands + have been turned into built-in. * A handful of places in in-tree code still relied on being able to execute the git subcommands, especially built-ins, in "git-foo" diff --git a/Documentation/RelNotes/2.33.0.txt b/Documentation/RelNotes/2.33.0.txt new file mode 100644 index 0000000000..57443c7466 --- /dev/null +++ b/Documentation/RelNotes/2.33.0.txt @@ -0,0 +1,78 @@ +Git 2.33 Release Notes +====================== + +Backward compatibility notes +---------------------------- + + * The "-m" option in "git log -m" that does not specify which format, + if any, of diff is desired did not have any visible effect; it now + implies some form of diff (by default "--patch") is produced. + + You can disable the diff output with "git log -m --no-patch", but + then there probably isn't much point in passing "-m" in the first + place ;-). + + +Updates since Git 2.32 +---------------------- + +UI, Workflows & Features + + * "git send-email" learned the "--sendmail-cmd" command line option + and the "sendemail.sendmailCmd" configuration variable, which is a + more sensible approach than the current way of repurposing the + "smtp-server" that is meant to name the server to instead name the + command to talk to the server. + + * The "-m" option in "git log -m" that does not specify which format, + if any, of diff is desired did not have any visible effect; it now + implies some form of diff (by default "--patch") is produced. + + +Performance, Internal Implementation, Development Support etc. + + * The code to handle the "--format" option in "for-each-ref" and + friends made too many string comparisons on %(atom)s used in the + format string, which has been corrected by converting them into + enum when the format string is parsed. + + * Use the hashfile API in the codepath that writes the index file to + reduce code duplication. + + * Repeated rename detections in a sequence of mergy operations have + been optimize out. + + +Fixes since v2.32 +----------------- + + * We historically rejected a very short string as an author name + while accepting a patch e-mail, which has been loosened. + (merge 72ee47ceeb ef/mailinfo-short-name later to maint). + + * The parallel checkout codepath did not initialize object ID field + used to talk to the worker processes in a futureproof way. + + * Rewrite code that triggers undefined behaviour warning. + (merge aafa5df0df jn/size-t-casted-to-off-t-fix later to maint). + + * The description of "fast-forward" in the glossary has been updated. + (merge e22f2daed0 ry/clarify-fast-forward-in-glossary later to maint). + + * Recent "git clone" left a temporary directory behind when the + transport layer returned an failure. + (merge 6aacb7d861 jk/clone-clean-upon-transport-error later to maint). + + * "git fetch" over protocol v2 left its side of the socket open after + it finished speaking, which unnecessarily wasted the resource on + the other side. + (merge ae1a7eefff jk/fetch-pack-v2-half-close-early later to maint). + + * Other code cleanup, docfix, build fix, etc. + (merge bfe35a6165 ah/doc-describe later to maint). + (merge f302c1e4aa jc/clarify-revision-range later to maint). + (merge 3127ff90ea tl/fix-packfile-uri-doc later to maint). + (merge a84216c684 jk/doc-color-pager later to maint). + (merge 4e0a64a713 ab/trace2-squelch-gcc-warning later to maint). + (merge 225f7fa847 ps/rev-list-object-type-filter later to maint). + (merge 5317dfeaed dd/honor-users-tar-in-tests later to maint). diff --git a/Documentation/RelNotes/2.8.0.txt b/Documentation/RelNotes/2.8.0.txt index 27320b6a9f..38453281b8 100644 --- a/Documentation/RelNotes/2.8.0.txt +++ b/Documentation/RelNotes/2.8.0.txt @@ -377,7 +377,7 @@ notes for details). on that order. * "git show 'HEAD:Foo[BAR]Baz'" did not interpret the argument as a - rev, i.e. the object named by the the pathname with wildcard + rev, i.e. the object named by the pathname with wildcard characters in a tree object. (merge aac4fac nd/dwim-wildcards-as-pathspecs later to maint). diff --git a/Documentation/config/color.txt b/Documentation/config/color.txt index d5daacb13a..e05d520a86 100644 --- a/Documentation/config/color.txt +++ b/Documentation/config/color.txt @@ -127,8 +127,9 @@ color.interactive.<slot>:: interactive commands. color.pager:: - A boolean to enable/disable colored output when the pager is in - use (default is true). + A boolean to specify whether `auto` color modes should colorize + output going to the pager. Defaults to true; set this to false + if your pager does not understand ANSI color codes. color.push:: A boolean to enable/disable color in push errors. May be set to diff --git a/Documentation/config/merge.txt b/Documentation/config/merge.txt index cb2ed58907..6b66c83eab 100644 --- a/Documentation/config/merge.txt +++ b/Documentation/config/merge.txt @@ -14,7 +14,7 @@ merge.defaultToUpstream:: branches at the remote named by `branch.<current branch>.remote` are consulted, and then they are mapped via `remote.<remote>.fetch` to their corresponding remote-tracking branches, and the tips of - these tracking branches are merged. + these tracking branches are merged. Defaults to true. merge.ff:: By default, Git does not create an extra merge commit when merging diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt index 530d115914..32e6dee5ac 100644 --- a/Documentation/diff-options.txt +++ b/Documentation/diff-options.txt @@ -49,10 +49,9 @@ ifdef::git-log[] --diff-merges=m::: -m::: This option makes diff output for merge commits to be shown in - the default format. `-m` will produce the output only if `-p` - is given as well. The default format could be changed using + the default format. The default format could be changed using `log.diffMerges` configuration parameter, which default value - is `separate`. + is `separate`. `-m` implies `-p`. + --diff-merges=first-parent::: --diff-merges=1::: @@ -62,7 +61,8 @@ ifdef::git-log[] --diff-merges=separate::: This makes merge commits show the full diff with respect to each of the parents. Separate log entry and diff is generated - for each parent. + for each parent. This is the format that `-m` produced + historically. + --diff-merges=combined::: --diff-merges=c::: diff --git a/Documentation/git-describe.txt b/Documentation/git-describe.txt index a88f6ae2c6..c6a79c2a0f 100644 --- a/Documentation/git-describe.txt +++ b/Documentation/git-describe.txt @@ -63,9 +63,10 @@ OPTIONS Automatically implies --tags. --abbrev=<n>:: - Instead of using the default 7 hexadecimal digits as the - abbreviated object name, use <n> digits, or as many digits - as needed to form a unique object name. An <n> of 0 + Instead of using the default number of hexadecimal digits (which + will vary according to the number of objects in the repository with + a default of 7) of the abbreviated object name, use <n> digits, or + as many digits as needed to form a unique object name. An <n> of 0 will suppress long format, only showing the closest tag. --candidates=<n>:: @@ -139,8 +140,11 @@ at the end. The number of additional commits is the number of commits which would be displayed by "git log v1.0.4..parent". -The hash suffix is "-g" + unambiguous abbreviation for the tip commit -of parent (which was `2414721b194453f058079d897d13c4e377f92dc6`). +The hash suffix is "-g" + an unambigous abbreviation for the tip commit +of parent (which was `2414721b194453f058079d897d13c4e377f92dc6`). The +length of the abbreviation scales as the repository grows, using the +approximate number of objects in the repository and a bit of math +around the birthday paradox, and defaults to a minimum of 7. The "g" prefix stands for "git" and is used to allow describing the version of a software depending on the SCM the software is managed with. This is useful in an environment where people may use different SCMs. diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt index 93708aefea..3db4eab4ba 100644 --- a/Documentation/git-send-email.txt +++ b/Documentation/git-send-email.txt @@ -167,6 +167,14 @@ Sending `sendemail.envelopeSender` configuration variable; if that is unspecified, choosing the envelope sender is left to your MTA. +--sendmail-cmd=<command>:: + Specify a command to run to send the email. The command should + be sendmail-like; specifically, it must support the `-i` option. + The command will be executed in the shell if necessary. Default + is the value of `sendemail.sendmailcmd`. If unspecified, and if + --smtp-server is also unspecified, git-send-email will search + for `sendmail` in `/usr/sbin`, `/usr/lib` and $PATH. + --smtp-encryption=<encryption>:: Specify the encryption to use, either 'ssl' or 'tls'. Any other value reverts to plain SMTP. Default is the value of @@ -211,13 +219,16 @@ a password is obtained using 'git-credential'. --smtp-server=<host>:: If set, specifies the outgoing SMTP server to use (e.g. - `smtp.example.com` or a raw IP address). Alternatively it can - specify a full pathname of a sendmail-like program instead; - the program must support the `-i` option. Default value can - be specified by the `sendemail.smtpServer` configuration - option; the built-in default is to search for `sendmail` in - `/usr/sbin`, `/usr/lib` and $PATH if such program is - available, falling back to `localhost` otherwise. + `smtp.example.com` or a raw IP address). If unspecified, and if + `--sendmail-cmd` is also unspecified, the default is to search + for `sendmail` in `/usr/sbin`, `/usr/lib` and $PATH if such a + program is available, falling back to `localhost` otherwise. ++ +For backward compatibility, this option can also specify a full pathname +of a sendmail-like program instead; the program must support the `-i` +option. This method does not support passing arguments or using plain +command names. For those use cases, consider using `--sendmail-cmd` +instead. --smtp-server-port=<port>:: Specifies a port different from the default port (SMTP diff --git a/Documentation/glossary-content.txt b/Documentation/glossary-content.txt index 67c7a50b96..c077971335 100644 --- a/Documentation/glossary-content.txt +++ b/Documentation/glossary-content.txt @@ -146,8 +146,8 @@ current branch integrates with) obviously do not work, as there is no <<def_revision,revision>> and you are "merging" another <<def_branch,branch>>'s changes that happen to be a descendant of what you have. In such a case, you do not make a new <<def_merge,merge>> - <<def_commit,commit>> but instead just update to his - revision. This will happen frequently on a + <<def_commit,commit>> but instead just update your branch to point at the same + revision as the branch you are merging. This will happen frequently on a <<def_remote_tracking_branch,remote-tracking branch>> of a remote <<def_repository,repository>>. diff --git a/Documentation/revisions.txt b/Documentation/revisions.txt index d9169c062e..f5f17b65a1 100644 --- a/Documentation/revisions.txt +++ b/Documentation/revisions.txt @@ -260,6 +260,9 @@ any of the given commits. A commit's reachable set is the commit itself and the commits in its ancestry chain. +There are several notations to specify a set of connected commits +(called a "revision range"), illustrated below. + Commit Exclusions ~~~~~~~~~~~~~~~~~ @@ -294,6 +297,26 @@ is a shorthand for 'HEAD..origin' and asks "What did the origin do since I forked from them?" Note that '..' would mean 'HEAD..HEAD' which is an empty range that is both reachable and unreachable from HEAD. +Commands that are specifically designed to take two distinct ranges +(e.g. "git range-diff R1 R2" to compare two ranges) do exist, but +they are exceptions. Unless otherwise noted, all "git" commands +that operate on a set of commits work on a single revision range. +In other words, writing two "two-dot range notation" next to each +other, e.g. + + $ git log A..B C..D + +does *not* specify two revision ranges for most commands. Instead +it will name a single connected set of commits, i.e. those that are +reachable from either B or D but are reachable from neither A or C. +In a linear history like this: + + ---A---B---o---o---C---D + +because A and B are reachable from C, the revision range specified +by these two dotted ranges is a single commit D. + + Other <rev>{caret} Parent Shorthand Notations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Three other shorthands exist, particularly useful for merge commits, diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt index 3f52f981a2..037a91cbca 100644 --- a/Documentation/technical/api-trace2.txt +++ b/Documentation/technical/api-trace2.txt @@ -396,14 +396,14 @@ only present on the "start" and "atexit" events. } ------------ -`"discard"`:: +`"too_many_files"`:: This event is written to the git-trace2-discard sentinel file if there are too many files in the target trace directory (see the trace2.maxFiles config option). + ------------ { - "event":"discard", + "event":"too_many_files", ... } ------------ diff --git a/Documentation/technical/hash-function-transition.txt b/Documentation/technical/hash-function-transition.txt index 7c1630bf83..260224b033 100644 --- a/Documentation/technical/hash-function-transition.txt +++ b/Documentation/technical/hash-function-transition.txt @@ -599,7 +599,7 @@ supports four different modes of operation: convert any object names written to output to SHA-1, but store objects using SHA-256. This allows users to test the code with no visible behavior change except for performance. This allows - allows running even tests that assume the SHA-1 hash function, to + running even tests that assume the SHA-1 hash function, to sanity-check the behavior of the new mode. 2. ("early transition") Allow both SHA-1 and SHA-256 object names in diff --git a/Documentation/technical/packfile-uri.txt b/Documentation/technical/packfile-uri.txt index f7eabc6c76..1eb525fe76 100644 --- a/Documentation/technical/packfile-uri.txt +++ b/Documentation/technical/packfile-uri.txt @@ -35,13 +35,14 @@ include some sort of non-trivial implementation in the Minimum Viable Product, at least so that we can test the client. This is the implementation: a feature, marked experimental, that allows the -server to be configured by one or more `uploadpack.blobPackfileUri=<sha1> -<uri>` entries. Whenever the list of objects to be sent is assembled, all such -blobs are excluded, replaced with URIs. As noted in "Future work" below, the -server can evolve in the future to support excluding other objects (or other -implementations of servers could be made that support excluding other objects) -without needing a protocol change, so clients should not expect that packfiles -downloaded in this way only contain single blobs. +server to be configured by one or more `uploadpack.blobPackfileUri= +<object-hash> <pack-hash> <uri>` entries. Whenever the list of objects to be +sent is assembled, all such blobs are excluded, replaced with URIs. As noted +in "Future work" below, the server can evolve in the future to support +excluding other objects (or other implementations of servers could be made +that support excluding other objects) without needing a protocol change, so +clients should not expect that packfiles downloaded in this way only contain +single blobs. Client design ------------- diff --git a/Documentation/technical/partial-clone.txt b/Documentation/technical/partial-clone.txt index 0780d30cac..a0dd7c66f2 100644 --- a/Documentation/technical/partial-clone.txt +++ b/Documentation/technical/partial-clone.txt @@ -242,8 +242,7 @@ remote in a specific order. repository and can satisfy all such requests. - Repack essentially treats promisor and non-promisor packfiles as 2 - distinct partitions and does not mix them. Repack currently only works - on non-promisor packfiles and loose objects. + distinct partitions and does not mix them. - Dynamic object fetching invokes fetch-pack once *for each item* because most algorithms stumble upon a missing object and need to have @@ -273,9 +272,6 @@ to use those promisor remotes in that order." The user might want to work in a triangular work flow with multiple promisor remotes that each have an incomplete view of the repository. -- Allow repack to work on promisor packfiles (while keeping them distinct - from non-promisor packfiles). - - Allow non-pathname-based filters to make use of packfile bitmaps (when present). This was just an omission during the initial implementation. diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index a1e31367f4..1040d85319 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -540,7 +540,7 @@ An `object-info` request takes the following arguments: Indicates to the server an object which the client wants to obtain information for. -The response of `object-info` is a list of the the requested object ids +The response of `object-info` is a list of the requested object ids and associated requested information, each separated by a single space. output = info flush-pkt diff --git a/Documentation/technical/remembering-renames.txt b/Documentation/technical/remembering-renames.txt new file mode 100644 index 0000000000..2fd5cc88e0 --- /dev/null +++ b/Documentation/technical/remembering-renames.txt @@ -0,0 +1,671 @@ +Rebases and cherry-picks involve a sequence of merges whose results are +recorded as new single-parent commits. The first parent side of those +merges represent the "upstream" side, and often include a far larger set of +changes than the second parent side. Traditionally, the renames on the +first-parent side of that sequence of merges were repeatedly re-detected +for every merge. This file explains why it is safe and effective during +rebases and cherry-picks to remember renames on the upstream side of +history as an optimization, assuming all merges are automatic and clean +(i.e. no conflicts and not interrupted for user input or editing). + +Outline: + + 0. Assumptions + + 1. How rebasing and cherry-picking work + + 2. Why the renames on MERGE_SIDE1 in any given pick are *always* a + superset of the renames on MERGE_SIDE1 for the next pick. + + 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ always also + a rename on MERGE_SIDE1 for the next pick + + 4. A detailed description of the the counter-examples to #3. + + 5. Why the special cases in #4 are still fully reasonable to use to pair + up files for three-way content merging in the merge machinery, and why + they do not affect the correctness of the merge. + + 6. Interaction with skipping of "irrelevant" renames + + 7. Additional items that need to be cached + + 8. How directory rename detection interacts with the above and why this + optimization is still safe even if merge.directoryRenames is set to + "true". + + +=== 0. Assumptions === + +There are two assumptions that will hold throughout this document: + + * The upstream side where commits are transplanted to is treated as the + first parent side when rebase/cherry-pick call the merge machinery + + * All merges are fully automatic + +and a third that will hold in sections 2-5 for simplicity, that I'll later +address in section 8: + + * No directory renames occur + + +Let me explain more about each assumption and why I include it: + + +The first assumption is merely for the purposes of making this document +clearer; the optimization implementation does not actually depend upon it. +However, the assumption does hold in all cases because it reflects the way +that both rebase and cherry-pick were implemented; and the implementation +of cherry-pick and rebase are not readily changeable for backwards +compatibility reasons (see for example the discussion of the --ours and +--theirs flag in the documentation of `git checkout`, particularly the +comments about how they behave with rebase). The optimization avoids +checking first-parent-ness, though. It checks the conditions that make the +optimization valid instead, so it would still continue working if someone +changed the parent ordering that cherry-pick and rebase use. But making +this assumption does make this document much clearer and prevents me from +having to repeat every example twice. + +If the second assumption is violated, then the optimization simply is +turned off and thus isn't relevant to consider. The second assumption can +also be stated as "there is no interruption for a user to resolve conflicts +or to just further edit or tweak files". While real rebases and +cherry-picks are often interrupted (either because it's an interactive +rebase where the user requested to stop and edit, or because there were +conflicts that the user needs to resolve), the cache of renames is not +stored on disk, and thus is thrown away as soon as the rebase or cherry +pick stops for the user to resolve the operation. + +The third assumption makes sections 2-5 simpler, and allows people to +understand the basics of why this optimization is safe and effective, and +then I can go back and address the specifics in section 8. It is probably +also worth noting that if directory renames do occur, then the default of +merge.directoryRenames being set to "conflict" means that the operation +will stop for users to resolve the conflicts and the cache will be thrown +away, and thus that there won't be an optimization to apply. So, the only +reason we need to address directory renames specifically, is that some +users will have set merge.directoryRenames to "true" to allow the merges to +continue to proceed automatically. The optimization is still safe with +this config setting, but we have to discuss a few more cases to show why; +this discussion is deferred until section 8. + + +=== 1. How rebasing and cherry-picking work === + +Consider the following setup (from the git-rebase manpage): + + A---B---C topic + / + D---E---F---G main + +After rebasing or cherry-picking topic onto main, this will appear as: + + A'--B'--C' topic + / + D---E---F---G main + +The way the commits A', B', and C' are created is through a series of +merges, where rebase or cherry-pick sequentially uses each of the three +A-B-C commits in a special merge operation. Let's label the three commits +in the merge operation as MERGE_BASE, MERGE_SIDE1, and MERGE_SIDE2. For +this picture, the three commits for each of the three merges would be: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +To create C': + MERGE_BASE: B + MERGE_SIDE1: B' + MERGE_SIDE2: C + +Sometimes, folks are surprised that these three-way merges are done. It +can be useful in understanding these three-way merges to view them in a +slightly different light. For example, in creating C', you can view it as +either: + + * Apply the changes between B & C to B' + * Apply the changes between B & B' to C + +Conceptually the two statements above are the same as a three-way merge of +B, B', and C, at least the parts before you decide to record a commit. + + +=== 2. Why the renames on MERGE_SIDE1 in any given pick are always a === +=== superset of the renames on MERGE_SIDE1 for the next pick. === + +The merge machinery uses the filenames it is fed from MERGE_BASE, +MERGE_SIDE1, and MERGE_SIDE2. It will only move content to a different +filename under one of three conditions: + + * To make both pieces of a conflict available to a user during conflict + resolution (examples: directory/file conflict, add/add type conflict + such as symlink vs. regular file) + + * When MERGE_SIDE1 renames the file. + + * When MERGE_SIDE2 renames the file. + +First, let's remember what commits are involved in the first and second +picks of the cherry-pick or rebase sequence: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +So, in particular, we need to show that the renames between E and G are a +superset of those between A and A'. + +A' is created by the first merge. A' will only have renames for one of the +three reasons listed above. The first case, a conflict, results in a +situation where the cache is dropped and thus this optimization doesn't +take effect, so we need not consider that case. The third case, a rename +on MERGE_SIDE2 (i.e. from G to A), will show up in A' but it also shows up +in A -- therefore when diffing A and A' that path does not show up as a +rename. The only remaining way for renames to show up in A' is for the +rename to come from MERGE_SIDE1. Therefore, all renames between A and A' +are a subset of those between E and G. Equivalently, all renames between E +and G are a superset of those between A and A'. + + +=== 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ === +=== always also a rename on MERGE_SIDE1 for the next pick. === + +Let's again look at the first two picks: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +Now let's look at any given rename from MERGE_SIDE1 of the first pick, i.e. +any given rename from E to G. Let's use the filenames 'oldfile' and +'newfile' for demonstration purposes. That first pick will function as +follows; when the rename is detected, the merge machinery will do a +three-way content merge of the following: + E:oldfile + G:newfile + A:oldfile +and produce a new result: + A':newfile + +Note above that I've assumed that E->A did not rename oldfile. If that +side did rename, then we most likely have a rename/rename(1to2) conflict +that will cause the rebase or cherry-pick operation to halt and drop the +in-memory cache of renames and thus doesn't need to be considered further. +In the special case that E->A does rename the file but also renames it to +newfile, then there is no conflict from the renaming and the merge can +succeed. In this special case, the rename is not valid to cache because +the second merge will find A:newfile in the MERGE_BASE (see also the new +testcases in t6429 with "rename same file identically" in their +description). So a rename/rename(1to1) needs to be specially handled by +pruning renames from the cache and decrementing the dir_rename_counts in +the current and leading directories associated with those renames. Or, +since these are really rare, one could just take the easy way out and +disable the remembering renames optimization when a rename/rename(1to1) +happens. + +The previous paragraph handled the cases for E->A renaming oldfile, let's +continue assuming that oldfile is not renamed in A. + +As per the diagram for creating B', MERGE_SIDE1 involves the changes from A +to A'. So, we are curious whether A:oldfile and A':newfile will be viewed +as renames. Note that: + + * There will be no A':oldfile (because there could not have been a + G:oldfile as we do not do break detection in the merge machinery and + G:newfile was detected as a rename, and by the construction of the + rename above that merged cleanly, the merge machinery will ensure there + is no 'oldfile' in the result). + + * There will be no A:newfile (if there had been, we would have had a + rename/add conflict). + + * Clearly A:oldfile and A':newfile are "related" (A':newfile came from a + clean three-way content merge involving A:oldfile). + +We can also expound on the third point above, by noting that three-way +content merges can also be viewed as applying the differences between the +base and one side to the other side. Thus we can view A':newfile as +having been created by taking the changes between E:oldfile and G:newfile +(which were detected as being related, i.e. <50% changed) to A:oldfile. + +Thus A:oldfile and A':newfile are just as related as E:oldfile and +G:newfile are -- they have exactly identical differences. Since the latter +were detected as renames, A:oldfile and A':newfile should also be +detectable as renames almost always. + + +=== 4. A detailed description of the counter-examples to #3. === + +We already noted in section 3 that rename/rename(1to1) (i.e. both sides +renaming a file the same way) was one counter-example. The more +interesting bit, though, is why did we need to use the "almost" qualifier +when stating that A:oldfile and A':newfile are "almost" always detectable +as renames? + +Let's repeat an earlier point that section 3 made: + + A':newfile was created by applying the changes between E:oldfile and + G:newfile to A:oldfile. The changes between E:oldfile and G:newfile were + <50% of the size of E:oldfile. + +If those changes that were <50% of the size of E:oldfile are also <50% of +the size of A:oldfile, then A:oldfile and A':newfile will be detectable as +renames. However, if there is a dramatic size reduction between E:oldfile +and A:oldfile (but the changes between E:oldfile, G:newfile, and A:oldfile +still somehow merge cleanly), then traditional rename detection would not +detect A:oldfile and A':newfile as renames. + +Here's an example where that can happen: + * E:oldfile had 20 lines + * G:newfile added 10 new lines at the beginning of the file + * A:oldfile kept the first 3 lines of the file, and deleted all the rest +then + => A':newfile would have 13 lines, 3 of which matches those in A:oldfile. +E:oldfile -> G:newfile would be detected as a rename, but A:oldfile and +A':newfile would not be. + + +=== 5. Why the special cases in #4 are still fully reasonable to use to === +=== pair up files for three-way content merging in the merge machinery, === +=== and why they do not affect the correctness of the merge. === + +In the rename/rename(1to1) case, A:newfile and A':newfile are not renames +since they use the *same* filename. However, files with the same filename +are obviously fine to pair up for three-way content merging (the merge +machinery has never employed break detection). The interesting +counter-example case is thus not the rename/rename(1to1) case, but the case +where A did not rename oldfile. That was the case that we spent most of +the time discussing in sections 3 and 4. The remainder of this section +will be devoted to that case as well. + +So, even if A:oldfile and A':newfile aren't detectable as renames, why is +it still reasonable to pair them up for three-way content merging in the +merge machinery? There are multiple reasons: + + * As noted in sections 3 and 4, the diff between A:oldfile and A':newfile + is *exactly* the same as the diff between E:oldfile and G:newfile. The + latter pair were detected as renames, so it seems unlikely to surprise + users for us to treat A:oldfile and A':newfile as renames. + + * In fact, "oldfile" and "newfile" were at one point detected as renames + due to how they were constructed in the E..G chain. And we used that + information once already in this rebase/cherry-pick. I think users + would be unlikely to be surprised at us continuing to treat the files + as renames and would quickly understand why we had done so. + + * Marking or declaring files as renames is *not* the end goal for merges. + Merges use renames to determine which files make sense to be paired up + for three-way content merges. + + * A:oldfile and A':newfile were _already_ paired up in a three-way + content merge; that is how A':newfile was created. In fact, that + three-way content merge was clean. So using them again in a later + three-way content merge seems very reasonable. + +However, the above is focusing on the common scenarios. Let's try to look +at all possible unusual scenarios and compare without the optimization to +with the optimization. Consider the following theoretical cases; we will +then dive into each to determine which of them are possible, +and if so, what they mean: + + 1. Without the optimization, the second merge results in a conflict. + With the optimization, the second merge also results in a conflict. + Questions: Are the conflicts confusingly different? Better in one case? + + 2. Without the optimization, the second merge results in NO conflict. + With the optimization, the second merge also results in NO conflict. + Questions: Are the merges the same? + + 3. Without the optimization, the second merge results in a conflict. + With the optimization, the second merge results in NO conflict. + Questions: Possible? Bug, bugfix, or something else? + + 4. Without the optimization, the second merge results in NO conflict. + With the optimization, the second merge results in a conflict. + Questions: Possible? Bug, bugfix, or something else? + +I'll consider all four cases, but out of order. + +The fourth case is impossible. For the code without the remembering +renames optimization to not get a conflict, B:oldfile would need to exactly +match A:oldfile -- if it doesn't, there would be a modify/delete conflict. +If A:oldfile matches B:oldfile exactly, then a three-way content merge +between A:oldfile, A':newfile, and B:oldfile would have no conflict and +just give us the version of newfile from A' as the result. + +From the same logic as the above paragraph, the second case would indeed +result in identical merges. When A:oldfile exactly matches B:oldfile, an +undetected rename would say, "Oh, I see one side didn't modify 'oldfile' +and the other side deleted it. I'll delete it. And I see you have this +brand new file named 'newfile' in A', so I'll keep it." That gives the +same results as three-way content merging A:oldfile, A':newfile, and +B:oldfile -- a removal of oldfile with the version of newfile from A' +showing up in the result. + +The third case is interesting. It means that A:oldfile and A':newfile were +not just similar enough, but that the changes between them did not conflict +with the changes between A:oldfile and B:oldfile. This would validate our +hunch that the files were similar enough to be used in a three-way content +merge, and thus seems entirely correct for us to have used them that way. +(Sidenote: One particular example here may be enlightening. Let's say that +B was an immediate revert of A. B clearly would have been a clean revert +of A, since A was B's immediate parent. One would assume that if you can +pick a commit, you should also be able to cherry-pick its immediate revert. +However, this is one of those funny corner cases; without this +optimization, we just successfully picked a commit cleanly, but we are +unable to cherry-pick its immediate revert due to the size differences +between E:oldfile and A:oldfile.) + +That leaves only the first case to consider -- when we get conflicts both +with or without the optimization. Without the optimization, we'll have a +modify/delete conflict, where both A':newfile and B:oldfile are left in the +tree for the user to deal with and no hints about the potential similarity +between the two. With the optimization, we'll have a three-way content +merged A:oldfile, A':newfile, and B:oldfile with conflict markers +suggesting we thought the files were related but giving the user the chance +to resolve. As noted above, I don't think users will find us treating +'oldfile' and 'newfile' as related as a surprise since they were between E +and G. In any event, though, this case shouldn't be concerning since we +hit a conflict in both cases, told the user what we know, and asked them to +resolve it. + +So, in summary, case 4 is impossible, case 2 yields the same behavior, and +cases 1 and 3 seem to provide as good or better behavior with the +optimization than without. + + +=== 6. Interaction with skipping of "irrelevant" renames === + +Previous optimizations involved skipping rename detection for paths +considered to be "irrelevant". See for example the following commits: + + * 32a56dfb99 ("merge-ort: precompute subset of sources for which we + need rename detection", 2021-03-11) + * 2fd9eda462 ("merge-ort: precompute whether directory rename + detection is needed", 2021-03-11) + * 9bd342137e ("diffcore-rename: determine which relevant_sources are + no longer relevant", 2021-03-13) + +Relevance is always determined by what the _other_ side of history has +done, in terms of modifing a file that our side renamed, or adding a +file to a directory which our side renamed. This means that a path +that is "irrelevant" when picking the first commit of a series in a +rebase or cherry-pick, may suddenly become "relevant" when picking the +next commit. + +The upshot of this is that we can only cache rename detection results +for relevant paths, and need to re-check relevance in subsequent +commits. If those subsequent commits have additional paths that are +relevant for rename detection, then we will need to redo rename +detection -- though we can limit it to the paths for which we have not +already detected renames. + + +=== 7. Additional items that need to be cached === + +It turns out we have to cache more than just renames; we also cache: + + A) non-renames (i.e. unpaired deletes) + B) counts of renames within directories + C) sources that were marked as RELEVANT_LOCATION, but which were + downgraded to RELEVANT_NO_MORE + D) the toplevel trees involved in the merge + +These are all stored in struct rename_info, and respectively appear in + * cached_pairs (along side actual renames, just with a value of NULL) + * dir_rename_counts + * cached_irrelevant + * merge_trees + +The reason for (A) comes from the irrelevant renames skipping +optimization discussed in section 6. The fact that irrelevant renames +are skipped means we only get a subset of the potential renames +detected and subsequent commits may need to run rename detection on +the upstream side on a subset of the remaining renames (to get the +renames that are relevant for that later commit). Since unpaired +deletes are involved in rename detection too, we don't want to +repeatedly check that those paths remain unpaired on the upstream side +with every commit we are transplanting. + +The reason for (B) is that diffcore_rename_extended() is what +generates the counts of renames by directory which is needed in +directory rename detection, and if we don't run +diffcore_rename_extended() again then we need to have the output from +it, including dir_rename_counts, from the previous run. + +The reason for (C) is that merge-ort's tree traversal will again think +those paths are relevant (marking them as RELEVANT_LOCATION), but the +fact that they were downgraded to RELEVANT_NO_MORE means that +dir_rename_counts already has the information we need for directory +rename detection. (A path which becomes RELEVANT_CONTENT in a +subsequent commit will be removed from cached_irrelevant.) + +The reason for (D) is that is how we determine whether the remember +renames optimization can be used. In particular, remembering that our +sequence of merges looks like: + + Merge 1: + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + => Creates A' + + Merge 2: + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + => Creates B' + +It is the fact that the trees A and A' appear both in Merge 1 and in +Merge 2, with A as a parent of A' that allows this optimization. So +we store the trees to compare with what we are asked to merge next +time. + + +=== 8. How directory rename detection interacts with the above and === +=== why this optimization is still safe even if === +=== merge.directoryRenames is set to "true". === + +As noted in the assumptions section: + + """ + ...if directory renames do occur, then the default of + merge.directoryRenames being set to "conflict" means that the operation + will stop for users to resolve the conflicts and the cache will be + thrown away, and thus that there won't be an optimization to apply. + So, the only reason we need to address directory renames specifically, + is that some users will have set merge.directoryRenames to "true" to + allow the merges to continue to proceed automatically. + """ + +Let's remember that we need to look at how any given pick affects the next +one. So let's again use the first two picks from the diagram in section +one: + + First pick does this three-way merge: + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + => creates A' + + Second pick does this three-way merge: + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + => creates B' + +Now, directory rename detection exists so that if one side of history +renames a directory, and the other side adds a new file to the old +directory, then the merge (with merge.directoryRenames=true) can move the +file into the new directory. There are two qualitatively different ways to +add a new file to an old directory: create a new file, or rename a file +into that directory. Also, directory renames can be done on either side of +history, so there are four cases to consider: + + * MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir + * MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir + * MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir + * MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir + +One last note before we consider these four cases: There are some +important properties about how we implement this optimization with +respect to directory rename detection that we need to bear in mind +while considering all of these cases: + + * rename caching occurs *after* applying directory renames + + * a rename created by directory rename detection is recorded for the side + of history that did the directory rename. + + * dir_rename_counts, the nested map of + {oldname => {newname => count}}, + is cached between runs as well. This basically means that directory + rename detection is also cached, though only on the side of history + that we cache renames for (MERGE_SIDE1 as far as this document is + concerned; see the assumptions section). Two interesting sub-notes + about these counts: + + * If we need to perform rename-detection again on the given side (e.g. + some paths are relevant for rename detection that weren't before), + then we clear dir_rename_counts and recompute it, making use of + cached_pairs. The reason it is important to do this is optimizations + around RELEVANT_LOCATION exist to prevent us from computing + unnecessary renames for directory rename detection and from computing + dir_rename_counts for irrelevant directories; but those same renames + or directories may become necessary for subsequent merges. The + easiest way to "fix up" dir_rename_counts in such cases is to just + recompute it. + + * If we prune rename/rename(1to1) entries from the cache, then we also + need to update dir_rename_counts to decrement the counts for the + involved directory and any relevant parent directories (to undo what + update_dir_rename_counts() in diffcore-rename.c incremented when the + rename was initially found). If we instead just disable the + remembering renames optimization when the exceedingly rare + rename/rename(1to1) cases occur, then dir_rename_counts will get + re-computed the next time rename detection occurs, as noted above. + + * the side with multiple commits to pick, is the side of history that we + do NOT cache renames for. Thus, there are no additional commits to + change the number of renames in a directory, except for those done by + directory rename detection (which always pad the majority). + + * the "renames" we cache are modified slightly by any directory rename, + as noted below. + +Now, with those notes out of the way, let's go through the four cases +in order: + +Case 1: MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Renames olddir/ -> newdir/ + MERGE_SIDE2: A, Adds olddir/newfile + => creates A', With newdir/newfile + + MERGE_BASE: A, Has olddir/newfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Modifies olddir/newfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers olddir/ -> newdir/ + * MERGE_SIDE1 has cached olddir/newfile -> newdir/newfile + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Case 2: MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir + + This case looks like this: + MERGE_BASE: E oldfile, olddir/ + MERGE_SIDE1: G oldfile, olddir/ -> newdir/ + MERGE_SIDE2: A oldfile -> olddir/newfile + => creates A', With newdir/newfile representing original oldfile + + MERGE_BASE: A olddir/newfile + MERGE_SIDE1: A' newdir/newfile + MERGE_SIDE2: B modify olddir/newfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers olddir/ -> newdir/ + * MERGE_SIDE1 has cached olddir/newfile -> newdir/newfile + (NOT oldfile -> newdir/newfile; compare to case with + (p->status == 'R' && new_path) in possibly_cache_new_pair()) + + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Case 3: MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Adds olddir/newfile + MERGE_SIDE2: A, Renames olddir/ -> newdir/ + => creates A', With newdir/newfile + + MERGE_BASE: A, Has newdir/, but no notion of newdir/newfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Has newdir/, but no notion of newdir/newfile + => expected B', with newdir/newfile from A' + + In this case, with the optimization, note that after the first commit there + were no renames on MERGE_SIDE1, and any renames on MERGE_SIDE2 are tossed. + But the second merge didn't need any renames so this is fine. + +Case 4: MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Renames oldfile -> olddir/newfile + MERGE_SIDE2: A, Renames olddir/ -> newdir/ + => creates A', With newdir/newfile representing original oldfile + + MERGE_BASE: A, Has oldfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Modifies oldfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers oldfile -> newdir/newfile + (NOT oldfile -> olddir/newfile; compare to case of second + block under p->status == 'R' in possibly_cache_new_pair()) + * MERGE_SIDE2 renames are tossed because only MERGE_SIDE1 is remembered + + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Finally, I'll just note here that interactions with the +skip-irrelevant-renames optimization means we sometimes don't detect +renames for any files within a directory that was renamed, in which +case we will not have been able to detect any rename for the directory +itself. In such a case, we do not know whether the directory was +renamed; we want to be careful to avoid cacheing some kind of "this +directory was not renamed" statement. If we did, then a subsequent +commit being rebased could add a file to the old directory, and the +user would expect it to end up in the correct directory -- something +our erroneous "this directory was not renamed" cache would preclude. @@ -398,6 +398,10 @@ all:: # with a different indexfile format version. If it isn't set the index # file format used is index-v[23]. # +# Define GIT_TEST_UTF8_LOCALE to preferred utf-8 locale for testing. +# If it isn't set, fallback to $LC_ALL, $LANG or use the first utf-8 +# locale returned by "locale -a". +# # Define HAVE_CLOCK_GETTIME if your platform has clock_gettime. # # Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC. @@ -2802,6 +2806,9 @@ endif ifdef GIT_TEST_CMP_USE_COPIED_CONTEXT @echo GIT_TEST_CMP_USE_COPIED_CONTEXT=YesPlease >>$@+ endif +ifdef GIT_TEST_UTF8_LOCALE + @echo GIT_TEST_UTF8_LOCALE=\''$(subst ','\'',$(subst ','\'',$(GIT_TEST_UTF8_LOCALE)))'\' >>$@+ +endif @echo NO_GETTEXT=\''$(subst ','\'',$(subst ','\'',$(NO_GETTEXT)))'\' >>$@+ ifdef GIT_PERF_REPEAT_COUNT @echo GIT_PERF_REPEAT_COUNT=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_REPEAT_COUNT)))'\' >>$@+ @@ -1 +1 @@ -Documentation/RelNotes/2.32.0.txt
\ No newline at end of file +Documentation/RelNotes/2.33.0.txt
\ No newline at end of file @@ -685,7 +685,7 @@ static struct attr_stack *read_attr_from_array(const char **list) * Callers into the attribute system assume there is a single, system-wide * global state where attributes are read from and when the state is flipped by * calling git_attr_set_direction(), the stack frames that have been - * constructed need to be discarded so so that subsequent calls into the + * constructed need to be discarded so that subsequent calls into the * attribute system will lazily read from the right place. Since changing * direction causes a global paradigm shift, it should not ever be called while * another thread could potentially be calling into the attribute system. diff --git a/builtin/checkout--worker.c b/builtin/checkout--worker.c index 289a9b8f89..fb9fd13b73 100644 --- a/builtin/checkout--worker.c +++ b/builtin/checkout--worker.c @@ -53,7 +53,7 @@ static void packet_to_pc_item(const char *buffer, int len, static void report_result(struct parallel_checkout_item *pc_item) { - struct pc_item_result res; + struct pc_item_result res = { 0 }; size_t size; res.id = pc_item->id; diff --git a/builtin/clone.c b/builtin/clone.c index eeb74c0217..66fe66679c 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1320,9 +1320,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } if (!is_local && !complete_refs_before_fetch) { - err = transport_fetch_refs(transport, mapped_refs); - if (err) - goto cleanup; + if (transport_fetch_refs(transport, mapped_refs)) + die(_("remote transport reported error")); } remote_head = find_ref_by_name(refs, "HEAD"); @@ -1380,9 +1379,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (is_local) clone_local(path, git_dir); else if (refs && complete_refs_before_fetch) { - err = transport_fetch_refs(transport, mapped_refs); - if (err) - goto cleanup; + if (transport_fetch_refs(transport, mapped_refs)) + die(_("remote transport reported error")); } update_remote_refs(refs, mapped_refs, remote_head_points_at, @@ -1410,7 +1408,6 @@ int cmd_clone(int argc, const char **argv, const char *prefix) junk_mode = JUNK_LEAVE_REPO; err = checkout(submodule_progress); -cleanup: free(remote_name); strbuf_release(&reflog_msg); strbuf_release(&branch_top); diff --git a/builtin/diff-index.c b/builtin/diff-index.c index 176fe7ff2b..cf09559e42 100644 --- a/builtin/diff-index.c +++ b/builtin/diff-index.c @@ -2,6 +2,7 @@ #include "cache.h" #include "config.h" #include "diff.h" +#include "diff-merges.h" #include "commit.h" #include "revision.h" #include "builtin.h" @@ -27,6 +28,12 @@ int cmd_diff_index(int argc, const char **argv, const char *prefix) rev.abbrev = 0; prefix = precompose_argv_prefix(argc, argv, prefix); + /* + * We need no diff for merges options, and we need to avoid conflict + * with our own meaning of "-m". + */ + diff_merges_suppress_options_parsing(); + argc = setup_revisions(argc, argv, &rev, NULL); for (i = 1; i < argc; i++) { const char *arg = argv[i]; @@ -35,6 +42,8 @@ int cmd_diff_index(int argc, const char **argv, const char *prefix) option |= DIFF_INDEX_CACHED; else if (!strcmp(arg, "--merge-base")) option |= DIFF_INDEX_MERGE_BASE; + else if (!strcmp(arg, "-m")) + rev.match_missing = 1; else usage(diff_cache_usage); } diff --git a/builtin/fetch.c b/builtin/fetch.c index dfde96a435..9191620e50 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1126,7 +1126,7 @@ static int store_updated_refs(const char *raw_url, const char *remote_name, if (rm->status == REF_STATUS_REJECT_SHALLOW) { if (want_status == FETCH_HEAD_MERGE) - warning(_("reject %s because shallow roots are not allowed to be updated"), + warning(_("rejected %s because shallow roots are not allowed to be updated"), rm->peer_ref ? rm->peer_ref->name : rm->name); continue; } diff --git a/builtin/log.c b/builtin/log.c index 6102893fcc..516a1142dd 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -1968,8 +1968,7 @@ int cmd_format_patch(int argc, const char **argv, const char *prefix) } else if (rev.diffopt.close_file) { /* * The diff code parsed --output; it has already opened the - * file, but but we must instruct it not to close after each - * diff. + * file, but we must instruct it not to close after each diff. */ rev.diffopt.no_free = 1; } else { diff --git a/builtin/merge.c b/builtin/merge.c index eddb8ae70d..a8a843b1f5 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -56,8 +56,8 @@ struct strategy { static const char * const builtin_merge_usage[] = { N_("git merge [<options>] [<commit>...]"), - N_("git merge --abort"), - N_("git merge --continue"), + "git merge --abort", + "git merge --continue", NULL }; diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 7af8dab8bc..22c4e1a4ff 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -435,11 +435,11 @@ static int cmd_parseopt(int argc, const char **argv, const char *prefix) /* get the usage up to the first line with a -- on it */ for (;;) { if (strbuf_getline(&sb, stdin) == EOF) - die("premature end of input"); + die(_("premature end of input")); ALLOC_GROW(usage, unb + 1, usz); if (!strcmp("--", sb.buf)) { if (unb < 1) - die("no usage string given before the `--' separator"); + die(_("no usage string given before the `--' separator")); usage[unb] = NULL; break; } @@ -545,7 +545,7 @@ static void die_no_single_rev(int quiet) if (quiet) exit(1); else - die("Needed a single revision"); + die(_("Needed a single revision")); } static const char builtin_rev_parse_usage[] = @@ -709,10 +709,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (!strcmp(arg, "--resolve-git-dir")) { const char *gitdir = argv[++i]; if (!gitdir) - die("--resolve-git-dir requires an argument"); + die(_("--resolve-git-dir requires an argument")); gitdir = resolve_gitdir(gitdir); if (!gitdir) - die("not a gitdir '%s'", argv[i]); + die(_("not a gitdir '%s'"), argv[i]); puts(gitdir); continue; } @@ -736,7 +736,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (!seen_end_of_options && *arg == '-') { if (!strcmp(arg, "--git-path")) { if (!argv[i + 1]) - die("--git-path requires an argument"); + die(_("--git-path requires an argument")); strbuf_reset(&buf); print_path(git_path("%s", argv[i + 1]), prefix, format, @@ -746,7 +746,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) } if (!strcmp(arg,"-n")) { if (++i >= argc) - die("-n requires an argument"); + die(_("-n requires an argument")); if ((filter & DO_FLAGS) && (filter & DO_REVS)) { show(arg); show(argv[i]); @@ -760,26 +760,26 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) } if (opt_with_value(arg, "--path-format", &arg)) { if (!arg) - die("--path-format requires an argument"); + die(_("--path-format requires an argument")); if (!strcmp(arg, "absolute")) { format = FORMAT_CANONICAL; } else if (!strcmp(arg, "relative")) { format = FORMAT_RELATIVE; } else { - die("unknown argument to --path-format: %s", arg); + die(_("unknown argument to --path-format: %s"), arg); } continue; } if (!strcmp(arg, "--default")) { def = argv[++i]; if (!def) - die("--default requires an argument"); + die(_("--default requires an argument")); continue; } if (!strcmp(arg, "--prefix")) { prefix = argv[++i]; if (!prefix) - die("--prefix requires an argument"); + die(_("--prefix requires an argument")); startup_info->prefix = prefix; output_prefix = 1; continue; @@ -848,7 +848,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) else if (!strcmp(arg, "loose")) abbrev_ref_strict = 0; else - die("unknown mode for --abbrev-ref: %s", + die(_("unknown mode for --abbrev-ref: %s"), arg); } continue; @@ -892,7 +892,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (work_tree) print_path(work_tree, prefix, format, DEFAULT_UNMODIFIED); else - die("this operation must be run in a work tree"); + die(_("this operation must be run in a work tree")); continue; } if (!strcmp(arg, "--show-superproject-working-tree")) { @@ -1020,7 +1020,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (strcmp(val, "storage") && strcmp(val, "input") && strcmp(val, "output")) - die("unknown mode for --show-object-format: %s", + die(_("unknown mode for --show-object-format: %s"), arg); puts(the_hash_algo->name); continue; @@ -1058,7 +1058,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (verify) die_no_single_rev(quiet); if (has_dashdash) - die("bad revision '%s'", arg); + die(_("bad revision '%s'"), arg); as_is = 1; if (!show_file(arg, output_prefix)) continue; diff --git a/builtin/stash.c b/builtin/stash.c index 01066d7085..9c72e4b125 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -26,7 +26,7 @@ static const char * const git_stash_usage[] = { N_("git stash drop [-q|--quiet] [<stash>]"), N_("git stash ( pop | apply ) [--index] [-q|--quiet] [<stash>]"), N_("git stash branch <branchname> [<stash>]"), - N_("git stash clear"), + "git stash clear", N_("git stash [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [-m|--message <message>]\n" " [--pathspec-from-file=<file> [--pathspec-file-nul]]\n" @@ -67,7 +67,7 @@ static const char * const git_stash_branch_usage[] = { }; static const char * const git_stash_clear_usage[] = { - N_("git stash clear"), + "git stash clear", NULL }; @@ -761,7 +761,7 @@ static int list_stash(int argc, const char **argv, const char *prefix) cp.git_cmd = 1; strvec_pushl(&cp.args, "log", "--format=%gd: %gs", "-g", - "--first-parent", "-m", NULL); + "--first-parent", NULL); strvec_pushv(&cp.args, argv); strvec_push(&cp.args, ref_stash); strvec_push(&cp.args, "--"); diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index d55f6262e9..ae6174ab05 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1300,7 +1300,7 @@ static int module_summary(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "cached", &cached, N_("use the commit stored in the index instead of the submodule HEAD")), OPT_BOOL(0, "files", &files, - N_("to compare the commit in the index with that in the submodule HEAD")), + N_("compare the commit in the index with that in the submodule HEAD")), OPT_BOOL(0, "for-status", &for_status, N_("skip submodules with 'ignore_config' value set to 'all'")), OPT_INTEGER('n', "summary-limit", &summary_limit, diff --git a/bulk-checkin.c b/bulk-checkin.c index 127312acd1..b023d9959a 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -100,6 +100,7 @@ static int stream_to_pack(struct bulk_checkin_state *state, const char *path, unsigned flags) { git_zstream s; + unsigned char ibuf[16384]; unsigned char obuf[16384]; unsigned hdrlen; int status = Z_OK; @@ -113,8 +114,6 @@ static int stream_to_pack(struct bulk_checkin_state *state, s.avail_out = sizeof(obuf) - hdrlen; while (status != Z_STREAM_END) { - unsigned char ibuf[16384]; - if (size && !s.avail_in) { ssize_t rsize = size < sizeof(ibuf) ? size : sizeof(ibuf); ssize_t read_result = read_in_full(fd, ibuf, rsize); diff --git a/chunk-format.c b/chunk-format.c index da191e59a2..1c3dca62e2 100644 --- a/chunk-format.c +++ b/chunk-format.c @@ -58,9 +58,11 @@ void add_chunk(struct chunkfile *cf, int write_chunkfile(struct chunkfile *cf, void *data) { - int i; + int i, result = 0; uint64_t cur_offset = hashfile_total(cf->f); + trace2_region_enter("chunkfile", "write", the_repository); + /* Add the table of contents to the current offset */ cur_offset += (cf->chunks_nr + 1) * CHUNK_TOC_ENTRY_SIZE; @@ -77,10 +79,10 @@ int write_chunkfile(struct chunkfile *cf, void *data) for (i = 0; i < cf->chunks_nr; i++) { off_t start_offset = hashfile_total(cf->f); - int result = cf->chunks[i].write_fn(cf->f, data); + result = cf->chunks[i].write_fn(cf->f, data); if (result) - return result; + goto cleanup; if (hashfile_total(cf->f) - start_offset != cf->chunks[i].size) BUG("expected to write %"PRId64" bytes to chunk %"PRIx32", but wrote %"PRId64" instead", @@ -88,7 +90,9 @@ int write_chunkfile(struct chunkfile *cf, void *data) hashfile_total(cf->f) - start_offset); } - return 0; +cleanup: + trace2_region_leave("chunkfile", "write", the_repository); + return result; } int read_table_of_contents(struct chunkfile *cf, @@ -229,6 +229,7 @@ linux-musl) CC=gcc MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" + MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" ;; esac diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index b50c5d0ea3..4073d67f3b 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1729,6 +1729,7 @@ __git_diff_common_options="--stat --numstat --shortstat --summary --indent-heuristic --no-indent-heuristic --textconv --no-textconv --patch --no-patch + --anchored= " __git_diff_difftool_options="--cached --staged --pickaxe-all --pickaxe-regex diff --git a/contrib/hooks/multimail/CHANGES b/contrib/hooks/multimail/CHANGES deleted file mode 100644 index 35791fd02c..0000000000 --- a/contrib/hooks/multimail/CHANGES +++ /dev/null @@ -1,285 +0,0 @@ -Release 1.5.0 -============= - -Backward-incompatible change ----------------------------- - -The name of classes for environment was misnamed as `*Environement`. -It is now `*Environment`. - -New features ------------- - -* A Thread-Index header is now added to each email sent (except for - combined emails where it would not make sense), so that MS Outlook - properly groups messages by threads even though they have a - different subject line. Unfortunately, even adding this header the - threading still seems to be unreliable, but it is unclear whether - this is an issue on our side or on MS Outlook's side (see discussion - here: https://github.com/git-multimail/git-multimail/pull/194). - -* A new variable multimailhook.ExcludeMergeRevisions was added to send - notification emails only for non-merge commits. - -* For gitolite environment, it is now possible to specify the mail map - in a separate file in addition to gitolite.conf, using the variable - multimailhook.MailaddressMap. - -Internal changes ----------------- - -* The testsuite now uses GIT_PRINT_SHA1_ELLIPSIS where needed for - compatibility with recent Git versions. Only tests are affected. - -* We don't try to install pyflakes in the continuous integration job - for old Python versions where it's no longer available. - -* Stop using the deprecated cgi.escape in Python 3. - -* New flake8 warnings have been fixed. - -* Python 3.6 is now tested against on Travis-CI. - -* A bunch of lgtm.com warnings have been fixed. - -Bug fixes ---------- - -* SMTPMailer logs in only once now. It used to re-login for each email - sent which triggered errors for some SMTP servers. - -* migrate-mailhook-config was broken by internal refactoring, it - should now work again. - -This version was tested with Python 2.6 to 3.7. It was tested with Git -1.7.10.406.gdc801, 2.15.1 and 2.20.1.98.gecbdaf0. - -Release 1.4.0 -============= - -New features to troubleshoot a git-multimail installation ---------------------------------------------------------- - -* One can now perform a basic check of git-multimail's setup by - running the hook with the environment variable - GIT_MULTIMAIL_CHECK_SETUP set to a non-empty string. See - doc/troubleshooting.rst for details. - -* A new log files system was added. See the multimailhook.logFile, - multimailhook.errorLogFile and multimailhook.debugLogFile variables. - -* git_multimail.py can now be made more verbose using - multimailhook.verbose. - -* A new option --check-ref-filter is now available to help debugging - the refFilter* options. - -Formatting emails ------------------ - -* Formatting of emails was made slightly more compact, to reduce the - odds of having long subject lines truncated or wrapped in short list - of commits. - -* multimailhook.emailPrefix may now use the '%(repo_shortname)s' - placeholder for the repository's short name. - -* A new option multimailhook.subjectMaxLength is available to truncate - overly long subject lines. - -Bug fixes and minor changes ---------------------------- - -* Options refFilterDoSendRegex and refFilterDontSendRegex were - essentially broken. They should work now. - -* The behavior when both refFilter{Do,Dont}SendRegex and - refFilter{Exclusion,Inclusion}Regex are set have been slightly - changed. Exclusion/Inclusion is now strictly stronger than - DoSend/DontSend. - -* The management of precedence when a setting can be computed in - multiple ways has been considerably refactored and modified. - multimailhook.from and multimailhook.reponame now have precedence - over the environment-specific settings ($GL_REPO/$GL_USER for - gitolite, --stash-user/repo for Stash, --submitter/--project for - Gerrit). - -* The coverage of the testsuite has been considerably improved. All - configuration variables now appear at least once in the testsuite. - -This version was tested with Python 2.6 to 3.5. It also mostly works -with Python 2.4, but there is one known breakage in the testsuite -related to non-ascii characters. It was tested with Git -1.7.10.406.gdc801, 1.8.5.6, 2.1.4, and 2.10.0.rc0.1.g07c9292. - -Release 1.3.1 (bugfix-only release) -=================================== - -* Generate links to commits in combined emails (it was done only for - commit emails in 1.3.0). - -* Fix broken links on PyPi. - -Release 1.3.0 -============= - -* New options multimailhook.htmlInIntro and multimailhook.htmlInFooter - now allow using HTML in the introduction and footer of emails (e.g. - for a more pleasant formatting or to insert a link to the commit on - a web interface). - -* A new option multimailhook.commitBrowseURL gives a simpler (and less - flexible) way to add a link to a web interface for commit emails - than multimailhook.htmlInIntro and multimailhook.htmlInFooter. - -* A new public function config.add_config_parameters was added to - allow custom hooks to set specific Git configuration variables - without modifying the configuration files. See an example in - post-receive.example. - -* Error handling for SMTP has been improved (we used to print Python - backtraces for legitimate errors). - -* The SMTP mailer can now check TLS certificates when the newly added - configuration variable multimailhook.smtpCACerts. - -* Python 3 portability has been improved. - -* The documentation's formatting has been improved. - -* The testsuite has been improved (we now use pyflakes to check for - errors in the code). - -This version has been tested with Python 2.4 and 2.6 to 3.5, and Git -v1.7.10-406-gdc801e7, 2.1.4 and 2.8.1.339.g3ad15fd. - -No change since 1.3 RC1. - -Release 1.2.0 -============= - -* It is now possible to exclude some refs (e.g. exclude some branches - or tags). See refFilterDoSendRegex, refFilterDontSendRegex, - refFilterInclusionRegex and refFilterExclusionRegex. - -* New commitEmailFormat option which can be set to "html" to generate - simple colorized diffs using HTML for the commit emails. - -* git-multimail can now be ran as a Gerrit ref-updated hook, or from - Atlassian BitBucket Server (formerly known as Atlassian Stash). - -* The From: field is now more customizeable. It can be set - independently for refchange emails and commit emails (see - fromCommit, fromRefChange). The special values pusher and author can - be used in these configuration variable. - -* A new command-line option, --version, was added. The version is also - available in the X-Git-Multimail-Version header of sent emails. - -* Set X-Git-NotificationType header to differentiate the various types - of notifications. Current values are: diff, ref_changed_plus_diff, - ref_changed. - -* Preliminary support for Python 3. The testsuite passes with Python 3, - but it has not received as much testing as the Python 2 version yet. - -* Several encoding-related fixes. UTF-8 characters work in more - situations (but non-ascii characters in email address are still not - supported). - -* The testsuite and its documentation has been greatly improved. - -Plus all the bugfixes from version 1.1.1. - -This version has been tested with Python 2.4 and 2.6 to 3.5, and Git -v1.7.10-406-gdc801e7, git-1.8.2.3 and 2.6.0. Git versions prior to -v1.7.10-406-gdc801e7 probably work, but cannot run the testsuite -properly. - -Release 1.1.1 (bugfix-only release) -=================================== - -* The SMTP mailer was not working with Python 2.4. - -Release 1.1.0 -============= - -* When a single commit is pushed, omit the reference changed email. - Set multimailhook.combineWhenSingleCommit to false to disable this - new feature. - -* In gitolite environments, the pusher's email address can be used as - the From address by creating a specially formatted comment block in - gitolite.conf (see multimailhook.from in README). - -* Support for SMTP authentication and SSL/TLS encryption was added, - see smtpUser, smtpPass, smtpEncryption in README. - -* A new option scanCommitForCc was added to allow git-multimail to - search the commit message for 'Cc: ...' lines, and add the - corresponding emails in Cc. - -* If $USER is not set, use the variable $USERNAME. This is needed on - Windows platform to recognize the pusher. - -* The emailPrefix variable can now be set to an empty string to remove - the prefix. - -* A short tutorial was added in doc/gitolite.rst to set up - git-multimail with gitolite. - -* The post-receive file was renamed to post-receive.example. It has - always been an example (the standard way to call git-multimail is to - call git_multimail.py), but it was unclear to many users. - -* A new refchangeShowGraph option was added to make it possible to - include both a graph and a log in the summary emails. The options - to control the graph formatting can be set via the new graphOpts - option. - -* New option --force-send was added to disable new commit detection - for update hook. One use-case is to run git_multimail.py after - running "git fetch" to send emails about commits that have just been - fetched (the detection of new commits was unreliable in this mode). - -* The testing infrastructure was considerably improved (continuous - integration with travis-ci, automatic check of PEP8 and RST syntax, - many improvements to the test scripts). - -This version has been tested with Python 2.4 to 2.7, and Git 1.7.1 to -2.4. - -Release 1.0.0 -============= - -* Fix encoding of non-ASCII email addresses in email headers. - -* Fix backwards-compatibility bugs for older Python 2.x versions. - -* Fix a backwards-compatibility bug for Git 1.7.1. - -* Add an option commitDiffOpts to customize logs for revisions. - -* Pass "-oi" to sendmail by default to prevent premature termination - on a line containing only ".". - -* Stagger email "Date:" values in an attempt to help mail clients - thread the emails in the right order. - -* If a mailing list setting is missing, just skip sending the - corresponding email (with a warning) instead of failing. - -* Add a X-Git-Host header that can be used for email filtering. - -* Allow the sender's fully-qualified domain name to be configured. - -* Minor documentation improvements. - -* Add this CHANGES file. - - -Release 0.9.0 -============= - -* Initial release. diff --git a/contrib/hooks/multimail/CONTRIBUTING.rst b/contrib/hooks/multimail/CONTRIBUTING.rst deleted file mode 100644 index de20a54287..0000000000 --- a/contrib/hooks/multimail/CONTRIBUTING.rst +++ /dev/null @@ -1,60 +0,0 @@ -Contributing -============ - -git-multimail is an open-source project, built by volunteers. We would -welcome your help! - -The current maintainers are `Matthieu Moy <http://matthieu-moy.fr>`__ and -`Michael Haggerty <https://github.com/mhagger>`__. - -Please note that although a copy of git-multimail is distributed in -the "contrib" section of the main Git project, development takes place -in a separate `git-multimail repository on GitHub`_. - -Whenever enough changes to git-multimail have accumulated, a new -code-drop of git-multimail will be submitted for inclusion in the Git -project. - -We use the GitHub issue tracker to keep track of bugs and feature -requests, and we use GitHub pull requests to exchange patches (though, -if you prefer, you can send patches via the Git mailing list with CC -to the maintainers). Please sign off your patches as per the `Git -project practice -<https://github.com/git/git/blob/master/Documentation/SubmittingPatches#L234>`__. - -Please vote for issues you would like to be addressed in priority -(click "add your reaction" and then the "+1" thumbs-up button on the -GitHub issue). - -General discussion of git-multimail can take place on the main `Git -mailing list`_. - -Please CC emails regarding git-multimail to the maintainers so that we -don't overlook them. - -Help needed: testers/maintainer for specific environments/OS ------------------------------------------------------------- - -The current maintainer uses and tests git-multimail on Linux with the -Generic environment. More testers, or better contributors are needed -to test git-multimail on other real-life setups: - -* Mac OS X, Windows: git-multimail is currently not supported on these - platforms. But since we have no external dependencies and try to - write code as portable as possible, it is possible that - git-multimail already runs there and if not, it is likely that it - could be ported easily. - - Patches to improve support for Windows and OS X are welcome. - Ideally, there would be a sub-maintainer for each OS who would test - at least once before each release (around twice a year). - -* Gerrit, Stash, Gitolite environments: although the testsuite - contains tests for these environments, a tester/maintainer for each - environment would be welcome to test and report failure (or success) - on real-life environments periodically (here also, feedback before - each release would be highly appreciated). - - -.. _`git-multimail repository on GitHub`: https://github.com/git-multimail/git-multimail -.. _`Git mailing list`: git@vger.kernel.org diff --git a/contrib/hooks/multimail/README.Git b/contrib/hooks/multimail/README.Git index 044444245d..c427efc7bd 100644 --- a/contrib/hooks/multimail/README.Git +++ b/contrib/hooks/multimail/README.Git @@ -1,15 +1,7 @@ -This copy of git-multimail is distributed as part of the "contrib" -section of the Git project as a convenience to Git users. git-multimail is developed as an independent project at the following website: https://github.com/git-multimail/git-multimail -The version in this directory was obtained from the upstream project -on January 07 2019 and consists of the "git-multimail" subdirectory from -revision - - 04e80e6c40be465cc62b6c246f0fcb8fd2cfd454 refs/tags/1.5.0 - -Please see the README file in this directory for information about how -to report bugs or contribute to git-multimail. +Please refer to that project page for information about how to report +bugs or contribute to git-multimail. diff --git a/contrib/hooks/multimail/README.migrate-from-post-receive-email b/contrib/hooks/multimail/README.migrate-from-post-receive-email deleted file mode 100644 index 1e6a976699..0000000000 --- a/contrib/hooks/multimail/README.migrate-from-post-receive-email +++ /dev/null @@ -1,145 +0,0 @@ -git-multimail is close to, but not exactly, a plug-in replacement for -the old Git project script contrib/hooks/post-receive-email. This -document describes the differences and explains how to configure -git-multimail to get behavior closest to that of post-receive-email. - -If you are in a hurry -===================== - -A script called migrate-mailhook-config is included with -git-multimail. If you run this script within a Git repository that is -configured to use post-receive-email, it will convert the -configuration settings into the approximate equivalent settings for -git-multimail. For more information, run - - migrate-mailhook-config --help - - -Configuration differences -========================= - -* The names of the config options for git-multimail are in namespace - "multimailhook.*" instead of "hooks.*". (Editorial comment: - post-receive-email should never have used such a generic top-level - namespace.) - -* In emails about new annotated tags, post-receive-email includes a - shortlog of all changes since the previous annotated tag. To get - this behavior with git-multimail, you need to set - multimailhook.announceshortlog to true: - - git config multimailhook.announceshortlog true - -* multimailhook.commitlist -- This is a new configuration variable. - Recipients listed here will receive a separate email for each new - commit. However, if this variable is *not* set, it defaults to the - value of multimailhook.mailinglist. Therefore, if you *don't* want - the members of multimailhook.mailinglist to receive one email per - commit, then set this value to the empty string: - - git config multimailhook.commitlist '' - -* multimailhook.emailprefix -- If this value is not set, then the - subjects of generated emails are prefixed with the short name of the - repository enclosed in square brackets; e.g., "[myrepo]". - post-receive-email defaults to prefix "[SCM]" if this option is not - set. So if you were using the old default and want to retain it - (for example, to avoid having to change your email filters), set - this variable explicitly to the old value: - - git config multimailhook.emailprefix "[SCM]" - -* The "multimailhook.showrev" configuration option is not supported. - Its main use is obsoleted by the one-email-per-commit feature of - git-multimail. - - -Other differences -================= - -This section describes other differences in the behavior of -git-multimail vs. post-receive-email. For full details, please refer -to the main README file: - -* One email per commit. For each reference change, the script first - outputs one email summarizing the reference change (including - one-line summaries of the new commits), then it outputs a separate - email for each new commit that was introduced, including patches. - These one-email-per-commit emails go to the addresses listed in - multimailhook.commitlist. post-receive-email sends only one email - for each *reference* that is changed, no matter how many commits - were added to the reference. - -* Better algorithm for detecting new commits. post-receive-email - processes one reference change at a time, which causes it to fail to - describe new commits that were included in multiple branches. For - example, if a single push adds the "*" commits in the diagram below, - then post-receive-email would never include the details of the two - commits that are common to "master" and "branch" in its - notifications. - - o---o---o---*---*---* <-- master - \ - *---* <-- branch - - git-multimail analyzes all reference modifications to determine - which commits were not present before the change, therefore avoiding - that error. - -* In reference change emails, git-multimail tells which commits have - been added to the reference vs. are entirely new to the repository, - and which commits that have been omitted from the reference - vs. entirely discarded from the repository. - -* The environment in which Git is running can be configured via an - "Environment" abstraction. - -* Built-in support for Gitolite-managed repositories. - -* Instead of using full SHA1 object names in emails, git-multimail - mostly uses abbreviated SHA1s, plus one-line log message summaries - where appropriate. - -* In the schematic diagrams that explain non-fast-forward commits, - git-multimail shows the names of the branches involved. - -* The emails generated by git-multimail include the name of the Git - repository that was modified; this is convenient for recipients who - are monitoring multiple repositories. - -* git-multimail allows the email "From" addresses to be configured. - -* The recipients lists (multimailhook.mailinglist, - multimailhook.refchangelist, multimailhook.announcelist, and - multimailhook.commitlist) can be comma-separated values and/or - multivalued settings in the config file; e.g., - - [multimailhook] - mailinglist = mr.brown@example.com, mr.black@example.com - announcelist = Him <him@example.com> - announcelist = Jim <jim@example.com> - announcelist = pop@example.com - - This might make it easier to maintain short recipients lists without - requiring full-fledged mailing list software. - -* By default, git-multimail sets email "Reply-To" headers to reply to - the pusher (for reference updates) and to the author (for commit - notifications). By default, the pusher's email address is - constructed by appending "multimailhook.emaildomain" to the pusher's - username. - -* The generated emails contain a configurable footer. By default, it - lists the name of the administrator who should be contacted to - unsubscribe from notification emails. - -* New option multimailhook.emailmaxlinelength to limit the length of - lines in the main part of the email body. The default limit is 500 - characters. - -* New option multimailhook.emailstrictutf8 to ensure that the main - part of the email body is valid UTF-8. Invalid characters are - turned into the Unicode replacement character, U+FFFD. By default - this option is turned on. - -* Written in Python. Easier to add new features. diff --git a/contrib/hooks/multimail/README.rst b/contrib/hooks/multimail/README.rst deleted file mode 100644 index 7c0fc4a6ef..0000000000 --- a/contrib/hooks/multimail/README.rst +++ /dev/null @@ -1,774 +0,0 @@ -git-multimail version 1.5.0 -=========================== - -.. image:: https://travis-ci.org/git-multimail/git-multimail.svg?branch=master - :target: https://travis-ci.org/git-multimail/git-multimail - -git-multimail is a tool for sending notification emails on pushes to a -Git repository. It includes a Python module called ``git_multimail.py``, -which can either be used as a hook script directly or can be imported -as a Python module into another script. - -git-multimail is derived from the Git project's old -contrib/hooks/post-receive-email, and is mostly compatible with that -script. See README.migrate-from-post-receive-email for details about -the differences and for how to migrate from post-receive-email to -git-multimail. - -git-multimail, like the rest of the Git project, is licensed under -GPLv2 (see the COPYING file for details). - -Please note: although, as a convenience, git-multimail may be -distributed along with the main Git project, development of -git-multimail takes place in its own, separate project. Please, read -`<CONTRIBUTING.rst>`__ for more information. - - -By default, for each push received by the repository, git-multimail: - -1. Outputs one email summarizing each reference that was changed. - These "reference change" (called "refchange" below) emails describe - the nature of the change (e.g., was the reference created, deleted, - fast-forwarded, etc.) and include a one-line summary of each commit - that was added to the reference. - -2. Outputs one email for each new commit that was introduced by the - reference change. These "commit" emails include a list of the - files changed by the commit, followed by the diffs of files - modified by the commit. The commit emails are threaded to the - corresponding reference change email via "In-Reply-To". This style - (similar to the "git format-patch" style used on the Git mailing - list) makes it easy to scan through the emails, jump to patches - that need further attention, and write comments about specific - commits. Commits are handled in reverse topological order (i.e., - parents shown before children). For example:: - - [git] branch master updated - + [git] 01/08: doc: fix xref link from api docs to manual pages - + [git] 02/08: api-credentials.txt: show the big picture first - + [git] 03/08: api-credentials.txt: mention credential.helper explicitly - + [git] 04/08: api-credentials.txt: add "see also" section - + [git] 05/08: t3510 (cherry-pick-sequence): add missing '&&' - + [git] 06/08: Merge branch 'rr/maint-t3510-cascade-fix' - + [git] 07/08: Merge branch 'mm/api-credentials-doc' - + [git] 08/08: Git 1.7.11-rc2 - - By default, each commit appears in exactly one commit email, the - first time that it is pushed to the repository. If a commit is later - merged into another branch, then a one-line summary of the commit - is included in the reference change email (as usual), but no - additional commit email is generated. See - `multimailhook.refFilter(Inclusion|Exclusion|DoSend|DontSend)Regex` - below to configure which branches and tags are watched by the hook. - - By default, reference change emails have their "Reply-To" field set - to the person who pushed the change, and commit emails have their - "Reply-To" field set to the author of the commit. - -3. Output one "announce" mail for each new annotated tag, including - information about the tag and optionally a shortlog describing the - changes since the previous tag. Such emails might be useful if you - use annotated tags to mark releases of your project. - - -Requirements ------------- - -* Python 2.x, version 2.4 or later. No non-standard Python modules - are required. git-multimail has preliminary support for Python 3 - (but it has been better tested with Python 2). - -* The ``git`` command must be in your PATH. git-multimail is known to - work with Git versions back to 1.7.1. (Earlier versions have not - been tested; if you do so, please report your results.) - -* To send emails using the default configuration, a standard sendmail - program must be located at '/usr/sbin/sendmail' or - '/usr/lib/sendmail' and must be configured correctly to send emails. - If this is not the case, set multimailhook.sendmailCommand, or see - the multimailhook.mailer configuration variable below for how to - configure git-multimail to send emails via an SMTP server. - -* git-multimail is currently tested only on Linux. It may or may not - work on other platforms such as Windows and Mac OS. See - `<CONTRIBUTING.rst>`__ to improve the situation. - - -Invocation ----------- - -``git_multimail.py`` is designed to be used as a ``post-receive`` hook in a -Git repository (see githooks(5)). Link or copy it to -$GIT_DIR/hooks/post-receive within the repository for which email -notifications are desired. Usually it should be installed on the -central repository for a project, to which all commits are eventually -pushed. - -For use on pre-v1.5.1 Git servers, ``git_multimail.py`` can also work as -an ``update`` hook, taking its arguments on the command line. To use -this script in this manner, link or copy it to $GIT_DIR/hooks/update. -Please note that the script is not completely reliable in this mode -[1]_. - -Alternatively, ``git_multimail.py`` can be imported as a Python module -into your own Python post-receive script. This method is a bit more -work, but allows the behavior of the hook to be customized using -arbitrary Python code. For example, you can use a custom environment -(perhaps inheriting from GenericEnvironment or GitoliteEnvironment) to - -* change how the user who did the push is determined - -* read users' email addresses from an LDAP server or from a database - -* decide which users should be notified about which commits based on - the contents of the commits (e.g., for users who want to be notified - only about changes affecting particular files or subdirectories) - -Or you can change how emails are sent by writing your own Mailer -class. The ``post-receive`` script in this directory demonstrates how -to use ``git_multimail.py`` as a Python module. (If you make interesting -changes of this type, please consider sharing them with the -community.) - - -Troubleshooting/FAQ -------------------- - -Please read `<doc/troubleshooting.rst>`__ for frequently asked -questions and common issues with git-multimail. - - -Configuration -------------- - -By default, git-multimail mostly takes its configuration from the -following ``git config`` settings: - -multimailhook.environment - This describes the general environment of the repository. In most - cases, you do not need to specify a value for this variable: - `git-multimail` will autodetect which environment to use. - Currently supported values: - - generic - the username of the pusher is read from $USER or $USERNAME and - the repository name is derived from the repository's path. - - gitolite - Environment to use when ``git-multimail`` is ran as a gitolite_ - hook. - - The username of the pusher is read from $GL_USER, the repository - name is read from $GL_REPO, and the From: header value is - optionally read from gitolite.conf (see multimailhook.from). - - For more information about gitolite and git-multimail, read - `<doc/gitolite.rst>`__ - - stash - Environment to use when ``git-multimail`` is ran as an Atlassian - BitBucket Server (formerly known as Atlassian Stash) hook. - - **Warning:** this mode was provided by a third-party contributor - and never tested by the git-multimail maintainers. It is - provided as-is and may or may not work for you. - - This value is automatically assumed when the stash-specific - flags (``--stash-user`` and ``--stash-repo``) are specified on - the command line. When this environment is active, the username - and repo come from these two command line flags, which must be - specified. - - gerrit - Environment to use when ``git-multimail`` is ran as a - ``ref-updated`` Gerrit hook. - - This value is used when the gerrit-specific command line flags - (``--oldrev``, ``--newrev``, ``--refname``, ``--project``) for - gerrit's ref-updated hook are present. When this environment is - active, the username of the pusher is taken from the - ``--submitter`` argument if that command line option is passed, - otherwise 'Gerrit' is used. The repository name is taken from - the ``--project`` option on the command line, which must be passed. - - For more information about gerrit and git-multimail, read - `<doc/gerrit.rst>`__ - - If none of these environments is suitable for your setup, then you - can implement a Python class that inherits from Environment and - instantiate it via a script that looks like the example - post-receive script. - - The environment value can be specified on the command line using - the ``--environment`` option. If it is not specified on the - command line or by ``multimailhook.environment``, the value is - guessed as follows: - - * If stash-specific (respectively gerrit-specific) command flags - are present on the command-line, then ``stash`` (respectively - ``gerrit``) is used. - - * If the environment variables $GL_USER and $GL_REPO are set, then - ``gitolite`` is used. - - * If none of the above apply, then ``generic`` is used. - -multimailhook.repoName - A short name of this Git repository, to be used in various places - in the notification email text. The default is to use $GL_REPO - for gitolite repositories, or otherwise to derive this value from - the repository path name. - -multimailhook.mailingList - The list of email addresses to which notification emails should be - sent, as RFC 2822 email addresses separated by commas. This - configuration option can be multivalued. Leave it unset or set it - to the empty string to not send emails by default. The next few - settings can be used to configure specific address lists for - specific types of notification email. - -multimailhook.refchangeList - The list of email addresses to which summary emails about - reference changes should be sent, as RFC 2822 email addresses - separated by commas. This configuration option can be - multivalued. The default is the value in - multimailhook.mailingList. Set this value to "none" (or the empty - string) to prevent reference change emails from being sent even if - multimailhook.mailingList is set. - -multimailhook.announceList - The list of email addresses to which emails about new annotated - tags should be sent, as RFC 2822 email addresses separated by - commas. This configuration option can be multivalued. The - default is the value in multimailhook.refchangeList or - multimailhook.mailingList. Set this value to "none" (or the empty - string) to prevent annotated tag announcement emails from being sent - even if one of the other values is set. - -multimailhook.commitList - The list of email addresses to which emails about individual new - commits should be sent, as RFC 2822 email addresses separated by - commas. This configuration option can be multivalued. The - default is the value in multimailhook.mailingList. Set this value - to "none" (or the empty string) to prevent notification emails about - individual commits from being sent even if - multimailhook.mailingList is set. - -multimailhook.announceShortlog - If this option is set to true, then emails about changes to - annotated tags include a shortlog of changes since the previous - tag. This can be useful if the annotated tags represent releases; - then the shortlog will be a kind of rough summary of what has - happened since the last release. But if your tagging policy is - not so straightforward, then the shortlog might be confusing - rather than useful. Default is false. - -multimailhook.commitEmailFormat - The format of email messages for the individual commits, can be "text" or - "html". In the latter case, the emails will include diffs using colorized - HTML instead of plain text used by default. Note that this currently the - ref change emails are always sent in plain text. - - Note that when using "html", the formatting is done by parsing the - output of ``git log`` with ``-p``. When using - ``multimailhook.commitLogOpts`` to specify a ``--format`` for - ``git log``, one may get false positive (e.g. lines in the body of - the message starting with ``+++`` or ``---`` colored in red or - green). - - By default, all the message is HTML-escaped. See - ``multimailhook.htmlInIntro`` to change this behavior. - -multimailhook.commitBrowseURL - Used to generate a link to an online repository browser in commit - emails. This variable must be a string. Format directives like - ``%(<variable>)s`` will be expanded the same way as template - strings. In particular, ``%(id)s`` will be replaced by the full - Git commit identifier (40-chars hexadecimal). - - If the string does not contain any format directive, then - ``%(id)s`` will be automatically added to the string. If you don't - want ``%(id)s`` to be automatically added, use the empty format - directive ``%()s`` anywhere in the string. - - For example, a suitable value for the git-multimail project itself - would be - ``https://github.com/git-multimail/git-multimail/commit/%(id)s``. - -multimailhook.htmlInIntro, multimailhook.htmlInFooter - When generating an HTML message, git-multimail escapes any HTML - sequence by default. This means that if a template contains HTML - like ``<a href="foo">link</a>``, the reader will see the HTML - source code and not a proper link. - - Set ``multimailhook.htmlInIntro`` to true to allow writing HTML - formatting in introduction templates. Similarly, set - ``multimailhook.htmlInFooter`` for HTML in the footer. - - Variables expanded in the template are still escaped. For example, - if a repository's path contains a ``<``, it will be rendered as - such in the message. - - Read `<doc/customizing-emails.rst>`__ for more details and - examples. - -multimailhook.refchangeShowGraph - If this option is set to true, then summary emails about reference - changes will additionally include: - - * a graph of the added commits (if any) - - * a graph of the discarded commits (if any) - - The log is generated by running ``git log --graph`` with the options - specified in graphOpts. The default is false. - -multimailhook.refchangeShowLog - If this option is set to true, then summary emails about reference - changes will include a detailed log of the added commits in - addition to the one line summary. The log is generated by running - ``git log`` with the options specified in multimailhook.logOpts. - Default is false. - -multimailhook.mailer - This option changes the way emails are sent. Accepted values are: - - * **sendmail (the default)**: use the command ``/usr/sbin/sendmail`` or - ``/usr/lib/sendmail`` (or sendmailCommand, if configured). This - mode can be further customized via the following options: - - multimailhook.sendmailCommand - The command used by mailer ``sendmail`` to send emails. Shell - quoting is allowed in the value of this setting, but remember that - Git requires double-quotes to be escaped; e.g.:: - - git config multimailhook.sendmailcommand '/usr/sbin/sendmail -oi -t -F \"Git Repo\"' - - Default is '/usr/sbin/sendmail -oi -t' or - '/usr/lib/sendmail -oi -t' (depending on which file is - present and executable). - - multimailhook.envelopeSender - If set then pass this value to sendmail via the -f option to set - the envelope sender address. - - * **smtp**: use Python's smtplib. This is useful when the sendmail - command is not available on the system. This mode can be - further customized via the following options: - - multimailhook.smtpServer - The name of the SMTP server to connect to. The value can - also include a colon and a port number; e.g., - ``mail.example.com:25``. Default is 'localhost' using port 25. - - multimailhook.smtpUser, multimailhook.smtpPass - Server username and password. Required if smtpEncryption is 'ssl'. - Note that the username and password currently need to be - set cleartext in the configuration file, which is not - recommended. If you need to use this option, be sure your - configuration file is read-only. - - multimailhook.envelopeSender - The sender address to be passed to the SMTP server. If - unset, then the value of multimailhook.from is used. - - multimailhook.smtpServerTimeout - Timeout in seconds. Default is 10. - - multimailhook.smtpEncryption - Set the security type. Allowed values: ``none``, ``ssl``, ``tls`` (starttls). - Default is ``none``. - - multimailhook.smtpCACerts - Set the path to a list of trusted CA certificate to verify the - server certificate, only supported when ``smtpEncryption`` is - ``tls``. If unset or empty, the server certificate is not - verified. If it targets a file containing a list of trusted CA - certificates (PEM format) these CAs will be used to verify the - server certificate. For debian, you can set - ``/etc/ssl/certs/ca-certificates.crt`` for using the system - trusted CAs. For self-signed server, you can add your server - certificate to the system store:: - - cd /usr/local/share/ca-certificates/ - openssl s_client -starttls smtp \ - -connect mail.example.net:587 -showcerts \ - </dev/null 2>/dev/null \ - | openssl x509 -outform PEM >mail.example.net.crt - update-ca-certificates - - and used the updated ``/etc/ssl/certs/ca-certificates.crt``. Or - directly use your ``/path/to/mail.example.net.crt``. Default is - unset. - - multimailhook.smtpServerDebugLevel - Integer number. Set to greater than 0 to activate debugging. - -multimailhook.from, multimailhook.fromCommit, multimailhook.fromRefchange - If set, use this value in the From: field of generated emails. - ``fromCommit`` is used for commit emails, ``fromRefchange`` is - used for refchange emails, and ``from`` is used as fall-back in - all cases. - - The value for these variables can be either: - - - An email address, which will be used directly. - - - The value ``pusher``, in which case the pusher's address (if - available) will be used. - - - The value ``author`` (meaningful only for ``fromCommit``), in which - case the commit author's address will be used. - - If config values are unset, the value of the From: header is - determined as follows: - - 1. (gitolite environment only) - 1.a) If ``multimailhook.MailaddressMap`` is set, and is a path - to an existing file (if relative, it is considered relative to - the place where ``gitolite.conf`` is located), then this file - should contain lines like:: - - username Firstname Lastname <email@example.com> - - git-multimail will then look for a line where ``$GL_USER`` - matches the ``username`` part, and use the rest of the line for - the ``From:`` header. - - 1.b) Parse gitolite.conf, looking for a block of comments that - looks like this:: - - # BEGIN USER EMAILS - # username Firstname Lastname <email@example.com> - # END USER EMAILS - - If that block exists, and there is a line between the BEGIN - USER EMAILS and END USER EMAILS lines where the first field - matches the gitolite username ($GL_USER), use the rest of the - line for the From: header. - - 2. If the user.email configuration setting is set, use its value - (and the value of user.name, if set). - - 3. Use the value of multimailhook.envelopeSender. - -multimailhook.MailaddressMap - (gitolite environment only) - File to look for a ``From:`` address based on the user doing the - push. Defaults to unset. See ``multimailhook.from`` for details. - -multimailhook.administrator - The name and/or email address of the administrator of the Git - repository; used in FOOTER_TEMPLATE. Default is - multimailhook.envelopesender if it is set; otherwise a generic - string is used. - -multimailhook.emailPrefix - All emails have this string prepended to their subjects, to aid - email filtering (though filtering based on the X-Git-* email - headers is probably more robust). Default is the short name of - the repository in square brackets; e.g., ``[myrepo]``. Set this - value to the empty string to suppress the email prefix. You may - use the placeholder ``%(repo_shortname)s`` for the short name of - the repository. - -multimailhook.emailMaxLines - The maximum number of lines that should be included in the body of - a generated email. If not specified, there is no limit. Lines - beyond the limit are suppressed and counted, and a final line is - added indicating the number of suppressed lines. - -multimailhook.emailMaxLineLength - The maximum length of a line in the email body. Lines longer than - this limit are truncated to this length with a trailing ``[...]`` - added to indicate the missing text. The default is 500, because - (a) diffs with longer lines are probably from binary files, for - which a diff is useless, and (b) even if a text file has such long - lines, the diffs are probably unreadable anyway. To disable line - truncation, set this option to 0. - -multimailhook.subjectMaxLength - The maximum length of the subject line (i.e. the ``oneline`` field - in templates, not including the prefix). Lines longer than this - limit are truncated to this length with a trailing ``[...]`` added - to indicate the missing text. This option The default is to use - ``multimailhook.emailMaxLineLength``. This option avoids sending - emails with overly long subject lines, but should not be needed if - the commit messages follow the Git convention (one short subject - line, then a blank line, then the message body). To disable line - truncation, set this option to 0. - -multimailhook.maxCommitEmails - The maximum number of commit emails to send for a given change. - When the number of patches is larger that this value, only the - summary refchange email is sent. This can avoid accidental - mailbombing, for example on an initial push. To disable commit - emails limit, set this option to 0. The default is 500. - -multimailhook.excludeMergeRevisions - When sending out revision emails, do not consider merge commits (the - functional equivalent of `rev-list --no-merges`). - The default is `false` (send merge commit emails). - -multimailhook.emailStrictUTF8 - If this boolean option is set to `true`, then the main part of the - email body is forced to be valid UTF-8. Any characters that are - not valid UTF-8 are converted to the Unicode replacement - character, U+FFFD. The default is `true`. - - This option is ineffective with Python 3, where non-UTF-8 - characters are unconditionally replaced. - -multimailhook.diffOpts - Options passed to ``git diff-tree`` when generating the summary - information for ReferenceChange emails. Default is ``--stat - --summary --find-copies-harder``. Add -p to those options to - include a unified diff of changes in addition to the usual summary - output. Shell quoting is allowed; see ``multimailhook.logOpts`` for - details. - -multimailhook.graphOpts - Options passed to ``git log --graph`` when generating graphs for the - reference change summary emails (used only if refchangeShowGraph - is true). The default is '--oneline --decorate'. - - Shell quoting is allowed; see logOpts for details. - -multimailhook.logOpts - Options passed to ``git log`` to generate additional info for - reference change emails (used only if refchangeShowLog is set). - For example, adding -p will show each commit's complete diff. The - default is empty. - - Shell quoting is allowed; for example, a log format that contains - spaces can be specified using something like:: - - git config multimailhook.logopts '--pretty=format:"%h %aN <%aE>%n%s%n%n%b%n"' - - If you want to set this by editing your configuration file - directly, remember that Git requires double-quotes to be escaped - (see git-config(1) for more information):: - - [multimailhook] - logopts = --pretty=format:\"%h %aN <%aE>%n%s%n%n%b%n\" - -multimailhook.commitLogOpts - Options passed to ``git log`` to generate additional info for - revision change emails. For example, adding --ignore-all-spaces - will suppress whitespace changes. The default options are ``-C - --stat -p --cc``. Shell quoting is allowed; see - multimailhook.logOpts for details. - -multimailhook.dateSubstitute - String to use as a substitute for ``Date:`` in the output of ``git - log`` while formatting commit messages. This is useful to avoid - emitting a line that can be interpreted by mailers as the start of - a cited message (Zimbra webmail in particular). Defaults to - ``CommitDate:``. Set to an empty string or ``none`` to deactivate - the behavior. - -multimailhook.emailDomain - Domain name appended to the username of the person doing the push - to convert it into an email address - (via ``"%s@%s" % (username, emaildomain)``). More complicated - schemes can be implemented by overriding Environment and - overriding its get_pusher_email() method. - -multimailhook.replyTo, multimailhook.replyToCommit, multimailhook.replyToRefchange - Addresses to use in the Reply-To: field for commit emails - (replyToCommit) and refchange emails (replyToRefchange). - multimailhook.replyTo is used as default when replyToCommit or - replyToRefchange is not set. The shortcuts ``pusher`` and - ``author`` are allowed with the same semantics as for - ``multimailhook.from``. In addition, the value ``none`` can be - used to omit the ``Reply-To:`` field. - - The default is ``pusher`` for refchange emails, and ``author`` for - commit emails. - -multimailhook.quiet - Do not output the list of email recipients from the hook - -multimailhook.stdout - For debugging, send emails to stdout rather than to the - mailer. Equivalent to the --stdout command line option - -multimailhook.scanCommitForCc - If this option is set to true, than recipients from lines in commit body - that starts with ``CC:`` will be added to CC list. - Default: false - -multimailhook.combineWhenSingleCommit - If this option is set to true and a single new commit is pushed to - a branch, combine the summary and commit email messages into a - single email. - Default: true - -multimailhook.refFilterInclusionRegex, multimailhook.refFilterExclusionRegex, multimailhook.refFilterDoSendRegex, multimailhook.refFilterDontSendRegex - **Warning:** these options are experimental. They should work, but - the user-interface is not stable yet (in particular, the option - names may change). If you want to participate in stabilizing the - feature, please contact the maintainers and/or send pull-requests. - If you are happy with the current shape of the feature, please - report it too. - - Regular expressions that can be used to limit refs for which email - updates will be sent. It is an error to specify both an inclusion - and an exclusion regex. If a ``refFilterInclusionRegex`` is - specified, emails will only be sent for refs which match this - regex. If a ``refFilterExclusionRegex`` regex is specified, - emails will be sent for all refs except those that match this - regex (or that match a predefined regex specific to the - environment, such as "^refs/notes" for most environments and - "^refs/notes|^refs/changes" for the gerrit environment). - - The expressions are matched against the complete refname, and is - considered to match if any substring matches. For example, to - filter-out all tags, set ``refFilterExclusionRegex`` to - ``^refs/tags/`` (note the leading ``^`` but no trailing ``$``). If - you set ``refFilterExclusionRegex`` to ``master``, then any ref - containing ``master`` will be excluded (the ``master`` branch, but - also ``refs/tags/master`` or ``refs/heads/foo-master-bar``). - - ``refFilterDoSendRegex`` and ``refFilterDontSendRegex`` are - analogous to ``refFilterInclusionRegex`` and - ``refFilterExclusionRegex`` with one difference: with - ``refFilterDoSendRegex`` and ``refFilterDontSendRegex``, commits - introduced by one excluded ref will not be considered as new when - they reach an included ref. Typically, if you add a branch ``foo`` - to ``refFilterDontSendRegex``, push commits to this branch, and - later merge branch ``foo`` into ``master``, then the notification - email for ``master`` will contain a commit email only for the - merge commit. If you include ``foo`` in - ``refFilterExclusionRegex``, then at the time of merge, you will - receive one commit email per commit in the branch. - - These variables can be multi-valued, like:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/ - refFilterExclusionRegex = ^refs/heads/master$ - - You can also provide a whitespace-separated list like:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/ ^refs/heads/master$ - - Both examples exclude tags and the master branch, and are - equivalent to:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/|^refs/heads/master$ - - ``refFilterInclusionRegex`` and ``refFilterExclusionRegex`` are - strictly stronger than ``refFilterDoSendRegex`` and - ``refFilterDontSendRegex``. In other words, adding a ref to a - DoSend/DontSend regex has no effect if it is already excluded by a - Exclusion/Inclusion regex. - -multimailhook.logFile, multimailhook.errorLogFile, multimailhook.debugLogFile - - When set, these variable designate path to files where - git-multimail will log some messages. Normal messages and error - messages are sent to ``logFile``, and error messages are also sent - to ``errorLogFile``. Debug messages and all other messages are - sent to ``debugLogFile``. The recommended way is to set only one - of these variables, but it is also possible to set several of them - (part of the information is then duplicated in several log files, - for example errors are duplicated to all log files). - - Relative path are relative to the Git repository where the push is - done. - -multimailhook.verbose - - Verbosity level of git-multimail on its standard output. By - default, show only error and info messages. If set to true, show - also debug messages. - -Email filtering aids --------------------- - -All emails include extra headers to enable fine tuned filtering and -give information for debugging. All emails include the headers -``X-Git-Host``, ``X-Git-Repo``, ``X-Git-Refname``, and ``X-Git-Reftype``. -ReferenceChange emails also include headers ``X-Git-Oldrev`` and ``X-Git-Newrev``; -Revision emails also include header ``X-Git-Rev``. - - -Customizing email contents --------------------------- - -git-multimail mostly generates emails by expanding templates. The -templates can be customized. To avoid the need to edit -``git_multimail.py`` directly, the preferred way to change the templates -is to write a separate Python script that imports ``git_multimail.py`` as -a module, then replaces the templates in place. See the provided -post-receive script for an example of how this is done. - - -Customizing git-multimail for your environment ----------------------------------------------- - -git-multimail is mostly customized via an "environment" that describes -the local environment in which Git is running. Two types of -environment are built in: - -GenericEnvironment - a stand-alone Git repository. - -GitoliteEnvironment - a Git repository that is managed by gitolite_. For such - repositories, the identity of the pusher is read from - environment variable $GL_USER, the name of the repository is read - from $GL_REPO (if it is not overridden by multimailhook.reponame), - and the From: header value is optionally read from gitolite.conf - (see multimailhook.from). - -By default, git-multimail assumes GitoliteEnvironment if $GL_USER and -$GL_REPO are set, and otherwise assumes GenericEnvironment. -Alternatively, you can choose one of these two environments explicitly -by setting a ``multimailhook.environment`` config setting (which can -have the value `generic` or `gitolite`) or by passing an --environment -option to the script. - -If you need to customize the script in ways that are not supported by -the existing environments, you can define your own environment class -class using arbitrary Python code. To do so, you need to import -``git_multimail.py`` as a Python module, as demonstrated by the example -post-receive script. Then implement your environment class; it should -usually inherit from one of the existing Environment classes and -possibly one or more of the EnvironmentMixin classes. Then set the -``environment`` variable to an instance of your own environment class -and pass it to ``run_as_post_receive_hook()``. - -The standard environment classes, GenericEnvironment and -GitoliteEnvironment, are in fact themselves put together out of a -number of mixin classes, each of which handles one aspect of the -customization. For the finest control over your configuration, you -can specify exactly which mixin classes your own environment class -should inherit from, and override individual methods (or even add your -own mixin classes) to implement entirely new behaviors. If you -implement any mixins that might be useful to other people, please -consider sharing them with the community! - - -Getting involved ----------------- - -Please, read `<CONTRIBUTING.rst>`__ for instructions on how to -contribute to git-multimail. - - -Footnotes ---------- - -.. [1] Because of the way information is passed to update hooks, the - script's method of determining whether a commit has already - been seen does not work when it is used as an ``update`` script. - In particular, no notification email will be generated for a - new commit that is added to multiple references in the same - push. A workaround is to use --force-send to force sending the - emails. - -.. _gitolite: https://github.com/sitaramc/gitolite diff --git a/contrib/hooks/multimail/doc/customizing-emails.rst b/contrib/hooks/multimail/doc/customizing-emails.rst deleted file mode 100644 index 3f5b67f768..0000000000 --- a/contrib/hooks/multimail/doc/customizing-emails.rst +++ /dev/null @@ -1,56 +0,0 @@ -Customizing the content and formatting of emails -================================================ - -Overloading template strings ----------------------------- - -The content of emails is generated based on template strings defined -in ``git_multimail.py``. You can customize these template strings -without changing the script itself, by defining a Python wrapper -around it. The python wrapper should ``import git_multimail`` and then -override the ``git_multimail.*`` strings like this:: - - import sys # needed for sys.argv - - # Import and customize git_multimail: - import git_multimail - git_multimail.REVISION_INTRO_TEMPLATE = """...""" - git_multimail.COMBINED_INTRO_TEMPLATE = git_multimail.REVISION_INTRO_TEMPLATE - - # start git_multimail itself: - git_multimail.main(sys.argv[1:]) - -The template strings can use any value already used in the existing -templates (read the source code). - -Using HTML in template strings ------------------------------- - -If ``multimailhook.commitEmailFormat`` is set to HTML, then -git-multimail will generate HTML emails for commit notifications. The -log and diff will be formatted automatically by git-multimail. By -default, any HTML special character in the templates will be escaped. - -To use HTML formatting in the introduction of the email, set -``multimailhook.htmlInIntro`` to ``true``. Then, the template can -contain any HTML tags, that will be sent as-is in the email. For -example, to add some formatting and a link to the online commit, use -a format like:: - - git_multimail.REVISION_INTRO_TEMPLATE = """\ - <span style="color:#808080">This is an automated email from the git hooks/post-receive script.</span><br /><br /> - - <strong>%(pusher)s</strong> pushed a commit to %(refname_type)s %(short_refname)s - in repository %(repo_shortname)s.<br /> - - <a href="https://github.com/git-multimail/git-multimail/commit/%(newrev)s">View on GitHub</a>. - """ - -Note that the values expanded from ``%(variable)s`` in the format -strings will still be escaped. - -For a less flexible but easier to set up way to add a link to commit -emails, see ``multimailhook.commitBrowseURL``. - -Similarly, one can set ``multimailhook.htmlInFooter`` and override any -of the ``*_FOOTER*`` template strings. diff --git a/contrib/hooks/multimail/doc/gerrit.rst b/contrib/hooks/multimail/doc/gerrit.rst deleted file mode 100644 index 8011d05dec..0000000000 --- a/contrib/hooks/multimail/doc/gerrit.rst +++ /dev/null @@ -1,56 +0,0 @@ -Setting up git-multimail on Gerrit -================================== - -Gerrit has its own email-sending system, but you may prefer using -``git-multimail`` instead. It supports Gerrit natively as a Gerrit -``ref-updated`` hook (Warning: `Gerrit hooks -<https://gerrit-review.googlesource.com/Documentation/config-hooks.html>`__ -are distinct from Git hooks). Setting up ``git-multimail`` on a Gerrit -installation can be done following the instructions below. - -The explanations show an easy way to set up ``git-multimail``, -but leave ``git-multimail`` installed and unconfigured for a while. If -you run Gerrit on a production server, it is advised that you -execute the step "Set up the hook" last to avoid confusing your users -in the meantime. - -Set up the hook ---------------- - -Create a directory ``$site_path/hooks/`` if it does not exist (if you -don't know what ``$site_path`` is, run ``gerrit.sh status`` and look -for a ``GERRIT_SITE`` line). Either copy ``git_multimail.py`` to -``$site_path/hooks/ref-updated`` or create a wrapper script like -this:: - - #! /bin/sh - exec /path/to/git_multimail.py "$@" - -In both cases, make sure the file is named exactly -``$site_path/hooks/ref-updated`` and is executable. - -(Alternatively, you may configure the ``[hooks]`` section of -gerrit.config) - -Configuration -------------- - -Log on the gerrit server and edit ``$site_path/git/$project/config`` -to configure ``git-multimail``. - -Troubleshooting ---------------- - -Warning: this will disable ``git-multimail`` during the debug, and -could confuse your users. Don't run on a production server. - -To debug configuration issues with ``git-multimail``, you can add the -``--stdout`` option when calling ``git_multimail.py`` like this:: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py \ - --stdout "$@" >> /tmp/log.txt - -and try pushing from a test repository. You should see the source of -the email that would have been sent in the output of ``git push`` in -the file ``/tmp/log.txt``. diff --git a/contrib/hooks/multimail/doc/gitolite.rst b/contrib/hooks/multimail/doc/gitolite.rst deleted file mode 100644 index 5054833105..0000000000 --- a/contrib/hooks/multimail/doc/gitolite.rst +++ /dev/null @@ -1,118 +0,0 @@ -Setting up git-multimail on gitolite -==================================== - -``git-multimail`` supports gitolite 3 natively. -The explanations below show an easy way to set up ``git-multimail``, -but leave ``git-multimail`` installed and unconfigured for a while. If -you run gitolite on a production server, it is advised that you -execute the step "Set up the hook" last to avoid confusing your users -in the meantime. - -Set up the hook ---------------- - -Log in as your gitolite user. - -Create a file ``.gitolite/hooks/common/post-receive`` on your gitolite -account containing (adapt the path, obviously):: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py "$@" - -Make sure it's executable (``chmod +x``). Record the hook in -gitolite:: - - gitolite setup - -Configuration -------------- - -First, you have to allow the admin to set Git configuration variables. - -As gitolite user, edit the line containing ``GIT_CONFIG_KEYS`` in file -``.gitolite.rc``, to make it look like:: - - GIT_CONFIG_KEYS => 'multimailhook\..*', - -You can now log out and return to your normal user. - -In the ``gitolite-admin`` clone, edit the file ``conf/gitolite.conf`` -and add:: - - repo @all - # Not strictly needed as git_multimail.py will chose gitolite if - # $GL_USER is set. - config multimailhook.environment = gitolite - config multimailhook.mailingList = # Where emails should be sent - config multimailhook.from = # From address to use - -Note that by default, gitolite forbids ``<`` and ``>`` in variable -values (for security/paranoia reasons, see -`compensating for UNSAFE_PATT -<http://gitolite.com/gitolite/git-config/index.html#compensating-for-unsafe95patt>`__ -in gitolite's documentation for explanations and a way to disable -this). As a consequence, you will not be able to use ``First Last -<First.Last@example.com>`` as recipient email, but specifying -``First.Last@example.com`` alone works. - -Obviously, you can customize all parameters on a per-repository basis by -adding these ``config multimailhook.*`` lines in the section -corresponding to a repository or set of repositories. - -To activate ``git-multimail`` on a per-repository basis, do not set -``multimailhook.mailingList`` in the ``@all`` section and set it only -for repositories for which you want ``git-multimail``. - -Alternatively, you can set up the ``From:`` field on a per-user basis -by adding a ``BEGIN USER EMAILS``/``END USER EMAILS`` section (see -``../README``). - -Specificities of Gitolite for Configuration -------------------------------------------- - -Empty configuration variables -............................. - -With gitolite, the syntax ``config multimailhook.commitList = ""`` -unsets the variable instead of setting it to an empty string (see -`here -<http://gitolite.com/gitolite/git-config.html#an-important-warning-about-deleting-a-config-line>`__). -As a result, there is no way to set a variable to the empty string. -In all most places where an empty value is required, git-multimail -now allows to specify special ``"none"`` value (case-sensitive) to -mean the same. - -Alternatively, one can use ``" "`` (a single space) instead of ``""``. -In most cases (in particular ``multimailhook.*List`` variables), this -will be equivalent to an empty string. - -If you have a use-case where ``"none"`` is not an acceptable value and -you need ``" "`` or ``""`` instead, please report it as a bug to -git-multimail. - -Allowing Regular Expressions in Configuration -............................................. - -gitolite has a mechanism to prevent unsafe configuration variable -values, which prevent characters like ``|`` commonly used in regular -expressions. If you do not need the safety feature of gitolite and -need to use regular expressions in your configuration (e.g. for -``multimailhook.refFilter*`` variables), set -`UNSAFE_PATT -<http://gitolite.com/gitolite/git-config.html#unsafe-patt>`__ to a -less restrictive value. - -Troubleshooting ---------------- - -Warning: this will disable ``git-multimail`` during the debug, and -could confuse your users. Don't run on a production server. - -To debug configuration issues with ``git-multimail``, you can add the -``--stdout`` option when calling ``git_multimail.py`` like this:: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py --stdout "$@" - -and try pushing from a test repository. You should see the source of -the email that would have been sent in the output of ``git push``. diff --git a/contrib/hooks/multimail/doc/troubleshooting.rst b/contrib/hooks/multimail/doc/troubleshooting.rst deleted file mode 100644 index 651b509ee6..0000000000 --- a/contrib/hooks/multimail/doc/troubleshooting.rst +++ /dev/null @@ -1,78 +0,0 @@ -Troubleshooting issues with git-multimail: a FAQ -================================================ - -How to check that git-multimail is properly set up? ---------------------------------------------------- - -Since version 1.4.0, git-multimail allows a simple self-checking of -its configuration: run it with the environment variable -``GIT_MULTIMAIL_CHECK_SETUP`` set to a non-empty string. You should -get something like this:: - - $ GIT_MULTIMAIL_CHECK_SETUP=true /home/moy/dev/git-multimail/git-multimail/git_multimail.py - Environment values: - administrator : 'the administrator of this repository' - charset : 'utf-8' - emailprefix : '[git-multimail] ' - fqdn : 'anie' - projectdesc : 'UNNAMED PROJECT' - pusher : 'moy' - repo_path : '/home/moy/dev/git-multimail' - repo_shortname : 'git-multimail' - - Now, checking that git-multimail's standard input is properly set ... - Please type some text and then press Return - foo - You have just entered: - foo - git-multimail seems properly set up. - -If you forgot to set an important variable, you may get instead:: - - $ GIT_MULTIMAIL_CHECK_SETUP=true /home/moy/dev/git-multimail/git-multimail/git_multimail.py - No email recipients configured! - -Do not set ``$GIT_MULTIMAIL_CHECK_SETUP`` other than for testing your -configuration: it would disable the hook completely. - -Git is not using the right address in the From/To/Reply-To field ----------------------------------------------------------------- - -First, make sure that git-multimail actually uses what you think it is -using. A lot happens to your email (especially when posting to a -mailing-list) between the time `git_multimail.py` sends it and the -time it reaches your inbox. - -A simple test (to do on a test repository, do not use in production as -it would disable email sending): change your post-receive hook to call -`git_multimail.py` with the `--stdout` option, and try to push to the -repository. You should see something like:: - - Counting objects: 3, done. - Writing objects: 100% (3/3), 263 bytes | 0 bytes/s, done. - Total 3 (delta 0), reused 0 (delta 0) - remote: Sending notification emails to: foo.bar@example.com - remote: =========================================================================== - remote: Date: Mon, 25 Apr 2016 18:39:59 +0200 - remote: To: foo.bar@example.com - remote: Subject: [git] branch master updated: foo - remote: MIME-Version: 1.0 - remote: Content-Type: text/plain; charset=utf-8 - remote: Content-Transfer-Encoding: 8bit - remote: Message-ID: <20160425163959.2311.20498@anie> - remote: From: Auth Or <Foo.Bar@example.com> - remote: Reply-To: Auth Or <Foo.Bar@example.com> - remote: X-Git-Host: example - ... - remote: -- - remote: To stop receiving notification emails like this one, please contact - remote: the administrator of this repository. - remote: =========================================================================== - To /path/to/repo - 6278f04..e173f20 master -> master - -Note: this does not include the sender (Return-Path: header), as it is -not part of the message content but passed to the mailer. Some mailer -show the ``Sender:`` field instead of the ``From:`` field (for -example, Zimbra Webmail shows ``From: <sender-field> on behalf of -<from-field>``). diff --git a/contrib/hooks/multimail/git_multimail.py b/contrib/hooks/multimail/git_multimail.py deleted file mode 100755 index f563be82fc..0000000000 --- a/contrib/hooks/multimail/git_multimail.py +++ /dev/null @@ -1,4346 +0,0 @@ -#! /usr/bin/env python - -__version__ = '1.5.0' - -# Copyright (c) 2015-2016 Matthieu Moy and others -# Copyright (c) 2012-2014 Michael Haggerty and others -# Derived from contrib/hooks/post-receive-email, which is -# Copyright (c) 2007 Andy Parkins -# and also includes contributions by other authors. -# -# This file is part of git-multimail. -# -# git-multimail is free software: you can redistribute it and/or -# modify it under the terms of the GNU General Public License version -# 2 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# <http://www.gnu.org/licenses/>. - -"""Generate notification emails for pushes to a git repository. - -This hook sends emails describing changes introduced by pushes to a -git repository. For each reference that was changed, it emits one -ReferenceChange email summarizing how the reference was changed, -followed by one Revision email for each new commit that was introduced -by the reference change. - -Each commit is announced in exactly one Revision email. If the same -commit is merged into another branch in the same or a later push, then -the ReferenceChange email will list the commit's SHA1 and its one-line -summary, but no new Revision email will be generated. - -This script is designed to be used as a "post-receive" hook in a git -repository (see githooks(5)). It can also be used as an "update" -script, but this usage is not completely reliable and is deprecated. - -To help with debugging, this script accepts a --stdout option, which -causes the emails to be written to standard output rather than sent -using sendmail. - -See the accompanying README file for the complete documentation. - -""" - -import sys -import os -import re -import bisect -import socket -import subprocess -import shlex -import optparse -import logging -import smtplib -try: - import ssl -except ImportError: - # Python < 2.6 do not have ssl, but that's OK if we don't use it. - pass -import time - -import uuid -import base64 - -PYTHON3 = sys.version_info >= (3, 0) - -if sys.version_info <= (2, 5): - def all(iterable): - for element in iterable: - if not element: - return False - return True - - -def is_ascii(s): - return all(ord(c) < 128 and ord(c) > 0 for c in s) - - -if PYTHON3: - def is_string(s): - return isinstance(s, str) - - def str_to_bytes(s): - return s.encode(ENCODING) - - def bytes_to_str(s, errors='strict'): - return s.decode(ENCODING, errors) - - unicode = str - - def write_str(f, msg): - # Try outputting with the default encoding. If it fails, - # try UTF-8. - try: - f.buffer.write(msg.encode(sys.getdefaultencoding())) - except UnicodeEncodeError: - f.buffer.write(msg.encode(ENCODING)) - - def read_line(f): - # Try reading with the default encoding. If it fails, - # try UTF-8. - out = f.buffer.readline() - try: - return out.decode(sys.getdefaultencoding()) - except UnicodeEncodeError: - return out.decode(ENCODING) - - import html - - def html_escape(s): - return html.escape(s) - -else: - def is_string(s): - try: - return isinstance(s, basestring) - except NameError: # Silence Pyflakes warning - raise - - def str_to_bytes(s): - return s - - def bytes_to_str(s, errors='strict'): - return s - - def write_str(f, msg): - f.write(msg) - - def read_line(f): - return f.readline() - - def next(it): - return it.next() - - import cgi - - def html_escape(s): - return cgi.escape(s, True) - -try: - from email.charset import Charset - from email.utils import make_msgid - from email.utils import getaddresses - from email.utils import formataddr - from email.utils import formatdate - from email.header import Header -except ImportError: - # Prior to Python 2.5, the email module used different names: - from email.Charset import Charset - from email.Utils import make_msgid - from email.Utils import getaddresses - from email.Utils import formataddr - from email.Utils import formatdate - from email.Header import Header - - -DEBUG = False - -ZEROS = '0' * 40 -LOGBEGIN = '- Log -----------------------------------------------------------------\n' -LOGEND = '-----------------------------------------------------------------------\n' - -ADDR_HEADERS = set(['from', 'to', 'cc', 'bcc', 'reply-to', 'sender']) - -# It is assumed in many places that the encoding is uniformly UTF-8, -# so changing these constants is unsupported. But define them here -# anyway, to make it easier to find (at least most of) the places -# where the encoding is important. -(ENCODING, CHARSET) = ('UTF-8', 'utf-8') - - -REF_CREATED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s created' - ' (now %(newrev_short)s)' - ) -REF_UPDATED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s updated' - ' (%(oldrev_short)s -> %(newrev_short)s)' - ) -REF_DELETED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s deleted' - ' (was %(oldrev_short)s)' - ) - -COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s updated: %(oneline)s' - ) - -REFCHANGE_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Subject: %(subject)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -Message-ID: %(msgid)s -From: %(fromaddr)s -Reply-To: %(reply_to)s -Thread-Index: %(thread_index)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Oldrev: %(oldrev)s -X-Git-Newrev: %(newrev)s -X-Git-NotificationType: ref_changed -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -REFCHANGE_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a change to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - - -FOOTER_TEMPLATE = """\ - --- \n\ -To stop receiving notification emails like this one, please contact -%(administrator)s. -""" - - -REWIND_ONLY_TEMPLATE = """\ -This update removed existing revisions from the reference, leaving the -reference pointing at a previous point in the repository history. - - * -- * -- N %(refname)s (%(newrev_short)s) - \\ - O -- O -- O (%(oldrev_short)s) - -Any revisions marked "omit" are not gone; other references still -refer to them. Any revisions marked "discard" are gone forever. -""" - - -NON_FF_TEMPLATE = """\ -This update added new revisions after undoing existing revisions. -That is to say, some revisions that were in the old version of the -%(refname_type)s are not in the new version. This situation occurs -when a user --force pushes a change and generates a repository -containing something like this: - - * -- * -- B -- O -- O -- O (%(oldrev_short)s) - \\ - N -- N -- N %(refname)s (%(newrev_short)s) - -You should already have received notification emails for all of the O -revisions, and so the following emails describe only the N revisions -from the common base, B. - -Any revisions marked "omit" are not gone; other references still -refer to them. Any revisions marked "discard" are gone forever. -""" - - -NO_NEW_REVISIONS_TEMPLATE = """\ -No new revisions were added by this update. -""" - - -DISCARDED_REVISIONS_TEMPLATE = """\ -This change permanently discards the following revisions: -""" - - -NO_DISCARDED_REVISIONS_TEMPLATE = """\ -The revisions that were on this %(refname_type)s are still contained in -other references; therefore, this change does not discard any commits -from the repository. -""" - - -NEW_REVISIONS_TEMPLATE = """\ -The %(tot)s revisions listed above as "new" are entirely new to this -repository and will be described in separate emails. The revisions -listed as "add" were already present in the repository and have only -been added to this reference. - -""" - - -TAG_CREATED_TEMPLATE = """\ - at %(newrev_short)-8s (%(newrev_type)s) -""" - - -TAG_UPDATED_TEMPLATE = """\ -*** WARNING: tag %(short_refname)s was modified! *** - - from %(oldrev_short)-8s (%(oldrev_type)s) - to %(newrev_short)-8s (%(newrev_type)s) -""" - - -TAG_DELETED_TEMPLATE = """\ -*** WARNING: tag %(short_refname)s was deleted! *** - -""" - - -# The template used in summary tables. It looks best if this uses the -# same alignment as TAG_CREATED_TEMPLATE and TAG_UPDATED_TEMPLATE. -BRIEF_SUMMARY_TEMPLATE = """\ -%(action)8s %(rev_short)-8s %(text)s -""" - - -NON_COMMIT_UPDATE_TEMPLATE = """\ -This is an unusual reference change because the reference did not -refer to a commit either before or after the change. We do not know -how to provide full information about this reference change. -""" - - -REVISION_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Cc: %(cc_recipients)s -Subject: %(emailprefix)s%(num)02d/%(tot)02d: %(oneline)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -From: %(fromaddr)s -Reply-To: %(reply_to)s -In-Reply-To: %(reply_to_msgid)s -References: %(reply_to_msgid)s -Thread-Index: %(thread_index)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Rev: %(rev)s -X-Git-NotificationType: diff -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -REVISION_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a commit to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - -LINK_TEXT_TEMPLATE = """\ -View the commit online: -%(browse_url)s - -""" - -LINK_HTML_TEMPLATE = """\ -<p><a href="%(browse_url)s">View the commit online</a>.</p> -""" - - -REVISION_FOOTER_TEMPLATE = FOOTER_TEMPLATE - - -# Combined, meaning refchange+revision email (for single-commit additions) -COMBINED_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Subject: %(subject)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -Message-ID: %(msgid)s -From: %(fromaddr)s -Reply-To: %(reply_to)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Oldrev: %(oldrev)s -X-Git-Newrev: %(newrev)s -X-Git-Rev: %(rev)s -X-Git-NotificationType: ref_changed_plus_diff -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -COMBINED_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a commit to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - -COMBINED_FOOTER_TEMPLATE = FOOTER_TEMPLATE - - -class CommandError(Exception): - def __init__(self, cmd, retcode): - self.cmd = cmd - self.retcode = retcode - Exception.__init__( - self, - 'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,) - ) - - -class ConfigurationException(Exception): - pass - - -# The "git" program (this could be changed to include a full path): -GIT_EXECUTABLE = 'git' - - -# How "git" should be invoked (including global arguments), as a list -# of words. This variable is usually initialized automatically by -# read_git_output() via choose_git_command(), but if a value is set -# here then it will be used unconditionally. -GIT_CMD = None - - -def choose_git_command(): - """Decide how to invoke git, and record the choice in GIT_CMD.""" - - global GIT_CMD - - if GIT_CMD is None: - try: - # Check to see whether the "-c" option is accepted (it was - # only added in Git 1.7.2). We don't actually use the - # output of "git --version", though if we needed more - # specific version information this would be the place to - # do it. - cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version'] - read_output(cmd) - GIT_CMD = [GIT_EXECUTABLE, '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)] - except CommandError: - GIT_CMD = [GIT_EXECUTABLE] - - -def read_git_output(args, input=None, keepends=False, **kw): - """Read the output of a Git command.""" - - if GIT_CMD is None: - choose_git_command() - - return read_output(GIT_CMD + args, input=input, keepends=keepends, **kw) - - -def read_output(cmd, input=None, keepends=False, **kw): - if input: - stdin = subprocess.PIPE - input = str_to_bytes(input) - else: - stdin = None - errors = 'strict' - if 'errors' in kw: - errors = kw['errors'] - del kw['errors'] - p = subprocess.Popen( - tuple(str_to_bytes(w) for w in cmd), - stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw - ) - (out, err) = p.communicate(input) - out = bytes_to_str(out, errors=errors) - retcode = p.wait() - if retcode: - raise CommandError(cmd, retcode) - if not keepends: - out = out.rstrip('\n\r') - return out - - -def read_git_lines(args, keepends=False, **kw): - """Return the lines output by Git command. - - Return as single lines, with newlines stripped off.""" - - return read_git_output(args, keepends=True, **kw).splitlines(keepends) - - -def git_rev_list_ish(cmd, spec, args=None, **kw): - """Common functionality for invoking a 'git rev-list'-like command. - - Parameters: - * cmd is the Git command to run, e.g., 'rev-list' or 'log'. - * spec is a list of revision arguments to pass to the named - command. If None, this function returns an empty list. - * args is a list of extra arguments passed to the named command. - * All other keyword arguments (if any) are passed to the - underlying read_git_lines() function. - - Return the output of the Git command in the form of a list, one - entry per output line. - """ - if spec is None: - return [] - if args is None: - args = [] - args = [cmd, '--stdin'] + args - spec_stdin = ''.join(s + '\n' for s in spec) - return read_git_lines(args, input=spec_stdin, **kw) - - -def git_rev_list(spec, **kw): - """Run 'git rev-list' with the given list of revision arguments. - - See git_rev_list_ish() for parameter and return value - documentation. - """ - return git_rev_list_ish('rev-list', spec, **kw) - - -def git_log(spec, **kw): - """Run 'git log' with the given list of revision arguments. - - See git_rev_list_ish() for parameter and return value - documentation. - """ - return git_rev_list_ish('log', spec, **kw) - - -def header_encode(text, header_name=None): - """Encode and line-wrap the value of an email header field.""" - - # Convert to unicode, if required. - if not isinstance(text, unicode): - text = unicode(text, 'utf-8') - - if is_ascii(text): - charset = 'ascii' - else: - charset = 'utf-8' - - return Header(text, header_name=header_name, charset=Charset(charset)).encode() - - -def addr_header_encode(text, header_name=None): - """Encode and line-wrap the value of an email header field containing - email addresses.""" - - # Convert to unicode, if required. - if not isinstance(text, unicode): - text = unicode(text, 'utf-8') - - text = ', '.join( - formataddr((header_encode(name), emailaddr)) - for name, emailaddr in getaddresses([text]) - ) - - if is_ascii(text): - charset = 'ascii' - else: - charset = 'utf-8' - - return Header(text, header_name=header_name, charset=Charset(charset)).encode() - - -class Config(object): - def __init__(self, section, git_config=None): - """Represent a section of the git configuration. - - If git_config is specified, it is passed to "git config" in - the GIT_CONFIG environment variable, meaning that "git config" - will read the specified path rather than the Git default - config paths.""" - - self.section = section - if git_config: - self.env = os.environ.copy() - self.env['GIT_CONFIG'] = git_config - else: - self.env = None - - @staticmethod - def _split(s): - """Split NUL-terminated values.""" - - words = s.split('\0') - assert words[-1] == '' - return words[:-1] - - @staticmethod - def add_config_parameters(c): - """Add configuration parameters to Git. - - c is either an str or a list of str, each element being of the - form 'var=val' or 'var', with the same syntax and meaning as - the argument of 'git -c var=val'. - """ - if isinstance(c, str): - c = (c,) - parameters = os.environ.get('GIT_CONFIG_PARAMETERS', '') - if parameters: - parameters += ' ' - # git expects GIT_CONFIG_PARAMETERS to be of the form - # "'name1=value1' 'name2=value2' 'name3=value3'" - # including everything inside the double quotes (but not the double - # quotes themselves). Spacing is critical. Also, if a value contains - # a literal single quote that quote must be represented using the - # four character sequence: '\'' - parameters += ' '.join("'" + x.replace("'", "'\\''") + "'" for x in c) - os.environ['GIT_CONFIG_PARAMETERS'] = parameters - - def get(self, name, default=None): - try: - values = self._split(read_git_output( - ['config', '--get', '--null', '%s.%s' % (self.section, name)], - env=self.env, keepends=True, - )) - assert len(values) == 1 - return values[0] - except CommandError: - return default - - def get_bool(self, name, default=None): - try: - value = read_git_output( - ['config', '--get', '--bool', '%s.%s' % (self.section, name)], - env=self.env, - ) - except CommandError: - return default - return value == 'true' - - def get_all(self, name, default=None): - """Read a (possibly multivalued) setting from the configuration. - - Return the result as a list of values, or default if the name - is unset.""" - - try: - return self._split(read_git_output( - ['config', '--get-all', '--null', '%s.%s' % (self.section, name)], - env=self.env, keepends=True, - )) - except CommandError: - t, e, traceback = sys.exc_info() - if e.retcode == 1: - # "the section or key is invalid"; i.e., there is no - # value for the specified key. - return default - else: - raise - - def set(self, name, value): - read_git_output( - ['config', '%s.%s' % (self.section, name), value], - env=self.env, - ) - - def add(self, name, value): - read_git_output( - ['config', '--add', '%s.%s' % (self.section, name), value], - env=self.env, - ) - - def __contains__(self, name): - return self.get_all(name, default=None) is not None - - # We don't use this method anymore internally, but keep it here in - # case somebody is calling it from their own code: - def has_key(self, name): - return name in self - - def unset_all(self, name): - try: - read_git_output( - ['config', '--unset-all', '%s.%s' % (self.section, name)], - env=self.env, - ) - except CommandError: - t, e, traceback = sys.exc_info() - if e.retcode == 5: - # The name doesn't exist, which is what we wanted anyway... - pass - else: - raise - - def set_recipients(self, name, value): - self.unset_all(name) - for pair in getaddresses([value]): - self.add(name, formataddr(pair)) - - -def generate_summaries(*log_args): - """Generate a brief summary for each revision requested. - - log_args are strings that will be passed directly to "git log" as - revision selectors. Iterate over (sha1_short, subject) for each - commit specified by log_args (subject is the first line of the - commit message as a string without EOLs).""" - - cmd = [ - 'log', '--abbrev', '--format=%h %s', - ] + list(log_args) + ['--'] - for line in read_git_lines(cmd): - yield tuple(line.split(' ', 1)) - - -def limit_lines(lines, max_lines): - for (index, line) in enumerate(lines): - if index < max_lines: - yield line - - if index >= max_lines: - yield '... %d lines suppressed ...\n' % (index + 1 - max_lines,) - - -def limit_linelength(lines, max_linelength): - for line in lines: - # Don't forget that lines always include a trailing newline. - if len(line) > max_linelength + 1: - line = line[:max_linelength - 7] + ' [...]\n' - yield line - - -class CommitSet(object): - """A (constant) set of object names. - - The set should be initialized with full SHA1 object names. The - __contains__() method returns True iff its argument is an - abbreviation of any the names in the set.""" - - def __init__(self, names): - self._names = sorted(names) - - def __len__(self): - return len(self._names) - - def __contains__(self, sha1_abbrev): - """Return True iff this set contains sha1_abbrev (which might be abbreviated).""" - - i = bisect.bisect_left(self._names, sha1_abbrev) - return i < len(self) and self._names[i].startswith(sha1_abbrev) - - -class GitObject(object): - def __init__(self, sha1, type=None): - if sha1 == ZEROS: - self.sha1 = self.type = self.commit_sha1 = None - else: - self.sha1 = sha1 - self.type = type or read_git_output(['cat-file', '-t', self.sha1]) - - if self.type == 'commit': - self.commit_sha1 = self.sha1 - elif self.type == 'tag': - try: - self.commit_sha1 = read_git_output( - ['rev-parse', '--verify', '%s^0' % (self.sha1,)] - ) - except CommandError: - # Cannot deref tag to determine commit_sha1 - self.commit_sha1 = None - else: - self.commit_sha1 = None - - self.short = read_git_output(['rev-parse', '--short', sha1]) - - def get_summary(self): - """Return (sha1_short, subject) for this commit.""" - - if not self.sha1: - raise ValueError('Empty commit has no summary') - - return next(iter(generate_summaries('--no-walk', self.sha1))) - - def __eq__(self, other): - return isinstance(other, GitObject) and self.sha1 == other.sha1 - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash(self.sha1) - - def __nonzero__(self): - return bool(self.sha1) - - def __bool__(self): - """Python 2 backward compatibility""" - return self.__nonzero__() - - def __str__(self): - return self.sha1 or ZEROS - - -class Change(object): - """A Change that has been made to the Git repository. - - Abstract class from which both Revisions and ReferenceChanges are - derived. A Change knows how to generate a notification email - describing itself.""" - - def __init__(self, environment): - self.environment = environment - self._values = None - self._contains_html_diff = False - - def _contains_diff(self): - # We do contain a diff, should it be rendered in HTML? - if self.environment.commit_email_format == "html": - self._contains_html_diff = True - - def _compute_values(self): - """Return a dictionary {keyword: expansion} for this Change. - - Derived classes overload this method to add more entries to - the return value. This method is used internally by - get_values(). The return value should always be a new - dictionary.""" - - values = self.environment.get_values() - fromaddr = self.environment.get_fromaddr(change=self) - if fromaddr is not None: - values['fromaddr'] = fromaddr - values['multimail_version'] = get_version() - return values - - # Aliases usable in template strings. Tuple of pairs (destination, - # source). - VALUES_ALIAS = ( - ("id", "newrev"), - ) - - def get_values(self, **extra_values): - """Return a dictionary {keyword: expansion} for this Change. - - Return a dictionary mapping keywords to the values that they - should be expanded to for this Change (used when interpolating - template strings). If any keyword arguments are supplied, add - those to the return value as well. The return value is always - a new dictionary.""" - - if self._values is None: - self._values = self._compute_values() - - values = self._values.copy() - if extra_values: - values.update(extra_values) - - for alias, val in self.VALUES_ALIAS: - values[alias] = values[val] - return values - - def expand(self, template, **extra_values): - """Expand template. - - Expand the template (which should be a string) using string - interpolation of the values for this Change. If any keyword - arguments are provided, also include those in the keywords - available for interpolation.""" - - return template % self.get_values(**extra_values) - - def expand_lines(self, template, html_escape_val=False, **extra_values): - """Break template into lines and expand each line.""" - - values = self.get_values(**extra_values) - if html_escape_val: - for k in values: - if is_string(values[k]): - values[k] = html_escape(values[k]) - for line in template.splitlines(True): - yield line % values - - def expand_header_lines(self, template, **extra_values): - """Break template into lines and expand each line as an RFC 2822 header. - - Encode values and split up lines that are too long. Silently - skip lines that contain references to unknown variables.""" - - values = self.get_values(**extra_values) - if self._contains_html_diff: - self._content_type = 'html' - else: - self._content_type = 'plain' - values['contenttype'] = self._content_type - - for line in template.splitlines(): - (name, value) = line.split(': ', 1) - - try: - value = value % values - except KeyError: - t, e, traceback = sys.exc_info() - if DEBUG: - self.environment.log_warning( - 'Warning: unknown variable %r in the following line; line skipped:\n' - ' %s\n' - % (e.args[0], line,) - ) - else: - if name.lower() in ADDR_HEADERS: - value = addr_header_encode(value, name) - else: - value = header_encode(value, name) - for splitline in ('%s: %s\n' % (name, value)).splitlines(True): - yield splitline - - def generate_email_header(self): - """Generate the RFC 2822 email headers for this Change, a line at a time. - - The output should not include the trailing blank line.""" - - raise NotImplementedError() - - def generate_browse_link(self, base_url): - """Generate a link to an online repository browser.""" - return iter(()) - - def generate_email_intro(self, html_escape_val=False): - """Generate the email intro for this Change, a line at a time. - - The output will be used as the standard boilerplate at the top - of the email body.""" - - raise NotImplementedError() - - def generate_email_body(self, push): - """Generate the main part of the email body, a line at a time. - - The text in the body might be truncated after a specified - number of lines (see multimailhook.emailmaxlines).""" - - raise NotImplementedError() - - def generate_email_footer(self, html_escape_val): - """Generate the footer of the email, a line at a time. - - The footer is always included, irrespective of - multimailhook.emailmaxlines.""" - - raise NotImplementedError() - - def _wrap_for_html(self, lines): - """Wrap the lines in HTML <pre> tag when using HTML format. - - Escape special HTML characters and add <pre> and </pre> tags around - the given lines if we should be generating HTML as indicated by - self._contains_html_diff being set to true. - """ - if self._contains_html_diff: - yield "<pre style='margin:0'>\n" - - for line in lines: - yield html_escape(line) - - yield '</pre>\n' - else: - for line in lines: - yield line - - def generate_email(self, push, body_filter=None, extra_header_values={}): - """Generate an email describing this change. - - Iterate over the lines (including the header lines) of an - email describing this change. If body_filter is not None, - then use it to filter the lines that are intended for the - email body. - - The extra_header_values field is received as a dict and not as - **kwargs, to allow passing other keyword arguments in the - future (e.g. passing extra values to generate_email_intro()""" - - for line in self.generate_email_header(**extra_header_values): - yield line - yield '\n' - html_escape_val = (self.environment.html_in_intro and - self._contains_html_diff) - intro = self.generate_email_intro(html_escape_val) - if not self.environment.html_in_intro: - intro = self._wrap_for_html(intro) - for line in intro: - yield line - - if self.environment.commitBrowseURL: - for line in self.generate_browse_link(self.environment.commitBrowseURL): - yield line - - body = self.generate_email_body(push) - if body_filter is not None: - body = body_filter(body) - - diff_started = False - if self._contains_html_diff: - # "white-space: pre" is the default, but we need to - # specify it again in case the message is viewed in a - # webmail which wraps it in an element setting white-space - # to something else (Zimbra does this and sets - # white-space: pre-line). - yield '<pre style="white-space: pre; background: #F8F8F8">' - for line in body: - if self._contains_html_diff: - # This is very, very naive. It would be much better to really - # parse the diff, i.e. look at how many lines do we have in - # the hunk headers instead of blindly highlighting everything - # that looks like it might be part of a diff. - bgcolor = '' - fgcolor = '' - if line.startswith('--- a/'): - diff_started = True - bgcolor = 'e0e0ff' - elif line.startswith('diff ') or line.startswith('index '): - diff_started = True - fgcolor = '808080' - elif diff_started: - if line.startswith('+++ '): - bgcolor = 'e0e0ff' - elif line.startswith('@@'): - bgcolor = 'e0e0e0' - elif line.startswith('+'): - bgcolor = 'e0ffe0' - elif line.startswith('-'): - bgcolor = 'ffe0e0' - elif line.startswith('commit '): - fgcolor = '808000' - elif line.startswith(' '): - fgcolor = '404040' - - # Chop the trailing LF, we don't want it inside <pre>. - line = html_escape(line[:-1]) - - if bgcolor or fgcolor: - style = 'display:block; white-space:pre;' - if bgcolor: - style += 'background:#' + bgcolor + ';' - if fgcolor: - style += 'color:#' + fgcolor + ';' - # Use a <span style='display:block> to color the - # whole line. The newline must be inside the span - # to display properly both in Firefox and in - # text-based browser. - line = "<span style='%s'>%s\n</span>" % (style, line) - else: - line = line + '\n' - - yield line - if self._contains_html_diff: - yield '</pre>' - html_escape_val = (self.environment.html_in_footer and - self._contains_html_diff) - footer = self.generate_email_footer(html_escape_val) - if not self.environment.html_in_footer: - footer = self._wrap_for_html(footer) - for line in footer: - yield line - - def get_specific_fromaddr(self): - """For kinds of Changes which specify it, return the kind-specific - From address to use.""" - return None - - -class Revision(Change): - """A Change consisting of a single git commit.""" - - CC_RE = re.compile(r'^\s*C[Cc]:\s*(?P<to>[^#]+@[^\s#]*)\s*(#.*)?$') - - def __init__(self, reference_change, rev, num, tot): - Change.__init__(self, reference_change.environment) - self.reference_change = reference_change - self.rev = rev - self.change_type = self.reference_change.change_type - self.refname = self.reference_change.refname - self.num = num - self.tot = tot - self.author = read_git_output(['log', '--no-walk', '--format=%aN <%aE>', self.rev.sha1]) - self.recipients = self.environment.get_revision_recipients(self) - - # -s is short for --no-patch, but -s works on older git's (e.g. 1.7) - self.parents = read_git_lines(['show', '-s', '--format=%P', - self.rev.sha1])[0].split() - - self.cc_recipients = '' - if self.environment.get_scancommitforcc(): - self.cc_recipients = ', '.join(to.strip() for to in self._cc_recipients()) - if self.cc_recipients: - self.environment.log_msg( - 'Add %s to CC for %s' % (self.cc_recipients, self.rev.sha1)) - - def _cc_recipients(self): - cc_recipients = [] - message = read_git_output(['log', '--no-walk', '--format=%b', self.rev.sha1]) - lines = message.strip().split('\n') - for line in lines: - m = re.match(self.CC_RE, line) - if m: - cc_recipients.append(m.group('to')) - - return cc_recipients - - def _compute_values(self): - values = Change._compute_values(self) - - oneline = read_git_output( - ['log', '--format=%s', '--no-walk', self.rev.sha1] - ) - - max_subject_length = self.environment.get_max_subject_length() - if max_subject_length > 0 and len(oneline) > max_subject_length: - oneline = oneline[:max_subject_length - 6] + ' [...]' - - values['rev'] = self.rev.sha1 - values['parents'] = ' '.join(self.parents) - values['rev_short'] = self.rev.short - values['change_type'] = self.change_type - values['refname'] = self.refname - values['newrev'] = self.rev.sha1 - values['short_refname'] = self.reference_change.short_refname - values['refname_type'] = self.reference_change.refname_type - values['reply_to_msgid'] = self.reference_change.msgid - values['thread_index'] = self.reference_change.thread_index - values['num'] = self.num - values['tot'] = self.tot - values['recipients'] = self.recipients - if self.cc_recipients: - values['cc_recipients'] = self.cc_recipients - values['oneline'] = oneline - values['author'] = self.author - - reply_to = self.environment.get_reply_to_commit(self) - if reply_to: - values['reply_to'] = reply_to - - return values - - def generate_email_header(self, **extra_values): - for line in self.expand_header_lines( - REVISION_HEADER_TEMPLATE, **extra_values - ): - yield line - - def generate_browse_link(self, base_url): - if '%(' not in base_url: - base_url += '%(id)s' - url = "".join(self.expand_lines(base_url)) - if self._content_type == 'html': - for line in self.expand_lines(LINK_HTML_TEMPLATE, - html_escape_val=True, - browse_url=url): - yield line - elif self._content_type == 'plain': - for line in self.expand_lines(LINK_TEXT_TEMPLATE, - html_escape_val=False, - browse_url=url): - yield line - else: - raise NotImplementedError("Content-type %s unsupported. Please report it as a bug.") - - def generate_email_intro(self, html_escape_val=False): - for line in self.expand_lines(REVISION_INTRO_TEMPLATE, - html_escape_val=html_escape_val): - yield line - - def generate_email_body(self, push): - """Show this revision.""" - - for line in read_git_lines( - ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1], - keepends=True, - errors='replace'): - if line.startswith('Date: ') and self.environment.date_substitute: - yield self.environment.date_substitute + line[len('Date: '):] - else: - yield line - - def generate_email_footer(self, html_escape_val): - return self.expand_lines(REVISION_FOOTER_TEMPLATE, - html_escape_val=html_escape_val) - - def generate_email(self, push, body_filter=None, extra_header_values={}): - self._contains_diff() - return Change.generate_email(self, push, body_filter, extra_header_values) - - def get_specific_fromaddr(self): - return self.environment.from_commit - - -class ReferenceChange(Change): - """A Change to a Git reference. - - An abstract class representing a create, update, or delete of a - Git reference. Derived classes handle specific types of reference - (e.g., tags vs. branches). These classes generate the main - reference change email summarizing the reference change and - whether it caused any any commits to be added or removed. - - ReferenceChange objects are usually created using the static - create() method, which has the logic to decide which derived class - to instantiate.""" - - REF_RE = re.compile(r'^refs\/(?P<area>[^\/]+)\/(?P<shortname>.*)$') - - @staticmethod - def create(environment, oldrev, newrev, refname): - """Return a ReferenceChange object representing the change. - - Return an object that represents the type of change that is being - made. oldrev and newrev should be SHA1s or ZEROS.""" - - old = GitObject(oldrev) - new = GitObject(newrev) - rev = new or old - - # The revision type tells us what type the commit is, combined with - # the location of the ref we can decide between - # - working branch - # - tracking branch - # - unannotated tag - # - annotated tag - m = ReferenceChange.REF_RE.match(refname) - if m: - area = m.group('area') - short_refname = m.group('shortname') - else: - area = '' - short_refname = refname - - if rev.type == 'tag': - # Annotated tag: - klass = AnnotatedTagChange - elif rev.type == 'commit': - if area == 'tags': - # Non-annotated tag: - klass = NonAnnotatedTagChange - elif area == 'heads': - # Branch: - klass = BranchChange - elif area == 'remotes': - # Tracking branch: - environment.log_warning( - '*** Push-update of tracking branch %r\n' - '*** - incomplete email generated.' - % (refname,) - ) - klass = OtherReferenceChange - else: - # Some other reference namespace: - environment.log_warning( - '*** Push-update of strange reference %r\n' - '*** - incomplete email generated.' - % (refname,) - ) - klass = OtherReferenceChange - else: - # Anything else (is there anything else?) - environment.log_warning( - '*** Unknown type of update to %r (%s)\n' - '*** - incomplete email generated.' - % (refname, rev.type,) - ) - klass = OtherReferenceChange - - return klass( - environment, - refname=refname, short_refname=short_refname, - old=old, new=new, rev=rev, - ) - - @staticmethod - def make_thread_index(): - """Return a string appropriate for the Thread-Index header, - needed by MS Outlook to get threading right. - - The format is (base64-encoded): - - 1 byte must be 1 - - 5 bytes encode a date (hardcoded here) - - 16 bytes for a globally unique identifier - - FIXME: Unfortunately, even with the Thread-Index field, MS - Outlook doesn't seem to do the threading reliably (see - https://github.com/git-multimail/git-multimail/pull/194). - """ - thread_index = b'\x01\x00\x00\x12\x34\x56' + uuid.uuid4().bytes - return base64.standard_b64encode(thread_index).decode('ascii') - - def __init__(self, environment, refname, short_refname, old, new, rev): - Change.__init__(self, environment) - self.change_type = { - (False, True): 'create', - (True, True): 'update', - (True, False): 'delete', - }[bool(old), bool(new)] - self.refname = refname - self.short_refname = short_refname - self.old = old - self.new = new - self.rev = rev - self.msgid = make_msgid() - self.thread_index = self.make_thread_index() - self.diffopts = environment.diffopts - self.graphopts = environment.graphopts - self.logopts = environment.logopts - self.commitlogopts = environment.commitlogopts - self.showgraph = environment.refchange_showgraph - self.showlog = environment.refchange_showlog - - self.header_template = REFCHANGE_HEADER_TEMPLATE - self.intro_template = REFCHANGE_INTRO_TEMPLATE - self.footer_template = FOOTER_TEMPLATE - - def _compute_values(self): - values = Change._compute_values(self) - - values['change_type'] = self.change_type - values['refname_type'] = self.refname_type - values['refname'] = self.refname - values['short_refname'] = self.short_refname - values['msgid'] = self.msgid - values['thread_index'] = self.thread_index - values['recipients'] = self.recipients - values['oldrev'] = str(self.old) - values['oldrev_short'] = self.old.short - values['newrev'] = str(self.new) - values['newrev_short'] = self.new.short - - if self.old: - values['oldrev_type'] = self.old.type - if self.new: - values['newrev_type'] = self.new.type - - reply_to = self.environment.get_reply_to_refchange(self) - if reply_to: - values['reply_to'] = reply_to - - return values - - def send_single_combined_email(self, known_added_sha1s): - """Determine if a combined refchange/revision email should be sent - - If there is only a single new (non-merge) commit added by a - change, it is useful to combine the ReferenceChange and - Revision emails into one. In such a case, return the single - revision; otherwise, return None. - - This method is overridden in BranchChange.""" - - return None - - def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}): - """Generate an email describing this change AND specified revision. - - Iterate over the lines (including the header lines) of an - email describing this change. If body_filter is not None, - then use it to filter the lines that are intended for the - email body. - - The extra_header_values field is received as a dict and not as - **kwargs, to allow passing other keyword arguments in the - future (e.g. passing extra values to generate_email_intro() - - This method is overridden in BranchChange.""" - - raise NotImplementedError - - def get_subject(self): - template = { - 'create': REF_CREATED_SUBJECT_TEMPLATE, - 'update': REF_UPDATED_SUBJECT_TEMPLATE, - 'delete': REF_DELETED_SUBJECT_TEMPLATE, - }[self.change_type] - return self.expand(template) - - def generate_email_header(self, **extra_values): - if 'subject' not in extra_values: - extra_values['subject'] = self.get_subject() - - for line in self.expand_header_lines( - self.header_template, **extra_values - ): - yield line - - def generate_email_intro(self, html_escape_val=False): - for line in self.expand_lines(self.intro_template, - html_escape_val=html_escape_val): - yield line - - def generate_email_body(self, push): - """Call the appropriate body-generation routine. - - Call one of generate_create_summary() / - generate_update_summary() / generate_delete_summary().""" - - change_summary = { - 'create': self.generate_create_summary, - 'delete': self.generate_delete_summary, - 'update': self.generate_update_summary, - }[self.change_type](push) - for line in change_summary: - yield line - - for line in self.generate_revision_change_summary(push): - yield line - - def generate_email_footer(self, html_escape_val): - return self.expand_lines(self.footer_template, - html_escape_val=html_escape_val) - - def generate_revision_change_graph(self, push): - if self.showgraph: - args = ['--graph'] + self.graphopts - for newold in ('new', 'old'): - has_newold = False - spec = push.get_commits_spec(newold, self) - for line in git_log(spec, args=args, keepends=True): - if not has_newold: - has_newold = True - yield '\n' - yield 'Graph of %s commits:\n\n' % ( - {'new': 'new', 'old': 'discarded'}[newold],) - yield ' ' + line - if has_newold: - yield '\n' - - def generate_revision_change_log(self, new_commits_list): - if self.showlog: - yield '\n' - yield 'Detailed log of new commits:\n\n' - for line in read_git_lines( - ['log', '--no-walk'] + - self.logopts + - new_commits_list + - ['--'], - keepends=True, - ): - yield line - - def generate_new_revision_summary(self, tot, new_commits_list, push): - for line in self.expand_lines(NEW_REVISIONS_TEMPLATE, tot=tot): - yield line - for line in self.generate_revision_change_graph(push): - yield line - for line in self.generate_revision_change_log(new_commits_list): - yield line - - def generate_revision_change_summary(self, push): - """Generate a summary of the revisions added/removed by this change.""" - - if self.new.commit_sha1 and not self.old.commit_sha1: - # A new reference was created. List the new revisions - # brought by the new reference (i.e., those revisions that - # were not in the repository before this reference - # change). - sha1s = list(push.get_new_commits(self)) - sha1s.reverse() - tot = len(sha1s) - new_revisions = [ - Revision(self, GitObject(sha1), num=i + 1, tot=tot) - for (i, sha1) in enumerate(sha1s) - ] - - if new_revisions: - yield self.expand('This %(refname_type)s includes the following new commits:\n') - yield '\n' - for r in new_revisions: - (sha1, subject) = r.rev.get_summary() - yield r.expand( - BRIEF_SUMMARY_TEMPLATE, action='new', text=subject, - ) - yield '\n' - for line in self.generate_new_revision_summary( - tot, [r.rev.sha1 for r in new_revisions], push): - yield line - else: - for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE): - yield line - - elif self.new.commit_sha1 and self.old.commit_sha1: - # A reference was changed to point at a different commit. - # List the revisions that were removed and/or added *from - # that reference* by this reference change, along with a - # diff between the trees for its old and new values. - - # List of the revisions that were added to the branch by - # this update. Note this list can include revisions that - # have already had notification emails; we want such - # revisions in the summary even though we will not send - # new notification emails for them. - adds = list(generate_summaries( - '--topo-order', '--reverse', '%s..%s' - % (self.old.commit_sha1, self.new.commit_sha1,) - )) - - # List of the revisions that were removed from the branch - # by this update. This will be empty except for - # non-fast-forward updates. - discards = list(generate_summaries( - '%s..%s' % (self.new.commit_sha1, self.old.commit_sha1,) - )) - - if adds: - new_commits_list = push.get_new_commits(self) - else: - new_commits_list = [] - new_commits = CommitSet(new_commits_list) - - if discards: - discarded_commits = CommitSet(push.get_discarded_commits(self)) - else: - discarded_commits = CommitSet([]) - - if discards and adds: - for (sha1, subject) in discards: - if sha1 in discarded_commits: - action = 'discard' - else: - action = 'omit' - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action=action, - rev_short=sha1, text=subject, - ) - for (sha1, subject) in adds: - if sha1 in new_commits: - action = 'new' - else: - action = 'add' - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action=action, - rev_short=sha1, text=subject, - ) - yield '\n' - for line in self.expand_lines(NON_FF_TEMPLATE): - yield line - - elif discards: - for (sha1, subject) in discards: - if sha1 in discarded_commits: - action = 'discard' - else: - action = 'omit' - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action=action, - rev_short=sha1, text=subject, - ) - yield '\n' - for line in self.expand_lines(REWIND_ONLY_TEMPLATE): - yield line - - elif adds: - (sha1, subject) = self.old.get_summary() - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action='from', - rev_short=sha1, text=subject, - ) - for (sha1, subject) in adds: - if sha1 in new_commits: - action = 'new' - else: - action = 'add' - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action=action, - rev_short=sha1, text=subject, - ) - - yield '\n' - - if new_commits: - for line in self.generate_new_revision_summary( - len(new_commits), new_commits_list, push): - yield line - else: - for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE): - yield line - for line in self.generate_revision_change_graph(push): - yield line - - # The diffstat is shown from the old revision to the new - # revision. This is to show the truth of what happened in - # this change. There's no point showing the stat from the - # base to the new revision because the base is effectively a - # random revision at this point - the user will be interested - # in what this revision changed - including the undoing of - # previous revisions in the case of non-fast-forward updates. - yield '\n' - yield 'Summary of changes:\n' - for line in read_git_lines( - ['diff-tree'] + - self.diffopts + - ['%s..%s' % (self.old.commit_sha1, self.new.commit_sha1,)], - keepends=True, - ): - yield line - - elif self.old.commit_sha1 and not self.new.commit_sha1: - # A reference was deleted. List the revisions that were - # removed from the repository by this reference change. - - sha1s = list(push.get_discarded_commits(self)) - tot = len(sha1s) - discarded_revisions = [ - Revision(self, GitObject(sha1), num=i + 1, tot=tot) - for (i, sha1) in enumerate(sha1s) - ] - - if discarded_revisions: - for line in self.expand_lines(DISCARDED_REVISIONS_TEMPLATE): - yield line - yield '\n' - for r in discarded_revisions: - (sha1, subject) = r.rev.get_summary() - yield r.expand( - BRIEF_SUMMARY_TEMPLATE, action='discard', text=subject, - ) - for line in self.generate_revision_change_graph(push): - yield line - else: - for line in self.expand_lines(NO_DISCARDED_REVISIONS_TEMPLATE): - yield line - - elif not self.old.commit_sha1 and not self.new.commit_sha1: - for line in self.expand_lines(NON_COMMIT_UPDATE_TEMPLATE): - yield line - - def generate_create_summary(self, push): - """Called for the creation of a reference.""" - - # This is a new reference and so oldrev is not valid - (sha1, subject) = self.new.get_summary() - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action='at', - rev_short=sha1, text=subject, - ) - yield '\n' - - def generate_update_summary(self, push): - """Called for the change of a pre-existing branch.""" - - return iter([]) - - def generate_delete_summary(self, push): - """Called for the deletion of any type of reference.""" - - (sha1, subject) = self.old.get_summary() - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action='was', - rev_short=sha1, text=subject, - ) - yield '\n' - - def get_specific_fromaddr(self): - return self.environment.from_refchange - - -class BranchChange(ReferenceChange): - refname_type = 'branch' - - def __init__(self, environment, refname, short_refname, old, new, rev): - ReferenceChange.__init__( - self, environment, - refname=refname, short_refname=short_refname, - old=old, new=new, rev=rev, - ) - self.recipients = environment.get_refchange_recipients(self) - self._single_revision = None - - def send_single_combined_email(self, known_added_sha1s): - if not self.environment.combine_when_single_commit: - return None - - # In the sadly-all-too-frequent usecase of people pushing only - # one of their commits at a time to a repository, users feel - # the reference change summary emails are noise rather than - # important signal. This is because, in this particular - # usecase, there is a reference change summary email for each - # new commit, and all these summaries do is point out that - # there is one new commit (which can readily be inferred by - # the existence of the individual revision email that is also - # sent). In such cases, our users prefer there to be a combined - # reference change summary/new revision email. - # - # So, if the change is an update and it doesn't discard any - # commits, and it adds exactly one non-merge commit (gerrit - # forces a workflow where every commit is individually merged - # and the git-multimail hook fired off for just this one - # change), then we send a combined refchange/revision email. - try: - # If this change is a reference update that doesn't discard - # any commits... - if self.change_type != 'update': - return None - - if read_git_lines( - ['merge-base', self.old.sha1, self.new.sha1] - ) != [self.old.sha1]: - return None - - # Check if this update introduced exactly one non-merge - # commit: - - def split_line(line): - """Split line into (sha1, [parent,...]).""" - - words = line.split() - return (words[0], words[1:]) - - # Get the new commits introduced by the push as a list of - # (sha1, [parent,...]) - new_commits = [ - split_line(line) - for line in read_git_lines( - [ - 'log', '-3', '--format=%H %P', - '%s..%s' % (self.old.sha1, self.new.sha1), - ] - ) - ] - - if not new_commits: - return None - - # If the newest commit is a merge, save it for a later check - # but otherwise ignore it - merge = None - tot = len(new_commits) - if len(new_commits[0][1]) > 1: - merge = new_commits[0][0] - del new_commits[0] - - # Our primary check: we can't combine if more than one commit - # is introduced. We also currently only combine if the new - # commit is a non-merge commit, though it may make sense to - # combine if it is a merge as well. - if not ( - len(new_commits) == 1 and - len(new_commits[0][1]) == 1 and - new_commits[0][0] in known_added_sha1s - ): - return None - - # We do not want to combine revision and refchange emails if - # those go to separate locations. - rev = Revision(self, GitObject(new_commits[0][0]), 1, tot) - if rev.recipients != self.recipients: - return None - - # We ignored the newest commit if it was just a merge of the one - # commit being introduced. But we don't want to ignore that - # merge commit it it involved conflict resolutions. Check that. - if merge and merge != read_git_output(['diff-tree', '--cc', merge]): - return None - - # We can combine the refchange and one new revision emails - # into one. Return the Revision that a combined email should - # be sent about. - return rev - except CommandError: - # Cannot determine number of commits in old..new or new..old; - # don't combine reference/revision emails: - return None - - def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}): - values = revision.get_values() - if extra_header_values: - values.update(extra_header_values) - if 'subject' not in extra_header_values: - values['subject'] = self.expand(COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE, **values) - - self._single_revision = revision - self._contains_diff() - self.header_template = COMBINED_HEADER_TEMPLATE - self.intro_template = COMBINED_INTRO_TEMPLATE - self.footer_template = COMBINED_FOOTER_TEMPLATE - - def revision_gen_link(base_url): - # revision is used only to generate the body, and - # _content_type is set while generating headers. Get it - # from the BranchChange object. - revision._content_type = self._content_type - return revision.generate_browse_link(base_url) - self.generate_browse_link = revision_gen_link - for line in self.generate_email(push, body_filter, values): - yield line - - def generate_email_body(self, push): - '''Call the appropriate body generation routine. - - If this is a combined refchange/revision email, the special logic - for handling this combined email comes from this function. For - other cases, we just use the normal handling.''' - - # If self._single_revision isn't set; don't override - if not self._single_revision: - for line in super(BranchChange, self).generate_email_body(push): - yield line - return - - # This is a combined refchange/revision email; we first provide - # some info from the refchange portion, and then call the revision - # generate_email_body function to handle the revision portion. - adds = list(generate_summaries( - '--topo-order', '--reverse', '%s..%s' - % (self.old.commit_sha1, self.new.commit_sha1,) - )) - - yield self.expand("The following commit(s) were added to %(refname)s by this push:\n") - for (sha1, subject) in adds: - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action='new', - rev_short=sha1, text=subject, - ) - - yield self._single_revision.rev.short + " is described below\n" - yield '\n' - - for line in self._single_revision.generate_email_body(push): - yield line - - -class AnnotatedTagChange(ReferenceChange): - refname_type = 'annotated tag' - - def __init__(self, environment, refname, short_refname, old, new, rev): - ReferenceChange.__init__( - self, environment, - refname=refname, short_refname=short_refname, - old=old, new=new, rev=rev, - ) - self.recipients = environment.get_announce_recipients(self) - self.show_shortlog = environment.announce_show_shortlog - - ANNOTATED_TAG_FORMAT = ( - '%(*objectname)\n' - '%(*objecttype)\n' - '%(taggername)\n' - '%(taggerdate)' - ) - - def describe_tag(self, push): - """Describe the new value of an annotated tag.""" - - # Use git for-each-ref to pull out the individual fields from - # the tag - [tagobject, tagtype, tagger, tagged] = read_git_lines( - ['for-each-ref', '--format=%s' % (self.ANNOTATED_TAG_FORMAT,), self.refname], - ) - - yield self.expand( - BRIEF_SUMMARY_TEMPLATE, action='tagging', - rev_short=tagobject, text='(%s)' % (tagtype,), - ) - if tagtype == 'commit': - # If the tagged object is a commit, then we assume this is a - # release, and so we calculate which tag this tag is - # replacing - try: - prevtag = read_git_output(['describe', '--abbrev=0', '%s^' % (self.new,)]) - except CommandError: - prevtag = None - if prevtag: - yield ' replaces %s\n' % (prevtag,) - else: - prevtag = None - yield ' length %s bytes\n' % (read_git_output(['cat-file', '-s', tagobject]),) - - yield ' by %s\n' % (tagger,) - yield ' on %s\n' % (tagged,) - yield '\n' - - # Show the content of the tag message; this might contain a - # change log or release notes so is worth displaying. - yield LOGBEGIN - contents = list(read_git_lines(['cat-file', 'tag', self.new.sha1], keepends=True)) - contents = contents[contents.index('\n') + 1:] - if contents and contents[-1][-1:] != '\n': - contents.append('\n') - for line in contents: - yield line - - if self.show_shortlog and tagtype == 'commit': - # Only commit tags make sense to have rev-list operations - # performed on them - yield '\n' - if prevtag: - # Show changes since the previous release - revlist = read_git_output( - ['rev-list', '--pretty=short', '%s..%s' % (prevtag, self.new,)], - keepends=True, - ) - else: - # No previous tag, show all the changes since time - # began - revlist = read_git_output( - ['rev-list', '--pretty=short', '%s' % (self.new,)], - keepends=True, - ) - for line in read_git_lines(['shortlog'], input=revlist, keepends=True): - yield line - - yield LOGEND - yield '\n' - - def generate_create_summary(self, push): - """Called for the creation of an annotated tag.""" - - for line in self.expand_lines(TAG_CREATED_TEMPLATE): - yield line - - for line in self.describe_tag(push): - yield line - - def generate_update_summary(self, push): - """Called for the update of an annotated tag. - - This is probably a rare event and may not even be allowed.""" - - for line in self.expand_lines(TAG_UPDATED_TEMPLATE): - yield line - - for line in self.describe_tag(push): - yield line - - def generate_delete_summary(self, push): - """Called when a non-annotated reference is updated.""" - - for line in self.expand_lines(TAG_DELETED_TEMPLATE): - yield line - - yield self.expand(' tag was %(oldrev_short)s\n') - yield '\n' - - -class NonAnnotatedTagChange(ReferenceChange): - refname_type = 'tag' - - def __init__(self, environment, refname, short_refname, old, new, rev): - ReferenceChange.__init__( - self, environment, - refname=refname, short_refname=short_refname, - old=old, new=new, rev=rev, - ) - self.recipients = environment.get_refchange_recipients(self) - - def generate_create_summary(self, push): - """Called for the creation of an annotated tag.""" - - for line in self.expand_lines(TAG_CREATED_TEMPLATE): - yield line - - def generate_update_summary(self, push): - """Called when a non-annotated reference is updated.""" - - for line in self.expand_lines(TAG_UPDATED_TEMPLATE): - yield line - - def generate_delete_summary(self, push): - """Called when a non-annotated reference is updated.""" - - for line in self.expand_lines(TAG_DELETED_TEMPLATE): - yield line - - for line in ReferenceChange.generate_delete_summary(self, push): - yield line - - -class OtherReferenceChange(ReferenceChange): - refname_type = 'reference' - - def __init__(self, environment, refname, short_refname, old, new, rev): - # We use the full refname as short_refname, because otherwise - # the full name of the reference would not be obvious from the - # text of the email. - ReferenceChange.__init__( - self, environment, - refname=refname, short_refname=refname, - old=old, new=new, rev=rev, - ) - self.recipients = environment.get_refchange_recipients(self) - - -class Mailer(object): - """An object that can send emails.""" - - def __init__(self, environment): - self.environment = environment - - def close(self): - pass - - def send(self, lines, to_addrs): - """Send an email consisting of lines. - - lines must be an iterable over the lines constituting the - header and body of the email. to_addrs is a list of recipient - addresses (can be needed even if lines already contains a - "To:" field). It can be either a string (comma-separated list - of email addresses) or a Python list of individual email - addresses. - - """ - - raise NotImplementedError() - - -class SendMailer(Mailer): - """Send emails using 'sendmail -oi -t'.""" - - SENDMAIL_CANDIDATES = [ - '/usr/sbin/sendmail', - '/usr/lib/sendmail', - ] - - @staticmethod - def find_sendmail(): - for path in SendMailer.SENDMAIL_CANDIDATES: - if os.access(path, os.X_OK): - return path - else: - raise ConfigurationException( - 'No sendmail executable found. ' - 'Try setting multimailhook.sendmailCommand.' - ) - - def __init__(self, environment, command=None, envelopesender=None): - """Construct a SendMailer instance. - - command should be the command and arguments used to invoke - sendmail, as a list of strings. If an envelopesender is - provided, it will also be passed to the command, via '-f - envelopesender'.""" - super(SendMailer, self).__init__(environment) - if command: - self.command = command[:] - else: - self.command = [self.find_sendmail(), '-oi', '-t'] - - if envelopesender: - self.command.extend(['-f', envelopesender]) - - def send(self, lines, to_addrs): - try: - p = subprocess.Popen(self.command, stdin=subprocess.PIPE) - except OSError: - self.environment.get_logger().error( - '*** Cannot execute command: %s\n' % ' '.join(self.command) + - '*** %s\n' % sys.exc_info()[1] + - '*** Try setting multimailhook.mailer to "smtp"\n' + - '*** to send emails without using the sendmail command.\n' - ) - sys.exit(1) - try: - lines = (str_to_bytes(line) for line in lines) - p.stdin.writelines(lines) - except Exception: - self.environment.get_logger().error( - '*** Error while generating commit email\n' - '*** - mail sending aborted.\n' - ) - if hasattr(p, 'terminate'): - # subprocess.terminate() is not available in Python 2.4 - p.terminate() - else: - import signal - os.kill(p.pid, signal.SIGTERM) - raise - else: - p.stdin.close() - retcode = p.wait() - if retcode: - raise CommandError(self.command, retcode) - - -class SMTPMailer(Mailer): - """Send emails using Python's smtplib.""" - - def __init__(self, environment, - envelopesender, smtpserver, - smtpservertimeout=10.0, smtpserverdebuglevel=0, - smtpencryption='none', - smtpuser='', smtppass='', - smtpcacerts='' - ): - super(SMTPMailer, self).__init__(environment) - if not envelopesender: - self.environment.get_logger().error( - 'fatal: git_multimail: cannot use SMTPMailer without a sender address.\n' - 'please set either multimailhook.envelopeSender or user.email\n' - ) - sys.exit(1) - if smtpencryption == 'ssl' and not (smtpuser and smtppass): - raise ConfigurationException( - 'Cannot use SMTPMailer with security option ssl ' - 'without options username and password.' - ) - self.envelopesender = envelopesender - self.smtpserver = smtpserver - self.smtpservertimeout = smtpservertimeout - self.smtpserverdebuglevel = smtpserverdebuglevel - self.security = smtpencryption - self.username = smtpuser - self.password = smtppass - self.smtpcacerts = smtpcacerts - self.loggedin = False - try: - def call(klass, server, timeout): - try: - return klass(server, timeout=timeout) - except TypeError: - # Old Python versions do not have timeout= argument. - return klass(server) - if self.security == 'none': - self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout) - elif self.security == 'ssl': - if self.smtpcacerts: - raise smtplib.SMTPException( - "Checking certificate is not supported for ssl, prefer starttls" - ) - self.smtp = call(smtplib.SMTP_SSL, self.smtpserver, timeout=self.smtpservertimeout) - elif self.security == 'tls': - if 'ssl' not in sys.modules: - self.environment.get_logger().error( - '*** Your Python version does not have the ssl library installed\n' - '*** smtpEncryption=tls is not available.\n' - '*** Either upgrade Python to 2.6 or later\n' - ' or use git_multimail.py version 1.2.\n') - if ':' not in self.smtpserver: - self.smtpserver += ':587' # default port for TLS - self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout) - # start: ehlo + starttls - # equivalent to - # self.smtp.ehlo() - # self.smtp.starttls() - # with access to the ssl layer - self.smtp.ehlo() - if not self.smtp.has_extn("starttls"): - raise smtplib.SMTPException("STARTTLS extension not supported by server") - resp, reply = self.smtp.docmd("STARTTLS") - if resp != 220: - raise smtplib.SMTPException("Wrong answer to the STARTTLS command") - if self.smtpcacerts: - self.smtp.sock = ssl.wrap_socket( - self.smtp.sock, - ca_certs=self.smtpcacerts, - cert_reqs=ssl.CERT_REQUIRED - ) - else: - self.smtp.sock = ssl.wrap_socket( - self.smtp.sock, - cert_reqs=ssl.CERT_NONE - ) - self.environment.get_logger().error( - '*** Warning, the server certificate is not verified (smtp) ***\n' - '*** set the option smtpCACerts ***\n' - ) - if not hasattr(self.smtp.sock, "read"): - # using httplib.FakeSocket with Python 2.5.x or earlier - self.smtp.sock.read = self.smtp.sock.recv - self.smtp.file = smtplib.SSLFakeFile(self.smtp.sock) - self.smtp.helo_resp = None - self.smtp.ehlo_resp = None - self.smtp.esmtp_features = {} - self.smtp.does_esmtp = 0 - # end: ehlo + starttls - self.smtp.ehlo() - else: - sys.stdout.write('*** Error: Control reached an invalid option. ***') - sys.exit(1) - if self.smtpserverdebuglevel > 0: - sys.stdout.write( - "*** Setting debug on for SMTP server connection (%s) ***\n" - % self.smtpserverdebuglevel) - self.smtp.set_debuglevel(self.smtpserverdebuglevel) - except Exception: - self.environment.get_logger().error( - '*** Error establishing SMTP connection to %s ***\n' - '*** %s\n' - % (self.smtpserver, sys.exc_info()[1])) - sys.exit(1) - - def close(self): - if hasattr(self, 'smtp'): - self.smtp.quit() - del self.smtp - - def __del__(self): - self.close() - - def send(self, lines, to_addrs): - try: - if self.username or self.password: - if not self.loggedin: - self.smtp.login(self.username, self.password) - self.loggedin = True - msg = ''.join(lines) - # turn comma-separated list into Python list if needed. - if is_string(to_addrs): - to_addrs = [email for (name, email) in getaddresses([to_addrs])] - self.smtp.sendmail(self.envelopesender, to_addrs, msg) - except socket.timeout: - self.environment.get_logger().error( - '*** Error sending email ***\n' - '*** SMTP server timed out (timeout is %s)\n' - % self.smtpservertimeout) - except smtplib.SMTPResponseException: - err = sys.exc_info()[1] - self.environment.get_logger().error( - '*** Error sending email ***\n' - '*** Error %d: %s\n' - % (err.smtp_code, bytes_to_str(err.smtp_error))) - try: - smtp = self.smtp - # delete the field before quit() so that in case of - # error, self.smtp is deleted anyway. - del self.smtp - smtp.quit() - except: - self.environment.get_logger().error( - '*** Error closing the SMTP connection ***\n' - '*** Exiting anyway ... ***\n' - '*** %s\n' % sys.exc_info()[1]) - sys.exit(1) - - -class OutputMailer(Mailer): - """Write emails to an output stream, bracketed by lines of '=' characters. - - This is intended for debugging purposes.""" - - SEPARATOR = '=' * 75 + '\n' - - def __init__(self, f, environment=None): - super(OutputMailer, self).__init__(environment=environment) - self.f = f - - def send(self, lines, to_addrs): - write_str(self.f, self.SEPARATOR) - for line in lines: - write_str(self.f, line) - write_str(self.f, self.SEPARATOR) - - -def get_git_dir(): - """Determine GIT_DIR. - - Determine GIT_DIR either from the GIT_DIR environment variable or - from the working directory, using Git's usual rules.""" - - try: - return read_git_output(['rev-parse', '--git-dir']) - except CommandError: - sys.stderr.write('fatal: git_multimail: not in a git directory\n') - sys.exit(1) - - -class Environment(object): - """Describes the environment in which the push is occurring. - - An Environment object encapsulates information about the local - environment. For example, it knows how to determine: - - * the name of the repository to which the push occurred - - * what user did the push - - * what users want to be informed about various types of changes. - - An Environment object is expected to have the following methods: - - get_repo_shortname() - - Return a short name for the repository, for display - purposes. - - get_repo_path() - - Return the absolute path to the Git repository. - - get_emailprefix() - - Return a string that will be prefixed to every email's - subject. - - get_pusher() - - Return the username of the person who pushed the changes. - This value is used in the email body to indicate who - pushed the change. - - get_pusher_email() (may return None) - - Return the email address of the person who pushed the - changes. The value should be a single RFC 2822 email - address as a string; e.g., "Joe User <user@example.com>" - if available, otherwise "user@example.com". If set, the - value is used as the Reply-To address for refchange - emails. If it is impossible to determine the pusher's - email, this attribute should be set to None (in which case - no Reply-To header will be output). - - get_sender() - - Return the address to be used as the 'From' email address - in the email envelope. - - get_fromaddr(change=None) - - Return the 'From' email address used in the email 'From:' - headers. If the change is known when this function is - called, it is passed in as the 'change' parameter. (May - be a full RFC 2822 email address like 'Joe User - <user@example.com>'.) - - get_administrator() - - Return the name and/or email of the repository - administrator. This value is used in the footer as the - person to whom requests to be removed from the - notification list should be sent. Ideally, it should - include a valid email address. - - get_reply_to_refchange() - get_reply_to_commit() - - Return the address to use in the email "Reply-To" header, - as a string. These can be an RFC 2822 email address, or - None to omit the "Reply-To" header. - get_reply_to_refchange() is used for refchange emails; - get_reply_to_commit() is used for individual commit - emails. - - get_ref_filter_regex() - - Return a tuple -- a compiled regex, and a boolean indicating - whether the regex picks refs to include (if False, the regex - matches on refs to exclude). - - get_default_ref_ignore_regex() - - Return a regex that should be ignored for both what emails - to send and when computing what commits are considered new - to the repository. Default is "^refs/notes/". - - get_max_subject_length() - - Return an int giving the maximal length for the subject - (git log --oneline). - - They should also define the following attributes: - - announce_show_shortlog (bool) - - True iff announce emails should include a shortlog. - - commit_email_format (string) - - If "html", generate commit emails in HTML instead of plain text - used by default. - - html_in_intro (bool) - html_in_footer (bool) - - When generating HTML emails, the introduction (respectively, - the footer) will be HTML-escaped iff html_in_intro (respectively, - the footer) is true. When false, only the values used to expand - the template are escaped. - - refchange_showgraph (bool) - - True iff refchanges emails should include a detailed graph. - - refchange_showlog (bool) - - True iff refchanges emails should include a detailed log. - - diffopts (list of strings) - - The options that should be passed to 'git diff' for the - summary email. The value should be a list of strings - representing words to be passed to the command. - - graphopts (list of strings) - - Analogous to diffopts, but contains options passed to - 'git log --graph' when generating the detailed graph for - a set of commits (see refchange_showgraph) - - logopts (list of strings) - - Analogous to diffopts, but contains options passed to - 'git log' when generating the detailed log for a set of - commits (see refchange_showlog) - - commitlogopts (list of strings) - - The options that should be passed to 'git log' for each - commit mail. The value should be a list of strings - representing words to be passed to the command. - - date_substitute (string) - - String to be used in substitution for 'Date:' at start of - line in the output of 'git log'. - - quiet (bool) - On success do not write to stderr - - stdout (bool) - Write email to stdout rather than emailing. Useful for debugging - - combine_when_single_commit (bool) - - True if a combined email should be produced when a single - new commit is pushed to a branch, False otherwise. - - from_refchange, from_commit (strings) - - Addresses to use for the From: field for refchange emails - and commit emails respectively. Set from - multimailhook.fromRefchange and multimailhook.fromCommit - by ConfigEnvironmentMixin. - - log_file, error_log_file, debug_log_file (string) - - Name of a file to which logs should be sent. - - verbose (int) - - How verbose the system should be. - - 0 (default): show info, errors, ... - - 1 : show basic debug info - """ - - REPO_NAME_RE = re.compile(r'^(?P<name>.+?)(?:\.git)$') - - def __init__(self, osenv=None): - self.osenv = osenv or os.environ - self.announce_show_shortlog = False - self.commit_email_format = "text" - self.html_in_intro = False - self.html_in_footer = False - self.commitBrowseURL = None - self.maxcommitemails = 500 - self.excludemergerevisions = False - self.diffopts = ['--stat', '--summary', '--find-copies-harder'] - self.graphopts = ['--oneline', '--decorate'] - self.logopts = [] - self.refchange_showgraph = False - self.refchange_showlog = False - self.commitlogopts = ['-C', '--stat', '-p', '--cc'] - self.date_substitute = 'AuthorDate: ' - self.quiet = False - self.stdout = False - self.combine_when_single_commit = True - self.logger = None - - self.COMPUTED_KEYS = [ - 'administrator', - 'charset', - 'emailprefix', - 'pusher', - 'pusher_email', - 'repo_path', - 'repo_shortname', - 'sender', - ] - - self._values = None - - def get_logger(self): - """Get (possibly creates) the logger associated to this environment.""" - if self.logger is None: - self.logger = Logger(self) - return self.logger - - def get_repo_shortname(self): - """Use the last part of the repo path, with ".git" stripped off if present.""" - - basename = os.path.basename(os.path.abspath(self.get_repo_path())) - m = self.REPO_NAME_RE.match(basename) - if m: - return m.group('name') - else: - return basename - - def get_pusher(self): - raise NotImplementedError() - - def get_pusher_email(self): - return None - - def get_fromaddr(self, change=None): - config = Config('user') - fromname = config.get('name', default='') - fromemail = config.get('email', default='') - if fromemail: - return formataddr([fromname, fromemail]) - return self.get_sender() - - def get_administrator(self): - return 'the administrator of this repository' - - def get_emailprefix(self): - return '' - - def get_repo_path(self): - if read_git_output(['rev-parse', '--is-bare-repository']) == 'true': - path = get_git_dir() - else: - path = read_git_output(['rev-parse', '--show-toplevel']) - return os.path.abspath(path) - - def get_charset(self): - return CHARSET - - def get_values(self): - """Return a dictionary {keyword: expansion} for this Environment. - - This method is called by Change._compute_values(). The keys - in the returned dictionary are available to be used in any of - the templates. The dictionary is created by calling - self.get_NAME() for each of the attributes named in - COMPUTED_KEYS and recording those that do not return None. - The return value is always a new dictionary.""" - - if self._values is None: - values = {'': ''} # %()s expands to the empty string. - - for key in self.COMPUTED_KEYS: - value = getattr(self, 'get_%s' % (key,))() - if value is not None: - values[key] = value - - self._values = values - - return self._values.copy() - - def get_refchange_recipients(self, refchange): - """Return the recipients for notifications about refchange. - - Return the list of email addresses to which notifications - about the specified ReferenceChange should be sent.""" - - raise NotImplementedError() - - def get_announce_recipients(self, annotated_tag_change): - """Return the recipients for notifications about annotated_tag_change. - - Return the list of email addresses to which notifications - about the specified AnnotatedTagChange should be sent.""" - - raise NotImplementedError() - - def get_reply_to_refchange(self, refchange): - return self.get_pusher_email() - - def get_revision_recipients(self, revision): - """Return the recipients for messages about revision. - - Return the list of email addresses to which notifications - about the specified Revision should be sent. This method - could be overridden, for example, to take into account the - contents of the revision when deciding whom to notify about - it. For example, there could be a scheme for users to express - interest in particular files or subdirectories, and only - receive notification emails for revisions that affecting those - files.""" - - raise NotImplementedError() - - def get_reply_to_commit(self, revision): - return revision.author - - def get_default_ref_ignore_regex(self): - # The commit messages of git notes are essentially meaningless - # and "filenames" in git notes commits are an implementational - # detail that might surprise users at first. As such, we - # would need a completely different method for handling emails - # of git notes in order for them to be of benefit for users, - # which we simply do not have right now. - return "^refs/notes/" - - def get_max_subject_length(self): - """Return the maximal subject line (git log --oneline) length. - Longer subject lines will be truncated.""" - raise NotImplementedError() - - def filter_body(self, lines): - """Filter the lines intended for an email body. - - lines is an iterable over the lines that would go into the - email body. Filter it (e.g., limit the number of lines, the - line length, character set, etc.), returning another iterable. - See FilterLinesEnvironmentMixin and MaxlinesEnvironmentMixin - for classes implementing this functionality.""" - - return lines - - def log_msg(self, msg): - """Write the string msg on a log file or on stderr. - - Sends the text to stderr by default, override to change the behavior.""" - self.get_logger().info(msg) - - def log_warning(self, msg): - """Write the string msg on a log file or on stderr. - - Sends the text to stderr by default, override to change the behavior.""" - self.get_logger().warning(msg) - - def log_error(self, msg): - """Write the string msg on a log file or on stderr. - - Sends the text to stderr by default, override to change the behavior.""" - self.get_logger().error(msg) - - def check(self): - pass - - -class ConfigEnvironmentMixin(Environment): - """A mixin that sets self.config to its constructor's config argument. - - This class's constructor consumes the "config" argument. - - Mixins that need to inspect the config should inherit from this - class (1) to make sure that "config" is still in the constructor - arguments with its own constructor runs and/or (2) to be sure that - self.config is set after construction.""" - - def __init__(self, config, **kw): - super(ConfigEnvironmentMixin, self).__init__(**kw) - self.config = config - - -class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin): - """An Environment that reads most of its information from "git config".""" - - @staticmethod - def forbid_field_values(name, value, forbidden): - for forbidden_val in forbidden: - if value is not None and value.lower() == forbidden: - raise ConfigurationException( - '"%s" is not an allowed setting for %s' % (value, name) - ) - - def __init__(self, config, **kw): - super(ConfigOptionsEnvironmentMixin, self).__init__( - config=config, **kw - ) - - for var, cfg in ( - ('announce_show_shortlog', 'announceshortlog'), - ('refchange_showgraph', 'refchangeShowGraph'), - ('refchange_showlog', 'refchangeshowlog'), - ('quiet', 'quiet'), - ('stdout', 'stdout'), - ): - val = config.get_bool(cfg) - if val is not None: - setattr(self, var, val) - - commit_email_format = config.get('commitEmailFormat') - if commit_email_format is not None: - if commit_email_format != "html" and commit_email_format != "text": - self.log_warning( - '*** Unknown value for multimailhook.commitEmailFormat: %s\n' % - commit_email_format + - '*** Expected either "text" or "html". Ignoring.\n' - ) - else: - self.commit_email_format = commit_email_format - - html_in_intro = config.get_bool('htmlInIntro') - if html_in_intro is not None: - self.html_in_intro = html_in_intro - - html_in_footer = config.get_bool('htmlInFooter') - if html_in_footer is not None: - self.html_in_footer = html_in_footer - - self.commitBrowseURL = config.get('commitBrowseURL') - - self.excludemergerevisions = config.get('excludeMergeRevisions') - - maxcommitemails = config.get('maxcommitemails') - if maxcommitemails is not None: - try: - self.maxcommitemails = int(maxcommitemails) - except ValueError: - self.log_warning( - '*** Malformed value for multimailhook.maxCommitEmails: %s\n' - % maxcommitemails + - '*** Expected a number. Ignoring.\n' - ) - - diffopts = config.get('diffopts') - if diffopts is not None: - self.diffopts = shlex.split(diffopts) - - graphopts = config.get('graphOpts') - if graphopts is not None: - self.graphopts = shlex.split(graphopts) - - logopts = config.get('logopts') - if logopts is not None: - self.logopts = shlex.split(logopts) - - commitlogopts = config.get('commitlogopts') - if commitlogopts is not None: - self.commitlogopts = shlex.split(commitlogopts) - - date_substitute = config.get('dateSubstitute') - if date_substitute == 'none': - self.date_substitute = None - elif date_substitute is not None: - self.date_substitute = date_substitute - - reply_to = config.get('replyTo') - self.__reply_to_refchange = config.get('replyToRefchange', default=reply_to) - self.forbid_field_values('replyToRefchange', - self.__reply_to_refchange, - ['author']) - self.__reply_to_commit = config.get('replyToCommit', default=reply_to) - - self.from_refchange = config.get('fromRefchange') - self.forbid_field_values('fromRefchange', - self.from_refchange, - ['author', 'none']) - self.from_commit = config.get('fromCommit') - self.forbid_field_values('fromCommit', - self.from_commit, - ['none']) - - combine = config.get_bool('combineWhenSingleCommit') - if combine is not None: - self.combine_when_single_commit = combine - - self.log_file = config.get('logFile', default=None) - self.error_log_file = config.get('errorLogFile', default=None) - self.debug_log_file = config.get('debugLogFile', default=None) - if config.get_bool('Verbose', default=False): - self.verbose = 1 - else: - self.verbose = 0 - - def get_administrator(self): - return ( - self.config.get('administrator') or - self.get_sender() or - super(ConfigOptionsEnvironmentMixin, self).get_administrator() - ) - - def get_repo_shortname(self): - return ( - self.config.get('reponame') or - super(ConfigOptionsEnvironmentMixin, self).get_repo_shortname() - ) - - def get_emailprefix(self): - emailprefix = self.config.get('emailprefix') - if emailprefix is not None: - emailprefix = emailprefix.strip() - if emailprefix: - emailprefix += ' ' - else: - emailprefix = '[%(repo_shortname)s] ' - short_name = self.get_repo_shortname() - try: - return emailprefix % {'repo_shortname': short_name} - except: - self.get_logger().error( - '*** Invalid multimailhook.emailPrefix: %s\n' % emailprefix + - '*** %s\n' % sys.exc_info()[1] + - "*** Only the '%(repo_shortname)s' placeholder is allowed\n" - ) - raise ConfigurationException( - '"%s" is not an allowed setting for emailPrefix' % emailprefix - ) - - def get_sender(self): - return self.config.get('envelopesender') - - def process_addr(self, addr, change): - if addr.lower() == 'author': - if hasattr(change, 'author'): - return change.author - else: - return None - elif addr.lower() == 'pusher': - return self.get_pusher_email() - elif addr.lower() == 'none': - return None - else: - return addr - - def get_fromaddr(self, change=None): - fromaddr = self.config.get('from') - if change: - specific_fromaddr = change.get_specific_fromaddr() - if specific_fromaddr: - fromaddr = specific_fromaddr - if fromaddr: - fromaddr = self.process_addr(fromaddr, change) - if fromaddr: - return fromaddr - return super(ConfigOptionsEnvironmentMixin, self).get_fromaddr(change) - - def get_reply_to_refchange(self, refchange): - if self.__reply_to_refchange is None: - return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_refchange(refchange) - else: - return self.process_addr(self.__reply_to_refchange, refchange) - - def get_reply_to_commit(self, revision): - if self.__reply_to_commit is None: - return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_commit(revision) - else: - return self.process_addr(self.__reply_to_commit, revision) - - def get_scancommitforcc(self): - return self.config.get('scancommitforcc') - - -class FilterLinesEnvironmentMixin(Environment): - """Handle encoding and maximum line length of body lines. - - email_max_line_length (int or None) - - The maximum length of any single line in the email body. - Longer lines are truncated at that length with ' [...]' - appended. - - strict_utf8 (bool) - - If this field is set to True, then the email body text is - expected to be UTF-8. Any invalid characters are - converted to U+FFFD, the Unicode replacement character - (encoded as UTF-8, of course). - - """ - - def __init__(self, strict_utf8=True, - email_max_line_length=500, max_subject_length=500, - **kw): - super(FilterLinesEnvironmentMixin, self).__init__(**kw) - self.__strict_utf8 = strict_utf8 - self.__email_max_line_length = email_max_line_length - self.__max_subject_length = max_subject_length - - def filter_body(self, lines): - lines = super(FilterLinesEnvironmentMixin, self).filter_body(lines) - if self.__strict_utf8: - if not PYTHON3: - lines = (line.decode(ENCODING, 'replace') for line in lines) - # Limit the line length in Unicode-space to avoid - # splitting characters: - if self.__email_max_line_length > 0: - lines = limit_linelength(lines, self.__email_max_line_length) - if not PYTHON3: - lines = (line.encode(ENCODING, 'replace') for line in lines) - elif self.__email_max_line_length: - lines = limit_linelength(lines, self.__email_max_line_length) - - return lines - - def get_max_subject_length(self): - return self.__max_subject_length - - -class ConfigFilterLinesEnvironmentMixin( - ConfigEnvironmentMixin, - FilterLinesEnvironmentMixin, - ): - """Handle encoding and maximum line length based on config.""" - - def __init__(self, config, **kw): - strict_utf8 = config.get_bool('emailstrictutf8', default=None) - if strict_utf8 is not None: - kw['strict_utf8'] = strict_utf8 - - email_max_line_length = config.get('emailmaxlinelength') - if email_max_line_length is not None: - kw['email_max_line_length'] = int(email_max_line_length) - - max_subject_length = config.get('subjectMaxLength', default=email_max_line_length) - if max_subject_length is not None: - kw['max_subject_length'] = int(max_subject_length) - - super(ConfigFilterLinesEnvironmentMixin, self).__init__( - config=config, **kw - ) - - -class MaxlinesEnvironmentMixin(Environment): - """Limit the email body to a specified number of lines.""" - - def __init__(self, emailmaxlines, **kw): - super(MaxlinesEnvironmentMixin, self).__init__(**kw) - self.__emailmaxlines = emailmaxlines - - def filter_body(self, lines): - lines = super(MaxlinesEnvironmentMixin, self).filter_body(lines) - if self.__emailmaxlines > 0: - lines = limit_lines(lines, self.__emailmaxlines) - return lines - - -class ConfigMaxlinesEnvironmentMixin( - ConfigEnvironmentMixin, - MaxlinesEnvironmentMixin, - ): - """Limit the email body to the number of lines specified in config.""" - - def __init__(self, config, **kw): - emailmaxlines = int(config.get('emailmaxlines', default='0')) - super(ConfigMaxlinesEnvironmentMixin, self).__init__( - config=config, - emailmaxlines=emailmaxlines, - **kw - ) - - -class FQDNEnvironmentMixin(Environment): - """A mixin that sets the host's FQDN to its constructor argument.""" - - def __init__(self, fqdn, **kw): - super(FQDNEnvironmentMixin, self).__init__(**kw) - self.COMPUTED_KEYS += ['fqdn'] - self.__fqdn = fqdn - - def get_fqdn(self): - """Return the fully-qualified domain name for this host. - - Return None if it is unavailable or unwanted.""" - - return self.__fqdn - - -class ConfigFQDNEnvironmentMixin( - ConfigEnvironmentMixin, - FQDNEnvironmentMixin, - ): - """Read the FQDN from the config.""" - - def __init__(self, config, **kw): - fqdn = config.get('fqdn') - super(ConfigFQDNEnvironmentMixin, self).__init__( - config=config, - fqdn=fqdn, - **kw - ) - - -class ComputeFQDNEnvironmentMixin(FQDNEnvironmentMixin): - """Get the FQDN by calling socket.getfqdn().""" - - def __init__(self, **kw): - super(ComputeFQDNEnvironmentMixin, self).__init__( - fqdn=socket.getfqdn(), - **kw - ) - - -class PusherDomainEnvironmentMixin(ConfigEnvironmentMixin): - """Deduce pusher_email from pusher by appending an emaildomain.""" - - def __init__(self, **kw): - super(PusherDomainEnvironmentMixin, self).__init__(**kw) - self.__emaildomain = self.config.get('emaildomain') - - def get_pusher_email(self): - if self.__emaildomain: - # Derive the pusher's full email address in the default way: - return '%s@%s' % (self.get_pusher(), self.__emaildomain) - else: - return super(PusherDomainEnvironmentMixin, self).get_pusher_email() - - -class StaticRecipientsEnvironmentMixin(Environment): - """Set recipients statically based on constructor parameters.""" - - def __init__( - self, - refchange_recipients, announce_recipients, revision_recipients, scancommitforcc, - **kw - ): - super(StaticRecipientsEnvironmentMixin, self).__init__(**kw) - - # The recipients for various types of notification emails, as - # RFC 2822 email addresses separated by commas (or the empty - # string if no recipients are configured). Although there is - # a mechanism to choose the recipient lists based on on the - # actual *contents* of the change being reported, we only - # choose based on the *type* of the change. Therefore we can - # compute them once and for all: - self.__refchange_recipients = refchange_recipients - self.__announce_recipients = announce_recipients - self.__revision_recipients = revision_recipients - - def check(self): - if not (self.get_refchange_recipients(None) or - self.get_announce_recipients(None) or - self.get_revision_recipients(None) or - self.get_scancommitforcc()): - raise ConfigurationException('No email recipients configured!') - super(StaticRecipientsEnvironmentMixin, self).check() - - def get_refchange_recipients(self, refchange): - if self.__refchange_recipients is None: - return super(StaticRecipientsEnvironmentMixin, - self).get_refchange_recipients(refchange) - return self.__refchange_recipients - - def get_announce_recipients(self, annotated_tag_change): - if self.__announce_recipients is None: - return super(StaticRecipientsEnvironmentMixin, - self).get_refchange_recipients(annotated_tag_change) - return self.__announce_recipients - - def get_revision_recipients(self, revision): - if self.__revision_recipients is None: - return super(StaticRecipientsEnvironmentMixin, - self).get_refchange_recipients(revision) - return self.__revision_recipients - - -class CLIRecipientsEnvironmentMixin(Environment): - """Mixin storing recipients information coming from the - command-line.""" - - def __init__(self, cli_recipients=None, **kw): - super(CLIRecipientsEnvironmentMixin, self).__init__(**kw) - self.__cli_recipients = cli_recipients - - def get_refchange_recipients(self, refchange): - if self.__cli_recipients is None: - return super(CLIRecipientsEnvironmentMixin, - self).get_refchange_recipients(refchange) - return self.__cli_recipients - - def get_announce_recipients(self, annotated_tag_change): - if self.__cli_recipients is None: - return super(CLIRecipientsEnvironmentMixin, - self).get_announce_recipients(annotated_tag_change) - return self.__cli_recipients - - def get_revision_recipients(self, revision): - if self.__cli_recipients is None: - return super(CLIRecipientsEnvironmentMixin, - self).get_revision_recipients(revision) - return self.__cli_recipients - - -class ConfigRecipientsEnvironmentMixin( - ConfigEnvironmentMixin, - StaticRecipientsEnvironmentMixin - ): - """Determine recipients statically based on config.""" - - def __init__(self, config, **kw): - super(ConfigRecipientsEnvironmentMixin, self).__init__( - config=config, - refchange_recipients=self._get_recipients( - config, 'refchangelist', 'mailinglist', - ), - announce_recipients=self._get_recipients( - config, 'announcelist', 'refchangelist', 'mailinglist', - ), - revision_recipients=self._get_recipients( - config, 'commitlist', 'mailinglist', - ), - scancommitforcc=config.get('scancommitforcc'), - **kw - ) - - def _get_recipients(self, config, *names): - """Return the recipients for a particular type of message. - - Return the list of email addresses to which a particular type - of notification email should be sent, by looking at the config - value for "multimailhook.$name" for each of names. Use the - value from the first name that is configured. The return - value is a (possibly empty) string containing RFC 2822 email - addresses separated by commas. If no configuration could be - found, raise a ConfigurationException.""" - - for name in names: - lines = config.get_all(name) - if lines is not None: - lines = [line.strip() for line in lines] - # Single "none" is a special value equivalen to empty string. - if lines == ['none']: - lines = [''] - return ', '.join(lines) - else: - return '' - - -class StaticRefFilterEnvironmentMixin(Environment): - """Set branch filter statically based on constructor parameters.""" - - def __init__(self, ref_filter_incl_regex, ref_filter_excl_regex, - ref_filter_do_send_regex, ref_filter_dont_send_regex, - **kw): - super(StaticRefFilterEnvironmentMixin, self).__init__(**kw) - - if ref_filter_incl_regex and ref_filter_excl_regex: - raise ConfigurationException( - "Cannot specify both a ref inclusion and exclusion regex.") - self.__is_inclusion_filter = bool(ref_filter_incl_regex) - default_exclude = self.get_default_ref_ignore_regex() - if ref_filter_incl_regex: - ref_filter_regex = ref_filter_incl_regex - elif ref_filter_excl_regex: - ref_filter_regex = ref_filter_excl_regex + '|' + default_exclude - else: - ref_filter_regex = default_exclude - try: - self.__compiled_regex = re.compile(ref_filter_regex) - except Exception: - raise ConfigurationException( - 'Invalid Ref Filter Regex "%s": %s' % (ref_filter_regex, sys.exc_info()[1])) - - if ref_filter_do_send_regex and ref_filter_dont_send_regex: - raise ConfigurationException( - "Cannot specify both a ref doSend and dontSend regex.") - self.__is_do_send_filter = bool(ref_filter_do_send_regex) - if ref_filter_do_send_regex: - ref_filter_send_regex = ref_filter_do_send_regex - elif ref_filter_dont_send_regex: - ref_filter_send_regex = ref_filter_dont_send_regex - else: - ref_filter_send_regex = '.*' - self.__is_do_send_filter = True - try: - self.__send_compiled_regex = re.compile(ref_filter_send_regex) - except Exception: - raise ConfigurationException( - 'Invalid Ref Filter Regex "%s": %s' % - (ref_filter_send_regex, sys.exc_info()[1])) - - def get_ref_filter_regex(self, send_filter=False): - if send_filter: - return self.__send_compiled_regex, self.__is_do_send_filter - else: - return self.__compiled_regex, self.__is_inclusion_filter - - -class ConfigRefFilterEnvironmentMixin( - ConfigEnvironmentMixin, - StaticRefFilterEnvironmentMixin - ): - """Determine branch filtering statically based on config.""" - - def _get_regex(self, config, key): - """Get a list of whitespace-separated regex. The refFilter* config - variables are multivalued (hence the use of get_all), and we - allow each entry to be a whitespace-separated list (hence the - split on each line). The whole thing is glued into a single regex.""" - values = config.get_all(key) - if values is None: - return values - items = [] - for line in values: - for i in line.split(): - items.append(i) - if items == []: - return None - return '|'.join(items) - - def __init__(self, config, **kw): - super(ConfigRefFilterEnvironmentMixin, self).__init__( - config=config, - ref_filter_incl_regex=self._get_regex(config, 'refFilterInclusionRegex'), - ref_filter_excl_regex=self._get_regex(config, 'refFilterExclusionRegex'), - ref_filter_do_send_regex=self._get_regex(config, 'refFilterDoSendRegex'), - ref_filter_dont_send_regex=self._get_regex(config, 'refFilterDontSendRegex'), - **kw - ) - - -class ProjectdescEnvironmentMixin(Environment): - """Make a "projectdesc" value available for templates. - - By default, it is set to the first line of $GIT_DIR/description - (if that file is present and appears to be set meaningfully).""" - - def __init__(self, **kw): - super(ProjectdescEnvironmentMixin, self).__init__(**kw) - self.COMPUTED_KEYS += ['projectdesc'] - - def get_projectdesc(self): - """Return a one-line description of the project.""" - - git_dir = get_git_dir() - try: - projectdesc = open(os.path.join(git_dir, 'description')).readline().strip() - if projectdesc and not projectdesc.startswith('Unnamed repository'): - return projectdesc - except IOError: - pass - - return 'UNNAMED PROJECT' - - -class GenericEnvironmentMixin(Environment): - def get_pusher(self): - return self.osenv.get('USER', self.osenv.get('USERNAME', 'unknown user')) - - -class GitoliteEnvironmentHighPrecMixin(Environment): - def get_pusher(self): - return self.osenv.get('GL_USER', 'unknown user') - - -class GitoliteEnvironmentLowPrecMixin( - ConfigEnvironmentMixin, - Environment): - - def get_repo_shortname(self): - # The gitolite environment variable $GL_REPO is a pretty good - # repo_shortname (though it's probably not as good as a value - # the user might have explicitly put in his config). - return ( - self.osenv.get('GL_REPO', None) or - super(GitoliteEnvironmentLowPrecMixin, self).get_repo_shortname() - ) - - @staticmethod - def _compile_regex(re_template): - return ( - re.compile(re_template % x) - for x in ( - r'BEGIN\s+USER\s+EMAILS', - r'([^\s]+)\s+(.*)', - r'END\s+USER\s+EMAILS', - )) - - def get_fromaddr(self, change=None): - GL_USER = self.osenv.get('GL_USER') - if GL_USER is not None: - # Find the path to gitolite.conf. Note that gitolite v3 - # did away with the GL_ADMINDIR and GL_CONF environment - # variables (they are now hard-coded). - GL_ADMINDIR = self.osenv.get( - 'GL_ADMINDIR', - os.path.expanduser(os.path.join('~', '.gitolite'))) - GL_CONF = self.osenv.get( - 'GL_CONF', - os.path.join(GL_ADMINDIR, 'conf', 'gitolite.conf')) - - mailaddress_map = self.config.get('MailaddressMap') - # If relative, consider relative to GL_CONF: - if mailaddress_map: - mailaddress_map = os.path.join(os.path.dirname(GL_CONF), - mailaddress_map) - if os.path.isfile(mailaddress_map): - f = open(mailaddress_map, 'rU') - try: - # Leading '#' is optional - re_begin, re_user, re_end = self._compile_regex( - r'^(?:\s*#)?\s*%s\s*$') - for l in f: - l = l.rstrip('\n') - if re_begin.match(l) or re_end.match(l): - continue # Ignore these lines - m = re_user.match(l) - if m: - if m.group(1) == GL_USER: - return m.group(2) - else: - continue # Not this user, but not an error - raise ConfigurationException( - "Syntax error in mail address map.\n" - "Check file {}.\n" - "Line: {}".format(mailaddress_map, l)) - - finally: - f.close() - - if os.path.isfile(GL_CONF): - f = open(GL_CONF, 'rU') - try: - in_user_emails_section = False - re_begin, re_user, re_end = self._compile_regex( - r'^\s*#\s*%s\s*$') - for l in f: - l = l.rstrip('\n') - if not in_user_emails_section: - if re_begin.match(l): - in_user_emails_section = True - continue - if re_end.match(l): - break - m = re_user.match(l) - if m and m.group(1) == GL_USER: - return m.group(2) - finally: - f.close() - return super(GitoliteEnvironmentLowPrecMixin, self).get_fromaddr(change) - - -class IncrementalDateTime(object): - """Simple wrapper to give incremental date/times. - - Each call will result in a date/time a second later than the - previous call. This can be used to falsify email headers, to - increase the likelihood that email clients sort the emails - correctly.""" - - def __init__(self): - self.time = time.time() - self.next = self.__next__ # Python 2 backward compatibility - - def __next__(self): - formatted = formatdate(self.time, True) - self.time += 1 - return formatted - - -class StashEnvironmentHighPrecMixin(Environment): - def __init__(self, user=None, repo=None, **kw): - super(StashEnvironmentHighPrecMixin, - self).__init__(user=user, repo=repo, **kw) - self.__user = user - self.__repo = repo - - def get_pusher(self): - return re.match(r'(.*?)\s*<', self.__user).group(1) - - def get_pusher_email(self): - return self.__user - - -class StashEnvironmentLowPrecMixin(Environment): - def __init__(self, user=None, repo=None, **kw): - super(StashEnvironmentLowPrecMixin, self).__init__(**kw) - self.__repo = repo - self.__user = user - - def get_repo_shortname(self): - return self.__repo - - def get_fromaddr(self, change=None): - return self.__user - - -class GerritEnvironmentHighPrecMixin(Environment): - def __init__(self, project=None, submitter=None, update_method=None, **kw): - super(GerritEnvironmentHighPrecMixin, - self).__init__(submitter=submitter, project=project, **kw) - self.__project = project - self.__submitter = submitter - self.__update_method = update_method - "Make an 'update_method' value available for templates." - self.COMPUTED_KEYS += ['update_method'] - - def get_pusher(self): - if self.__submitter: - if self.__submitter.find('<') != -1: - # Submitter has a configured email, we transformed - # __submitter into an RFC 2822 string already. - return re.match(r'(.*?)\s*<', self.__submitter).group(1) - else: - # Submitter has no configured email, it's just his name. - return self.__submitter - else: - # If we arrive here, this means someone pushed "Submit" from - # the gerrit web UI for the CR (or used one of the programmatic - # APIs to do the same, such as gerrit review) and the - # merge/push was done by the Gerrit user. It was technically - # triggered by someone else, but sadly we have no way of - # determining who that someone else is at this point. - return 'Gerrit' # 'unknown user'? - - def get_pusher_email(self): - if self.__submitter: - return self.__submitter - else: - return super(GerritEnvironmentHighPrecMixin, self).get_pusher_email() - - def get_default_ref_ignore_regex(self): - default = super(GerritEnvironmentHighPrecMixin, self).get_default_ref_ignore_regex() - return default + '|^refs/changes/|^refs/cache-automerge/|^refs/meta/' - - def get_revision_recipients(self, revision): - # Merge commits created by Gerrit when users hit "Submit this patchset" - # in the Web UI (or do equivalently with REST APIs or the gerrit review - # command) are not something users want to see an individual email for. - # Filter them out. - committer = read_git_output(['log', '--no-walk', '--format=%cN', - revision.rev.sha1]) - if committer == 'Gerrit Code Review': - return [] - else: - return super(GerritEnvironmentHighPrecMixin, self).get_revision_recipients(revision) - - def get_update_method(self): - return self.__update_method - - -class GerritEnvironmentLowPrecMixin(Environment): - def __init__(self, project=None, submitter=None, **kw): - super(GerritEnvironmentLowPrecMixin, self).__init__(**kw) - self.__project = project - self.__submitter = submitter - - def get_repo_shortname(self): - return self.__project - - def get_fromaddr(self, change=None): - if self.__submitter and self.__submitter.find('<') != -1: - return self.__submitter - else: - return super(GerritEnvironmentLowPrecMixin, self).get_fromaddr(change) - - -class Push(object): - """Represent an entire push (i.e., a group of ReferenceChanges). - - It is easy to figure out what commits were added to a *branch* by - a Reference change: - - git rev-list change.old..change.new - - or removed from a *branch*: - - git rev-list change.new..change.old - - But it is not quite so trivial to determine which entirely new - commits were added to the *repository* by a push and which old - commits were discarded by a push. A big part of the job of this - class is to figure out these things, and to make sure that new - commits are only detailed once even if they were added to multiple - references. - - The first step is to determine the "other" references--those - unaffected by the current push. They are computed by listing all - references then removing any affected by this push. The results - are stored in Push._other_ref_sha1s. - - The commits contained in the repository before this push were - - git rev-list other1 other2 other3 ... change1.old change2.old ... - - Where "changeN.old" is the old value of one of the references - affected by this push. - - The commits contained in the repository after this push are - - git rev-list other1 other2 other3 ... change1.new change2.new ... - - The commits added by this push are the difference between these - two sets, which can be written - - git rev-list \ - ^other1 ^other2 ... \ - ^change1.old ^change2.old ... \ - change1.new change2.new ... - - The commits removed by this push can be computed by - - git rev-list \ - ^other1 ^other2 ... \ - ^change1.new ^change2.new ... \ - change1.old change2.old ... - - The last point is that it is possible that other pushes are - occurring simultaneously to this one, so reference values can - change at any time. It is impossible to eliminate all race - conditions, but we reduce the window of time during which problems - can occur by translating reference names to SHA1s as soon as - possible and working with SHA1s thereafter (because SHA1s are - immutable).""" - - # A map {(changeclass, changetype): integer} specifying the order - # that reference changes will be processed if multiple reference - # changes are included in a single push. The order is significant - # mostly because new commit notifications are threaded together - # with the first reference change that includes the commit. The - # following order thus causes commits to be grouped with branch - # changes (as opposed to tag changes) if possible. - SORT_ORDER = dict( - (value, i) for (i, value) in enumerate([ - (BranchChange, 'update'), - (BranchChange, 'create'), - (AnnotatedTagChange, 'update'), - (AnnotatedTagChange, 'create'), - (NonAnnotatedTagChange, 'update'), - (NonAnnotatedTagChange, 'create'), - (BranchChange, 'delete'), - (AnnotatedTagChange, 'delete'), - (NonAnnotatedTagChange, 'delete'), - (OtherReferenceChange, 'update'), - (OtherReferenceChange, 'create'), - (OtherReferenceChange, 'delete'), - ]) - ) - - def __init__(self, environment, changes, ignore_other_refs=False): - self.changes = sorted(changes, key=self._sort_key) - self.__other_ref_sha1s = None - self.__cached_commits_spec = {} - self.environment = environment - - if ignore_other_refs: - self.__other_ref_sha1s = set() - - @classmethod - def _sort_key(klass, change): - return (klass.SORT_ORDER[change.__class__, change.change_type], change.refname,) - - @property - def _other_ref_sha1s(self): - """The GitObjects referred to by references unaffected by this push. - """ - if self.__other_ref_sha1s is None: - # The refnames being changed by this push: - updated_refs = set( - change.refname - for change in self.changes - ) - - # The SHA-1s of commits referred to by all references in this - # repository *except* updated_refs: - sha1s = set() - fmt = ( - '%(objectname) %(objecttype) %(refname)\n' - '%(*objectname) %(*objecttype) %(refname)' - ) - ref_filter_regex, is_inclusion_filter = \ - self.environment.get_ref_filter_regex() - for line in read_git_lines( - ['for-each-ref', '--format=%s' % (fmt,)]): - (sha1, type, name) = line.split(' ', 2) - if (sha1 and type == 'commit' and - name not in updated_refs and - include_ref(name, ref_filter_regex, is_inclusion_filter)): - sha1s.add(sha1) - - self.__other_ref_sha1s = sha1s - - return self.__other_ref_sha1s - - def _get_commits_spec_incl(self, new_or_old, reference_change=None): - """Get new or old SHA-1 from one or each of the changed refs. - - Return a list of SHA-1 commit identifier strings suitable as - arguments to 'git rev-list' (or 'git log' or ...). The - returned identifiers are either the old or new values from one - or all of the changed references, depending on the values of - new_or_old and reference_change. - - new_or_old is either the string 'new' or the string 'old'. If - 'new', the returned SHA-1 identifiers are the new values from - each changed reference. If 'old', the SHA-1 identifiers are - the old values from each changed reference. - - If reference_change is specified and not None, only the new or - old reference from the specified reference is included in the - return value. - - This function returns None if there are no matching revisions - (e.g., because a branch was deleted and new_or_old is 'new'). - """ - - if not reference_change: - incl_spec = sorted( - getattr(change, new_or_old).sha1 - for change in self.changes - if getattr(change, new_or_old) - ) - if not incl_spec: - incl_spec = None - elif not getattr(reference_change, new_or_old).commit_sha1: - incl_spec = None - else: - incl_spec = [getattr(reference_change, new_or_old).commit_sha1] - return incl_spec - - def _get_commits_spec_excl(self, new_or_old): - """Get exclusion revisions for determining new or discarded commits. - - Return a list of strings suitable as arguments to 'git - rev-list' (or 'git log' or ...) that will exclude all - commits that, depending on the value of new_or_old, were - either previously in the repository (useful for determining - which commits are new to the repository) or currently in the - repository (useful for determining which commits were - discarded from the repository). - - new_or_old is either the string 'new' or the string 'old'. If - 'new', the commits to be excluded are those that were in the - repository before the push. If 'old', the commits to be - excluded are those that are currently in the repository. """ - - old_or_new = {'old': 'new', 'new': 'old'}[new_or_old] - excl_revs = self._other_ref_sha1s.union( - getattr(change, old_or_new).sha1 - for change in self.changes - if getattr(change, old_or_new).type in ['commit', 'tag'] - ) - return ['^' + sha1 for sha1 in sorted(excl_revs)] - - def get_commits_spec(self, new_or_old, reference_change=None): - """Get rev-list arguments for added or discarded commits. - - Return a list of strings suitable as arguments to 'git - rev-list' (or 'git log' or ...) that select those commits - that, depending on the value of new_or_old, are either new to - the repository or were discarded from the repository. - - new_or_old is either the string 'new' or the string 'old'. If - 'new', the returned list is used to select commits that are - new to the repository. If 'old', the returned value is used - to select the commits that have been discarded from the - repository. - - If reference_change is specified and not None, the new or - discarded commits are limited to those that are reachable from - the new or old value of the specified reference. - - This function returns None if there are no added (or discarded) - revisions. - """ - key = (new_or_old, reference_change) - if key not in self.__cached_commits_spec: - ret = self._get_commits_spec_incl(new_or_old, reference_change) - if ret is not None: - ret.extend(self._get_commits_spec_excl(new_or_old)) - self.__cached_commits_spec[key] = ret - return self.__cached_commits_spec[key] - - def get_new_commits(self, reference_change=None): - """Return a list of commits added by this push. - - Return a list of the object names of commits that were added - by the part of this push represented by reference_change. If - reference_change is None, then return a list of *all* commits - added by this push.""" - - spec = self.get_commits_spec('new', reference_change) - return git_rev_list(spec) - - def get_discarded_commits(self, reference_change): - """Return a list of commits discarded by this push. - - Return a list of the object names of commits that were - entirely discarded from the repository by the part of this - push represented by reference_change.""" - - spec = self.get_commits_spec('old', reference_change) - return git_rev_list(spec) - - def send_emails(self, mailer, body_filter=None): - """Use send all of the notification emails needed for this push. - - Use send all of the notification emails (including reference - change emails and commit emails) needed for this push. Send - the emails using mailer. If body_filter is not None, then use - it to filter the lines that are intended for the email - body.""" - - # The sha1s of commits that were introduced by this push. - # They will be removed from this set as they are processed, to - # guarantee that one (and only one) email is generated for - # each new commit. - unhandled_sha1s = set(self.get_new_commits()) - send_date = IncrementalDateTime() - for change in self.changes: - sha1s = [] - for sha1 in reversed(list(self.get_new_commits(change))): - if sha1 in unhandled_sha1s: - sha1s.append(sha1) - unhandled_sha1s.remove(sha1) - - # Check if we've got anyone to send to - if not change.recipients: - change.environment.log_warning( - '*** no recipients configured so no email will be sent\n' - '*** for %r update %s->%s' - % (change.refname, change.old.sha1, change.new.sha1,) - ) - else: - if not change.environment.quiet: - change.environment.log_msg( - 'Sending notification emails to: %s' % (change.recipients,)) - extra_values = {'send_date': next(send_date)} - - rev = change.send_single_combined_email(sha1s) - if rev: - mailer.send( - change.generate_combined_email(self, rev, body_filter, extra_values), - rev.recipients, - ) - # This change is now fully handled; no need to handle - # individual revisions any further. - continue - else: - mailer.send( - change.generate_email(self, body_filter, extra_values), - change.recipients, - ) - - max_emails = change.environment.maxcommitemails - if max_emails and len(sha1s) > max_emails: - change.environment.log_warning( - '*** Too many new commits (%d), not sending commit emails.\n' % len(sha1s) + - '*** Try setting multimailhook.maxCommitEmails to a greater value\n' + - '*** Currently, multimailhook.maxCommitEmails=%d' % max_emails - ) - return - - for (num, sha1) in enumerate(sha1s): - rev = Revision(change, GitObject(sha1), num=num + 1, tot=len(sha1s)) - if len(rev.parents) > 1 and change.environment.excludemergerevisions: - # skipping a merge commit - continue - if not rev.recipients and rev.cc_recipients: - change.environment.log_msg('*** Replacing Cc: with To:') - rev.recipients = rev.cc_recipients - rev.cc_recipients = None - if rev.recipients: - extra_values = {'send_date': next(send_date)} - mailer.send( - rev.generate_email(self, body_filter, extra_values), - rev.recipients, - ) - - # Consistency check: - if unhandled_sha1s: - change.environment.log_error( - 'ERROR: No emails were sent for the following new commits:\n' - ' %s' - % ('\n '.join(sorted(unhandled_sha1s)),) - ) - - -def include_ref(refname, ref_filter_regex, is_inclusion_filter): - does_match = bool(ref_filter_regex.search(refname)) - if is_inclusion_filter: - return does_match - else: # exclusion filter -- we include the ref if the regex doesn't match - return not does_match - - -def run_as_post_receive_hook(environment, mailer): - environment.check() - send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True) - ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False) - changes = [] - while True: - line = read_line(sys.stdin) - if line == '': - break - (oldrev, newrev, refname) = line.strip().split(' ', 2) - environment.get_logger().debug( - "run_as_post_receive_hook: oldrev=%s, newrev=%s, refname=%s" % - (oldrev, newrev, refname)) - - if not include_ref(refname, ref_filter_regex, is_inclusion_filter): - continue - if not include_ref(refname, send_filter_regex, send_is_inclusion_filter): - continue - changes.append( - ReferenceChange.create(environment, oldrev, newrev, refname) - ) - if not changes: - mailer.close() - return - push = Push(environment, changes) - try: - push.send_emails(mailer, body_filter=environment.filter_body) - finally: - mailer.close() - - -def run_as_update_hook(environment, mailer, refname, oldrev, newrev, force_send=False): - environment.check() - send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True) - ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False) - if not include_ref(refname, ref_filter_regex, is_inclusion_filter): - return - if not include_ref(refname, send_filter_regex, send_is_inclusion_filter): - return - changes = [ - ReferenceChange.create( - environment, - read_git_output(['rev-parse', '--verify', oldrev]), - read_git_output(['rev-parse', '--verify', newrev]), - refname, - ), - ] - if not changes: - mailer.close() - return - push = Push(environment, changes, force_send) - try: - push.send_emails(mailer, body_filter=environment.filter_body) - finally: - mailer.close() - - -def check_ref_filter(environment): - send_filter_regex, send_is_inclusion = environment.get_ref_filter_regex(True) - ref_filter_regex, ref_is_inclusion = environment.get_ref_filter_regex(False) - - def inc_exc_lusion(b): - if b: - return 'inclusion' - else: - return 'exclusion' - - if send_filter_regex: - sys.stdout.write("DoSend/DontSend filter regex (" + - (inc_exc_lusion(send_is_inclusion)) + - '): ' + send_filter_regex.pattern + - '\n') - if send_filter_regex: - sys.stdout.write("Include/Exclude filter regex (" + - (inc_exc_lusion(ref_is_inclusion)) + - '): ' + ref_filter_regex.pattern + - '\n') - sys.stdout.write(os.linesep) - - sys.stdout.write( - "Refs marked as EXCLUDE are excluded by either refFilterInclusionRegex\n" - "or refFilterExclusionRegex. No emails will be sent for commits included\n" - "in these refs.\n" - "Refs marked as DONT-SEND are excluded by either refFilterDoSendRegex or\n" - "refFilterDontSendRegex, but not by either refFilterInclusionRegex or\n" - "refFilterExclusionRegex. Emails will be sent for commits included in these\n" - "refs only when the commit reaches a ref which isn't excluded.\n" - "Refs marked as DO-SEND are not excluded by any filter. Emails will\n" - "be sent normally for commits included in these refs.\n") - - sys.stdout.write(os.linesep) - - for refname in read_git_lines(['for-each-ref', '--format', '%(refname)']): - sys.stdout.write(refname) - if not include_ref(refname, ref_filter_regex, ref_is_inclusion): - sys.stdout.write(' EXCLUDE') - elif not include_ref(refname, send_filter_regex, send_is_inclusion): - sys.stdout.write(' DONT-SEND') - else: - sys.stdout.write(' DO-SEND') - - sys.stdout.write(os.linesep) - - -def show_env(environment, out): - out.write('Environment values:\n') - for (k, v) in sorted(environment.get_values().items()): - if k: # Don't show the {'' : ''} pair. - out.write(' %s : %r\n' % (k, v)) - out.write('\n') - # Flush to avoid interleaving with further log output - out.flush() - - -def check_setup(environment): - environment.check() - show_env(environment, sys.stdout) - sys.stdout.write("Now, checking that git-multimail's standard input " - "is properly set ..." + os.linesep) - sys.stdout.write("Please type some text and then press Return" + os.linesep) - stdin = sys.stdin.readline() - sys.stdout.write("You have just entered:" + os.linesep) - sys.stdout.write(stdin) - sys.stdout.write("git-multimail seems properly set up." + os.linesep) - - -def choose_mailer(config, environment): - mailer = config.get('mailer', default='sendmail') - - if mailer == 'smtp': - smtpserver = config.get('smtpserver', default='localhost') - smtpservertimeout = float(config.get('smtpservertimeout', default=10.0)) - smtpserverdebuglevel = int(config.get('smtpserverdebuglevel', default=0)) - smtpencryption = config.get('smtpencryption', default='none') - smtpuser = config.get('smtpuser', default='') - smtppass = config.get('smtppass', default='') - smtpcacerts = config.get('smtpcacerts', default='') - mailer = SMTPMailer( - environment, - envelopesender=(environment.get_sender() or environment.get_fromaddr()), - smtpserver=smtpserver, smtpservertimeout=smtpservertimeout, - smtpserverdebuglevel=smtpserverdebuglevel, - smtpencryption=smtpencryption, - smtpuser=smtpuser, - smtppass=smtppass, - smtpcacerts=smtpcacerts - ) - elif mailer == 'sendmail': - command = config.get('sendmailcommand') - if command: - command = shlex.split(command) - mailer = SendMailer(environment, - command=command, envelopesender=environment.get_sender()) - else: - environment.log_error( - 'fatal: multimailhook.mailer is set to an incorrect value: "%s"\n' % mailer + - 'please use one of "smtp" or "sendmail".' - ) - sys.exit(1) - return mailer - - -KNOWN_ENVIRONMENTS = { - 'generic': {'highprec': GenericEnvironmentMixin}, - 'gitolite': {'highprec': GitoliteEnvironmentHighPrecMixin, - 'lowprec': GitoliteEnvironmentLowPrecMixin}, - 'stash': {'highprec': StashEnvironmentHighPrecMixin, - 'lowprec': StashEnvironmentLowPrecMixin}, - 'gerrit': {'highprec': GerritEnvironmentHighPrecMixin, - 'lowprec': GerritEnvironmentLowPrecMixin}, - } - - -def choose_environment(config, osenv=None, env=None, recipients=None, - hook_info=None): - env_name = choose_environment_name(config, env, osenv) - environment_klass = build_environment_klass(env_name) - env = build_environment(environment_klass, env_name, config, - osenv, recipients, hook_info) - return env - - -def choose_environment_name(config, env, osenv): - if not osenv: - osenv = os.environ - - if not env: - env = config.get('environment') - - if not env: - if 'GL_USER' in osenv and 'GL_REPO' in osenv: - env = 'gitolite' - else: - env = 'generic' - return env - - -COMMON_ENVIRONMENT_MIXINS = [ - ConfigRecipientsEnvironmentMixin, - CLIRecipientsEnvironmentMixin, - ConfigRefFilterEnvironmentMixin, - ProjectdescEnvironmentMixin, - ConfigMaxlinesEnvironmentMixin, - ComputeFQDNEnvironmentMixin, - ConfigFilterLinesEnvironmentMixin, - PusherDomainEnvironmentMixin, - ConfigOptionsEnvironmentMixin, - ] - - -def build_environment_klass(env_name): - if 'class' in KNOWN_ENVIRONMENTS[env_name]: - return KNOWN_ENVIRONMENTS[env_name]['class'] - - environment_mixins = [] - known_env = KNOWN_ENVIRONMENTS[env_name] - if 'highprec' in known_env: - high_prec_mixin = known_env['highprec'] - environment_mixins.append(high_prec_mixin) - environment_mixins = environment_mixins + COMMON_ENVIRONMENT_MIXINS - if 'lowprec' in known_env: - low_prec_mixin = known_env['lowprec'] - environment_mixins.append(low_prec_mixin) - environment_mixins.append(Environment) - klass_name = env_name.capitalize() + 'Environment' - environment_klass = type( - klass_name, - tuple(environment_mixins), - {}, - ) - KNOWN_ENVIRONMENTS[env_name]['class'] = environment_klass - return environment_klass - - -GerritEnvironment = build_environment_klass('gerrit') -StashEnvironment = build_environment_klass('stash') -GitoliteEnvironment = build_environment_klass('gitolite') -GenericEnvironment = build_environment_klass('generic') - - -def build_environment(environment_klass, env, config, - osenv, recipients, hook_info): - environment_kw = { - 'osenv': osenv, - 'config': config, - } - - if env == 'stash': - environment_kw['user'] = hook_info['stash_user'] - environment_kw['repo'] = hook_info['stash_repo'] - elif env == 'gerrit': - environment_kw['project'] = hook_info['project'] - environment_kw['submitter'] = hook_info['submitter'] - environment_kw['update_method'] = hook_info['update_method'] - - environment_kw['cli_recipients'] = recipients - - return environment_klass(**environment_kw) - - -def get_version(): - oldcwd = os.getcwd() - try: - try: - os.chdir(os.path.dirname(os.path.realpath(__file__))) - git_version = read_git_output(['describe', '--tags', 'HEAD']) - if git_version == __version__: - return git_version - else: - return '%s (%s)' % (__version__, git_version) - except: - pass - finally: - os.chdir(oldcwd) - return __version__ - - -def compute_gerrit_options(options, args, required_gerrit_options, - raw_refname): - if None in required_gerrit_options: - raise SystemExit("Error: Specify all of --oldrev, --newrev, --refname, " - "and --project; or none of them.") - - if options.environment not in (None, 'gerrit'): - raise SystemExit("Non-gerrit environments incompatible with --oldrev, " - "--newrev, --refname, and --project") - options.environment = 'gerrit' - - if args: - raise SystemExit("Error: Positional parameters not allowed with " - "--oldrev, --newrev, and --refname.") - - # Gerrit oddly omits 'refs/heads/' in the refname when calling - # ref-updated hook; put it back. - git_dir = get_git_dir() - if (not os.path.exists(os.path.join(git_dir, raw_refname)) and - os.path.exists(os.path.join(git_dir, 'refs', 'heads', - raw_refname))): - options.refname = 'refs/heads/' + options.refname - - # New revisions can appear in a gerrit repository either due to someone - # pushing directly (in which case options.submitter will be set), or they - # can press "Submit this patchset" in the web UI for some CR (in which - # case options.submitter will not be set and gerrit will not have provided - # us the information about who pressed the button). - # - # Note for the nit-picky: I'm lumping in REST API calls and the ssh - # gerrit review command in with "Submit this patchset" button, since they - # have the same effect. - if options.submitter: - update_method = 'pushed' - # The submitter argument is almost an RFC 2822 email address; change it - # from 'User Name (email@domain)' to 'User Name <email@domain>' so it is - options.submitter = options.submitter.replace('(', '<').replace(')', '>') - else: - update_method = 'submitted' - # Gerrit knew who submitted this patchset, but threw that information - # away when it invoked this hook. However, *IF* Gerrit created a - # merge to bring the patchset in (project 'Submit Type' is either - # "Always Merge", or is "Merge if Necessary" and happens to be - # necessary for this particular CR), then it will have the committer - # of that merge be 'Gerrit Code Review' and the author will be the - # person who requested the submission of the CR. Since this is fairly - # likely for most gerrit installations (of a reasonable size), it's - # worth the extra effort to try to determine the actual submitter. - rev_info = read_git_lines(['log', '--no-walk', '--merges', - '--format=%cN%n%aN <%aE>', options.newrev]) - if rev_info and rev_info[0] == 'Gerrit Code Review': - options.submitter = rev_info[1] - - # We pass back refname, oldrev, newrev as args because then the - # gerrit ref-updated hook is much like the git update hook - return (options, - [options.refname, options.oldrev, options.newrev], - {'project': options.project, 'submitter': options.submitter, - 'update_method': update_method}) - - -def check_hook_specific_args(options, args): - raw_refname = options.refname - # Convert each string option unicode for Python3. - if PYTHON3: - opts = ['environment', 'recipients', 'oldrev', 'newrev', 'refname', - 'project', 'submitter', 'stash_user', 'stash_repo'] - for opt in opts: - if not hasattr(options, opt): - continue - obj = getattr(options, opt) - if obj: - enc = obj.encode('utf-8', 'surrogateescape') - dec = enc.decode('utf-8', 'replace') - setattr(options, opt, dec) - - # First check for stash arguments - if (options.stash_user is None) != (options.stash_repo is None): - raise SystemExit("Error: Specify both of --stash-user and " - "--stash-repo or neither.") - if options.stash_user: - options.environment = 'stash' - return options, args, {'stash_user': options.stash_user, - 'stash_repo': options.stash_repo} - - # Finally, check for gerrit specific arguments - required_gerrit_options = (options.oldrev, options.newrev, options.refname, - options.project) - if required_gerrit_options != (None,) * 4: - return compute_gerrit_options(options, args, required_gerrit_options, - raw_refname) - - # No special options in use, just return what we started with - return options, args, {} - - -class Logger(object): - def parse_verbose(self, verbose): - if verbose > 0: - return logging.DEBUG - else: - return logging.INFO - - def create_log_file(self, environment, name, path, verbosity): - log_file = logging.getLogger(name) - file_handler = logging.FileHandler(path) - log_fmt = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") - file_handler.setFormatter(log_fmt) - log_file.addHandler(file_handler) - log_file.setLevel(verbosity) - return log_file - - def __init__(self, environment): - self.environment = environment - self.loggers = [] - stderr_log = logging.getLogger('git_multimail.stderr') - - class EncodedStderr(object): - def write(self, x): - write_str(sys.stderr, x) - - def flush(self): - sys.stderr.flush() - - stderr_handler = logging.StreamHandler(EncodedStderr()) - stderr_log.addHandler(stderr_handler) - stderr_log.setLevel(self.parse_verbose(environment.verbose)) - self.loggers.append(stderr_log) - - if environment.debug_log_file is not None: - debug_log_file = self.create_log_file( - environment, 'git_multimail.debug', environment.debug_log_file, logging.DEBUG) - self.loggers.append(debug_log_file) - - if environment.log_file is not None: - log_file = self.create_log_file( - environment, 'git_multimail.file', environment.log_file, logging.INFO) - self.loggers.append(log_file) - - if environment.error_log_file is not None: - error_log_file = self.create_log_file( - environment, 'git_multimail.error', environment.error_log_file, logging.ERROR) - self.loggers.append(error_log_file) - - def info(self, msg, *args, **kwargs): - for l in self.loggers: - l.info(msg, *args, **kwargs) - - def debug(self, msg, *args, **kwargs): - for l in self.loggers: - l.debug(msg, *args, **kwargs) - - def warning(self, msg, *args, **kwargs): - for l in self.loggers: - l.warning(msg, *args, **kwargs) - - def error(self, msg, *args, **kwargs): - for l in self.loggers: - l.error(msg, *args, **kwargs) - - -def main(args): - parser = optparse.OptionParser( - description=__doc__, - usage='%prog [OPTIONS]\n or: %prog [OPTIONS] REFNAME OLDREV NEWREV', - ) - - parser.add_option( - '--environment', '--env', action='store', type='choice', - choices=list(KNOWN_ENVIRONMENTS.keys()), default=None, - help=( - 'Choose type of environment is in use. Default is taken from ' - 'multimailhook.environment if set; otherwise "generic".' - ), - ) - parser.add_option( - '--stdout', action='store_true', default=False, - help='Output emails to stdout rather than sending them.', - ) - parser.add_option( - '--recipients', action='store', default=None, - help='Set list of email recipients for all types of emails.', - ) - parser.add_option( - '--show-env', action='store_true', default=False, - help=( - 'Write to stderr the values determined for the environment ' - '(intended for debugging purposes), then proceed normally.' - ), - ) - parser.add_option( - '--force-send', action='store_true', default=False, - help=( - 'Force sending refchange email when using as an update hook. ' - 'This is useful to work around the unreliable new commits ' - 'detection in this mode.' - ), - ) - parser.add_option( - '-c', metavar="<name>=<value>", action='append', - help=( - 'Pass a configuration parameter through to git. The value given ' - 'will override values from configuration files. See the -c option ' - 'of git(1) for more details. (Only works with git >= 1.7.3)' - ), - ) - parser.add_option( - '--version', '-v', action='store_true', default=False, - help=( - "Display git-multimail's version" - ), - ) - - parser.add_option( - '--python-version', action='store_true', default=False, - help=( - "Display the version of Python used by git-multimail" - ), - ) - - parser.add_option( - '--check-ref-filter', action='store_true', default=False, - help=( - 'List refs and show information on how git-multimail ' - 'will process them.' - ) - ) - - # The following options permit this script to be run as a gerrit - # ref-updated hook. See e.g. - # code.google.com/p/gerrit/source/browse/Documentation/config-hooks.txt - # We suppress help for these items, since these are specific to gerrit, - # and we don't want users directly using them any way other than how the - # gerrit ref-updated hook is called. - parser.add_option('--oldrev', action='store', help=optparse.SUPPRESS_HELP) - parser.add_option('--newrev', action='store', help=optparse.SUPPRESS_HELP) - parser.add_option('--refname', action='store', help=optparse.SUPPRESS_HELP) - parser.add_option('--project', action='store', help=optparse.SUPPRESS_HELP) - parser.add_option('--submitter', action='store', help=optparse.SUPPRESS_HELP) - - # The following allow this to be run as a stash asynchronous post-receive - # hook (almost identical to a git post-receive hook but triggered also for - # merges of pull requests from the UI). We suppress help for these items, - # since these are specific to stash. - parser.add_option('--stash-user', action='store', help=optparse.SUPPRESS_HELP) - parser.add_option('--stash-repo', action='store', help=optparse.SUPPRESS_HELP) - - (options, args) = parser.parse_args(args) - (options, args, hook_info) = check_hook_specific_args(options, args) - - if options.version: - sys.stdout.write('git-multimail version ' + get_version() + '\n') - return - - if options.python_version: - sys.stdout.write('Python version ' + sys.version + '\n') - return - - if options.c: - Config.add_config_parameters(options.c) - - config = Config('multimailhook') - - environment = None - try: - environment = choose_environment( - config, osenv=os.environ, - env=options.environment, - recipients=options.recipients, - hook_info=hook_info, - ) - - if options.show_env: - show_env(environment, sys.stderr) - - if options.stdout or environment.stdout: - mailer = OutputMailer(sys.stdout, environment) - else: - mailer = choose_mailer(config, environment) - - must_check_setup = os.environ.get('GIT_MULTIMAIL_CHECK_SETUP') - if must_check_setup == '': - must_check_setup = False - if options.check_ref_filter: - check_ref_filter(environment) - elif must_check_setup: - check_setup(environment) - # Dual mode: if arguments were specified on the command line, run - # like an update hook; otherwise, run as a post-receive hook. - elif args: - if len(args) != 3: - parser.error('Need zero or three non-option arguments') - (refname, oldrev, newrev) = args - environment.get_logger().debug( - "run_as_update_hook: refname=%s, oldrev=%s, newrev=%s, force_send=%s" % - (refname, oldrev, newrev, options.force_send)) - run_as_update_hook(environment, mailer, refname, oldrev, newrev, options.force_send) - else: - run_as_post_receive_hook(environment, mailer) - except ConfigurationException: - sys.exit(sys.exc_info()[1]) - except SystemExit: - raise - except Exception: - t, e, tb = sys.exc_info() - import traceback - sys.stderr.write('\n') # Avoid mixing message with previous output - msg = ( - 'Exception \'' + t.__name__ + - '\' raised. Please report this as a bug to\n' - 'https://github.com/git-multimail/git-multimail/issues\n' - 'with the information below:\n\n' - 'git-multimail version ' + get_version() + '\n' - 'Python version ' + sys.version + '\n' + - traceback.format_exc()) - try: - environment.get_logger().error(msg) - except: - sys.stderr.write(msg) - sys.exit(1) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/contrib/hooks/multimail/migrate-mailhook-config b/contrib/hooks/multimail/migrate-mailhook-config deleted file mode 100755 index 241ba22fa3..0000000000 --- a/contrib/hooks/multimail/migrate-mailhook-config +++ /dev/null @@ -1,274 +0,0 @@ -#! /usr/bin/env python - -"""Migrate a post-receive-email configuration to be usable with git_multimail.py. - -See README.migrate-from-post-receive-email for more information. - -""" - -import sys -import optparse - -from git_multimail import CommandError -from git_multimail import Config -from git_multimail import read_output - - -OLD_NAMES = [ - 'mailinglist', - 'announcelist', - 'envelopesender', - 'emailprefix', - 'showrev', - 'emailmaxlines', - 'diffopts', - 'scancommitforcc', - ] - -NEW_NAMES = [ - 'environment', - 'reponame', - 'mailinglist', - 'refchangelist', - 'commitlist', - 'announcelist', - 'announceshortlog', - 'envelopesender', - 'administrator', - 'emailprefix', - 'emailmaxlines', - 'diffopts', - 'emaildomain', - 'scancommitforcc', - ] - - -INFO = """\ - -SUCCESS! - -Your post-receive-email configuration has been converted to -git-multimail format. Please see README and -README.migrate-from-post-receive-email to learn about other -git-multimail configuration possibilities. - -For example, git-multimail has the following new options with no -equivalent in post-receive-email. You might want to read about them -to see if they would be useful in your situation: - -""" - - -def _check_old_config_exists(old): - """Check that at least one old configuration value is set.""" - - for name in OLD_NAMES: - if name in old: - return True - - return False - - -def _check_new_config_clear(new): - """Check that none of the new configuration names are set.""" - - retval = True - for name in NEW_NAMES: - if name in new: - if retval: - sys.stderr.write('INFO: The following configuration values already exist:\n\n') - sys.stderr.write(' "%s.%s"\n' % (new.section, name)) - retval = False - - return retval - - -def erase_values(config, names): - for name in names: - if name in config: - try: - sys.stderr.write('...unsetting "%s.%s"\n' % (config.section, name)) - config.unset_all(name) - except CommandError: - sys.stderr.write( - '\nWARNING: could not unset "%s.%s". ' - 'Perhaps it is not set at the --local level?\n\n' - % (config.section, name) - ) - - -def is_section_empty(section, local): - """Return True iff the specified configuration section is empty. - - Iff local is True, use the --local option when invoking 'git - config'.""" - - if local: - local_option = ['--local'] - else: - local_option = [] - - try: - read_output( - ['git', 'config'] + - local_option + - ['--get-regexp', '^%s\.' % (section,)] - ) - except CommandError: - t, e, traceback = sys.exc_info() - if e.retcode == 1: - # This means that no settings were found. - return True - else: - raise - else: - return False - - -def remove_section_if_empty(section): - """If the specified configuration section is empty, delete it.""" - - try: - empty = is_section_empty(section, local=True) - except CommandError: - # Older versions of git do not support the --local option, so - # if the first attempt fails, try without --local. - try: - empty = is_section_empty(section, local=False) - except CommandError: - sys.stderr.write( - '\nINFO: If configuration section "%s.*" is empty, you might want ' - 'to delete it.\n\n' - % (section,) - ) - return - - if empty: - sys.stderr.write('...removing section "%s.*"\n' % (section,)) - read_output(['git', 'config', '--remove-section', section]) - else: - sys.stderr.write( - '\nINFO: Configuration section "%s.*" still has contents. ' - 'It will not be deleted.\n\n' - % (section,) - ) - - -def migrate_config(strict=False, retain=False, overwrite=False): - old = Config('hooks') - new = Config('multimailhook') - if not _check_old_config_exists(old): - sys.exit( - 'Your repository has no post-receive-email configuration. ' - 'Nothing to do.' - ) - if not _check_new_config_clear(new): - if overwrite: - sys.stderr.write('\nWARNING: Erasing the above values...\n\n') - erase_values(new, NEW_NAMES) - else: - sys.exit( - '\nERROR: Refusing to overwrite existing values. Use the --overwrite\n' - 'option to continue anyway.' - ) - - name = 'showrev' - if name in old: - msg = 'git-multimail does not support "%s.%s"' % (old.section, name,) - if strict: - sys.exit( - 'ERROR: %s.\n' - 'Please unset that value then try again, or run without --strict.' - % (msg,) - ) - else: - sys.stderr.write('\nWARNING: %s (ignoring).\n\n' % (msg,)) - - for name in ['mailinglist', 'announcelist']: - if name in old: - sys.stderr.write( - '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name) - ) - old_recipients = old.get_all(name, default=None) - old_recipients = ', '.join(o.strip() for o in old_recipients) - new.set_recipients(name, old_recipients) - - if strict: - sys.stderr.write( - '...setting "%s.commitlist" to the empty string\n' % (new.section,) - ) - new.set_recipients('commitlist', '') - sys.stderr.write( - '...setting "%s.announceshortlog" to "true"\n' % (new.section,) - ) - new.set('announceshortlog', 'true') - - for name in ['envelopesender', 'emailmaxlines', 'diffopts', 'scancommitforcc']: - if name in old: - sys.stderr.write( - '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name) - ) - new.set(name, old.get(name)) - - name = 'emailprefix' - if name in old: - sys.stderr.write( - '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name) - ) - new.set(name, old.get(name)) - elif strict: - sys.stderr.write( - '...setting "%s.%s" to "[SCM]" to preserve old subject lines\n' - % (new.section, name) - ) - new.set(name, '[SCM]') - - if not retain: - erase_values(old, OLD_NAMES) - remove_section_if_empty(old.section) - - sys.stderr.write(INFO) - for name in NEW_NAMES: - if name not in OLD_NAMES: - sys.stderr.write(' "%s.%s"\n' % (new.section, name,)) - sys.stderr.write('\n') - - -def main(args): - parser = optparse.OptionParser( - description=__doc__, - usage='%prog [OPTIONS]', - ) - - parser.add_option( - '--strict', action='store_true', default=False, - help=( - 'Slavishly configure git-multimail as closely as possible to ' - 'the post-receive-email configuration. Default is to turn ' - 'on some new features that have no equivalent in post-receive-email.' - ), - ) - parser.add_option( - '--retain', action='store_true', default=False, - help=( - 'Retain the post-receive-email configuration values. ' - 'Default is to delete them after the new values are set.' - ), - ) - parser.add_option( - '--overwrite', action='store_true', default=False, - help=( - 'Overwrite any existing git-multimail configuration settings. ' - 'Default is to abort if such settings already exist.' - ), - ) - - (options, args) = parser.parse_args(args) - - if args: - parser.error('Unexpected arguments: %s' % (' '.join(args),)) - - migrate_config(strict=options.strict, retain=options.retain, overwrite=options.overwrite) - - -main(sys.argv[1:]) diff --git a/contrib/hooks/multimail/post-receive.example b/contrib/hooks/multimail/post-receive.example deleted file mode 100755 index 0f98c5a23d..0000000000 --- a/contrib/hooks/multimail/post-receive.example +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python - -"""Example post-receive hook based on git-multimail. - -The simplest way to use git-multimail is to use the script -git_multimail.py directly as a post-receive hook, and to configure it -using Git's configuration files and command-line parameters. You can -also write your own Python wrapper for more advanced configurability, -using git_multimail.py as a Python module. - -This script is a simple example of such a post-receive hook. It is -intended to be customized before use; see the comments in the script -to help you get started. - -Using git-multimail as a Python module as done here provides more -flexibility. It has the following advantages: - -* The tool's behavior can be customized using arbitrary Python code, - without having to edit git_multimail.py. - -* Configuration settings can be read from other sources; for example, - user names and email addresses could be read from LDAP or from a - database. Or the settings can even be hardcoded in the importing - Python script, if this is preferred. - -This script is a very basic example of how to use git_multimail.py as -a module. The comments below explain some of the points at which the -script's behavior could be changed or customized. - -""" - -import sys - -# If necessary, add the path to the directory containing -# git_multimail.py to the Python path as follows. (This is not -# necessary if git_multimail.py is in the same directory as this -# script): - -#LIBDIR = 'path/to/directory/containing/module' -#sys.path.insert(0, LIBDIR) - -import git_multimail - -# It is possible to modify the output templates here; e.g.: - -#git_multimail.FOOTER_TEMPLATE = """\ -# -#-- \n\ -#This email was generated by the wonderful git-multimail tool. -#""" - - -# Specify which "git config" section contains the configuration for -# git-multimail: -config = git_multimail.Config('multimailhook') - -# Set some Git configuration variables. Equivalent to passing var=val -# to "git -c var=val" each time git is called, or to adding the -# configuration in .git/config (must come before instantiating the -# environment) : -#git_multimail.Config.add_config_parameters('multimailhook.commitEmailFormat=html') -#git_multimail.Config.add_config_parameters(('user.name=foo', 'user.email=foo@example.com')) - -# Select the type of environment: -try: - environment = git_multimail.GenericEnvironment(config=config) - #environment = git_multimail.GitoliteEnvironment(config=config) -except git_multimail.ConfigurationException: - sys.stderr.write('*** %s\n' % sys.exc_info()[1]) - sys.exit(1) - - -# Choose the method of sending emails based on the git config: -mailer = git_multimail.choose_mailer(config, environment) - -# Alternatively, you may hardcode the mailer using code like one of -# the following: - -# Use "/usr/sbin/sendmail -oi -t" to send emails. The envelopesender -# argument is optional: -#mailer = git_multimail.SendMailer( -# command=['/usr/sbin/sendmail', '-oi', '-t'], -# envelopesender='git-repo@example.com', -# ) - -# Use Python's smtplib to send emails. Both arguments are required. -#mailer = git_multimail.SMTPMailer( -# environment=environment, -# envelopesender='git-repo@example.com', -# # The smtpserver argument can also include a port number; e.g., -# # smtpserver='mail.example.com:25' -# smtpserver='mail.example.com', -# ) - -# OutputMailer is intended only for testing; it writes the emails to -# the specified file stream. -#mailer = git_multimail.OutputMailer(sys.stdout) - - -# Read changes from stdin and send notification emails: -git_multimail.run_as_post_receive_hook(environment, mailer) diff --git a/contrib/subtree/git-subtree.sh b/contrib/subtree/git-subtree.sh index b06782bc79..7f767b5c38 100755 --- a/contrib/subtree/git-subtree.sh +++ b/contrib/subtree/git-subtree.sh @@ -5,8 +5,12 @@ # Copyright (C) 2009 Avery Pennarun <apenwarr@gmail.com> # -if test -z "$GIT_EXEC_PATH" || test "${PATH#"${GIT_EXEC_PATH}:"}" = "$PATH" || ! test -f "$GIT_EXEC_PATH/git-sh-setup" +if test -z "$GIT_EXEC_PATH" || ! test -f "$GIT_EXEC_PATH/git-sh-setup" || { + test "${PATH#"${GIT_EXEC_PATH}:"}" = "$PATH" && + test ! "$GIT_EXEC_PATH" -ef "${PATH%%:*}" 2>/dev/null +} then + basename=${0##*[/\\]} echo >&2 'It looks like either your git installation or your' echo >&2 'git-subtree installation is broken.' echo >&2 @@ -14,10 +18,10 @@ then echo >&2 " - If \`git --exec-path\` does not print the correct path to" echo >&2 " your git install directory, then set the GIT_EXEC_PATH" echo >&2 " environment variable to the correct directory." - echo >&2 " - Make sure that your \`${0##*/}\` file is either in your" + echo >&2 " - Make sure that your \`$basename\` file is either in your" echo >&2 " PATH or in your git exec path (\`$(git --exec-path)\`)." - echo >&2 " - You should run git-subtree as \`git ${0##*/git-}\`," - echo >&2 " not as \`${0##*/}\`." >&2 + echo >&2 " - You should run git-subtree as \`git ${basename#git-}\`," + echo >&2 " not as \`$basename\`." >&2 exit 126 fi diff --git a/csum-file.c b/csum-file.c index 7510950fa3..3487d28ed7 100644 --- a/csum-file.c +++ b/csum-file.c @@ -11,35 +11,33 @@ #include "progress.h" #include "csum-file.h" +static void verify_buffer_or_die(struct hashfile *f, + const void *buf, + unsigned int count) +{ + ssize_t ret = read_in_full(f->check_fd, f->check_buffer, count); + + if (ret < 0) + die_errno("%s: sha1 file read error", f->name); + if (ret != count) + die("%s: sha1 file truncated", f->name); + if (memcmp(buf, f->check_buffer, count)) + die("sha1 file '%s' validation error", f->name); +} + static void flush(struct hashfile *f, const void *buf, unsigned int count) { - if (0 <= f->check_fd && count) { - unsigned char check_buffer[8192]; - ssize_t ret = read_in_full(f->check_fd, check_buffer, count); - - if (ret < 0) - die_errno("%s: sha1 file read error", f->name); - if (ret != count) - die("%s: sha1 file truncated", f->name); - if (memcmp(buf, check_buffer, count)) - die("sha1 file '%s' validation error", f->name); - } + if (0 <= f->check_fd && count) + verify_buffer_or_die(f, buf, count); - for (;;) { - int ret = xwrite(f->fd, buf, count); - if (ret > 0) { - f->total += ret; - display_throughput(f->tp, f->total); - buf = (char *) buf + ret; - count -= ret; - if (count) - continue; - return; - } - if (!ret) + if (write_in_full(f->fd, buf, count) < 0) { + if (errno == ENOSPC) die("sha1 file '%s' write error. Out of diskspace", f->name); die_errno("sha1 file '%s' write error", f->name); } + + f->total += count; + display_throughput(f->tp, f->total); } void hashflush(struct hashfile *f) @@ -53,6 +51,13 @@ void hashflush(struct hashfile *f) } } +static void free_hashfile(struct hashfile *f) +{ + free(f->buffer); + free(f->check_buffer); + free(f); +} + int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int flags) { int fd; @@ -82,20 +87,20 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int fl if (close(f->check_fd)) die_errno("%s: sha1 file error on close", f->name); } - free(f); + free_hashfile(f); return fd; } void hashwrite(struct hashfile *f, const void *buf, unsigned int count) { while (count) { - unsigned left = sizeof(f->buffer) - f->offset; + unsigned left = f->buffer_len - f->offset; unsigned nr = count > left ? left : count; if (f->do_crc) f->crc32 = crc32(f->crc32, buf, nr); - if (nr == sizeof(f->buffer)) { + if (nr == f->buffer_len) { /* * Flush a full batch worth of data directly * from the input, skipping the memcpy() to @@ -121,11 +126,6 @@ void hashwrite(struct hashfile *f, const void *buf, unsigned int count) } } -struct hashfile *hashfd(int fd, const char *name) -{ - return hashfd_throughput(fd, name, NULL); -} - struct hashfile *hashfd_check(const char *name) { int sink, check; @@ -139,10 +139,14 @@ struct hashfile *hashfd_check(const char *name) die_errno("unable to open '%s'", name); f = hashfd(sink, name); f->check_fd = check; + f->check_buffer = xmalloc(f->buffer_len); + return f; } -struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp) +static struct hashfile *hashfd_internal(int fd, const char *name, + struct progress *tp, + size_t buffer_len) { struct hashfile *f = xmalloc(sizeof(*f)); f->fd = fd; @@ -153,9 +157,35 @@ struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp f->name = name; f->do_crc = 0; the_hash_algo->init_fn(&f->ctx); + + f->buffer_len = buffer_len; + f->buffer = xmalloc(buffer_len); + f->check_buffer = NULL; + return f; } +struct hashfile *hashfd(int fd, const char *name) +{ + /* + * Since we are not going to use a progress meter to + * measure the rate of data passing through this hashfile, + * use a larger buffer size to reduce fsync() calls. + */ + return hashfd_internal(fd, name, NULL, 128 * 1024); +} + +struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp) +{ + /* + * Since we are expecting to report progress of the + * write into this hashfile, use a smaller buffer + * size so the progress indicators arrive at a more + * frequent rate. + */ + return hashfd_internal(fd, name, tp, 8 * 1024); +} + void hashfile_checkpoint(struct hashfile *f, struct hashfile_checkpoint *checkpoint) { hashflush(f); diff --git a/csum-file.h b/csum-file.h index e54d53d1d0..3044bd19ab 100644 --- a/csum-file.h +++ b/csum-file.h @@ -16,7 +16,9 @@ struct hashfile { const char *name; int do_crc; uint32_t crc32; - unsigned char buffer[8192]; + size_t buffer_len; + unsigned char *buffer; + unsigned char *check_buffer; }; /* Checkpoint */ diff --git a/diff-merges.c b/diff-merges.c index f3a9daed7e..0dfcaa1b11 100644 --- a/diff-merges.c +++ b/diff-merges.c @@ -6,6 +6,7 @@ typedef void (*diff_merges_setup_func_t)(struct rev_info *); static void set_separate(struct rev_info *revs); static diff_merges_setup_func_t set_to_default = set_separate; +static int suppress_parsing; static void suppress(struct rev_info *revs) { @@ -14,7 +15,7 @@ static void suppress(struct rev_info *revs) revs->combine_merges = 0; revs->dense_combined_merges = 0; revs->combined_all_paths = 0; - revs->combined_imply_patch = 0; + revs->merges_imply_patch = 0; revs->merges_need_diff = 0; } @@ -30,17 +31,6 @@ static void set_first_parent(struct rev_info *revs) revs->first_parent_merges = 1; } -static void set_m(struct rev_info *revs) -{ - /* - * To "diff-index", "-m" means "match missing", and to the "log" - * family of commands, it means "show default diff for merges". Set - * both fields appropriately. - */ - set_to_default(revs); - revs->match_missing = 1; -} - static void set_combined(struct rev_info *revs) { suppress(revs); @@ -101,20 +91,29 @@ int diff_merges_config(const char *value) return 0; } +void diff_merges_suppress_options_parsing(void) +{ + suppress_parsing = 1; +} + int diff_merges_parse_opts(struct rev_info *revs, const char **argv) { int argcount = 1; const char *optarg; const char *arg = argv[0]; + if (suppress_parsing) + return 0; + if (!strcmp(arg, "-m")) { - set_m(revs); + set_to_default(revs); + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "-c")) { set_combined(revs); - revs->combined_imply_patch = 1; + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "--cc")) { set_dense_combined(revs); - revs->combined_imply_patch = 1; + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "--no-diff-merges")) { suppress(revs); } else if (!strcmp(arg, "--combined-all-paths")) { @@ -155,15 +154,18 @@ void diff_merges_set_dense_combined_if_unset(struct rev_info *revs) void diff_merges_setup_revs(struct rev_info *revs) { + if (suppress_parsing) + return; + if (revs->combine_merges == 0) revs->dense_combined_merges = 0; if (revs->separate_merges == 0) revs->first_parent_merges = 0; if (revs->combined_all_paths && !revs->combine_merges) die("--combined-all-paths makes no sense without -c or --cc"); - if (revs->combined_imply_patch) + if (revs->merges_imply_patch) revs->diff = 1; - if (revs->combined_imply_patch || revs->merges_need_diff) { + if (revs->merges_imply_patch || revs->merges_need_diff) { if (!revs->diffopt.output_format) revs->diffopt.output_format = DIFF_FORMAT_PATCH; } diff --git a/diff-merges.h b/diff-merges.h index 09d9a6c9a4..b5d57f6563 100644 --- a/diff-merges.h +++ b/diff-merges.h @@ -11,6 +11,8 @@ struct rev_info; int diff_merges_config(const char *value); +void diff_merges_suppress_options_parsing(void); + int diff_merges_parse_opts(struct rev_info *revs, const char **argv); void diff_merges_suppress(struct rev_info *revs); diff --git a/diffcore-rename.c b/diffcore-rename.c index 963ca58221..3375e24659 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -568,7 +568,8 @@ static void update_dir_rename_counts(struct dir_rename_info *info, static void initialize_dir_rename_info(struct dir_rename_info *info, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count) + struct strmap *dir_rename_count, + struct strmap *cached_pairs) { struct hashmap_iter iter; struct strmap_entry *entry; @@ -633,6 +634,17 @@ static void initialize_dir_rename_info(struct dir_rename_info *info, rename_dst[i].p->two->path); } + /* Add cached_pairs to counts */ + strmap_for_each_entry(cached_pairs, &iter, entry) { + const char *old_name = entry->key; + const char *new_name = entry->value; + if (!new_name) + /* known delete; ignore it */ + continue; + + update_dir_rename_counts(info, dirs_removed, old_name, new_name); + } + /* * Now we collapse * dir_rename_count: old_directory -> {new_directory -> count} @@ -1247,7 +1259,8 @@ static void handle_early_known_dir_renames(struct dir_rename_info *info, void diffcore_rename_extended(struct diff_options *options, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count) + struct strmap *dir_rename_count, + struct strmap *cached_pairs) { int detect_rename = options->detect_rename; int minimum_score = options->rename_score; @@ -1363,7 +1376,8 @@ void diffcore_rename_extended(struct diff_options *options, /* Preparation for basename-driven matching. */ trace2_region_enter("diff", "dir rename setup", options->repo); initialize_dir_rename_info(&info, relevant_sources, - dirs_removed, dir_rename_count); + dirs_removed, dir_rename_count, + cached_pairs); trace2_region_leave("diff", "dir rename setup", options->repo); /* Utilize file basenames to quickly find renames. */ @@ -1560,5 +1574,5 @@ void diffcore_rename_extended(struct diff_options *options, void diffcore_rename(struct diff_options *options) { - diffcore_rename_extended(options, NULL, NULL, NULL); + diffcore_rename_extended(options, NULL, NULL, NULL, NULL); } diff --git a/diffcore.h b/diffcore.h index f5c6de4841..533b30e21e 100644 --- a/diffcore.h +++ b/diffcore.h @@ -181,7 +181,8 @@ void diffcore_rename(struct diff_options *); void diffcore_rename_extended(struct diff_options *options, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count); + struct strmap *dir_rename_count, + struct strmap *cached_pairs); void diffcore_merge_broken(void); void diffcore_pickaxe(struct diff_options *); void diffcore_order(const char *orderfile); diff --git a/fetch-pack.c b/fetch-pack.c index c135635e34..b0c7be717c 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1645,6 +1645,15 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (process_section_header(&reader, "packfile-uris", 1)) receive_packfile_uris(&reader, &packfile_uris); process_section_header(&reader, "packfile", 0); + + /* + * this is the final request we'll make of the server; + * do a half-duplex shutdown to indicate that they can + * hang up as soon as the pack is sent. + */ + close(fd[1]); + fd[1] = -1; + if (get_pack(args, fd, pack_lockfiles, packfile_uris.nr ? &index_pack_args : NULL, sought, nr_sought, &fsck_options.gitmodules_found)) diff --git a/git-compat-util.h b/git-compat-util.h index a508dbe5a3..dca72cba29 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -986,11 +986,9 @@ static inline char *xstrdup_or_null(const char *str) static inline size_t xsize_t(off_t len) { - size_t size = (size_t) len; - - if (len != (off_t) size) + if (len < 0 || (uintmax_t) len > SIZE_MAX) die("Cannot handle files this big"); - return size; + return (size_t) len; } __attribute__((format (printf, 3, 4))) @@ -1368,7 +1366,7 @@ static inline void *container_of_or_null_offset(void *ptr, size_t offset) (type *)container_of_or_null_offset(ptr, offsetof(type, member)) /* - * like offsetof(), but takes a pointer to a a variable of type which + * like offsetof(), but takes a pointer to a variable of type which * contains @member, instead of a specified type. * @ptr is subject to multiple evaluation since we can't rely on __typeof__ * everywhere. diff --git a/git-cvsserver.perl b/git-cvsserver.perl index f6f3fc192c..ed035f32c2 100755 --- a/git-cvsserver.perl +++ b/git-cvsserver.perl @@ -2149,7 +2149,7 @@ sub req_diff ( $meta2->{revision} or "workingcopy" )); # TODO: Use --label instead of -L because -L is no longer - # documented and may go away someday. Not sure if there there are + # documented and may go away someday. Not sure if there are # versions that only support -L, which would make this change risky? # http://osdir.com/ml/bug-gnu-utils-gnu/2010-12/msg00060.html # ("man diff" should actually document the best migration strategy, diff --git a/git-send-email.perl b/git-send-email.perl index 25be2ebd2a..7ba0b3433d 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -70,6 +70,7 @@ git send-email --dump-aliases Sending: --envelope-sender <str> * Email envelope sender. + --sendmail-cmd <str> * Command to run to send email. --smtp-server <str:int> * Outgoing SMTP server to use. The port is optional. Default 'localhost'. --smtp-server-option <str> * Outgoing SMTP server option to use. @@ -262,6 +263,7 @@ my ($confirm); my (@suppress_cc); my ($auto_8bit_encoding); my ($compose_encoding); +my ($sendmail_cmd); # Variables with corresponding config settings & hardcoded defaults my ($debug_net_smtp) = 0; # Net::SMTP, see send_message() my $thread = 1; @@ -309,6 +311,7 @@ my %config_settings = ( "assume8bitencoding" => \$auto_8bit_encoding, "composeencoding" => \$compose_encoding, "transferencoding" => \$target_xfer_encoding, + "sendmailcmd" => \$sendmail_cmd, ); my %config_path_settings = ( @@ -442,6 +445,7 @@ $rc = GetOptions( "no-bcc" => \$no_bcc, "chain-reply-to!" => \$chain_reply_to, "no-chain-reply-to" => sub {$chain_reply_to = 0}, + "sendmail-cmd=s" => \$sendmail_cmd, "smtp-server=s" => \$smtp_server, "smtp-server-option=s" => \@smtp_server_options, "smtp-server-port=s" => \$smtp_server_port, @@ -1013,16 +1017,19 @@ if (defined $reply_to) { $reply_to = sanitize_address($reply_to); } -if (!defined $smtp_server) { +if (!defined $sendmail_cmd && !defined $smtp_server) { my @sendmail_paths = qw( /usr/sbin/sendmail /usr/lib/sendmail ); push @sendmail_paths, map {"$_/sendmail"} split /:/, $ENV{PATH}; foreach (@sendmail_paths) { if (-x $_) { - $smtp_server = $_; + $sendmail_cmd = $_; last; } } - $smtp_server ||= 'localhost'; # could be 127.0.0.1, too... *shrug* + + if (!defined $sendmail_cmd) { + $smtp_server = 'localhost'; # could be 127.0.0.1, too... *shrug* + } } if ($compose && $compose > 0) { @@ -1502,11 +1509,17 @@ EOF if ($dry_run) { # We don't want to send the email. - } elsif (file_name_is_absolute($smtp_server)) { + } elsif (defined $sendmail_cmd || file_name_is_absolute($smtp_server)) { my $pid = open my $sm, '|-'; defined $pid or die $!; if (!$pid) { - exec($smtp_server, @sendmail_parameters) or die $!; + if (defined $sendmail_cmd) { + exec ("sh", "-c", "$sendmail_cmd \"\$@\"", "-", @sendmail_parameters) + or die $!; + } else { + exec ($smtp_server, @sendmail_parameters) + or die $!; + } } print $sm "$header\n$message"; close $sm or die $!; @@ -1602,14 +1615,21 @@ EOF printf($dry_run ? __("Dry-Sent %s\n") : __("Sent %s\n"), $subject); } else { print($dry_run ? __("Dry-OK. Log says:\n") : __("OK. Log says:\n")); - if (!file_name_is_absolute($smtp_server)) { + if (!defined $sendmail_cmd && !file_name_is_absolute($smtp_server)) { print "Server: $smtp_server\n"; print "MAIL FROM:<$raw_from>\n"; foreach my $entry (@recipients) { print "RCPT TO:<$entry>\n"; } } else { - print "Sendmail: $smtp_server ".join(' ',@sendmail_parameters)."\n"; + my $sm; + if (defined $sendmail_cmd) { + $sm = $sendmail_cmd; + } else { + $sm = $smtp_server; + } + + print "Sendmail: $sm ".join(' ',@sendmail_parameters)."\n"; } print $header, "\n"; if ($smtp) { @@ -95,7 +95,7 @@ static void parse_graph_colors_config(struct strvec *colors, const char *string) if (!color_parse_mem(start, comma - start, color)) strvec_push(colors, color); else - warning(_("ignore invalid color '%.*s' in log.graphColors"), + warning(_("ignored invalid color '%.*s' in log.graphColors"), (int)(comma - start), start); start = comma + 1; } @@ -263,6 +263,22 @@ static inline void oidcpy(struct object_id *dst, const struct object_id *src) dst->algo = src->algo; } +/* Like oidcpy() but zero-pads the unused bytes in dst's hash array. */ +static inline void oidcpy_with_padding(struct object_id *dst, + struct object_id *src) +{ + size_t hashsz; + + if (!src->algo) + hashsz = the_hash_algo->rawsz; + else + hashsz = hash_algos[src->algo].rawsz; + + memcpy(dst->hash, src->hash, hashsz); + memset(dst->hash + hashsz, 0, GIT_MAX_RAWSZ - hashsz); + dst->algo = src->algo; +} + static inline struct object_id *oiddup(const struct object_id *src) { struct object_id *dst = xmalloc(sizeof(struct object_id)); diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 96a605c8ad..fd8d59f653 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -102,7 +102,7 @@ static int gently_parse_list_objects_filter( } else if (skip_prefix(arg, "object:type=", &v0)) { int type = type_from_string_gently(v0, strlen(v0), 1); if (type < 0) { - strbuf_addf(errbuf, _("'%s' for 'object:type=<type>' is" + strbuf_addf(errbuf, _("'%s' for 'object:type=<type>' is " "not a valid object type"), v0); return 1; } diff --git a/mailinfo.c b/mailinfo.c index ccc6beb27e..184ed8d581 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -19,7 +19,7 @@ static void cleanup_space(struct strbuf *sb) static void get_sane_name(struct strbuf *out, struct strbuf *name, struct strbuf *email) { struct strbuf *src = name; - if (name->len < 3 || 60 < name->len || strpbrk(name->buf, "@<>")) + if (!name->len || 60 < name->len || strpbrk(name->buf, "@<>")) src = email; else if (name == out) return; diff --git a/merge-ort.c b/merge-ort.c index 4a9ce2a822..b954f7184a 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -53,6 +53,8 @@ enum merge_side { MERGE_SIDE2 = 2 }; +static unsigned RESULT_INITIALIZED = 0x1abe11ed; /* unlikely accidental value */ + struct traversal_callback_data { unsigned long mask; unsigned long dirmask; @@ -141,6 +143,72 @@ struct rename_info { char *callback_data_traverse_path; /* + * merge_trees: trees passed to the merge algorithm for the merge + * + * merge_trees records the trees passed to the merge algorithm. But, + * this data also is stored in merge_result->priv. If a sequence of + * merges are being done (such as when cherry-picking or rebasing), + * the next merge can look at this and re-use information from + * previous merges under certain circumstances. + * + * See also all the cached_* variables. + */ + struct tree *merge_trees[3]; + + /* + * cached_pairs_valid_side: which side's cached info can be reused + * + * See the description for merge_trees. For repeated merges, at most + * only one side's cached information can be used. Valid values: + * MERGE_SIDE2: cached data from side2 can be reused + * MERGE_SIDE1: cached data from side1 can be reused + * 0: no cached data can be reused + */ + int cached_pairs_valid_side; + + /* + * cached_pairs: Caching of renames and deletions. + * + * These are mappings recording renames and deletions of individual + * files (not directories). They are thus a map from an old + * filename to either NULL (for deletions) or a new filename (for + * renames). + */ + struct strmap cached_pairs[3]; + + /* + * cached_target_names: just the destinations from cached_pairs + * + * We sometimes want a fast lookup to determine if a given filename + * is one of the destinations in cached_pairs. cached_target_names + * is thus duplicative information, but it provides a fast lookup. + */ + struct strset cached_target_names[3]; + + /* + * cached_irrelevant: Caching of rename_sources that aren't relevant. + * + * If we try to detect a rename for a source path and succeed, it's + * part of a rename. If we try to detect a rename for a source path + * and fail, then it's a delete. If we do not try to detect a rename + * for a path, then we don't know if it's a rename or a delete. If + * merge-ort doesn't think the path is relevant, then we just won't + * cache anything for that path. But there's a slight problem in + * that merge-ort can think a path is RELEVANT_LOCATION, but due to + * commit 9bd342137e ("diffcore-rename: determine which + * relevant_sources are no longer relevant", 2021-03-13), + * diffcore-rename can downgrade the path to RELEVANT_NO_MORE. To + * avoid excessive calls to diffcore_rename_extended() we still need + * to cache such paths, though we cannot record them as either + * renames or deletes. So we cache them here as a "turned out to be + * irrelevant *for this commit*" as they are often also irrelevant + * for subsequent commits, though we will have to do some extra + * checking to see whether such paths become relevant for rename + * detection when cherry-picking/rebasing subsequent commits. + */ + struct strset cached_irrelevant[3]; + + /* * needed_limit: value needed for inexact rename detection to run * * If the current rename limit wasn't high enough for inexact @@ -382,6 +450,8 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, reinitialize ? strmap_partial_clear : strmap_clear; void (*strintmap_func)(struct strintmap *) = reinitialize ? strintmap_partial_clear : strintmap_clear; + void (*strset_func)(struct strset *) = + reinitialize ? strset_partial_clear : strset_clear; /* * We marked opti->paths with strdup_strings = 0, so that we @@ -417,15 +487,21 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, /* Free memory used by various renames maps */ for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) { strintmap_func(&renames->dirs_removed[i]); - - partial_clear_dir_rename_count(&renames->dir_rename_count[i]); - if (!reinitialize) - strmap_clear(&renames->dir_rename_count[i], 1); - strmap_func(&renames->dir_renames[i], 0); - strintmap_func(&renames->relevant_sources[i]); + if (!reinitialize) + assert(renames->cached_pairs_valid_side == 0); + if (i != renames->cached_pairs_valid_side) { + strset_func(&renames->cached_target_names[i]); + strmap_func(&renames->cached_pairs[i], 1); + strset_func(&renames->cached_irrelevant[i]); + partial_clear_dir_rename_count(&renames->dir_rename_count[i]); + if (!reinitialize) + strmap_clear(&renames->dir_rename_count[i], 1); + } } + renames->cached_pairs_valid_side = 0; + renames->dir_rename_mask = 0; if (!reinitialize) { struct hashmap_iter iter; @@ -448,8 +524,6 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_clear(&opti->output, 0); } - renames->dir_rename_mask = 0; - /* Clean out callback_data as well. */ FREE_AND_NULL(renames->callback_data); renames->callback_data_nr = renames->callback_data_alloc = 0; @@ -690,15 +764,48 @@ static void add_pair(struct merge_options *opt, struct rename_info *renames = &opt->priv->renames; int names_idx = is_add ? side : 0; - if (!is_add) { + if (is_add) { + if (strset_contains(&renames->cached_target_names[side], + pathname)) + return; + } else { unsigned content_relevant = (match_mask == 0); unsigned location_relevant = (dir_rename_mask == 0x07); + /* + * If pathname is found in cached_irrelevant[side] due to + * previous pick but for this commit content is relevant, + * then we need to remove it from cached_irrelevant. + */ + if (content_relevant) + /* strset_remove is no-op if strset doesn't have key */ + strset_remove(&renames->cached_irrelevant[side], + pathname); + + /* + * We do not need to re-detect renames for paths that we already + * know the pairing, i.e. for cached_pairs (or + * cached_irrelevant). However, handle_deferred_entries() needs + * to loop over the union of keys from relevant_sources[side] and + * cached_pairs[side], so for simplicity we set relevant_sources + * for all the cached_pairs too and then strip them back out in + * prune_cached_from_relevant() at the beginning of + * detect_regular_renames(). + */ if (content_relevant || location_relevant) { /* content_relevant trumps location_relevant */ strintmap_set(&renames->relevant_sources[side], pathname, content_relevant ? RELEVANT_CONTENT : RELEVANT_LOCATION); } + + /* + * Avoid creating pair if we've already cached rename results. + * Note that we do this after setting relevant_sources[side] + * as noted in the comment above. + */ + if (strmap_contains(&renames->cached_pairs[side], pathname) || + strset_contains(&renames->cached_irrelevant[side], pathname)) + return; } one = alloc_filespec(pathname); @@ -2037,6 +2144,9 @@ static int process_renames(struct merge_options *opt, VERIFY_CI(side2); if (!strcmp(pathnames[1], pathnames[2])) { + struct rename_info *ri = &opt->priv->renames; + int j; + /* Both sides renamed the same way */ assert(side1 == side2); memcpy(&side1->stages[0], &base->stages[0], @@ -2046,6 +2156,16 @@ static int process_renames(struct merge_options *opt, base->merged.is_null = 1; base->merged.clean = 1; + /* + * Disable remembering renames optimization; + * rename/rename(1to1) is incredibly rare, and + * just disabling the optimization is easier + * than purging cached_pairs, + * cached_target_names, and dir_rename_counts. + */ + for (j = 0; j < 3; j++) + ri->merge_trees[j] = NULL; + /* We handled both renames, i.e. i+1 handled */ i++; /* Move to next rename */ @@ -2273,7 +2393,9 @@ static inline int possible_side_renames(struct rename_info *renames, static inline int possible_renames(struct rename_info *renames) { return possible_side_renames(renames, 1) || - possible_side_renames(renames, 2); + possible_side_renames(renames, 2) || + !strmap_empty(&renames->cached_pairs[1]) || + !strmap_empty(&renames->cached_pairs[2]); } static void resolve_diffpair_statuses(struct diff_queue_struct *q) @@ -2297,6 +2419,112 @@ static void resolve_diffpair_statuses(struct diff_queue_struct *q) } } +static void prune_cached_from_relevant(struct rename_info *renames, + unsigned side) +{ + /* Reason for this function described in add_pair() */ + struct hashmap_iter iter; + struct strmap_entry *entry; + + /* Remove from relevant_sources all entries in cached_pairs[side] */ + strmap_for_each_entry(&renames->cached_pairs[side], &iter, entry) { + strintmap_remove(&renames->relevant_sources[side], + entry->key); + } + /* Remove from relevant_sources all entries in cached_irrelevant[side] */ + strset_for_each_entry(&renames->cached_irrelevant[side], &iter, entry) { + strintmap_remove(&renames->relevant_sources[side], + entry->key); + } +} + +static void use_cached_pairs(struct merge_options *opt, + struct strmap *cached_pairs, + struct diff_queue_struct *pairs) +{ + struct hashmap_iter iter; + struct strmap_entry *entry; + + /* + * Add to side_pairs all entries from renames->cached_pairs[side_index]. + * (Info in cached_irrelevant[side_index] is not relevant here.) + */ + strmap_for_each_entry(cached_pairs, &iter, entry) { + struct diff_filespec *one, *two; + const char *old_name = entry->key; + const char *new_name = entry->value; + if (!new_name) + new_name = old_name; + + /* We don't care about oid/mode, only filenames and status */ + one = alloc_filespec(old_name); + two = alloc_filespec(new_name); + diff_queue(pairs, one, two); + pairs->queue[pairs->nr-1]->status = entry->value ? 'R' : 'D'; + } +} + +static void cache_new_pair(struct rename_info *renames, + int side, + char *old_path, + char *new_path, + int free_old_value) +{ + char *old_value; + new_path = xstrdup(new_path); + old_value = strmap_put(&renames->cached_pairs[side], + old_path, new_path); + strset_add(&renames->cached_target_names[side], new_path); + if (free_old_value) + free(old_value); + else + assert(!old_value); +} + +static void possibly_cache_new_pair(struct rename_info *renames, + struct diff_filepair *p, + unsigned side, + char *new_path) +{ + int dir_renamed_side = 0; + + if (new_path) { + /* + * Directory renames happen on the other side of history from + * the side that adds new files to the old directory. + */ + dir_renamed_side = 3 - side; + } else { + int val = strintmap_get(&renames->relevant_sources[side], + p->one->path); + if (val == RELEVANT_NO_MORE) { + assert(p->status == 'D'); + strset_add(&renames->cached_irrelevant[side], + p->one->path); + } + if (val <= 0) + return; + } + + if (p->status == 'D') { + /* + * If we already had this delete, we'll just set it's value + * to NULL again, so no harm. + */ + strmap_put(&renames->cached_pairs[side], p->one->path, NULL); + } else if (p->status == 'R') { + if (!new_path) + new_path = p->two->path; + else + cache_new_pair(renames, dir_renamed_side, + p->two->path, new_path, 0); + cache_new_pair(renames, side, p->one->path, new_path, 1); + } else if (p->status == 'A' && new_path) { + cache_new_pair(renames, dir_renamed_side, + p->two->path, new_path, 0); + } +} + static int compare_pairs(const void *a_, const void *b_) { const struct diff_filepair *a = *((const struct diff_filepair **)a_); @@ -2312,6 +2540,7 @@ static void detect_regular_renames(struct merge_options *opt, struct diff_options diff_opts; struct rename_info *renames = &opt->priv->renames; + prune_cached_from_relevant(renames, side_index); if (!possible_side_renames(renames, side_index)) { /* * No rename detection needed for this side, but we still need @@ -2322,6 +2551,7 @@ static void detect_regular_renames(struct merge_options *opt, return; } + partial_clear_dir_rename_count(&renames->dir_rename_count[side_index]); repo_diff_setup(opt->repo, &diff_opts); diff_opts.flags.recursive = 1; diff_opts.flags.rename_empty = 0; @@ -2339,7 +2569,8 @@ static void detect_regular_renames(struct merge_options *opt, diffcore_rename_extended(&diff_opts, &renames->relevant_sources[side_index], &renames->dirs_removed[side_index], - &renames->dir_rename_count[side_index]); + &renames->dir_rename_count[side_index], + &renames->cached_pairs[side_index]); trace2_region_leave("diff", "diffcore_rename", opt->repo); resolve_diffpair_statuses(&diff_queued_diff); @@ -2379,6 +2610,7 @@ static int collect_renames(struct merge_options *opt, char *new_path; /* non-NULL only with directory renames */ if (p->status != 'A' && p->status != 'R') { + possibly_cache_new_pair(renames, p, side_index, NULL); diff_free_filepair(p); continue; } @@ -2390,6 +2622,7 @@ static int collect_renames(struct merge_options *opt, &collisions, &clean); + possibly_cache_new_pair(renames, p, side_index, new_path); if (p->status != 'R' && !new_path) { diff_free_filepair(p); continue; @@ -2445,6 +2678,8 @@ static int detect_and_process_renames(struct merge_options *opt, trace2_region_enter("merge", "regular renames", opt->repo); detect_regular_renames(opt, MERGE_SIDE1); detect_regular_renames(opt, MERGE_SIDE2); + use_cached_pairs(opt, &renames->cached_pairs[1], &renames->pairs[1]); + use_cached_pairs(opt, &renames->cached_pairs[2], &renames->pairs[2]); trace2_region_leave("merge", "regular renames", opt->repo); trace2_region_enter("merge", "directory renames", opt->repo); @@ -3635,6 +3870,10 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) assert(opt->obuf.len == 0); assert(opt->priv == NULL); + if (result->_properly_initialized != 0 && + result->_properly_initialized != RESULT_INITIALIZED) + BUG("struct merge_result passed to merge_incore_*recursive() must be zeroed or filled with values from a previous run"); + assert(!!result->priv == !!result->_properly_initialized); if (result->priv) { opt->priv = result->priv; result->priv = NULL; @@ -3674,8 +3913,22 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) NULL, 1); strmap_init_with_options(&renames->dir_renames[i], NULL, 0); + /* + * relevant_sources uses -1 for the default, because we need + * to be able to distinguish not-in-strintmap from valid + * relevant_source values from enum file_rename_relevance. + * In particular, possibly_cache_new_pair() expects a negative + * value for not-found entries. + */ strintmap_init_with_options(&renames->relevant_sources[i], - 0, NULL, 0); + -1 /* explicitly invalid */, + NULL, 0); + strmap_init_with_options(&renames->cached_pairs[i], + NULL, 1); + strset_init_with_options(&renames->cached_irrelevant[i], + NULL, 1); + strset_init_with_options(&renames->cached_target_names[i], + NULL, 0); } /* @@ -3701,6 +3954,50 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) trace2_region_leave("merge", "allocate/init", opt->repo); } +static void merge_check_renames_reusable(struct merge_options *opt, + struct merge_result *result, + struct tree *merge_base, + struct tree *side1, + struct tree *side2) +{ + struct rename_info *renames; + struct tree **merge_trees; + struct merge_options_internal *opti = result->priv; + + if (!opti) + return; + + renames = &opti->renames; + merge_trees = renames->merge_trees; + + /* + * Handle case where previous merge operation did not want cache to + * take effect, e.g. because rename/rename(1to1) makes it invalid. + */ + if (!merge_trees[0]) { + assert(!merge_trees[0] && !merge_trees[1] && !merge_trees[2]); + renames->cached_pairs_valid_side = 0; /* neither side valid */ + return; + } + + /* + * Handle other cases; note that merge_trees[0..2] will only + * be NULL if opti is, or if all three were manually set to + * NULL by e.g. rename/rename(1to1) handling. + */ + assert(merge_trees[0] && merge_trees[1] && merge_trees[2]); + + /* Check if we meet a condition for re-using cached_pairs */ + if (oideq(&merge_base->object.oid, &merge_trees[2]->object.oid) && + oideq(&side1->object.oid, &result->tree->object.oid)) + renames->cached_pairs_valid_side = MERGE_SIDE1; + else if (oideq(&merge_base->object.oid, &merge_trees[1]->object.oid) && + oideq(&side2->object.oid, &result->tree->object.oid)) + renames->cached_pairs_valid_side = MERGE_SIDE2; + else + renames->cached_pairs_valid_side = 0; /* neither side valid */ +} + /*** Function Grouping: merge_incore_*() and their internal variants ***/ /* @@ -3751,6 +4048,7 @@ static void merge_ort_nonrecursive_internal(struct merge_options *opt, result->clean &= strmap_empty(&opt->priv->conflicted); if (!opt->priv->call_depth) { result->priv = opt->priv; + result->_properly_initialized = RESULT_INITIALIZED; opt->priv = NULL; } } @@ -3848,7 +4146,16 @@ void merge_incore_nonrecursive(struct merge_options *opt, trace2_region_enter("merge", "merge_start", opt->repo); assert(opt->ancestor != NULL); + merge_check_renames_reusable(opt, result, merge_base, side1, side2); merge_start(opt, result); + /* + * Record the trees used in this merge, so if there's a next merge in + * a cherry-pick or rebase sequence it might be able to take advantage + * of the cached_pairs in that next merge. + */ + opt->priv->renames.merge_trees[0] = merge_base; + opt->priv->renames.merge_trees[1] = side1; + opt->priv->renames.merge_trees[2] = side2; trace2_region_leave("merge", "merge_start", opt->repo); merge_ort_nonrecursive_internal(opt, merge_base, side1, side2, result); diff --git a/merge-ort.h b/merge-ort.h index d53a0a339f..c011864ffe 100644 --- a/merge-ort.h +++ b/merge-ort.h @@ -29,6 +29,8 @@ struct merge_result { * !clean) and to print "CONFLICT" messages. Not for external use. */ void *priv; + /* Also private */ + unsigned _properly_initialized; }; /* diff --git a/merge-recursive.c b/merge-recursive.c index d146bb116f..4327e0cfa3 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -2152,7 +2152,7 @@ static char *handle_path_level_conflicts(struct merge_options *opt, * implicit renaming of files that should be left in place. (See * testcase 6b in t6043 for details.) * 2. Prune directory renames if there are still files left in the - * the original directory. These represent a partial directory rename, + * original directory. These represent a partial directory rename, * i.e. a rename where only some of the files within the directory * were renamed elsewhere. (Technically, this could be done earlier * in get_directory_renames(), except that would prevent us from diff --git a/parallel-checkout.c b/parallel-checkout.c index 6b1af32bb3..ddc0ff3c06 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -411,7 +411,7 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) len_data = sizeof(struct pc_item_fixed_portion) + name_len + working_tree_encoding_len; - data = xcalloc(1, len_data); + data = xmalloc(len_data); fixed_portion = (struct pc_item_fixed_portion *)data; fixed_portion->id = pc_item->id; @@ -421,13 +421,12 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) fixed_portion->name_len = name_len; fixed_portion->working_tree_encoding_len = working_tree_encoding_len; /* - * We use hashcpy() instead of oidcpy() because the hash[] positions - * after `the_hash_algo->rawsz` might not be initialized. And Valgrind - * would complain about passing uninitialized bytes to a syscall - * (write(2)). There is no real harm in this case, but the warning could - * hinder the detection of actual errors. + * We pad the unused bytes in the hash array because, otherwise, + * Valgrind would complain about passing uninitialized bytes to a + * write() syscall. The warning doesn't represent any real risk here, + * but it could hinder the detection of actual errors. */ - hashcpy(fixed_portion->oid.hash, pc_item->ce->oid.hash); + oidcpy_with_padding(&fixed_portion->oid, &pc_item->ce->oid); variant = data + sizeof(*fixed_portion); if (working_tree_encoding_len) { diff --git a/perl/Git/SVN.pm b/perl/Git/SVN.pm index f6f1dc03c6..35ff5a6896 100644 --- a/perl/Git/SVN.pm +++ b/perl/Git/SVN.pm @@ -1636,7 +1636,7 @@ sub has_no_changes { my $commit = shift; my @revs = split / /, command_oneline( - qw(rev-list --parents -1 -m), $commit); + qw(rev-list --parents -1), $commit); # Commits with no parents, e.g. the start of a partial branch, # have changes by definition. diff --git a/read-cache.c b/read-cache.c index 1b3c2eb408..77961a3885 100644 --- a/read-cache.c +++ b/read-cache.c @@ -26,6 +26,7 @@ #include "thread-utils.h" #include "progress.h" #include "sparse-index.h" +#include "csum-file.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -2521,80 +2522,23 @@ int repo_index_has_changes(struct repository *repo, } } -#define WRITE_BUFFER_SIZE (128 * 1024) -static unsigned char write_buffer[WRITE_BUFFER_SIZE]; -static unsigned long write_buffer_len; - -static int ce_write_flush(git_hash_ctx *context, int fd) +static int write_index_ext_header(struct hashfile *f, + git_hash_ctx *eoie_f, + unsigned int ext, + unsigned int sz) { - unsigned int buffered = write_buffer_len; - if (buffered) { - the_hash_algo->update_fn(context, write_buffer, buffered); - if (write_in_full(fd, write_buffer, buffered) < 0) - return -1; - write_buffer_len = 0; - } - return 0; -} + hashwrite_be32(f, ext); + hashwrite_be32(f, sz); -static int ce_write(git_hash_ctx *context, int fd, void *data, unsigned int len) -{ - while (len) { - unsigned int buffered = write_buffer_len; - unsigned int partial = WRITE_BUFFER_SIZE - buffered; - if (partial > len) - partial = len; - memcpy(write_buffer + buffered, data, partial); - buffered += partial; - if (buffered == WRITE_BUFFER_SIZE) { - write_buffer_len = buffered; - if (ce_write_flush(context, fd)) - return -1; - buffered = 0; - } - write_buffer_len = buffered; - len -= partial; - data = (char *) data + partial; + if (eoie_f) { + ext = htonl(ext); + sz = htonl(sz); + the_hash_algo->update_fn(eoie_f, &ext, sizeof(ext)); + the_hash_algo->update_fn(eoie_f, &sz, sizeof(sz)); } return 0; } -static int write_index_ext_header(git_hash_ctx *context, git_hash_ctx *eoie_context, - int fd, unsigned int ext, unsigned int sz) -{ - ext = htonl(ext); - sz = htonl(sz); - if (eoie_context) { - the_hash_algo->update_fn(eoie_context, &ext, 4); - the_hash_algo->update_fn(eoie_context, &sz, 4); - } - return ((ce_write(context, fd, &ext, 4) < 0) || - (ce_write(context, fd, &sz, 4) < 0)) ? -1 : 0; -} - -static int ce_flush(git_hash_ctx *context, int fd, unsigned char *hash) -{ - unsigned int left = write_buffer_len; - - if (left) { - write_buffer_len = 0; - the_hash_algo->update_fn(context, write_buffer, left); - } - - /* Flush first if not enough space for hash signature */ - if (left + the_hash_algo->rawsz > WRITE_BUFFER_SIZE) { - if (write_in_full(fd, write_buffer, left) < 0) - return -1; - left = 0; - } - - /* Append the hash signature at the end */ - the_hash_algo->final_fn(write_buffer + left, context); - hashcpy(hash, write_buffer + left); - left += the_hash_algo->rawsz; - return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0; -} - static void ce_smudge_racily_clean_entry(struct index_state *istate, struct cache_entry *ce) { @@ -2673,11 +2617,10 @@ static void copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk, } } -static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, +static int ce_write_entry(struct hashfile *f, struct cache_entry *ce, struct strbuf *previous_name, struct ondisk_cache_entry *ondisk) { int size; - int result; unsigned int saved_namelen; int stripped_name = 0; static unsigned char padding[8] = { 0x00 }; @@ -2693,11 +2636,9 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, if (!previous_name) { int len = ce_namelen(ce); copy_cache_entry_to_ondisk(ondisk, ce); - result = ce_write(c, fd, ondisk, size); - if (!result) - result = ce_write(c, fd, ce->name, len); - if (!result) - result = ce_write(c, fd, padding, align_padding_size(size, len)); + hashwrite(f, ondisk, size); + hashwrite(f, ce->name, len); + hashwrite(f, padding, align_padding_size(size, len)); } else { int common, to_remove, prefix_size; unsigned char to_remove_vi[16]; @@ -2711,13 +2652,10 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, prefix_size = encode_varint(to_remove, to_remove_vi); copy_cache_entry_to_ondisk(ondisk, ce); - result = ce_write(c, fd, ondisk, size); - if (!result) - result = ce_write(c, fd, to_remove_vi, prefix_size); - if (!result) - result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common); - if (!result) - result = ce_write(c, fd, padding, 1); + hashwrite(f, ondisk, size); + hashwrite(f, to_remove_vi, prefix_size); + hashwrite(f, ce->name + common, ce_namelen(ce) - common); + hashwrite(f, padding, 1); strbuf_splice(previous_name, common, to_remove, ce->name + common, ce_namelen(ce) - common); @@ -2727,7 +2665,7 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, ce->ce_flags &= ~CE_STRIP_NAME; } - return result; + return 0; } /* @@ -2839,8 +2777,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, int strip_extensions) { uint64_t start = getnanotime(); - int newfd = tempfile->fd; - git_hash_ctx c, eoie_c; + struct hashfile *f; + git_hash_ctx *eoie_c = NULL; struct cache_header hdr; int i, err = 0, removed, extended, hdr_version; struct cache_entry **cache = istate->cache; @@ -2854,6 +2792,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct index_entry_offset_table *ieot = NULL; int nr, nr_threads; + f = hashfd(tempfile->fd, tempfile->filename.buf); + for (i = removed = extended = 0; i < entries; i++) { if (cache[i]->ce_flags & CE_REMOVE) removed++; @@ -2882,9 +2822,7 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, hdr.hdr_version = htonl(hdr_version); hdr.hdr_entries = htonl(entries - removed); - the_hash_algo->init_fn(&c); - if (ce_write(&c, newfd, &hdr, sizeof(hdr)) < 0) - return -1; + hashwrite(f, &hdr, sizeof(hdr)); if (!HAVE_THREADS || git_config_get_index_threads(&nr_threads)) nr_threads = 1; @@ -2919,12 +2857,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, } } - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; - } - offset += write_buffer_len; + offset = hashfile_total(f); + nr = 0; previous_name = (hdr_version == 4) ? &previous_name_buf : NULL; @@ -2959,14 +2893,10 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, if (previous_name) previous_name->buf[0] = 0; nr = 0; - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; - } - offset += write_buffer_len; + + offset = hashfile_total(f); } - if (ce_write_entry(&c, newfd, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0) + if (ce_write_entry(f, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0) err = -1; if (err) @@ -2985,14 +2915,16 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, return err; } - /* Write extension data here */ - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; + offset = hashfile_total(f); + + /* + * The extension headers must be hashed on their own for the + * EOIE extension. Create a hashfile here to compute that hash. + */ + if (offset && record_eoie()) { + CALLOC_ARRAY(eoie_c, 1); + the_hash_algo->init_fn(eoie_c); } - offset += write_buffer_len; - the_hash_algo->init_fn(&eoie_c); /* * Lets write out CACHE_EXT_INDEXENTRYOFFSETTABLE first so that we @@ -3005,8 +2937,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_ieot_extension(&sb, ieot); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); free(ieot); if (err) @@ -3018,9 +2950,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; err = write_link_extension(&sb, istate) < 0 || - write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_LINK, - sb.len) < 0 || - ce_write(&c, newfd, sb.buf, sb.len) < 0; + write_index_ext_header(f, eoie_c, CACHE_EXT_LINK, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3029,8 +2961,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; cache_tree_write(&sb, istate->cache_tree); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_TREE, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_TREE, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3039,9 +2971,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; resolve_undo_write(&sb, istate->resolve_undo); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_RESOLVE_UNDO, - sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_RESOLVE_UNDO, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3050,9 +2982,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_untracked_extension(&sb, istate->untracked); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_UNTRACKED, - sb.len) < 0 || - ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_UNTRACKED, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3061,14 +2993,14 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_fsmonitor_extension(&sb, istate); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_FSMONITOR, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; } if (istate->sparse_index) { - if (write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) + if (write_index_ext_header(f, eoie_c, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) return -1; } @@ -3078,19 +3010,18 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, * read. Write it out regardless of the strip_extensions parameter as we need it * when loading the shared index. */ - if (offset && record_eoie()) { + if (eoie_c) { struct strbuf sb = STRBUF_INIT; - write_eoie_extension(&sb, &eoie_c, offset); - err = write_index_ext_header(&c, NULL, newfd, CACHE_EXT_ENDOFINDEXENTRIES, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + write_eoie_extension(&sb, eoie_c, offset); + err = write_index_ext_header(f, NULL, CACHE_EXT_ENDOFINDEXENTRIES, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; } - if (ce_flush(&c, newfd, istate->oid.hash)) - return -1; + finalize_hashfile(f, istate->oid.hash, CSUM_HASH_IN_STREAM); if (close_tempfile_gently(tempfile)) { error(_("could not close '%s'"), get_tempfile_path(tempfile)); return -1; diff --git a/ref-filter.c b/ref-filter.c index 97116e12d7..4db0e40ff4 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -109,6 +109,56 @@ static struct ref_to_worktree_map { } ref_to_worktree_map; /* + * The enum atom_type is used as the index of valid_atom array. + * In the atom parsing stage, it will be passed to used_atom.atom_type + * as the identifier of the atom type. We can check the type of used_atom + * entry by `if (used_atom[i].atom_type == ATOM_*)`. + */ +enum atom_type { + ATOM_REFNAME, + ATOM_OBJECTTYPE, + ATOM_OBJECTSIZE, + ATOM_OBJECTNAME, + ATOM_DELTABASE, + ATOM_TREE, + ATOM_PARENT, + ATOM_NUMPARENT, + ATOM_OBJECT, + ATOM_TYPE, + ATOM_TAG, + ATOM_AUTHOR, + ATOM_AUTHORNAME, + ATOM_AUTHOREMAIL, + ATOM_AUTHORDATE, + ATOM_COMMITTER, + ATOM_COMMITTERNAME, + ATOM_COMMITTEREMAIL, + ATOM_COMMITTERDATE, + ATOM_TAGGER, + ATOM_TAGGERNAME, + ATOM_TAGGEREMAIL, + ATOM_TAGGERDATE, + ATOM_CREATOR, + ATOM_CREATORDATE, + ATOM_SUBJECT, + ATOM_BODY, + ATOM_TRAILERS, + ATOM_CONTENTS, + ATOM_UPSTREAM, + ATOM_PUSH, + ATOM_SYMREF, + ATOM_FLAG, + ATOM_HEAD, + ATOM_COLOR, + ATOM_WORKTREEPATH, + ATOM_ALIGN, + ATOM_END, + ATOM_IF, + ATOM_THEN, + ATOM_ELSE, +}; + +/* * An atom is a valid field atom listed below, possibly prefixed with * a "*" to denote deref_tag(). * @@ -119,6 +169,7 @@ static struct ref_to_worktree_map { * array. */ static struct used_atom { + enum atom_type atom_type; const char *name; cmp_type type; info_source source; @@ -146,6 +197,9 @@ static struct used_atom { enum { O_FULL, O_LENGTH, O_SHORT } option; unsigned int length; } oid; + struct { + enum { O_SIZE, O_SIZE_DISK } option; + } objectsize; struct email_option { enum { EO_RAW, EO_TRIM, EO_LOCALPART } option; } email_option; @@ -269,11 +323,13 @@ static int objectsize_atom_parser(const struct ref_format *format, struct used_a const char *arg, struct strbuf *err) { if (!arg) { + atom->u.objectsize.option = O_SIZE; if (*atom->name == '*') oi_deref.info.sizep = &oi_deref.size; else oi.info.sizep = &oi.size; } else if (!strcmp(arg, "disk")) { + atom->u.objectsize.option = O_SIZE_DISK; if (*atom->name == '*') oi_deref.info.disk_sizep = &oi_deref.disk_size; else @@ -501,47 +557,47 @@ static struct { int (*parser)(const struct ref_format *format, struct used_atom *atom, const char *arg, struct strbuf *err); } valid_atom[] = { - { "refname", SOURCE_NONE, FIELD_STR, refname_atom_parser }, - { "objecttype", SOURCE_OTHER, FIELD_STR, objecttype_atom_parser }, - { "objectsize", SOURCE_OTHER, FIELD_ULONG, objectsize_atom_parser }, - { "objectname", SOURCE_OTHER, FIELD_STR, oid_atom_parser }, - { "deltabase", SOURCE_OTHER, FIELD_STR, deltabase_atom_parser }, - { "tree", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, - { "parent", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, - { "numparent", SOURCE_OBJ, FIELD_ULONG }, - { "object", SOURCE_OBJ }, - { "type", SOURCE_OBJ }, - { "tag", SOURCE_OBJ }, - { "author", SOURCE_OBJ }, - { "authorname", SOURCE_OBJ }, - { "authoremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "authordate", SOURCE_OBJ, FIELD_TIME }, - { "committer", SOURCE_OBJ }, - { "committername", SOURCE_OBJ }, - { "committeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "committerdate", SOURCE_OBJ, FIELD_TIME }, - { "tagger", SOURCE_OBJ }, - { "taggername", SOURCE_OBJ }, - { "taggeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "taggerdate", SOURCE_OBJ, FIELD_TIME }, - { "creator", SOURCE_OBJ }, - { "creatordate", SOURCE_OBJ, FIELD_TIME }, - { "subject", SOURCE_OBJ, FIELD_STR, subject_atom_parser }, - { "body", SOURCE_OBJ, FIELD_STR, body_atom_parser }, - { "trailers", SOURCE_OBJ, FIELD_STR, trailers_atom_parser }, - { "contents", SOURCE_OBJ, FIELD_STR, contents_atom_parser }, - { "upstream", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, - { "push", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, - { "symref", SOURCE_NONE, FIELD_STR, refname_atom_parser }, - { "flag", SOURCE_NONE }, - { "HEAD", SOURCE_NONE, FIELD_STR, head_atom_parser }, - { "color", SOURCE_NONE, FIELD_STR, color_atom_parser }, - { "worktreepath", SOURCE_NONE }, - { "align", SOURCE_NONE, FIELD_STR, align_atom_parser }, - { "end", SOURCE_NONE }, - { "if", SOURCE_NONE, FIELD_STR, if_atom_parser }, - { "then", SOURCE_NONE }, - { "else", SOURCE_NONE }, + [ATOM_REFNAME] = { "refname", SOURCE_NONE, FIELD_STR, refname_atom_parser }, + [ATOM_OBJECTTYPE] = { "objecttype", SOURCE_OTHER, FIELD_STR, objecttype_atom_parser }, + [ATOM_OBJECTSIZE] = { "objectsize", SOURCE_OTHER, FIELD_ULONG, objectsize_atom_parser }, + [ATOM_OBJECTNAME] = { "objectname", SOURCE_OTHER, FIELD_STR, oid_atom_parser }, + [ATOM_DELTABASE] = { "deltabase", SOURCE_OTHER, FIELD_STR, deltabase_atom_parser }, + [ATOM_TREE] = { "tree", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, + [ATOM_PARENT] = { "parent", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, + [ATOM_NUMPARENT] = { "numparent", SOURCE_OBJ, FIELD_ULONG }, + [ATOM_OBJECT] = { "object", SOURCE_OBJ }, + [ATOM_TYPE] = { "type", SOURCE_OBJ }, + [ATOM_TAG] = { "tag", SOURCE_OBJ }, + [ATOM_AUTHOR] = { "author", SOURCE_OBJ }, + [ATOM_AUTHORNAME] = { "authorname", SOURCE_OBJ }, + [ATOM_AUTHOREMAIL] = { "authoremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_AUTHORDATE] = { "authordate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_COMMITTER] = { "committer", SOURCE_OBJ }, + [ATOM_COMMITTERNAME] = { "committername", SOURCE_OBJ }, + [ATOM_COMMITTEREMAIL] = { "committeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_COMMITTERDATE] = { "committerdate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_TAGGER] = { "tagger", SOURCE_OBJ }, + [ATOM_TAGGERNAME] = { "taggername", SOURCE_OBJ }, + [ATOM_TAGGEREMAIL] = { "taggeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_TAGGERDATE] = { "taggerdate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_CREATOR] = { "creator", SOURCE_OBJ }, + [ATOM_CREATORDATE] = { "creatordate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_SUBJECT] = { "subject", SOURCE_OBJ, FIELD_STR, subject_atom_parser }, + [ATOM_BODY] = { "body", SOURCE_OBJ, FIELD_STR, body_atom_parser }, + [ATOM_TRAILERS] = { "trailers", SOURCE_OBJ, FIELD_STR, trailers_atom_parser }, + [ATOM_CONTENTS] = { "contents", SOURCE_OBJ, FIELD_STR, contents_atom_parser }, + [ATOM_UPSTREAM] = { "upstream", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, + [ATOM_PUSH] = { "push", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, + [ATOM_SYMREF] = { "symref", SOURCE_NONE, FIELD_STR, refname_atom_parser }, + [ATOM_FLAG] = { "flag", SOURCE_NONE }, + [ATOM_HEAD] = { "HEAD", SOURCE_NONE, FIELD_STR, head_atom_parser }, + [ATOM_COLOR] = { "color", SOURCE_NONE, FIELD_STR, color_atom_parser }, + [ATOM_WORKTREEPATH] = { "worktreepath", SOURCE_NONE }, + [ATOM_ALIGN] = { "align", SOURCE_NONE, FIELD_STR, align_atom_parser }, + [ATOM_END] = { "end", SOURCE_NONE }, + [ATOM_IF] = { "if", SOURCE_NONE, FIELD_STR, if_atom_parser }, + [ATOM_THEN] = { "then", SOURCE_NONE }, + [ATOM_ELSE] = { "else", SOURCE_NONE }, /* * Please update $__git_ref_fieldlist in git-completion.bash * when you add new atoms @@ -623,6 +679,7 @@ static int parse_ref_filter_atom(const struct ref_format *format, at = used_atom_cnt; used_atom_cnt++; REALLOC_ARRAY(used_atom, used_atom_cnt); + used_atom[at].atom_type = i; used_atom[at].name = xmemdupz(atom, ep - atom); used_atom[at].type = valid_atom[i].cmp_type; used_atom[at].source = valid_atom[i].source; @@ -647,7 +704,7 @@ static int parse_ref_filter_atom(const struct ref_format *format, return -1; if (*atom == '*') need_tagged = 1; - if (!strcmp(valid_atom[i].name, "symref")) + if (i == ATOM_SYMREF) need_symref = 1; return at; } @@ -960,22 +1017,25 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_ for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (!strcmp(name, "objecttype")) + if (atom_type == ATOM_OBJECTTYPE) v->s = xstrdup(type_name(oi->type)); - else if (!strcmp(name, "objectsize:disk")) { - v->value = oi->disk_size; - v->s = xstrfmt("%"PRIuMAX, (uintmax_t)oi->disk_size); - } else if (!strcmp(name, "objectsize")) { - v->value = oi->size; - v->s = xstrfmt("%"PRIuMAX , (uintmax_t)oi->size); - } else if (!strcmp(name, "deltabase")) + else if (atom_type == ATOM_OBJECTSIZE) { + if (used_atom[i].u.objectsize.option == O_SIZE_DISK) { + v->value = oi->disk_size; + v->s = xstrfmt("%"PRIuMAX, (uintmax_t)oi->disk_size); + } else if (used_atom[i].u.objectsize.option == O_SIZE) { + v->value = oi->size; + v->s = xstrfmt("%"PRIuMAX , (uintmax_t)oi->size); + } + } else if (atom_type == ATOM_DELTABASE) v->s = xstrdup(oid_to_hex(&oi->delta_base_oid)); - else if (deref) + else if (atom_type == ATOM_OBJECTNAME && deref) grab_oid(name, "objectname", &oi->oid, v, &used_atom[i]); } } @@ -988,16 +1048,17 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (!strcmp(name, "tag")) + if (atom_type == ATOM_TAG) v->s = xstrdup(tag->tag); - else if (!strcmp(name, "type") && tag->tagged) + else if (atom_type == ATOM_TYPE && tag->tagged) v->s = xstrdup(type_name(tag->tagged->type)); - else if (!strcmp(name, "object") && tag->tagged) + else if (atom_type == ATOM_OBJECT && tag->tagged) v->s = xstrdup(oid_to_hex(&tag->tagged->oid)); } } @@ -1010,18 +1071,20 @@ static void grab_commit_values(struct atom_value *val, int deref, struct object for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i])) + if (atom_type == ATOM_TREE && + grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i])) continue; - if (!strcmp(name, "numparent")) { + if (atom_type == ATOM_NUMPARENT) { v->value = commit_list_count(commit->parents); v->s = xstrfmt("%lu", (unsigned long)v->value); } - else if (starts_with(name, "parent")) { + else if (atom_type == ATOM_PARENT) { struct commit_list *parents; struct strbuf s = STRBUF_INIT; for (parents = commit->parents; parents; parents = parents->next) { @@ -1201,15 +1264,16 @@ static void grab_person(const char *who, struct atom_value *val, int deref, void return; for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (starts_with(name, "creatordate")) + if (atom_type == ATOM_CREATORDATE) grab_date(wholine, v, name); - else if (!strcmp(name, "creator")) + else if (atom_type == ATOM_CREATOR) v->s = copy_line(wholine); } } @@ -1689,6 +1753,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) /* Fill in specials first */ for (i = 0; i < used_atom_cnt; i++) { struct used_atom *atom = &used_atom[i]; + enum atom_type atom_type = atom->atom_type; const char *name = used_atom[i].name; struct atom_value *v = &ref->value[i]; int deref = 0; @@ -1703,18 +1768,18 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) name++; } - if (starts_with(name, "refname")) + if (atom_type == ATOM_REFNAME) refname = get_refname(atom, ref); - else if (!strcmp(name, "worktreepath")) { + else if (atom_type == ATOM_WORKTREEPATH) { if (ref->kind == FILTER_REFS_BRANCHES) v->s = get_worktree_path(atom, ref); else v->s = xstrdup(""); continue; } - else if (starts_with(name, "symref")) + else if (atom_type == ATOM_SYMREF) refname = get_symref(atom, ref); - else if (starts_with(name, "upstream")) { + else if (atom_type == ATOM_UPSTREAM) { const char *branch_name; /* only local branches may have an upstream */ if (!skip_prefix(ref->refname, "refs/heads/", @@ -1730,7 +1795,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) else v->s = xstrdup(""); continue; - } else if (!strcmp(atom->name, "push") || starts_with(atom->name, "push:")) { + } else if (atom_type == ATOM_PUSH && atom->u.remote_ref.push) { const char *branch_name; v->s = xstrdup(""); if (!skip_prefix(ref->refname, "refs/heads/", @@ -1749,10 +1814,10 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) free((char *)v->s); fill_remote_ref_details(atom, refname, branch, &v->s); continue; - } else if (starts_with(name, "color:")) { + } else if (atom_type == ATOM_COLOR) { v->s = xstrdup(atom->u.color); continue; - } else if (!strcmp(name, "flag")) { + } else if (atom_type == ATOM_FLAG) { char buf[256], *cp = buf; if (ref->flag & REF_ISSYMREF) cp = copy_advance(cp, ",symref"); @@ -1765,23 +1830,24 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) v->s = xstrdup(buf + 1); } continue; - } else if (!deref && grab_oid(name, "objectname", &ref->objectname, v, atom)) { - continue; - } else if (!strcmp(name, "HEAD")) { + } else if (!deref && atom_type == ATOM_OBJECTNAME && + grab_oid(name, "objectname", &ref->objectname, v, atom)) { + continue; + } else if (atom_type == ATOM_HEAD) { if (atom->u.head && !strcmp(ref->refname, atom->u.head)) v->s = xstrdup("*"); else v->s = xstrdup(" "); continue; - } else if (starts_with(name, "align")) { + } else if (atom_type == ATOM_ALIGN) { v->handler = align_atom_handler; v->s = xstrdup(""); continue; - } else if (!strcmp(name, "end")) { + } else if (atom_type == ATOM_END) { v->handler = end_atom_handler; v->s = xstrdup(""); continue; - } else if (starts_with(name, "if")) { + } else if (atom_type == ATOM_IF) { const char *s; if (skip_prefix(name, "if:", &s)) v->s = xstrdup(s); @@ -1789,11 +1855,11 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) v->s = xstrdup(""); v->handler = if_atom_handler; continue; - } else if (!strcmp(name, "then")) { + } else if (atom_type == ATOM_THEN) { v->handler = then_atom_handler; v->s = xstrdup(""); continue; - } else if (!strcmp(name, "else")) { + } else if (atom_type == ATOM_ELSE) { v->handler = else_atom_handler; v->s = xstrdup(""); continue; @@ -1592,7 +1592,7 @@ void set_ref_status_for_push(struct ref *remote_refs, int send_mirror, else /* * If the ref isn't stale, and is reachable - * from from one of the reflog entries of + * from one of the reflog entries of * the local branch, force the update. */ force_ref_update = 1; diff --git a/revision.c b/revision.c index 8140561b6c..cddd0542a6 100644 --- a/revision.c +++ b/revision.c @@ -316,9 +316,10 @@ static void add_pending_object_with_path(struct rev_info *revs, revs->no_walk = 0; if (revs->reflog_info && obj->type == OBJ_COMMIT) { struct strbuf buf = STRBUF_INIT; - int len = interpret_branch_name(name, 0, &buf, &options); + size_t namelen = strlen(name); + int len = interpret_branch_name(name, namelen, &buf, &options); - if (0 < len && name[len] && buf.len) + if (0 < len && len < namelen && buf.len) strbuf_addstr(&buf, name + len); add_reflog_for_walk(revs->reflog_info, (struct commit *)obj, diff --git a/revision.h b/revision.h index 7bb5fa4e7c..5c5510d422 100644 --- a/revision.h +++ b/revision.h @@ -193,10 +193,10 @@ struct rev_info { /* Diff-merge flags */ explicit_diff_merges: 1, merges_need_diff: 1, + merges_imply_patch:1, separate_merges: 1, combine_merges:1, combined_all_paths:1, - combined_imply_patch:1, dense_combined_merges:1, first_parent_merges:1; @@ -666,7 +666,9 @@ int verify_repository_format(const struct repository_format *format, if (format->version >= 1 && format->unknown_extensions.nr) { int i; - strbuf_addstr(err, _("unknown repository extensions found:")); + strbuf_addstr(err, Q_("unknown repository extension found:", + "unknown repository extensions found:", + format->unknown_extensions.nr)); for (i = 0; i < format->unknown_extensions.nr; i++) strbuf_addf(err, "\n\t%s", @@ -678,7 +680,9 @@ int verify_repository_format(const struct repository_format *format, int i; strbuf_addstr(err, - _("repo version is 0, but v1-only extensions found:")); + Q_("repo version is 0, but v1-only extension found:", + "repo version is 0, but v1-only extensions found:", + format->v1_only_extensions.nr)); for (i = 0; i < format->v1_only_extensions.nr; i++) strbuf_addf(err, "\n\t%s", diff --git a/split-index.c b/split-index.c index 4d6e52d46f..8e52e891c3 100644 --- a/split-index.c +++ b/split-index.c @@ -207,7 +207,8 @@ static int compare_ce_content(struct cache_entry *a, struct cache_entry *b) b->ce_flags &= ondisk_flags; ret = memcmp(&a->ce_stat_data, &b->ce_stat_data, offsetof(struct cache_entry, name) - - offsetof(struct cache_entry, ce_stat_data)); + offsetof(struct cache_entry, oid)) || + !oideq(&a->oid, &b->oid); a->ce_flags = ce_flags; b->ce_flags = base_flags; diff --git a/t/helper/test-fast-rebase.c b/t/helper/test-fast-rebase.c index 373212256a..fc2d460904 100644 --- a/t/helper/test-fast-rebase.c +++ b/t/helper/test-fast-rebase.c @@ -91,7 +91,6 @@ int cmd__fast_rebase(int argc, const char **argv) struct commit *last_commit = NULL, *last_picked_commit = NULL; struct object_id head; struct lock_file lock = LOCK_INIT; - int clean = 1; struct strvec rev_walk_args = STRVEC_INIT; struct rev_info revs; struct commit *commit; @@ -124,7 +123,8 @@ int cmd__fast_rebase(int argc, const char **argv) assert(oideq(&onto->object.oid, &head)); hold_locked_index(&lock, LOCK_DIE_ON_ERROR); - assert(repo_read_index(the_repository) >= 0); + if (repo_read_index(the_repository) < 0) + BUG("Could not read index"); repo_init_revisions(the_repository, &revs, NULL); revs.verbose_header = 1; @@ -175,11 +175,10 @@ int cmd__fast_rebase(int argc, const char **argv) free((char*)merge_opt.ancestor); merge_opt.ancestor = NULL; if (!result.clean) - die("Aborting: Hit a conflict and restarting is not implemented."); + break; last_picked_commit = commit; last_commit = create_commit(result.tree, commit, last_commit); } - fprintf(stderr, "\nDone.\n"); /* TODO: There should be some kind of rev_info_free(&revs) call... */ memset(&revs, 0, sizeof(revs)); @@ -188,24 +187,39 @@ int cmd__fast_rebase(int argc, const char **argv) if (result.clean < 0) exit(128); - strbuf_addf(&reflog_msg, "finish rebase %s onto %s", - oid_to_hex(&last_picked_commit->object.oid), - oid_to_hex(&last_commit->object.oid)); - if (update_ref(reflog_msg.buf, branch_name.buf, - &last_commit->object.oid, - &last_picked_commit->object.oid, - REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { - error(_("could not update %s"), argv[4]); - die("Failed to update %s", argv[4]); + if (result.clean) { + fprintf(stderr, "\nDone.\n"); + strbuf_addf(&reflog_msg, "finish rebase %s onto %s", + oid_to_hex(&last_picked_commit->object.oid), + oid_to_hex(&last_commit->object.oid)); + if (update_ref(reflog_msg.buf, branch_name.buf, + &last_commit->object.oid, + &last_picked_commit->object.oid, + REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { + error(_("could not update %s"), argv[4]); + die("Failed to update %s", argv[4]); + } + if (create_symref("HEAD", branch_name.buf, reflog_msg.buf) < 0) + die(_("unable to update HEAD")); + strbuf_release(&reflog_msg); + strbuf_release(&branch_name); + + prime_cache_tree(the_repository, the_repository->index, + result.tree); + } else { + fprintf(stderr, "\nAborting: Hit a conflict.\n"); + strbuf_addf(&reflog_msg, "rebase progress up to %s", + oid_to_hex(&last_picked_commit->object.oid)); + if (update_ref(reflog_msg.buf, "HEAD", + &last_commit->object.oid, + &head, + REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { + error(_("could not update %s"), argv[4]); + die("Failed to update %s", argv[4]); + } } - if (create_symref("HEAD", branch_name.buf, reflog_msg.buf) < 0) - die(_("unable to update HEAD")); - strbuf_release(&reflog_msg); - strbuf_release(&branch_name); - - prime_cache_tree(the_repository, the_repository->index, result.tree); if (write_locked_index(&the_index, &lock, COMMIT_LOCK | SKIP_IF_UNCHANGED)) die(_("unable to write %s"), get_index_file()); - return (clean == 0); + return (result.clean == 0); } diff --git a/t/lib-git-svn.sh b/t/lib-git-svn.sh index 547eb3c31a..2fde2353fd 100644 --- a/t/lib-git-svn.sh +++ b/t/lib-git-svn.sh @@ -121,12 +121,22 @@ start_svnserve () { --listen-host 127.0.0.1 & } -prepare_a_utf8_locale () { - a_utf8_locale=$(locale -a | sed -n '/\.[uU][tT][fF]-*8$/{ - p - q -}') - if test -n "$a_utf8_locale" +prepare_utf8_locale () { + if test -z "$GIT_TEST_UTF8_LOCALE" + then + case "${LC_ALL:-$LANG}" in + *.[Uu][Tt][Ff]8 | *.[Uu][Tt][Ff]-8) + GIT_TEST_UTF8_LOCALE="${LC_ALL:-$LANG}" + ;; + *) + GIT_TEST_UTF8_LOCALE=$(locale -a | sed -n '/\.[uU][tT][fF]-*8$/{ + p + q + }') + ;; + esac + fi + if test -n "$GIT_TEST_UTF8_LOCALE" then test_set_prereq UTF8 else diff --git a/t/lib-submodule-update.sh b/t/lib-submodule-update.sh index 4b714e9308..f7c7df0ca4 100644 --- a/t/lib-submodule-update.sh +++ b/t/lib-submodule-update.sh @@ -63,6 +63,7 @@ create_lib_submodule_repo () { git init submodule_update_repo && ( cd submodule_update_repo && + branch=$(git symbolic-ref --short HEAD) && echo "expect" >>.gitignore && echo "actual" >>.gitignore && echo "x" >file1 && @@ -144,7 +145,7 @@ create_lib_submodule_repo () { git checkout -b valid_sub1 && git revert HEAD && - git checkout "${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" + git checkout "$branch" ) } diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 705d62cc27..2c6e34b947 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -84,10 +84,6 @@ _run_sub_test_lib_test_common () { passing metrics ' - # Tell the framework that we are self-testing to make sure - # it yields a stable result. - GIT_TEST_FRAMEWORK_SELFTEST=t && - # Point to the t/test-lib.sh, which isn't in ../ as usual . "\$TEST_DIRECTORY"/test-lib.sh EOF diff --git a/t/t1307-config-blob.sh b/t/t1307-config-blob.sh index 002e6d3388..930dce06f0 100755 --- a/t/t1307-config-blob.sh +++ b/t/t1307-config-blob.sh @@ -65,9 +65,7 @@ test_expect_success 'parse errors in blobs are properly attributed' ' ' test_expect_success 'can parse blob ending with CR' ' - printf "[some]key = value\\r" >config && - git add config && - git commit -m CR && + test_commit --printf CR config "[some]key = value\\r" && echo value >expect && git config --blob=HEAD:config some.key >actual && test_cmp expect actual diff --git a/t/t1403-show-ref.sh b/t/t1403-show-ref.sh index 6ce62f878c..17d3cc1405 100755 --- a/t/t1403-show-ref.sh +++ b/t/t1403-show-ref.sh @@ -7,11 +7,9 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh test_expect_success setup ' - test_commit A && - git tag -f -a -m "annotated A" A && + test_commit --annotate A && git checkout -b side && - test_commit B && - git tag -f -a -m "annotated B" B && + test_commit --annotate B && git checkout main && test_commit C && git branch B A^0 diff --git a/t/t2030-unresolve-info.sh b/t/t2030-unresolve-info.sh index be6c84c52a..f691e6d903 100755 --- a/t/t2030-unresolve-info.sh +++ b/t/t2030-unresolve-info.sh @@ -179,8 +179,7 @@ test_expect_success 'rerere and rerere forget (subdirectory)' ' test_expect_success 'rerere forget (binary)' ' git checkout -f side && - printf "a\0c" >binary && - git commit -a -m binary && + test_commit --printf binary binary "a\0c" && test_must_fail git merge second && git rerere forget binary ' diff --git a/t/t3513-revert-submodule.sh b/t/t3513-revert-submodule.sh index 74cd96e582..8bfe3ed246 100755 --- a/t/t3513-revert-submodule.sh +++ b/t/t3513-revert-submodule.sh @@ -14,7 +14,7 @@ test_description='revert can handle submodules' git_revert () { git status -su >expect && ls -1pR * >>expect && - tar cf "$TRASH_DIRECTORY/tmp.tar" * && + "$TAR" cf "$TRASH_DIRECTORY/tmp.tar" * && may_only_be_test_must_fail "$2" && $2 git checkout "$1" && if test -n "$2" @@ -23,7 +23,7 @@ git_revert () { fi && git revert HEAD && rm -rf * && - tar xf "$TRASH_DIRECTORY/tmp.tar" && + "$TAR" xf "$TRASH_DIRECTORY/tmp.tar" && git status -su >actual && ls -1pR * >>actual && test_cmp expect actual && diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index 5f282ecf61..873aa56e35 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -859,7 +859,7 @@ test_expect_success 'setup stash with index and worktree changes' ' git stash ' -test_expect_success 'stash list implies --first-parent -m' ' +test_expect_success 'stash list -p shows simple diff' ' cat >expect <<-EOF && stash@{0} diff --git a/t/t4006-diff-mode.sh b/t/t4006-diff-mode.sh index 275ce5fa15..6cdee2a216 100755 --- a/t/t4006-diff-mode.sh +++ b/t/t4006-diff-mode.sh @@ -26,10 +26,8 @@ test_expect_success 'chmod' ' ' test_expect_success 'prepare binary file' ' - git commit -m rezrov && - printf "\00\01\02\03\04\05\06" >binbin && - git add binbin && - git commit -m binbin + git commit -m one && + test_commit --printf two binbin "\00\01\02\03\04\05\06" ' test_expect_success '--stat output after text chmod' ' diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 87def81699..7fadc985cc 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -293,6 +293,7 @@ diff-tree --stat initial mode diff-tree --summary initial mode diff-tree master +diff-tree -m master diff-tree -p master diff-tree -p -m master diff-tree -c master @@ -337,6 +338,8 @@ log -m -p --first-parent master log -m -p master log --cc -m -p master log -c -m -p master +log -m --raw master +log -m --stat master log -SF master log -S F master log -SF -p master @@ -452,6 +455,14 @@ diff-tree --stat --compact-summary initial mode diff-tree -R --stat --compact-summary initial mode EOF +test_expect_success 'log -m matches log -m -p' ' + git log -m -p master >result && + process_diffs result >expected && + git log -m >result && + process_diffs result >actual && + test_cmp expected actual +' + test_expect_success 'log --diff-merges=on matches --diff-merges=separate' ' git log -p --diff-merges=separate master >result && process_diffs result >expected && @@ -483,6 +494,19 @@ test_expect_success 'git config log.diffMerges first-parent vs -m' ' test_cmp expected actual ' +# -m in "git diff-index" means "match missing", that differs +# from its meaning in "git diff". Let's check it in diff-index. +# The line in the output for removed file should disappear when +# we provide -m in diff-index. +test_expect_success 'git diff-index -m' ' + rm -f file1 && + git diff-index HEAD >without-m && + lines_count=$(wc -l <without-m) && + git diff-index -m HEAD >with-m && + git restore file1 && + test_line_count = $((lines_count - 1)) with-m +' + test_expect_success 'log -S requires an argument' ' test_must_fail git log -S ' diff --git a/t/t4013/diff.diff-tree_-m_master b/t/t4013/diff.diff-tree_-m_master new file mode 100644 index 0000000000..6d0a2207fb --- /dev/null +++ b/t/t4013/diff.diff-tree_-m_master @@ -0,0 +1,11 @@ +$ git diff-tree -m master +59d314ad6f356dd08601a4cd5e530381da3e3c64 +:040000 040000 65f5c9dd60ce3b2b3324b618ac7accf8d912c113 0564e026437809817a64fff393079714b6dd4628 M dir +:100644 100644 b414108e81e5091fe0974a1858b4d0d22b107f70 10a8a9f3657f91a156b9f0184ed79a20adef9f7f M file0 +59d314ad6f356dd08601a4cd5e530381da3e3c64 +:040000 040000 f977ed46ae6873c1c30ab878e15a4accedc3618b 0564e026437809817a64fff393079714b6dd4628 M dir +:100644 100644 f4615da674c09df322d6ba8d6b21ecfb1b1ba510 10a8a9f3657f91a156b9f0184ed79a20adef9f7f M file0 +:000000 100644 0000000000000000000000000000000000000000 b1e67221afe8461efd244b487afca22d46b95eb8 A file1 +:100644 000000 01e79c32a8c99c557f0757da7cb6d65b3414466d 0000000000000000000000000000000000000000 D file2 +:100644 000000 7289e35bff32727c08dda207511bec138fdb9ea5 0000000000000000000000000000000000000000 D file3 +$ diff --git a/t/t4013/diff.log_-m_--raw_master b/t/t4013/diff.log_-m_--raw_master new file mode 100644 index 0000000000..cd2ecc4628 --- /dev/null +++ b/t/t4013/diff.log_-m_--raw_master @@ -0,0 +1,61 @@ +$ git log -m --raw master +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +:100644 100644 cead32e... 992913c... M dir/sub +:100644 100644 b414108... 10a8a9f... M file0 + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +:100644 100644 7289e35... 992913c... M dir/sub +:100644 100644 f4615da... 10a8a9f... M file0 +:000000 100644 0000000... b1e6722... A file1 +:100644 000000 01e79c3... 0000000... D file2 +:100644 000000 7289e35... 0000000... D file3 + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +:100644 100644 35d242b... 7289e35... M dir/sub +:100644 100644 01e79c3... f4615da... M file0 +:000000 100644 0000000... 7289e35... A file3 + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +:100644 100644 8422d40... cead32e... M dir/sub +:000000 100644 0000000... b1e6722... A file1 + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +:100644 100644 35d242b... 8422d40... M dir/sub +:100644 100644 01e79c3... b414108... M file0 +:100644 000000 01e79c3... 0000000... D file2 + +commit 444ac553ac7612cc88969031b02b3767fb8a353a +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4013/diff.log_-m_--stat_master b/t/t4013/diff.log_-m_--stat_master new file mode 100644 index 0000000000..c7db084fd9 --- /dev/null +++ b/t/t4013/diff.log_-m_--stat_master @@ -0,0 +1,66 @@ +$ git log -m --stat master +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + + dir/sub | 2 ++ + file0 | 3 +++ + 2 files changed, 5 insertions(+) + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + + dir/sub | 4 ++++ + file0 | 3 +++ + file1 | 3 +++ + file2 | 3 --- + file3 | 4 ---- + 5 files changed, 10 insertions(+), 7 deletions(-) + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + + dir/sub | 2 ++ + file0 | 3 +++ + file3 | 4 ++++ + 3 files changed, 9 insertions(+) + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + + dir/sub | 2 ++ + file1 | 3 +++ + 2 files changed, 5 insertions(+) + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + + dir/sub | 2 ++ + file0 | 3 +++ + file2 | 3 --- + 3 files changed, 5 insertions(+), 3 deletions(-) + +commit 444ac553ac7612cc88969031b02b3767fb8a353a +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4030-diff-textconv.sh b/t/t4030-diff-textconv.sh index c906320b60..a39a626664 100755 --- a/t/t4030-diff-textconv.sh +++ b/t/t4030-diff-textconv.sh @@ -26,12 +26,8 @@ EOF chmod +x hexdump test_expect_success 'setup binary file with history' ' - printf "\\0\\n" >file && - git add file && - git commit -m one && - printf "\\01\\n" >>file && - git add file && - git commit -m two + test_commit --printf one file "\\0\\n" && + test_commit --printf --append two file "\\01\\n" ' test_expect_success 'file is considered binary by porcelain' ' diff --git a/t/t5406-remote-rejects.sh b/t/t5406-remote-rejects.sh index ff06f99649..5c509db6fc 100755 --- a/t/t5406-remote-rejects.sh +++ b/t/t5406-remote-rejects.sh @@ -5,7 +5,6 @@ test_description='remote push rejects are reported by client' . ./test-lib.sh test_expect_success 'setup' ' - mkdir .git/hooks && write_script .git/hooks/update <<-\EOF && exit 1 EOF diff --git a/t/t5407-post-rewrite-hook.sh b/t/t5407-post-rewrite-hook.sh index 5bb23cc3a4..6da8d760e2 100755 --- a/t/t5407-post-rewrite-hook.sh +++ b/t/t5407-post-rewrite-hook.sh @@ -20,8 +20,6 @@ test_expect_success 'setup' ' git checkout main ' -mkdir .git/hooks - cat >.git/hooks/post-rewrite <<EOF #!/bin/sh echo \$@ > "$TRASH_DIRECTORY"/post-rewrite.args diff --git a/t/t5409-colorize-remote-messages.sh b/t/t5409-colorize-remote-messages.sh index 5d8f401d8e..9f1a483f42 100755 --- a/t/t5409-colorize-remote-messages.sh +++ b/t/t5409-colorize-remote-messages.sh @@ -5,7 +5,6 @@ test_description='remote messages are colorized on the client' . ./test-lib.sh test_expect_success 'setup' ' - mkdir .git/hooks && write_script .git/hooks/update <<-\EOF && echo error: error echo ERROR: also highlighted diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index c7b392794b..e6e3c8f552 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -182,7 +182,7 @@ test_expect_success 'rename errors out early when deleting non-existent branch' ) ' -test_expect_success 'rename errors out early when when new name is invalid' ' +test_expect_success 'rename errors out early when new name is invalid' ' test_config remote.foo.vcs bar && echo "fatal: '\''invalid...name'\'' is not a valid remote name" >expect && test_must_fail git remote rename foo invalid...name 2>actual && diff --git a/t/t5520-pull.sh b/t/t5520-pull.sh index a09411327f..e2c0c51022 100755 --- a/t/t5520-pull.sh +++ b/t/t5520-pull.sh @@ -746,14 +746,8 @@ test_expect_success 'pull --rebase fails on corrupt HEAD' ' ' test_expect_success 'setup for detecting upstreamed changes' ' - mkdir src && - ( - cd src && - git init && - printf "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" > stuff && - git add stuff && - git commit -m "Initial revision" - ) && + test_create_repo src && + test_commit -C src --printf one stuff "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" && git clone src dst && ( cd src && diff --git a/t/t5600-clone-fail-cleanup.sh b/t/t5600-clone-fail-cleanup.sh index 4a1a912e03..5bf10261d3 100755 --- a/t/t5600-clone-fail-cleanup.sh +++ b/t/t5600-clone-fail-cleanup.sh @@ -97,4 +97,11 @@ test_expect_success 'failed clone into empty leaves directory (separate, wt)' ' test_dir_is_empty empty-wt ' +test_expect_success 'transport failure cleans up directory' ' + test_must_fail git clone --no-local \ + -u "f() { git-upload-pack \"\$@\"; return 1; }; f" \ + foo broken-clone && + test_path_is_missing broken-clone +' + test_done diff --git a/t/t6020-bundle-misc.sh b/t/t6020-bundle-misc.sh index 881f72fd44..3140ca4fdc 100755 --- a/t/t6020-bundle-misc.sh +++ b/t/t6020-bundle-misc.sh @@ -80,30 +80,42 @@ test_commit_setvar () { eval $var=$oid } +get_abbrev_oid () { + oid=$1 && + suffix=${oid#???????} && + oid=${oid%$suffix} && + if test -n "$oid" + then + echo "$oid" + else + echo "undefined-oid" + fi +} + # Format the output of git commands to make a user-friendly and stable # text. We can easily prepare the expect text without having to worry # about future changes of the commit ID and spaces of the output. make_user_friendly_and_stable_output () { sed \ - -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ - -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ - -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ - -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ - -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ - -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ - -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ - -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ - -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ - -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ - -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \ - -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ - -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ - -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ - -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ - -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ - -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ - -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ - -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/$(get_abbrev_oid $C)[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/$(get_abbrev_oid $D)[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/$(get_abbrev_oid $E)[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/$(get_abbrev_oid $F)[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/$(get_abbrev_oid $G)[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/$(get_abbrev_oid $H)[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/$(get_abbrev_oid $I)[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/$(get_abbrev_oid $J)[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/$(get_abbrev_oid $K)[0-9a-f]*/<COMMIT-K>/g" \ + -e "s/$(get_abbrev_oid $L)[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/$(get_abbrev_oid $M)[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/$(get_abbrev_oid $N)[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/$(get_abbrev_oid $O)[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/$(get_abbrev_oid $P)[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*/<TAG-1>/g" \ + -e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*/<TAG-2>/g" \ + -e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*/<TAG-3>/g" \ -e "s/ *\$//" } diff --git a/t/t6041-bisect-submodule.sh b/t/t6041-bisect-submodule.sh index df1eff0fb8..82013fc903 100755 --- a/t/t6041-bisect-submodule.sh +++ b/t/t6041-bisect-submodule.sh @@ -8,7 +8,7 @@ test_description='bisect can handle submodules' git_bisect () { git status -su >expect && ls -1pR * >>expect && - tar cf "$TRASH_DIRECTORY/tmp.tar" * && + "$TAR" cf "$TRASH_DIRECTORY/tmp.tar" * && GOOD=$(git rev-parse --verify HEAD) && may_only_be_test_must_fail "$2" && $2 git checkout "$1" && @@ -25,7 +25,7 @@ git_bisect () { git bisect start && git bisect good $GOOD && rm -rf * && - tar xf "$TRASH_DIRECTORY/tmp.tar" && + "$TAR" xf "$TRASH_DIRECTORY/tmp.tar" && git status -su >actual && ls -1pR * >>actual && test_cmp expect actual && diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index e89b6747be..88fddc9142 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -31,64 +31,32 @@ check_describe () { } test_expect_success setup ' + test_commit initial file one && + test_commit second file two && + test_commit third file three && + test_commit --annotate A file A && + test_commit c file c && - test_tick && - echo one >file && git add file && git commit -m initial && - one=$(git rev-parse HEAD) && - - git describe --always HEAD && - - test_tick && - echo two >file && git add file && git commit -m second && - two=$(git rev-parse HEAD) && - - test_tick && - echo three >file && git add file && git commit -m third && - - test_tick && - echo A >file && git add file && git commit -m A && - test_tick && - git tag -a -m A A && - - test_tick && - echo c >file && git add file && git commit -m c && - test_tick && - git tag c && - - git reset --hard $two && - test_tick && - echo B >side && git add side && git commit -m B && - test_tick && - git tag -a -m B B && + git reset --hard second && + test_commit --annotate B side B && test_tick && git merge -m Merged c && merged=$(git rev-parse HEAD) && - git reset --hard $two && - test_tick && - echo D >another && git add another && git commit -m D && - test_tick && - git tag -a -m D D && - test_tick && - git tag -a -m R R && - - test_tick && - echo DD >another && git commit -a -m another && + git reset --hard second && + test_commit --no-tag D another D && test_tick && - git tag e && + git tag -a -m R R && - test_tick && - echo DDD >another && git commit -a -m "yet another" && + test_commit e another DD && + test_commit --no-tag "yet another" another DDD && test_tick && git merge -m Merged $merged && - test_tick && - echo X >file && echo X >side && git add file side && - git commit -m x - + test_commit --no-tag x file ' check_describe A-* HEAD diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh index 7134769149..be84d22419 100755 --- a/t/t6423-merge-rename-directories.sh +++ b/t/t6423-merge-rename-directories.sh @@ -4966,6 +4966,64 @@ test_expect_success '12g: Testcase with two kinds of "relevant" renames' ' ) ' +# Testcase 12h, Testcase with two kinds of "relevant" renames +# Commit O: olddir/{a_1, b} +# Commit A: newdir/{a_2, b} +# Commit B: olddir/{alpha_1, b} +# Expected: newdir/{alpha_2, b} + +test_setup_12h () { + test_create_repo 12h && + ( + cd 12h && + + mkdir olddir && + test_seq 3 8 >olddir/a && + >olddir/b && + git add olddir && + git commit -m orig && + + git branch O && + git branch A && + git branch B && + + git switch A && + test_seq 3 10 >olddir/a && + git add olddir/a && + git mv olddir newdir && + git commit -m A && + + git switch B && + + git mv olddir/a olddir/alpha && + git commit -m B + ) +} + +test_expect_failure '12h: renaming a file within a renamed directory' ' + test_setup_12h && + ( + cd 12h && + + git checkout A^0 && + + test_might_fail git -c merge.directoryRenames=true merge -s recursive B^0 && + + git ls-files >tracked && + test_line_count = 2 tracked && + + test_path_is_missing olddir/a && + test_path_is_file newdir/alpha && + test_path_is_file newdir/b && + + git rev-parse >actual \ + HEAD:newdir/alpha HEAD:newdir/b && + git rev-parse >expect \ + A:newdir/a O:oldir/b && + test_cmp expect actual + ) +' + ########################################################################### # SECTION 13: Checking informational and conflict messages # diff --git a/t/t6429-merge-sequence-rename-caching.sh b/t/t6429-merge-sequence-rename-caching.sh new file mode 100755 index 0000000000..035edc40b1 --- /dev/null +++ b/t/t6429-merge-sequence-rename-caching.sh @@ -0,0 +1,700 @@ +#!/bin/sh + +test_description="remember regular & dir renames in sequence of merges" + +. ./test-lib.sh + +# +# NOTE 1: this testfile tends to not only rename files, but modify on both +# sides; without modifying on both sides, optimizations can kick in +# which make rename detection irrelevant or trivial. We want to make +# sure that we are triggering rename caching rather than rename +# bypassing. +# +# NOTE 2: this testfile uses 'test-tool fast-rebase' instead of either +# cherry-pick or rebase. sequencer.c is only superficially +# integrated with merge-ort; it calls merge_switch_to_result() +# after EACH merge, which updates the index and working copy AND +# throws away the cached results (because merge_switch_to_result() +# is only supposed to be called at the end of the sequence). +# Integrating them more deeply is a big task, so for now the tests +# use 'test-tool fast-rebase'. +# + + +# +# In the following simple testcase: +# Base: numbers_1, values_1 +# Upstream: numbers_2, values_2 +# Topic_1: sequence_3 +# Topic_2: scruples_3 +# or, in english, rename numbers -> sequence in the first commit, and rename +# values -> scruples in the second commit. +# +# This shouldn't be a challenge, it's just verifying that cached renames isn't +# preventing us from finding new renames. +# +test_expect_success 'caching renames does not preclude finding new ones' ' + test_create_repo caching-renames-and-new-renames && + ( + cd caching-renames-and-new-renames && + + test_seq 2 10 >numbers && + test_seq 2 10 >values && + git add numbers values && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 10 >numbers && + test_seq 1 10 >values && + git add numbers values && + git commit -m "Tweaked both files" && + + git switch topic && + + test_seq 2 12 >numbers && + git add numbers && + git mv numbers sequence && + git commit -m A && + + test_seq 2 12 >values && + git add values && + git mv values scruples && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic + + git ls-files >tracked-files && + test_line_count = 2 tracked-files && + test_seq 1 12 >expect && + test_cmp expect sequence && + test_cmp expect scruples + ) +' + +# +# In the following testcase: +# Base: numbers_1 +# Upstream: rename numbers_1 -> sequence_2 +# Topic_1: numbers_3 +# Topic_2: numbers_1 +# or, in english, the first commit on the topic branch modifies numbers by +# shrinking it (dramatically) and the second commit on topic reverts its +# parent. +# +# Can git apply both patches? +# +# Traditional cherry-pick/rebase will fail to apply the second commit, the +# one that reverted its parent, because despite detecting the rename from +# 'numbers' to 'sequence' for the first commit, it fails to detect that +# rename when picking the second commit. That's "reasonable" given the +# dramatic change in size of the file, but remembering the rename and +# reusing it is reasonable too. +# +# We do test here that we expect rename detection to only be run once total +# (the topic side of history doesn't need renames, and with caching we +# should be able to only run rename detection on the upstream side one +# time.) +test_expect_success 'cherry-pick both a commit and its immediate revert' ' + test_create_repo pick-commit-and-its-immediate-revert && + ( + cd pick-commit-and-its-immediate-revert && + + test_seq 11 30 >numbers && + git add numbers && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 30 >numbers && + git add numbers && + git mv numbers sequence && + git commit -m "Renamed (and modified) numbers -> sequence" && + + git switch topic && + + test_seq 11 13 >numbers && + git add numbers && + git commit -m A && + + git revert HEAD && + + # + # Actual testing + # + + git switch upstream && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 1 calls + ) +' + +# +# In the following testcase: +# Base: sequence_1 +# Upstream: rename sequence_1 -> values_2 +# Topic_1: rename sequence_1 -> values_3 +# Topic_2: add unrelated sequence_4 +# or, in english, both sides rename sequence -> values, and then the second +# commit on the topic branch adds an unrelated file called sequence. +# +# This testcase presents no problems for git traditionally, but having both +# sides do the same rename in effect "uses it up" and if it remains cached, +# could cause a spurious rename/add conflict. +# +test_expect_success 'rename same file identically, then reintroduce it' ' + test_create_repo rename-rename-1to1-then-add-old-filename && + ( + cd rename-rename-1to1-then-add-old-filename && + + test_seq 3 8 >sequence && + git add sequence && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >sequence && + git add sequence && + git mv sequence values && + git commit -m "Renamed (and modified) sequence -> values" && + + git switch topic && + + test_seq 3 10 >sequence && + git add sequence && + git mv sequence values && + git commit -m A && + + test_write_lines A B C D E F G H I J >sequence && + git add sequence && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + git ls-files >tracked && + test_line_count = 2 tracked && + test_path_is_file values && + test_path_is_file sequence && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls + ) +' + +# +# In the following testcase: +# Base: olddir/{valuesZ_1, valuesY_1, valuesX_1} +# Upstream: rename olddir/valuesZ_1 -> dirA/valuesZ_2 +# rename olddir/valuesY_1 -> dirA/valuesY_2 +# rename olddir/valuesX_1 -> dirB/valuesX_2 +# Topic_1: rename olddir/valuesZ_1 -> dirA/valuesZ_3 +# rename olddir/valuesY_1 -> dirA/valuesY_3 +# Topic_2: add olddir/newfile +# Expected Pick1: dirA/{valuesZ, valuesY}, dirB/valuesX +# Expected Pick2: dirA/{valuesZ, valuesY}, dirB/{valuesX, newfile} +# +# This testcase presents no problems for git traditionally, but having both +# sides do the same renames in effect "use it up" but if the renames remain +# cached, the directory rename could put newfile in the wrong directory. +# +test_expect_success 'rename same file identically, then add file to old dir' ' + test_create_repo rename-rename-1to1-then-add-file-to-old-dir && + ( + cd rename-rename-1to1-then-add-file-to-old-dir && + + mkdir olddir/ && + test_seq 3 8 >olddir/valuesZ && + test_seq 3 8 >olddir/valuesY && + test_seq 3 8 >olddir/valuesX && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >olddir/valuesZ && + test_seq 1 8 >olddir/valuesY && + test_seq 1 8 >olddir/valuesX && + git add olddir && + mkdir dirA && + git mv olddir/valuesZ olddir/valuesY dirA && + git mv olddir/ dirB/ && + git commit -m "Renamed (and modified) values*" && + + git switch topic && + + test_seq 3 10 >olddir/valuesZ && + test_seq 3 10 >olddir/valuesY && + git add olddir && + mkdir dirA && + git mv olddir/valuesZ olddir/valuesY dirA && + git commit -m A && + + >olddir/newfile && + git add olddir/newfile && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + git ls-files >tracked && + test_line_count = 4 tracked && + test_path_is_file dirA/valuesZ && + test_path_is_file dirA/valuesY && + test_path_is_file dirB/valuesX && + test_path_is_file dirB/newfile && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls + ) +' + +# +# In the following testcase, upstream renames a directory, and the topic branch +# first adds a file to the directory, then later renames the directory +# differently: +# Base: olddir/a +# olddir/b +# Upstream: rename olddir/ -> newdir/ +# Topic_1: add olddir/newfile +# Topic_2: rename olddir/ -> otherdir/ +# +# Here we are just concerned that cached renames might prevent us from seeing +# the rename conflict, and we want to ensure that we do get a conflict. +# +# While at it, though, we do test that we only try to detect renames 2 +# times and not three. (The first merge needs to detect renames on the +# upstream side. Traditionally, the second merge would need to detect +# renames on both sides of history, but our caching of upstream renames +# should avoid the need to re-detect upstream renames.) +# +test_expect_success 'cached dir rename does not prevent noticing later conflict' ' + test_create_repo dir-rename-cache-not-occluding-later-conflict && + ( + cd dir-rename-cache-not-occluding-later-conflict && + + mkdir olddir && + test_seq 3 10 >olddir/a && + test_seq 3 10 >olddir/b && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 3 10 >olddir/a && + test_seq 3 10 >olddir/b && + git add olddir && + git mv olddir newdir && + git commit -m "Dir renamed" && + + git switch topic && + + >olddir/newfile && + git add olddir/newfile && + git commit -m A && + + test_seq 1 8 >olddir/a && + test_seq 1 8 >olddir/b && + git add olddir && + git mv olddir otherdir && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test_must_fail test-tool fast-rebase --onto HEAD upstream~1 topic >output && + #git cherry-pick upstream..topic && + + grep CONFLICT..rename/rename output && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls + ) +' + +# Helper for the next two tests +test_setup_upstream_rename () { + test_create_repo $1 && + ( + cd $1 && + + test_seq 3 8 >somefile && + test_seq 3 8 >relevant-rename && + git add somefile relevant-rename && + mkdir olddir && + test_write_lines a b c d e f g >olddir/a && + test_write_lines z y x w v u t >olddir/b && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >somefile && + test_seq 1 8 >relevant-rename && + git add somefile relevant-rename && + git mv relevant-rename renamed && + echo h >>olddir/a && + echo s >>olddir/b && + git add olddir && + git mv olddir newdir && + git commit -m "Dir renamed" + ) +} + +# +# In the following testcase, upstream renames a file in the toplevel directory +# as well as its only directory: +# Base: relevant-rename_1 +# somefile +# olddir/a +# olddir/b +# Upstream: rename relevant-rename_1 -> renamed_2 +# rename olddir/ -> newdir/ +# Topic_1: relevant-rename_3 +# Topic_2: olddir/newfile_1 +# Topic_3: olddir/newfile_2 +# +# In this testcase, since the first commit being picked only modifies a +# file in the toplevel directory, the directory rename is irrelevant for +# that first merge. However, we need to notice the directory rename for +# the merge that picks the second commit, and we don't want the third +# commit to mess up its location either. We want to make sure that +# olddir/newfile doesn't exist in the result and that newdir/newfile does. +# +# We also test that we only do rename detection twice. We never need +# rename detection on the topic side of history, but we do need it twice on +# the upstream side of history. For the first topic commit, we only need +# the +# relevant-rename -> renamed +# rename, because olddir is unmodified by Topic_1. For Topic_2, however, +# the new file being added to olddir means files that were previously +# irrelevant for rename detection are now relevant, forcing us to repeat +# rename detection for the paths we don't already have cached. Topic_3 also +# tweaks olddir/newfile, but the renames in olddir/ will have been cached +# from the second rename detection run. +# +test_expect_success 'dir rename unneeded, then add new file to old dir' ' + test_setup_upstream_rename dir-rename-unneeded-until-new-file && + ( + cd dir-rename-unneeded-until-new-file && + + git switch topic && + + test_seq 3 10 >relevant-rename && + git add relevant-rename && + git commit -m A && + + echo foo >olddir/newfile && + git add olddir/newfile && + git commit -m B && + + echo bar >>olddir/newfile && + git add olddir/newfile && + git commit -m C && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls && + + git ls-files >tracked && + test_line_count = 5 tracked && + test_path_is_missing olddir/newfile && + test_path_is_file newdir/newfile + ) +' + +# +# The following testcase is *very* similar to the last one, but instead of +# adding a new olddir/newfile, it renames somefile -> olddir/newfile: +# Base: relevant-rename_1 +# somefile_1 +# olddir/a +# olddir/b +# Upstream: rename relevant-rename_1 -> renamed_2 +# rename olddir/ -> newdir/ +# Topic_1: relevant-rename_3 +# Topic_2: rename somefile -> olddir/newfile_2 +# Topic_3: modify olddir/newfile_3 +# +# In this testcase, since the first commit being picked only modifies a +# file in the toplevel directory, the directory rename is irrelevant for +# that first merge. However, we need to notice the directory rename for +# the merge that picks the second commit, and we don't want the third +# commit to mess up its location either. We want to make sure that +# neither somefile or olddir/newfile exists in the result and that +# newdir/newfile does. +# +# This testcase needs one more call to rename detection than the last +# testcase, because of the somefile -> olddir/newfile rename in Topic_2. +test_expect_success 'dir rename unneeded, then rename existing file into old dir' ' + test_setup_upstream_rename dir-rename-unneeded-until-file-moved-inside && + ( + cd dir-rename-unneeded-until-file-moved-inside && + + git switch topic && + + test_seq 3 10 >relevant-rename && + git add relevant-rename && + git commit -m A && + + test_seq 1 10 >somefile && + git add somefile && + git mv somefile olddir/newfile && + git commit -m B && + + test_seq 1 12 >olddir/newfile && + git add olddir/newfile && + git commit -m C && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls && + + test_path_is_missing somefile && + test_path_is_missing olddir/newfile && + test_path_is_file newdir/newfile && + git ls-files >tracked && + test_line_count = 4 tracked + ) +' + +# Helper for the next two tests +test_setup_topic_rename () { + test_create_repo $1 && + ( + cd $1 && + + test_seq 3 8 >somefile && + mkdir olddir && + test_seq 3 8 >olddir/a && + echo b >olddir/b && + git add olddir somefile && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch topic && + test_seq 1 8 >somefile && + test_seq 1 8 >olddir/a && + git add somefile olddir/a && + git mv olddir newdir && + git commit -m "Dir renamed" && + + test_seq 1 10 >somefile && + git add somefile && + mkdir olddir && + >olddir/unrelated-file && + git add olddir && + git commit -m "Unrelated file in recreated old dir" + ) +} + +# +# In the following testcase, the first commit on the topic branch renames +# a directory, while the second recreates the old directory and places a +# file into it: +# Base: somefile +# olddir/a +# olddir/b +# Upstream: olddir/newfile +# Topic_1: somefile_2 +# rename olddir/ -> newdir/ +# Topic_2: olddir/unrelated-file +# +# Note that the first pick should merge: +# Base: somefile +# olddir/{a,b} +# Upstream: olddir/newfile +# Topic_1: rename olddir/ -> newdir/ +# For which the expected result (assuming merge.directoryRenames=true) is +# clearly: +# Result: somefile +# newdir/{a, b, newfile} +# +# While the second pick does the following three-way merge: +# Base (Topic_1): somefile +# newdir/{a,b} +# Upstream (Result from 1): same files as base, but adds newdir/newfile +# Topic_2: same files as base, but adds olddir/unrelated-file +# +# The second merge is pretty trivial; upstream adds newdir/newfile, and +# topic_2 adds olddir/unrelated-file. We're just testing that we don't +# accidentally cache directory renames somehow and rename +# olddir/unrelated-file to newdir/unrelated-file. +# +# This testcase should only need one call to diffcore_rename_extended(). +test_expect_success 'caching renames only on upstream side, part 1' ' + test_setup_topic_rename cache-renames-only-upstream-add-file && + ( + cd cache-renames-only-upstream-add-file && + + git switch upstream && + + >olddir/newfile && + git add olddir/newfile && + git commit -m "Add newfile" && + + # + # Actual testing + # + + git switch upstream && + + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 1 calls && + + git ls-files >tracked && + test_line_count = 5 tracked && + test_path_is_missing newdir/unrelated-file && + test_path_is_file olddir/unrelated-file && + test_path_is_file newdir/newfile && + test_path_is_file newdir/b && + test_path_is_file newdir/a && + test_path_is_file somefile + ) +' + +# +# The following testcase is *very* similar to the last one, but instead of +# adding a new olddir/newfile, it renames somefile -> olddir/newfile: +# Base: somefile +# olddir/a +# olddir/b +# Upstream: somefile_1 -> olddir/newfile +# Topic_1: rename olddir/ -> newdir/ +# somefile_2 +# Topic_2: olddir/unrelated-file +# somefile_3 +# +# Much like the previous test, this case is actually trivial and we are just +# making sure there isn't some spurious directory rename caching going on +# for the wrong side of history. +# +# +# This testcase should only need two calls to diffcore_rename_extended(), +# both for the first merge, one for each side of history. +# +test_expect_success 'caching renames only on upstream side, part 2' ' + test_setup_topic_rename cache-renames-only-upstream-rename-file && + ( + cd cache-renames-only-upstream-rename-file && + + git switch upstream && + + git mv somefile olddir/newfile && + git commit -m "Add newfile" && + + # + # Actual testing + # + + git switch upstream && + + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls && + + git ls-files >tracked && + test_line_count = 4 tracked && + test_path_is_missing newdir/unrelated-file && + test_path_is_file olddir/unrelated-file && + test_path_is_file newdir/newfile && + test_path_is_file newdir/b && + test_path_is_file newdir/a + ) +' + +test_done diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 3b7540050c..30eff725a9 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -2167,6 +2167,37 @@ test_expect_success $PREREQ 'leading and trailing whitespaces are removed' ' test_cmp expected-list actual-list ' +test_expect_success $PREREQ 'test using command name with --sendmail-cmd' ' + clean_fake_sendmail && + PATH="$(pwd):$PATH" \ + git send-email \ + --from="Example <nobody@example.com>" \ + --to=nobody@example.com \ + --sendmail-cmd="fake.sendmail" \ + HEAD^ && + test_path_is_file commandline1 +' + +test_expect_success $PREREQ 'test using arguments with --sendmail-cmd' ' + clean_fake_sendmail && + git send-email \ + --from="Example <nobody@example.com>" \ + --to=nobody@example.com \ + --sendmail-cmd='\''"$(pwd)/fake.sendmail" -f nobody@example.com'\'' \ + HEAD^ && + test_path_is_file commandline1 +' + +test_expect_success $PREREQ 'test shell expression with --sendmail-cmd' ' + clean_fake_sendmail && + git send-email \ + --from="Example <nobody@example.com>" \ + --to=nobody@example.com \ + --sendmail-cmd='\''f() { "$(pwd)/fake.sendmail" "$@"; };f'\'' \ + HEAD^ && + test_path_is_file commandline1 +' + test_expect_success $PREREQ 'invoke hook' ' mkdir -p .git/hooks && diff --git a/t/t9100-git-svn-basic.sh b/t/t9100-git-svn-basic.sh index 1d3fdcc997..fea41b3c36 100755 --- a/t/t9100-git-svn-basic.sh +++ b/t/t9100-git-svn-basic.sh @@ -4,21 +4,13 @@ # test_description='git svn basic tests' -GIT_SVN_LC_ALL=${LC_ALL:-$LANG} GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./lib-git-svn.sh -case "$GIT_SVN_LC_ALL" in -*.UTF-8) - test_set_prereq UTF8 - ;; -*) - say "# UTF-8 locale not set, some tests skipped ($GIT_SVN_LC_ALL)" - ;; -esac +prepare_utf8_locale test_expect_success 'git svn --version works anywhere' ' nongit git svn --version @@ -187,8 +179,8 @@ test_expect_success POSIXPERM,SYMLINKS "$name" ' test ! -h "$SVN_TREE"/exec-2.sh && test_cmp help "$SVN_TREE"/exec-2.sh' -name="commit with UTF-8 message: locale: $GIT_SVN_LC_ALL" -LC_ALL="$GIT_SVN_LC_ALL" +name="commit with UTF-8 message: locale: $GIT_TEST_UTF8_LOCALE" +LC_ALL="$GIT_TEST_UTF8_LOCALE" export LC_ALL # This test relies on the previous test, hence requires POSIXPERM,SYMLINKS test_expect_success UTF8,POSIXPERM,SYMLINKS "$name" " @@ -330,7 +322,7 @@ test_expect_success 'git-svn works in a bare repository' ' git svn fetch ) && rm -rf bare-repo ' -test_expect_success 'git-svn works in in a repository with a gitdir: link' ' +test_expect_success 'git-svn works in a repository with a gitdir: link' ' mkdir worktree gitdir && ( cd worktree && git svn init "$svnrepo" && diff --git a/t/t9115-git-svn-dcommit-funky-renames.sh b/t/t9115-git-svn-dcommit-funky-renames.sh index 9b44a44bc1..743fbe1fe4 100755 --- a/t/t9115-git-svn-dcommit-funky-renames.sh +++ b/t/t9115-git-svn-dcommit-funky-renames.sh @@ -93,9 +93,9 @@ test_expect_success 'git svn rebase works inside a fresh-cloned repository' ' # > ... All of the above characters, except for the backslash, are converted # > to special UNICODE characters in the range 0xf000 to 0xf0ff (the # > "Private use area") when creating or accessing files. -prepare_a_utf8_locale +prepare_utf8_locale test_expect_success UTF8,!MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 new file on dcommit' ' - LC_ALL=$a_utf8_locale && + LC_ALL=$GIT_TEST_UTF8_LOCALE && export LC_ALL && neq=$(printf "\201\202") && git config svn.pathnameencoding cp932 && @@ -107,7 +107,7 @@ test_expect_success UTF8,!MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 new # See the comment on the above test for setting of LC_ALL. test_expect_success !MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 rename on dcommit' ' - LC_ALL=$a_utf8_locale && + LC_ALL=$GIT_TEST_UTF8_LOCALE && export LC_ALL && inf=$(printf "\201\207") && git config svn.pathnameencoding cp932 && diff --git a/t/t9129-git-svn-i18n-commitencoding.sh b/t/t9129-git-svn-i18n-commitencoding.sh index 2c213ae654..01e1e8a8f7 100755 --- a/t/t9129-git-svn-i18n-commitencoding.sh +++ b/t/t9129-git-svn-i18n-commitencoding.sh @@ -14,12 +14,12 @@ compare_git_head_with () { test_cmp current "$1" } -prepare_a_utf8_locale +prepare_utf8_locale compare_svn_head_with () { # extract just the log message and strip out committer info. # don't use --limit here since svn 1.1.x doesn't have it, - LC_ALL="$a_utf8_locale" svn log $(git svn info --url) | perl -w -e ' + LC_ALL="$GIT_TEST_UTF8_LOCALE" svn log $(git svn info --url) | perl -w -e ' use bytes; $/ = ("-"x72) . "\n"; my @x = <STDIN>; diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index b823c14027..f0448daa74 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -172,12 +172,23 @@ debug () { # --notick # Do not call test_tick before making a commit # --append -# Use "echo >>" instead of "echo >" when writing "<contents>" to -# "<file>" +# Use ">>" instead of ">" when writing "<contents>" to "<file>" +# --printf +# Use "printf" instead of "echo" when writing "<contents>" to +# "<file>", use this to write escape sequences such as "\0", a +# trailing "\n" won't be added automatically. This option +# supports nothing but the FORMAT of printf(1), i.e. no custom +# ARGUMENT(s). # --signoff # Invoke "git commit" with --signoff # --author <author> # Invoke "git commit" with --author <author> +# --no-tag +# Do not tag the resulting commit +# --annotate +# Create an annotated tag with "--annotate -m <message>". Calls +# test_tick between making the commit and tag, unless --notick +# is given. # # This will commit a file with the given contents and the given commit # message, and tag the resulting commit with the given tag name. @@ -186,17 +197,21 @@ debug () { test_commit () { notick= && + echo=echo && append= && author= && signoff= && indir= && - no_tag= && + tag=light && while test $# != 0 do case "$1" in --notick) notick=yes ;; + --printf) + echo=printf + ;; --append) append=yes ;; @@ -218,7 +233,10 @@ test_commit () { shift ;; --no-tag) - no_tag=yes + tag=none + ;; + --annotate) + tag=annotate ;; *) break @@ -230,9 +248,9 @@ test_commit () { file=${2:-"$1.t"} && if test -n "$append" then - echo "${3-$1}" >>"$indir$file" + $echo "${3-$1}" >>"$indir$file" else - echo "${3-$1}" >"$indir$file" + $echo "${3-$1}" >"$indir$file" fi && git ${indir:+ -C "$indir"} add "$file" && if test -z "$notick" @@ -242,10 +260,20 @@ test_commit () { git ${indir:+ -C "$indir"} commit \ ${author:+ --author "$author"} \ $signoff -m "$1" && - if test -z "$no_tag" - then + case "$tag" in + none) + ;; + light) git ${indir:+ -C "$indir"} tag "${4:-$1}" - fi + ;; + annotate) + if test -z "$notick" + then + test_tick + fi && + git ${indir:+ -C "$indir"} tag -a -m "$1" "${4:-$1}" + ;; + esac } # Call test_merge with the arguments "<message> <commit>", where <commit> @@ -1215,22 +1243,10 @@ test_atexit () { } && (exit \"\$eval_ret\"); eval_ret=\$?; $test_atexit_cleanup" } -# Most tests can use the created repository, but some may need to create more. +# Deprecated wrapper for "git init", use "git init" directly instead # Usage: test_create_repo <directory> test_create_repo () { - test "$#" = 1 || - BUG "not 1 parameter to test-create-repo" - repo="$1" - mkdir -p "$repo" - ( - cd "$repo" || error "Cannot setup test environment" - "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" -c \ - init.defaultBranch="${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" \ - init \ - "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || - error "cannot run git init -- have you built things yet?" - mv .git/hooks .git/hooks-disabled - ) || exit + git init "$@" } # This function helps on symlink challenged file systems when it is not diff --git a/t/test-lib.sh b/t/test-lib.sh index adaf03543e..54938c6427 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -64,6 +64,11 @@ then export GIT_TEST_DISALLOW_ABBREVIATED_OPTIONS fi +# Explicitly set the default branch name for testing, to avoid the +# transitory "git init" warning under --verbose. +: ${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME:=master} +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + ################################################################ # It appears that people try to run tests without building... "${GIT_TEST_INSTALLED:-$GIT_BUILD_DIR}/git$X" >/dev/null @@ -1172,7 +1177,7 @@ test_done () { esac fi - if test -z "$debug" + if test -z "$debug" && test -n "$remove_trash" then test -d "$TRASH_DIRECTORY" || error "Tests passed but trash directory already removed before test cleanup; aborting" @@ -1337,6 +1342,22 @@ then exit 1 fi +# Are we running this test at all? +remove_trash= +this_test=${0##*/} +this_test=${this_test%%-*} +if match_pattern_list "$this_test" $GIT_SKIP_TESTS +then + say_color info >&3 "skipping test $this_test altogether" + skip_all="skip all tests in $this_test" + test_done +fi + +# Last-minute variable setup +HOME="$TRASH_DIRECTORY" +GNUPGHOME="$HOME/gnupg-home-not-used" +export HOME GNUPGHOME + # Test repository rm -fr "$TRASH_DIRECTORY" || { GIT_EXIT_OK=t @@ -1344,13 +1365,11 @@ rm -fr "$TRASH_DIRECTORY" || { exit 1 } -HOME="$TRASH_DIRECTORY" -GNUPGHOME="$HOME/gnupg-home-not-used" -export HOME GNUPGHOME - +remove_trash=t if test -z "$TEST_NO_CREATE_REPO" then - test_create_repo "$TRASH_DIRECTORY" + git init "$TRASH_DIRECTORY" >&3 2>&4 || + error "cannot run git init" else mkdir -p "$TRASH_DIRECTORY" fi @@ -1359,15 +1378,6 @@ fi # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 -this_test=${0##*/} -this_test=${this_test%%-*} -if match_pattern_list "$this_test" $GIT_SKIP_TESTS -then - say_color info >&3 "skipping test $this_test altogether" - skip_all="skip all tests in $this_test" - test_done -fi - if test -n "$write_junit_xml" then junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index ae052a07fe..bda283e7f4 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -204,15 +204,16 @@ static int tr2_dst_try_uds_connect(const char *path, int sock_type, int *out_fd) fd = socket(AF_UNIX, sock_type, 0); if (fd == -1) - return errno; + return -1; sa.sun_family = AF_UNIX; strlcpy(sa.sun_path, path, sizeof(sa.sun_path)); if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) == -1) { - int e = errno; + int saved_errno = errno; close(fd); - return e; + errno = saved_errno; + return -1; } *out_fd = fd; @@ -227,7 +228,6 @@ static int tr2_dst_try_unix_domain_socket(struct tr2_dst *dst, { unsigned int uds_try = 0; int fd; - int e; const char *path = NULL; /* @@ -271,15 +271,13 @@ static int tr2_dst_try_unix_domain_socket(struct tr2_dst *dst, } if (uds_try & TR2_DST_UDS_TRY_STREAM) { - e = tr2_dst_try_uds_connect(path, SOCK_STREAM, &fd); - if (!e) + if (!tr2_dst_try_uds_connect(path, SOCK_STREAM, &fd)) goto connected; - if (e != EPROTOTYPE) + if (errno != EPROTOTYPE) goto error; } if (uds_try & TR2_DST_UDS_TRY_DGRAM) { - e = tr2_dst_try_uds_connect(path, SOCK_DGRAM, &fd); - if (!e) + if (!tr2_dst_try_uds_connect(path, SOCK_DGRAM, &fd)) goto connected; } @@ -287,7 +285,7 @@ error: if (tr2_dst_want_warning()) warning("trace2: could not connect to socket '%s' for '%s' tracing: %s", path, tr2_sysenv_display_name(dst->sysenv_var), - strerror(e)); + strerror(errno)); tr2_dst_trace_disable(dst); return 0; diff --git a/transport.c b/transport.c index 6cf3da19eb..50f5830eb6 100644 --- a/transport.c +++ b/transport.c @@ -427,7 +427,8 @@ static int fetch_refs_via_pack(struct transport *transport, cleanup: close(data->fd[0]); - close(data->fd[1]); + if (data->fd[1] >= 0) + close(data->fd[1]); if (finish_connect(data->conn)) ret = -1; data->conn = NULL; @@ -869,7 +870,8 @@ static int disconnect_git(struct transport *transport) if (data->got_remote_heads && !transport->stateless_rpc) packet_flush(data->fd[1]); close(data->fd[0]); - close(data->fd[1]); + if (data->fd[1] >= 0) + close(data->fd[1]); finish_connect(data->conn); } |