diff options
438 files changed, 14420 insertions, 1492 deletions
diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml index 8c4358d805..ad3466ad16 100644 --- a/.github/workflows/check-whitespace.yml +++ b/.github/workflows/check-whitespace.yml @@ -1,8 +1,9 @@ name: check-whitespace -# Get the repo with the commits(+1) in the series. +# Get the repository with all commits to ensure that we can analyze +# all of the commits contributed via the Pull Request. # Process `git log --check` output to extract just the check errors. -# Add a comment to the pull request with the check errors. +# Exit with failure upon white-space issues. on: pull_request: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ed6a9e807..c35200defb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: CI/PR +name: CI on: [push, pull_request] @@ -7,6 +7,7 @@ env: jobs: ci-config: + name: config runs-on: ubuntu-latest outputs: enabled: ${{ steps.check-ref.outputs.enabled }}${{ steps.skip-if-redundant.outputs.enabled }} @@ -77,6 +78,7 @@ jobs: } windows-build: + name: win build needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' runs-on: windows-latest @@ -97,6 +99,7 @@ jobs: name: windows-artifacts path: artifacts windows-test: + name: win test runs-on: windows-latest needs: [windows-build] strategy: @@ -127,6 +130,7 @@ jobs: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} vs-build: + name: win+VS build needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' env: @@ -178,6 +182,7 @@ jobs: name: vs-artifacts path: artifacts vs-test: + name: win+VS test runs-on: windows-latest needs: vs-build strategy: @@ -210,6 +215,7 @@ jobs: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} regular: + name: ${{matrix.vector.jobname}} (${{matrix.vector.pool}}) needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' strategy: @@ -219,14 +225,25 @@ jobs: - jobname: linux-clang cc: clang pool: ubuntu-latest + - jobname: linux-sha256 + cc: clang + os: ubuntu + pool: ubuntu-latest - jobname: linux-gcc cc: gcc + cc_package: gcc-8 + pool: ubuntu-latest + - jobname: linux-TEST-vars + cc: gcc + os: ubuntu + cc_package: gcc-8 pool: ubuntu-latest - jobname: osx-clang cc: clang pool: macos-latest - jobname: osx-gcc cc: gcc + cc_package: gcc-9 pool: macos-latest - jobname: linux-gcc-default cc: gcc @@ -236,7 +253,9 @@ jobs: pool: ubuntu-latest env: CC: ${{matrix.vector.cc}} + CC_PACKAGE: ${{matrix.vector.cc_package}} jobname: ${{matrix.vector.jobname}} + runs_on_pool: ${{matrix.vector.pool}} runs-on: ${{matrix.vector.pool}} steps: - uses: actions/checkout@v2 @@ -251,6 +270,7 @@ jobs: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} dockerized: + name: ${{matrix.vector.jobname}} (${{matrix.vector.image}}) needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' strategy: @@ -259,7 +279,8 @@ jobs: vector: - jobname: linux-musl image: alpine - - jobname: Linux32 + - jobname: linux32 + os: ubuntu32 image: daald/ubuntu32:xenial - jobname: pedantic image: fedora @@ -289,6 +310,7 @@ jobs: - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh + - run: ci/check-directional-formatting.bash sparse: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' @@ -310,6 +332,7 @@ jobs: run: ci/install-dependencies.sh - run: make sparse documentation: + name: documentation needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' env: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 908330a0a3..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,60 +0,0 @@ -language: c - -cache: - directories: - - $HOME/travis-cache - -os: - - linux - - osx - -osx_image: xcode10.1 - -compiler: - - clang - - gcc - -matrix: - include: - - env: jobname=linux-gcc-default - os: linux - compiler: - addons: - before_install: - - env: jobname=linux-gcc-4.8 - os: linux - dist: trusty - compiler: - - env: jobname=Linux32 - os: linux - compiler: - addons: - services: - - docker - before_install: - script: ci/run-docker.sh - - env: jobname=linux-musl - os: linux - compiler: - addons: - services: - - docker - before_install: - script: ci/run-docker.sh - - env: jobname=StaticAnalysis - os: linux - compiler: - script: ci/run-static-analysis.sh - after_failure: - - env: jobname=Documentation - os: linux - compiler: - script: ci/test-documentation.sh - after_failure: - -before_install: ci/install-dependencies.sh -script: ci/run-build-and-tests.sh -after_failure: ci/print-test-failures.sh - -notifications: - email: false diff --git a/Documentation/CodingGuidelines b/Documentation/CodingGuidelines index 711cb9171e..0e27b5395d 100644 --- a/Documentation/CodingGuidelines +++ b/Documentation/CodingGuidelines @@ -499,6 +499,33 @@ For Python scripts: - Where required libraries do not restrict us to Python 2, we try to also be compatible with Python 3.1 and later. + +Program Output + + We make a distinction between a Git command's primary output and + output which is merely chatty feedback (for instance, status + messages, running transcript, or progress display), as well as error + messages. Roughly speaking, a Git command's primary output is that + which one might want to capture to a file or send down a pipe; its + chatty output should not interfere with these use-cases. + + As such, primary output should be sent to the standard output stream + (stdout), and chatty output should be sent to the standard error + stream (stderr). Examples of commands which produce primary output + include `git log`, `git show`, and `git branch --list` which generate + output on the stdout stream. + + Not all Git commands have primary output; this is often true of + commands whose main function is to perform an action. Some action + commands are silent, whereas others are chatty. An example of a + chatty action commands is `git clone` with its "Cloning into + '<path>'..." and "Checking connectivity..." status messages which it + sends to the stderr stream. + + Error messages from Git commands should always be sent to the stderr + stream. + + Error Messages - Do not end error messages with a full stop. diff --git a/Documentation/MyFirstContribution.txt b/Documentation/MyFirstContribution.txt index b20bc8e914..63a2ef5449 100644 --- a/Documentation/MyFirstContribution.txt +++ b/Documentation/MyFirstContribution.txt @@ -905,19 +905,34 @@ Sending emails with Git is a two-part process; before you can prepare the emails themselves, you'll need to prepare the patches. Luckily, this is pretty simple: ---- -$ git format-patch --cover-letter -o psuh/ master..psuh ----- - -The `--cover-letter` parameter tells `format-patch` to create a cover letter -template for you. You will need to fill in the template before you're ready -to send - but for now, the template will be next to your other patches. - -The `-o psuh/` parameter tells `format-patch` to place the patch files into a -directory. This is useful because `git send-email` can take a directory and -send out all the patches from there. - -`master..psuh` tells `format-patch` to generate patches for the difference -between `master` and `psuh`. It will make one patch file per commit. After you +$ git format-patch --cover-letter -o psuh/ --base=auto psuh@{u}..psuh +---- + + . The `--cover-letter` option tells `format-patch` to create a + cover letter template for you. You will need to fill in the + template before you're ready to send - but for now, the template + will be next to your other patches. + + . The `-o psuh/` option tells `format-patch` to place the patch + files into a directory. This is useful because `git send-email` + can take a directory and send out all the patches from there. + + . The `--base=auto` option tells the command to record the "base + commit", on which the recipient is expected to apply the patch + series. The `auto` value will cause `format-patch` to compute + the base commit automatically, which is the merge base of tip + commit of the remote-tracking branch and the specified revision + range. + + . The `psuh@{u}..psuh` option tells `format-patch` to generate + patches for the commits you created on the `psuh` branch since it + forked from its upstream (which is `origin/master` if you + followed the example in the "Set up your workspace" section). If + you are already on the `psuh` branch, you can just say `@{u}`, + which means "commits on the current branch since it forked from + its upstream", which is the same thing. + +The command will make one patch file per commit. After you run, you can go have a look at each of the patches with your favorite text editor and make sure everything looks alright; however, it's not recommended to make code fixups via the patch file. It's a better idea to make the change the diff --git a/Documentation/MyFirstObjectWalk.txt b/Documentation/MyFirstObjectWalk.txt index 45eb84d8b4..ca267941f3 100644 --- a/Documentation/MyFirstObjectWalk.txt +++ b/Documentation/MyFirstObjectWalk.txt @@ -58,14 +58,19 @@ running, enable trace output by setting the environment variable `GIT_TRACE`. Add usage text and `-h` handling, like all subcommands should consistently do (our test suite will notice and complain if you fail to do so). +We'll need to include the `parse-options.h` header. ---- +#include "parse-options.h" + +... + int cmd_walken(int argc, const char **argv, const char *prefix) { const char * const walken_usage[] = { N_("git walken"), NULL, - } + }; struct option options[] = { OPT_END() }; @@ -195,9 +200,14 @@ Similarly to the default values, we don't have anything to do here yet ourselves; however, we should call `git_default_config()` if we aren't calling any other existing config callbacks. -Add a new function to `builtin/walken.c`: +Add a new function to `builtin/walken.c`. +We'll also need to include the `config.h` header: ---- +#include "config.h" + +... + static int git_walken_config(const char *var, const char *value, void *cb) { /* @@ -229,8 +239,14 @@ typically done by calling `repo_init_revisions()` with the repository you intend to target, as well as the `prefix` argument of `cmd_walken` and your `rev_info` struct. -Add the `struct rev_info` and the `repo_init_revisions()` call: +Add the `struct rev_info` and the `repo_init_revisions()` call. +We'll also need to include the `revision.h` header: + ---- +#include "revision.h" + +... + int cmd_walken(int argc, const char **argv, const char *prefix) { /* This can go wherever you like in your declarations.*/ @@ -624,9 +640,14 @@ static void walken_object_walk(struct rev_info *rev) ---- Let's start by calling just the unfiltered walk and reporting our counts. -Complete your implementation of `walken_object_walk()`: +Complete your implementation of `walken_object_walk()`. +We'll also need to include the `list-objects.h` header. ---- +#include "list-objects.h" + +... + traverse_commit_list(rev, walken_show_commit, walken_show_object, NULL); printf("commits %d\nblobs %d\ntags %d\ntrees %d\n", commit_count, @@ -697,7 +718,7 @@ First, we'll need to `#include "list-objects-filter-options.h"` and set up the ---- static void walken_object_walk(struct rev_info *rev) { - struct list_objects_filter_options filter_options = {}; + struct list_objects_filter_options filter_options = { 0 }; ... ---- diff --git a/Documentation/RelNotes/2.35.0.txt b/Documentation/RelNotes/2.35.0.txt new file mode 100644 index 0000000000..34b5ffedfd --- /dev/null +++ b/Documentation/RelNotes/2.35.0.txt @@ -0,0 +1,164 @@ +Git 2.35 Release Notes +====================== + +Updates since Git 2.34 +---------------------- + +Backward compatibility warts + + * "_" is now treated as any other URL-valid characters in an URL when + matching the per-URL configuration variable names. + + +UI, Workflows & Features + + * "git status --porcelain=v2" now show the number of stash entries + with --show-stash like the normal output does. + + * "git stash" learned the "--staged" option to stash away what has + been added to the index (and nothing else). + + * "git var GIT_DEFAULT_BRANCH" is a way to see what name is used for + the newly created branch if "git init" is run. + + * Various operating modes of "git reset" have been made to work + better with the sparse index. + + * "git submodule deinit" for a submodule whose .git metadata + directory is embedded in its working tree refused to work, until + the submodule gets converted to use the "absorbed" form where the + metadata directory is stored in superproject, and a gitfile at the + top-level of the working tree of the submodule points at it. The + command is taught to convert such submodules to the absorbed form + as needed. + + * The completion script (in contrib/) learns that the "--date" + option of commands from the "git log" family takes "human" and + "auto" as valid values. + + +Performance, Internal Implementation, Development Support etc. + + * The use of errno as a means to carry the nature of error in the ref + API implementation has been reworked and reduced. + + * Teach and encourage first-time contributors to this project to + state the base commit when they submit their topic. + + * The command line complation for "git send-email" options have been + tweaked to make it easier to keep it in sync with the command itself. + + + * Ensure that the sparseness of the in-core index matches the + index.sparse configuration specified by the repository immediately + after the on-disk index file is read. + + * Code clean-up to eventually allow information on remotes defined + for an arbitrary repository to be read. + + * Build optimization. + + * Tighten code for testing pack-bitmap. + + * Weather balloon to break people with compilers that do not support + C99. + + +Fixes since v2.34 +----------------- + + * "git grep" looking in a blob that has non-UTF8 payload was + completely broken when linked with certain versions of PCREv2 + library in the latest release. + + * Other code cleanup, docfix, build fix, etc. + + * "git pull" with any strategy when the other side is behind us + should succeed as it is a no-op, but doesn't. + + * An earlier change in 2.34.0 caused JGit application (that abused + GIT_EDITOR mechanism when invoking "git config") to get stuck with + a SIGTTOU signal; it has been reverted. + + * An earlier change that broke .gitignore matching has been reverted. + + * Things like "git -c branch.sort=bogus branch new HEAD", i.e. the + operation modes of the "git branch" command that do not need the + sort key information, no longer errors out by seeing a bogus sort + key. + (merge 98e7ab6d42 jc/fix-ref-sorting-parse later to maint). + + * The compatibility implementation for unsetenv(3) were written to + mimic ancient, non-POSIX, variant seen in an old glibc; it has been + changed to return an integer to match the more modern era. + (merge a38989bd5b jc/unsetenv-returns-an-int later to maint). + + * The clean/smudge conversion code path has been prepared to better + work on platforms where ulong is narrower than size_t. + (merge 596b5e77c9 mc/clean-smudge-with-llp64 later to maint). + + * Redact the path part of packfile URI that appears in the trace output. + (merge 0ba558ffb1 if/redact-packfile-uri later to maint). + + * CI has been taught to catch some Unicode directional formatting + sequence that can be used in certain mischief. + (merge 0e7696c64d js/ci-no-directional-formatting later to maint). + + * The "--date=format:<strftime>" gained a workaround for the lack of + system support for a non-local timezone to handle "%s" placeholder. + (merge 9b591b9403 jk/strbuf-addftime-seconds-since-epoch later to maint). + + * The "merge" subcommand of "git jump" (in contrib/) silently ignored + pathspec and other parameters. + (merge 67ba13e5a4 jk/jump-merge-with-pathspec later to maint). + + * The code to decode the length of packed object size has been + corrected. + (merge 34de5b8eac jt/pack-header-lshift-overflow later to maint). + + * The advice message given by "git pull" when the user hasn't made a + choice between merge and rebase still said that the merge is the + default, which no longer is the case. This has been corrected. + (merge 71076d0edd ah/advice-pull-has-no-preference-between-rebase-and-merge later to maint). + + * "git fetch", when received a bad packfile, can fail with SIGPIPE. + This wasn't wrong per-se, but we now detect the situation and fail + in a more predictable way. + (merge 2a4aed42ec jk/fetch-pack-avoid-sigpipe-to-index-pack later to maint). + + * The function to cull a child process and determine the exit status + had two separate code paths for normal callers and callers in a + signal handler, and the latter did not yield correct value when the + child has caught a signal. The handling of the exit status has + been unified for these two code paths. An existing test with + flakiness has also been corrected. + (merge 5263e22cba jk/t7006-sigpipe-tests-fix later to maint). + + * When a non-existent program is given as the pager, we tried to + reuse an uninitialized child_process structure and crashed, which + has been fixed. + (merge f917f57f40 em/missing-pager later to maint). + + * The single-key-input mode in "git add -p" had some code to handle + keys that generate a sequence of input via ReadKey(), which did not + handle end-of-file correctly, which has been fixed. + (merge fc8a8126df cb/add-p-single-key-fix later to maint). + + * "git rebase -x" added an unnecessary 'exec' instructions before + 'noop', which has been corrected. + (merge cc9dcdee61 en/rebase-x-fix later to maint). + + * Other code cleanup, docfix, build fix, etc. + (merge 74db416c9c cw/protocol-v2-doc-fix later to maint). + (merge f9b2b6684d ja/doc-cleanup later to maint). + (merge 7d1b866778 jc/fix-first-object-walk later to maint). + (merge 538ac74604 js/trace2-avoid-recursive-errors later to maint). + (merge 152923b132 jk/t5319-midx-corruption-test-deflake later to maint). + (merge 9081a421a6 ab/checkout-branch-info-leakfix later to maint). + (merge 42c456ff81 rs/mergesort later to maint). + (merge ad506e6780 tl/midx-docfix later to maint). + (merge bf5b83fd8a hk/ci-checkwhitespace-commentfix later to maint). + (merge 49f1eb3b34 jk/refs-g11-workaround later to maint). + (merge 7d3fc7df70 jt/midx-doc-fix later to maint). + (merge 7b089120d9 hn/create-reflog-simplify later to maint). + (merge 9e12400da8 cb/mingw-gmtime-r later to maint). diff --git a/Documentation/config/merge.txt b/Documentation/config/merge.txt index e27cc63944..99e83dd36e 100644 --- a/Documentation/config/merge.txt +++ b/Documentation/config/merge.txt @@ -4,7 +4,14 @@ merge.conflictStyle:: shows a `<<<<<<<` conflict marker, changes made by one side, a `=======` marker, changes made by the other side, and then a `>>>>>>>` marker. An alternate style, "diff3", adds a `|||||||` - marker and the original text before the `=======` marker. + marker and the original text before the `=======` marker. The + "merge" style tends to produce smaller conflict regions than diff3, + both because of the exclusion of the original text, and because + when a subset of lines match on the two sides they are just pulled + out of the conflict region. Another alternate style, "zdiff3", is + similar to diff3 but removes matching lines on the two sides from + the conflict region when those matching lines appear near either + the beginning or end of a conflict region. merge.defaultToUpstream:: If merge is called without any commit argument, merge the upstream diff --git a/Documentation/date-formats.txt b/Documentation/date-formats.txt index 99c455f51c..67645cae64 100644 --- a/Documentation/date-formats.txt +++ b/Documentation/date-formats.txt @@ -5,9 +5,9 @@ The `GIT_AUTHOR_DATE` and `GIT_COMMITTER_DATE` environment variables support the following date formats: Git internal format:: - It is `<unix timestamp> <time zone offset>`, where `<unix - timestamp>` is the number of seconds since the UNIX epoch. - `<time zone offset>` is a positive or negative offset from UTC. + It is `<unix-timestamp> <time-zone-offset>`, where + `<unix-timestamp>` is the number of seconds since the UNIX epoch. + `<time-zone-offset>` is a positive or negative offset from UTC. For example CET (which is 1 hour ahead of UTC) is `+0100`. RFC 2822:: diff --git a/Documentation/git-archimport.txt b/Documentation/git-archimport.txt index a595a0ffee..847777fd17 100644 --- a/Documentation/git-archimport.txt +++ b/Documentation/git-archimport.txt @@ -9,14 +9,14 @@ git-archimport - Import a GNU Arch repository into Git SYNOPSIS -------- [verse] -'git archimport' [-h] [-v] [-o] [-a] [-f] [-T] [-D depth] [-t tempdir] - <archive/branch>[:<git-branch>] ... +'git archimport' [-h] [-v] [-o] [-a] [-f] [-T] [-D <depth>] [-t <tempdir>] + <archive>/<branch>[:<git-branch>]... DESCRIPTION ----------- Imports a project from one or more GNU Arch repositories. It will follow branches -and repositories within the namespaces defined by the <archive/branch> +and repositories within the namespaces defined by the <archive>/<branch> parameters supplied. If it cannot find the remote branch a merge comes from it will just import it as a regular commit. If it can find it, it will mark it as a merge whenever possible (see discussion below). @@ -27,7 +27,7 @@ import new branches within the provided roots. It expects to be dealing with one project only. If it sees branches that have different roots, it will refuse to run. In that case, -edit your <archive/branch> parameters to define clearly the scope of the +edit your <archive>/<branch> parameters to define clearly the scope of the import. 'git archimport' uses `tla` extensively in the background to access the @@ -42,7 +42,7 @@ incremental imports. While 'git archimport' will try to create sensible branch names for the archives that it imports, it is also possible to specify Git branch names -manually. To do so, write a Git branch name after each <archive/branch> +manually. To do so, write a Git branch name after each <archive>/<branch> parameter, separated by a colon. This way, you can shorten the Arch branch names and convert Arch jargon to Git jargon, for example mapping a "PROJECT{litdd}devo{litdd}VERSION" branch to "master". @@ -104,8 +104,8 @@ OPTIONS Override the default tempdir. -<archive/branch>:: - Archive/branch identifier in a format that `tla log` understands. +<archive>/<branch>:: + <archive>/<branch> identifier in a format that `tla log` understands. GIT diff --git a/Documentation/git-checkout.txt b/Documentation/git-checkout.txt index d473c9bf38..c497db7eae 100644 --- a/Documentation/git-checkout.txt +++ b/Documentation/git-checkout.txt @@ -11,7 +11,7 @@ SYNOPSIS 'git checkout' [-q] [-f] [-m] [<branch>] 'git checkout' [-q] [-f] [-m] --detach [<branch>] 'git checkout' [-q] [-f] [-m] [--detach] <commit> -'git checkout' [-q] [-f] [-m] [[-b|-B|--orphan] <new_branch>] [<start_point>] +'git checkout' [-q] [-f] [-m] [[-b|-B|--orphan] <new-branch>] [<start-point>] 'git checkout' [-f|--ours|--theirs|-m|--conflict=<style>] [<tree-ish>] [--] <pathspec>... 'git checkout' [-f|--ours|--theirs|-m|--conflict=<style>] [<tree-ish>] --pathspec-from-file=<file> [--pathspec-file-nul] 'git checkout' (-p|--patch) [<tree-ish>] [--] [<pathspec>...] @@ -43,7 +43,7 @@ You could omit `<branch>`, in which case the command degenerates to rather expensive side-effects to show only the tracking information, if exists, for the current branch. -'git checkout' -b|-B <new_branch> [<start point>]:: +'git checkout' -b|-B <new-branch> [<start-point>]:: Specifying `-b` causes a new branch to be created as if linkgit:git-branch[1] were called and then checked out. In @@ -52,11 +52,11 @@ if exists, for the current branch. `--track` without `-b` implies branch creation; see the description of `--track` below. + -If `-B` is given, `<new_branch>` is created if it doesn't exist; otherwise, it +If `-B` is given, `<new-branch>` is created if it doesn't exist; otherwise, it is reset. This is the transactional equivalent of + ------------ -$ git branch -f <branch> [<start point>] +$ git branch -f <branch> [<start-point>] $ git checkout <branch> ------------ + @@ -145,13 +145,13 @@ as `ours` (i.e. "our shared canonical history"), while what you did on your side branch as `theirs` (i.e. "one contributor's work on top of it"). --b <new_branch>:: - Create a new branch named `<new_branch>` and start it at - `<start_point>`; see linkgit:git-branch[1] for details. +-b <new-branch>:: + Create a new branch named `<new-branch>` and start it at + `<start-point>`; see linkgit:git-branch[1] for details. --B <new_branch>:: - Creates the branch `<new_branch>` and start it at `<start_point>`; - if it already exists, then reset it to `<start_point>`. This is +-B <new-branch>:: + Creates the branch `<new-branch>` and start it at `<start-point>`; + if it already exists, then reset it to `<start-point>`. This is equivalent to running "git branch" with "-f"; see linkgit:git-branch[1] for details. @@ -210,16 +210,16 @@ variable. `<commit>` is not a branch name. See the "DETACHED HEAD" section below for details. ---orphan <new_branch>:: - Create a new 'orphan' branch, named `<new_branch>`, started from - `<start_point>` and switch to it. The first commit made on this +--orphan <new-branch>:: + Create a new 'orphan' branch, named `<new-branch>`, started from + `<start-point>` and switch to it. The first commit made on this new branch will have no parents and it will be the root of a new history totally disconnected from all the other branches and commits. + The index and the working tree are adjusted as if you had previously run -`git checkout <start_point>`. This allows you to start a new history -that records a set of paths similar to `<start_point>` by easily running +`git checkout <start-point>`. This allows you to start a new history +that records a set of paths similar to `<start-point>` by easily running `git commit -a` to make the root commit. + This can be useful when you want to publish the tree from a commit @@ -229,7 +229,7 @@ whose full history contains proprietary or otherwise encumbered bits of code. + If you want to start a disconnected history that records a set of paths -that is totally different from the one of `<start_point>`, then you should +that is totally different from the one of `<start-point>`, then you should clear the index and the working tree right after creating the orphan branch by running `git rm -rf .` from the top level of the working tree. Afterwards you will be ready to prepare your new files, repopulating the @@ -266,8 +266,7 @@ When switching branches with `--merge`, staged changes may be lost. The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values are - "merge" (default) and "diff3" (in addition to what is shown by - "merge" style, shows the original contents). + "merge" (default), "diff3", and "zdiff3". -p:: --patch:: @@ -341,10 +340,10 @@ As a special case, you may use `A...B` as a shortcut for the merge base of `A` and `B` if there is exactly one merge base. You can leave out at most one of `A` and `B`, in which case it defaults to `HEAD`. -<new_branch>:: +<new-branch>:: Name for the new branch. -<start_point>:: +<start-point>:: The name of a commit at which to start the new branch; see linkgit:git-branch[1] for details. Defaults to `HEAD`. + diff --git a/Documentation/git-cherry-pick.txt b/Documentation/git-cherry-pick.txt index 5d750314b2..78dcc9171f 100644 --- a/Documentation/git-cherry-pick.txt +++ b/Documentation/git-cherry-pick.txt @@ -8,7 +8,7 @@ git-cherry-pick - Apply the changes introduced by some existing commits SYNOPSIS -------- [verse] -'git cherry-pick' [--edit] [-n] [-m parent-number] [-s] [-x] [--ff] +'git cherry-pick' [--edit] [-n] [-m <parent-number>] [-s] [-x] [--ff] [-S[<keyid>]] <commit>... 'git cherry-pick' (--continue | --skip | --abort | --quit) @@ -81,8 +81,8 @@ OPTIONS described above, and `-r` was to disable it. Now the default is not to do `-x` so this option is a no-op. --m parent-number:: ---mainline parent-number:: +-m <parent-number>:: +--mainline <parent-number>:: Usually you cannot cherry-pick a merge because you do not know which side of the merge should be considered the mainline. This option specifies the parent number (starting from 1) of diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index 3fe3810f1c..9685ea0691 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -9,10 +9,10 @@ git-clone - Clone a repository into a new directory SYNOPSIS -------- [verse] -'git clone' [--template=<template_directory>] +'git clone' [--template=<template-directory>] [-l] [-s] [--no-hardlinks] [-q] [-n] [--bare] [--mirror] [-o <name>] [-b <name>] [-u <upload-pack>] [--reference <repository>] - [--dissociate] [--separate-git-dir <git dir>] + [--dissociate] [--separate-git-dir <git-dir>] [--depth <depth>] [--[no-]single-branch] [--no-tags] [--recurse-submodules[=<pathspec>]] [--[no-]shallow-submodules] [--[no-]remote-submodules] [--jobs <n>] [--sparse] [--[no-]reject-shallow] @@ -211,7 +211,7 @@ objects from the source repository into a pack in the cloned repository. via ssh, this specifies a non-default path for the command run on the other end. ---template=<template_directory>:: +--template=<template-directory>:: Specify the directory from which templates will be used; (See the "TEMPLATE DIRECTORY" section of linkgit:git-init[1].) @@ -294,7 +294,7 @@ or `--mirror` is given) superproject's recorded SHA-1. Equivalent to passing `--remote` to `git submodule update`. ---separate-git-dir=<git dir>:: +--separate-git-dir=<git-dir>:: Instead of placing the cloned repository where it is supposed to be, place the cloned repository at the specified directory, then make a filesystem-agnostic Git symbolic link to there. diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt index 992225f612..2285effb36 100644 --- a/Documentation/git-config.txt +++ b/Documentation/git-config.txt @@ -9,20 +9,20 @@ git-config - Get and set repository or global options SYNOPSIS -------- [verse] -'git config' [<file-option>] [--type=<type>] [--fixed-value] [--show-origin] [--show-scope] [-z|--null] name [value [value-pattern]] -'git config' [<file-option>] [--type=<type>] --add name value -'git config' [<file-option>] [--type=<type>] [--fixed-value] --replace-all name value [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get name [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get-all name [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] [--name-only] --get-regexp name_regex [value-pattern] -'git config' [<file-option>] [--type=<type>] [-z|--null] --get-urlmatch name URL -'git config' [<file-option>] [--fixed-value] --unset name [value-pattern] -'git config' [<file-option>] [--fixed-value] --unset-all name [value-pattern] -'git config' [<file-option>] --rename-section old_name new_name -'git config' [<file-option>] --remove-section name +'git config' [<file-option>] [--type=<type>] [--fixed-value] [--show-origin] [--show-scope] [-z|--null] <name> [<value> [<value-pattern>]] +'git config' [<file-option>] [--type=<type>] --add <name> <value> +'git config' [<file-option>] [--type=<type>] [--fixed-value] --replace-all <name> <value> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get <name> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get-all <name> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] [--name-only] --get-regexp <name-regex> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [-z|--null] --get-urlmatch <name> <URL> +'git config' [<file-option>] [--fixed-value] --unset <name> [<value-pattern>] +'git config' [<file-option>] [--fixed-value] --unset-all <name> [<value-pattern>] +'git config' [<file-option>] --rename-section <old-name> <new-name> +'git config' [<file-option>] --remove-section <name> 'git config' [<file-option>] [--show-origin] [--show-scope] [-z|--null] [--name-only] -l | --list -'git config' [<file-option>] --get-color name [default] -'git config' [<file-option>] --get-colorbool name [stdout-is-tty] +'git config' [<file-option>] --get-color <name> [<default>] +'git config' [<file-option>] --get-colorbool <name> [<stdout-is-tty>] 'git config' [<file-option>] -e | --edit DESCRIPTION @@ -102,9 +102,9 @@ OPTIONS in which section and variable names are lowercased, but subsection names are not. ---get-urlmatch name URL:: +--get-urlmatch <name> <URL>:: When given a two-part name section.key, the value for - section.<url>.key whose <url> part matches the best to the + section.<URL>.key whose <URL> part matches the best to the given URL is returned (if no such key exists, the value for section.key is used as a fallback). When given just the section as name, do so for all the keys in the section and @@ -145,8 +145,8 @@ See also <<FILES>>. read from or written to if `extensions.worktreeConfig` is present. If not it's the same as `--local`. --f config-file:: ---file config-file:: +-f <config-file>:: +--file <config-file>:: For writing options: write to the specified file rather than the repository `.git/config`. + @@ -155,7 +155,7 @@ available files. + See also <<FILES>>. ---blob blob:: +--blob <blob>:: Similar to `--file` but use the given blob instead of a file. E.g. you can use 'master:.gitmodules' to read values from the file '.gitmodules' in the master branch. See "SPECIFYING REVISIONS" @@ -246,18 +246,18 @@ Valid `<type>`'s include: all queried config options with the scope of that value (local, global, system, command). ---get-colorbool name [stdout-is-tty]:: +--get-colorbool <name> [<stdout-is-tty>]:: - Find the color setting for `name` (e.g. `color.diff`) and output - "true" or "false". `stdout-is-tty` should be either "true" or + Find the color setting for `<name>` (e.g. `color.diff`) and output + "true" or "false". `<stdout-is-tty>` should be either "true" or "false", and is taken into account when configuration says - "auto". If `stdout-is-tty` is missing, then checks the standard + "auto". If `<stdout-is-tty>` is missing, then checks the standard output of the command itself, and exits with status 0 if color is to be used, or exits with status 1 otherwise. When the color setting for `name` is undefined, the command uses `color.ui` as fallback. ---get-color name [default]:: +--get-color <name> [<default>]:: Find the color configured for `name` (e.g. `color.diff.new`) and output it as the ANSI color escape sequence to the standard diff --git a/Documentation/git-credential.txt b/Documentation/git-credential.txt index 206e3c5f40..f18673017f 100644 --- a/Documentation/git-credential.txt +++ b/Documentation/git-credential.txt @@ -8,7 +8,7 @@ git-credential - Retrieve and store user credentials SYNOPSIS -------- ------------------ -git credential <fill|approve|reject> +'git credential' (fill|approve|reject) ------------------ DESCRIPTION diff --git a/Documentation/git-cvsexportcommit.txt b/Documentation/git-cvsexportcommit.txt index 00154b6c85..41c8a8a05c 100644 --- a/Documentation/git-cvsexportcommit.txt +++ b/Documentation/git-cvsexportcommit.txt @@ -9,8 +9,8 @@ git-cvsexportcommit - Export a single commit to a CVS checkout SYNOPSIS -------- [verse] -'git cvsexportcommit' [-h] [-u] [-v] [-c] [-P] [-p] [-a] [-d cvsroot] - [-w cvsworkdir] [-W] [-f] [-m msgprefix] [PARENTCOMMIT] COMMITID +'git cvsexportcommit' [-h] [-u] [-v] [-c] [-P] [-p] [-a] [-d <cvsroot>] + [-w <cvs-workdir>] [-W] [-f] [-m <msgprefix>] [<parent-commit>] <commit-id> DESCRIPTION diff --git a/Documentation/git-cvsimport.txt b/Documentation/git-cvsimport.txt index de1ebed67d..b3f27671a0 100644 --- a/Documentation/git-cvsimport.txt +++ b/Documentation/git-cvsimport.txt @@ -11,9 +11,9 @@ SYNOPSIS [verse] 'git cvsimport' [-o <branch-for-HEAD>] [-h] [-v] [-d <CVSROOT>] [-A <author-conv-file>] [-p <options-for-cvsps>] [-P <file>] - [-C <git_repository>] [-z <fuzz>] [-i] [-k] [-u] [-s <subst>] - [-a] [-m] [-M <regex>] [-S <regex>] [-L <commitlimit>] - [-r <remote>] [-R] [<CVS_module>] + [-C <git-repository>] [-z <fuzz>] [-i] [-k] [-u] [-s <subst>] + [-a] [-m] [-M <regex>] [-S <regex>] [-L <commit-limit>] + [-r <remote>] [-R] [<CVS-module>] DESCRIPTION @@ -59,7 +59,7 @@ OPTIONS from `CVS/Root`. If no such file exists, it checks for the `CVSROOT` environment variable. -<CVS_module>:: +<CVS-module>:: The CVS module you want to import. Relative to <CVSROOT>. If not given, 'git cvsimport' tries to read it from `CVS/Repository`. diff --git a/Documentation/git-diff-files.txt b/Documentation/git-diff-files.txt index 906774f0f7..bf1febb9ae 100644 --- a/Documentation/git-diff-files.txt +++ b/Documentation/git-diff-files.txt @@ -9,7 +9,7 @@ git-diff-files - Compares files in the working tree and the index SYNOPSIS -------- [verse] -'git diff-files' [-q] [-0|-1|-2|-3|-c|--cc] [<common diff options>] [<path>...] +'git diff-files' [-q] [-0|-1|-2|-3|-c|--cc] [<common-diff-options>] [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-diff-index.txt b/Documentation/git-diff-index.txt index 27acb31cbf..679cae27d9 100644 --- a/Documentation/git-diff-index.txt +++ b/Documentation/git-diff-index.txt @@ -9,7 +9,7 @@ git-diff-index - Compare a tree to the working tree or index SYNOPSIS -------- [verse] -'git diff-index' [-m] [--cached] [--merge-base] [<common diff options>] <tree-ish> [<path>...] +'git diff-index' [-m] [--cached] [--merge-base] [<common-diff-options>] <tree-ish> [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-diff-tree.txt b/Documentation/git-diff-tree.txt index 2fc24c542f..274d5eaba9 100644 --- a/Documentation/git-diff-tree.txt +++ b/Documentation/git-diff-tree.txt @@ -11,7 +11,7 @@ SYNOPSIS [verse] 'git diff-tree' [--stdin] [-m] [-s] [-v] [--no-commit-id] [--pretty] [-t] [-r] [-c | --cc] [--combined-all-paths] [--root] [--merge-base] - [<common diff options>] <tree-ish> [<tree-ish>] [<path>...] + [<common-diff-options>] <tree-ish> [<tree-ish>] [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-fsck.txt b/Documentation/git-fsck.txt index bd596619c0..5088783dcc 100644 --- a/Documentation/git-fsck.txt +++ b/Documentation/git-fsck.txt @@ -12,7 +12,7 @@ SYNOPSIS 'git fsck' [--tags] [--root] [--unreachable] [--cache] [--no-reflogs] [--[no-]full] [--strict] [--verbose] [--lost-found] [--[no-]dangling] [--[no-]progress] [--connectivity-only] - [--[no-]name-objects] [<object>*] + [--[no-]name-objects] [<object>...] DESCRIPTION ----------- diff --git a/Documentation/git-gui.txt b/Documentation/git-gui.txt index c9d7e96214..e8f3ccb433 100644 --- a/Documentation/git-gui.txt +++ b/Documentation/git-gui.txt @@ -8,7 +8,7 @@ git-gui - A portable graphical interface to Git SYNOPSIS -------- [verse] -'git gui' [<command>] [arguments] +'git gui' [<command>] [<arguments>] DESCRIPTION ----------- diff --git a/Documentation/git-help.txt b/Documentation/git-help.txt index 96d5f598b4..44ea63cc6d 100644 --- a/Documentation/git-help.txt +++ b/Documentation/git-help.txt @@ -9,14 +9,14 @@ SYNOPSIS -------- [verse] 'git help' [-a|--all [--[no-]verbose]] - [[-i|--info] [-m|--man] [-w|--web]] [COMMAND|GUIDE] + [[-i|--info] [-m|--man] [-w|--web]] [<command>|<guide>] 'git help' [-g|--guides] 'git help' [-c|--config] DESCRIPTION ----------- -With no options and no COMMAND or GUIDE given, the synopsis of the 'git' +With no options and no '<command>' or '<guide>' given, the synopsis of the 'git' command and a list of the most commonly used Git commands are printed on the standard output. @@ -33,7 +33,7 @@ variables. If an alias is given, git shows the definition of the alias on standard output. To get the manual page for the aliased command, use -`git COMMAND --help`. +`git <command> --help`. Note that `git --help ...` is identical to `git help ...` because the former is internally converted into the latter. diff --git a/Documentation/git-http-fetch.txt b/Documentation/git-http-fetch.txt index 9fa17b60e4..319062c021 100644 --- a/Documentation/git-http-fetch.txt +++ b/Documentation/git-http-fetch.txt @@ -9,7 +9,7 @@ git-http-fetch - Download from a remote Git repository via HTTP SYNOPSIS -------- [verse] -'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin | --packfile=<hash> | <commit>] <url> +'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w <filename>] [--recover] [--stdin | --packfile=<hash> | <commit>] <URL> DESCRIPTION ----------- diff --git a/Documentation/git-http-push.txt b/Documentation/git-http-push.txt index ea03a4eeb0..7c6a6dd7f6 100644 --- a/Documentation/git-http-push.txt +++ b/Documentation/git-http-push.txt @@ -9,7 +9,7 @@ git-http-push - Push objects over HTTP/DAV to another repository SYNOPSIS -------- [verse] -'git http-push' [--all] [--dry-run] [--force] [--verbose] <url> <ref> [<ref>...] +'git http-push' [--all] [--dry-run] [--force] [--verbose] <URL> <ref> [<ref>...] DESCRIPTION ----------- @@ -63,16 +63,15 @@ of such patterns separated by a colon ":" (this means that a ref name cannot have a colon in it). A single pattern '<name>' is just a shorthand for '<name>:<name>'. -Each pattern pair consists of the source side (before the colon) -and the destination side (after the colon). The ref to be -pushed is determined by finding a match that matches the source -side, and where it is pushed is determined by using the -destination side. +Each pattern pair '<src>:<dst>' consists of the source side (before +the colon) and the destination side (after the colon). The ref to be +pushed is determined by finding a match that matches the source side, +and where it is pushed is determined by using the destination side. - - It is an error if <src> does not match exactly one of the + - It is an error if '<src>' does not match exactly one of the local refs. - - If <dst> does not match any remote ref, either + - If '<dst>' does not match any remote ref, either * it has to start with "refs/"; <dst> is used as the destination literally in this case. diff --git a/Documentation/git-init-db.txt b/Documentation/git-init-db.txt index 648a6cd78a..18bf1a3c8c 100644 --- a/Documentation/git-init-db.txt +++ b/Documentation/git-init-db.txt @@ -9,7 +9,7 @@ git-init-db - Creates an empty Git repository SYNOPSIS -------- [verse] -'git init-db' [-q | --quiet] [--bare] [--template=<template_directory>] [--separate-git-dir <git dir>] [--shared[=<permissions>]] +'git init-db' [-q | --quiet] [--bare] [--template=<template-directory>] [--separate-git-dir <git-dir>] [--shared[=<permissions>]] DESCRIPTION diff --git a/Documentation/git-init.txt b/Documentation/git-init.txt index b611d80697..ad921fe782 100644 --- a/Documentation/git-init.txt +++ b/Documentation/git-init.txt @@ -9,10 +9,10 @@ git-init - Create an empty Git repository or reinitialize an existing one SYNOPSIS -------- [verse] -'git init' [-q | --quiet] [--bare] [--template=<template_directory>] - [--separate-git-dir <git dir>] [--object-format=<format>] +'git init' [-q | --quiet] [--bare] [--template=<template-directory>] + [--separate-git-dir <git-dir>] [--object-format=<format>] [-b <branch-name> | --initial-branch=<branch-name>] - [--shared[=<permissions>]] [directory] + [--shared[=<permissions>]] [<directory>] DESCRIPTION @@ -57,12 +57,12 @@ values are 'sha1' and (if enabled) 'sha256'. 'sha1' is the default. + include::object-format-disclaimer.txt[] ---template=<template_directory>:: +--template=<template-directory>:: Specify the directory from which templates will be used. (See the "TEMPLATE DIRECTORY" section below.) ---separate-git-dir=<git dir>:: +--separate-git-dir=<git-dir>:: Instead of initializing the repository as a directory to either `$GIT_DIR` or `./.git/`, create a text file there containing the path to the actual @@ -79,7 +79,7 @@ repository. If not specified, fall back to the default name (currently `master`, but this is subject to change in the future; the name can be customized via the `init.defaultBranch` configuration variable). ---shared[=(false|true|umask|group|all|world|everybody|0xxx)]:: +--shared[=(false|true|umask|group|all|world|everybody|<perm>)]:: Specify that the Git repository is to be shared amongst several users. This allows users belonging to the same group to push into that @@ -110,13 +110,16 @@ the repository permissions. Same as 'group', but make the repository readable by all users. -'0xxx':: +'<perm>':: -'0xxx' is an octal number and each file will have mode '0xxx'. '0xxx' will -override users' umask(2) value (and not only loosen permissions as 'group' and -'all' does). '0640' will create a repository which is group-readable, but not -group-writable or accessible to others. '0660' will create a repo that is -readable and writable to the current user and group, but inaccessible to others. +'<perm>' is a 3-digit octal number prefixed with `0` and each file +will have mode '<perm>'. '<perm>' will override users' umask(2) +value (and not only loosen permissions as 'group' and 'all' +does). '0640' will create a repository which is group-readable, but +not group-writable or accessible to others. '0660' will create a repo +that is readable and writable to the current user and group, but +inaccessible to others (directories and executable files get their +`x` bit from the `r` bit for corresponding classes of users). -- By default, the configuration flag `receive.denyNonFastForwards` is enabled diff --git a/Documentation/git-log.txt b/Documentation/git-log.txt index 0498e7bacb..20e87cecf4 100644 --- a/Documentation/git-log.txt +++ b/Documentation/git-log.txt @@ -9,7 +9,7 @@ git-log - Show commit logs SYNOPSIS -------- [verse] -'git log' [<options>] [<revision range>] [[--] <path>...] +'git log' [<options>] [<revision-range>] [[--] <path>...] DESCRIPTION ----------- @@ -81,13 +81,13 @@ produced by `--stat`, etc. include::line-range-options.txt[] -<revision range>:: +<revision-range>:: Show only commits in the specified revision range. When no - <revision range> is specified, it defaults to `HEAD` (i.e. the + <revision-range> is specified, it defaults to `HEAD` (i.e. the whole history leading to the current commit). `origin..HEAD` specifies all the commits reachable from the current commit (i.e. `HEAD`), but not from `origin`. For a complete list of - ways to spell <revision range>, see the 'Specifying Ranges' + ways to spell <revision-range>, see the 'Specifying Ranges' section of linkgit:gitrevisions[7]. [--] <path>...:: diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt index 6d11ab506b..2e3d695fa2 100644 --- a/Documentation/git-ls-files.txt +++ b/Documentation/git-ls-files.txt @@ -10,9 +10,9 @@ SYNOPSIS -------- [verse] 'git ls-files' [-z] [-t] [-v] [-f] - (--[cached|deleted|others|ignored|stage|unmerged|killed|modified])* - (-[c|d|o|i|s|u|k|m])* - [--eol] + [-c|--cached] [-d|--deleted] [-o|--others] [-i|--|ignored] + [-s|--stage] [-u|--unmerged] [-k|--|killed] [-m|--modified] + [--directory [--no-empty-directory]] [--eol] [--deduplicate] [-x <pattern>|--exclude=<pattern>] [-X <file>|--exclude-from=<file>] diff --git a/Documentation/git-merge-file.txt b/Documentation/git-merge-file.txt index f856032613..7e9093fab6 100644 --- a/Documentation/git-merge-file.txt +++ b/Documentation/git-merge-file.txt @@ -70,6 +70,9 @@ OPTIONS --diff3:: Show conflicts in "diff3" style. +--zdiff3:: + Show conflicts in "zdiff3" style. + --ours:: --theirs:: --union:: diff --git a/Documentation/git-merge-index.txt b/Documentation/git-merge-index.txt index 2ab84a91e5..eea56b3154 100644 --- a/Documentation/git-merge-index.txt +++ b/Documentation/git-merge-index.txt @@ -9,7 +9,7 @@ git-merge-index - Run a merge for files needing merging SYNOPSIS -------- [verse] -'git merge-index' [-o] [-q] <merge-program> (-a | [--] <file>*) +'git merge-index' [-o] [-q] <merge-program> (-a | ( [--] <file>...) ) DESCRIPTION ----------- diff --git a/Documentation/git-merge.txt b/Documentation/git-merge.txt index e4f3352eb5..e8cecf5a51 100644 --- a/Documentation/git-merge.txt +++ b/Documentation/git-merge.txt @@ -240,7 +240,8 @@ from the RCS suite to present such a conflicted hunk, like this: ------------ Here are lines that are either unchanged from the common -ancestor, or cleanly resolved because only one side changed. +ancestor, or cleanly resolved because only one side changed, +or cleanly resolved because both sides changed the same way. <<<<<<< yours:sample.txt Conflict resolution is hard; let's go shopping. @@ -261,16 +262,37 @@ side wants to say it is hard and you'd prefer to go shopping, while the other side wants to claim it is easy. An alternative style can be used by setting the "merge.conflictStyle" -configuration variable to "diff3". In "diff3" style, the above conflict -may look like this: +configuration variable to either "diff3" or "zdiff3". In "diff3" +style, the above conflict may look like this: ------------ Here are lines that are either unchanged from the common -ancestor, or cleanly resolved because only one side changed. +ancestor, or cleanly resolved because only one side changed, <<<<<<< yours:sample.txt +or cleanly resolved because both sides changed the same way. Conflict resolution is hard; let's go shopping. -||||||| +||||||| base:sample.txt +or cleanly resolved because both sides changed identically. +Conflict resolution is hard. +======= +or cleanly resolved because both sides changed the same way. +Git makes conflict resolution easy. +>>>>>>> theirs:sample.txt +And here is another line that is cleanly resolved or unmodified. +------------ + +while in "zdiff3" style, it may look like this: + +------------ +Here are lines that are either unchanged from the common +ancestor, or cleanly resolved because only one side changed, +or cleanly resolved because both sides changed the same way. +<<<<<<< yours:sample.txt +Conflict resolution is hard; +let's go shopping. +||||||| base:sample.txt +or cleanly resolved because both sides changed identically. Conflict resolution is hard. ======= Git makes conflict resolution easy. diff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt index 38e5257b2a..e21fcd8f71 100644 --- a/Documentation/git-p4.txt +++ b/Documentation/git-p4.txt @@ -9,10 +9,10 @@ git-p4 - Import from and submit to Perforce repositories SYNOPSIS -------- [verse] -'git p4 clone' [<sync options>] [<clone options>] <p4 depot path>... -'git p4 sync' [<sync options>] [<p4 depot path>...] +'git p4 clone' [<sync-options>] [<clone-options>] <p4-depot-path>... +'git p4 sync' [<sync-options>] [<p4-depot-path>...] 'git p4 rebase' -'git p4 submit' [<submit options>] [<master branch name>] +'git p4 submit' [<submit-options>] [<master-branch-name>] DESCRIPTION @@ -361,7 +361,7 @@ These options can be used to modify 'git p4 submit' behavior. p4/master. See the "Sync options" section above for more information. ---commit <sha1>|<sha1..sha1>:: +--commit (<sha1>|<sha1>..<sha1>):: Submit only the specified commit or range of commits, instead of the full list of changes that are in the current Git branch. diff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt index dbfd1f9017..f8344e1e5b 100644 --- a/Documentation/git-pack-objects.txt +++ b/Documentation/git-pack-objects.txt @@ -13,8 +13,8 @@ SYNOPSIS [--no-reuse-delta] [--delta-base-offset] [--non-empty] [--local] [--incremental] [--window=<n>] [--depth=<n>] [--revs [--unpacked | --all]] [--keep-pack=<pack-name>] - [--stdout [--filter=<filter-spec>] | base-name] - [--shallow] [--keep-true-parents] [--[no-]sparse] < object-list + [--stdout [--filter=<filter-spec>] | <base-name>] + [--shallow] [--keep-true-parents] [--[no-]sparse] < <object-list> DESCRIPTION diff --git a/Documentation/git-pack-redundant.txt b/Documentation/git-pack-redundant.txt index f2869da572..ee7034b5e5 100644 --- a/Documentation/git-pack-redundant.txt +++ b/Documentation/git-pack-redundant.txt @@ -9,7 +9,7 @@ git-pack-redundant - Find redundant pack files SYNOPSIS -------- [verse] -'git pack-redundant' [ --verbose ] [ --alt-odb ] < --all | .pack filename ... > +'git pack-redundant' [ --verbose ] [ --alt-odb ] ( --all | <pack-filename>... ) DESCRIPTION ----------- diff --git a/Documentation/git-rebase.txt b/Documentation/git-rebase.txt index a1af21fcef..9da4647061 100644 --- a/Documentation/git-rebase.txt +++ b/Documentation/git-rebase.txt @@ -714,9 +714,9 @@ information about the rebased commits and their parents (and instead generates new fake commits based off limited information in the generated patches), those commits cannot be identified; instead it has to fall back to a commit summary. Also, when merge.conflictStyle is -set to diff3, the apply backend will use "constructed merge base" to -label the content from the merge base, and thus provide no information -about the merge base commit whatsoever. +set to diff3 or zdiff3, the apply backend will use "constructed merge +base" to label the content from the merge base, and thus provide no +information about the merge base commit whatsoever. The merge backend works with the full commits on both sides of history and thus has no such limitations. diff --git a/Documentation/git-reflog.txt b/Documentation/git-reflog.txt index ff487ff77d..5ced7ad4f8 100644 --- a/Documentation/git-reflog.txt +++ b/Documentation/git-reflog.txt @@ -17,12 +17,12 @@ The command takes various subcommands, and different options depending on the subcommand: [verse] -'git reflog' ['show'] [log-options] [<ref>] +'git reflog' ['show'] [<log-options>] [<ref>] 'git reflog expire' [--expire=<time>] [--expire-unreachable=<time>] [--rewrite] [--updateref] [--stale-fix] [--dry-run | -n] [--verbose] [--all [--single-worktree] | <refs>...] 'git reflog delete' [--rewrite] [--updateref] - [--dry-run | -n] [--verbose] ref@\{specifier\}... + [--dry-run | -n] [--verbose] <ref>@\{<specifier>\}... 'git reflog exists' <ref> Reference logs, or "reflogs", record when the tips of branches and diff --git a/Documentation/git-remote.txt b/Documentation/git-remote.txt index 31c29c9b31..2bebc32566 100644 --- a/Documentation/git-remote.txt +++ b/Documentation/git-remote.txt @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] 'git remote' [-v | --verbose] -'git remote add' [-t <branch>] [-m <master>] [-f] [--[no-]tags] [--mirror=(fetch|push)] <name> <url> +'git remote add' [-t <branch>] [-m <master>] [-f] [--[no-]tags] [--mirror=(fetch|push)] <name> <URL> 'git remote rename' <old> <new> 'git remote remove' <name> 'git remote set-head' <name> (-a | --auto | -d | --delete | <branch>) @@ -18,7 +18,7 @@ SYNOPSIS 'git remote get-url' [--push] [--all] <name> 'git remote set-url' [--push] <name> <newurl> [<oldurl>] 'git remote set-url --add' [--push] <name> <newurl> -'git remote set-url --delete' [--push] <name> <url> +'git remote set-url --delete' [--push] <name> <URL> 'git remote' [-v | --verbose] 'show' [-n] <name>... 'git remote prune' [-n | --dry-run] <name>... 'git remote' [-v | --verbose] 'update' [-p | --prune] [(<group> | <remote>)...] @@ -47,7 +47,7 @@ subcommands are available to perform operations on the remotes. 'add':: Add a remote named <name> for the repository at -<url>. The command `git fetch <name>` can then be used to create and +<URL>. The command `git fetch <name>` can then be used to create and update remote-tracking branches <name>/<branch>. + With `-f` option, `git fetch <name>` is run immediately after @@ -152,7 +152,7 @@ With `--push`, push URLs are manipulated instead of fetch URLs. With `--add`, instead of changing existing URLs, new URL is added. + With `--delete`, instead of changing existing URLs, all URLs matching -regex <url> are deleted for remote <name>. Trying to delete all +regex <URL> are deleted for remote <name>. Trying to delete all non-push URLs is an error. + Note that the push URL and the fetch URL, even though they can diff --git a/Documentation/git-request-pull.txt b/Documentation/git-request-pull.txt index 4d4392d0f8..fa5a426709 100644 --- a/Documentation/git-request-pull.txt +++ b/Documentation/git-request-pull.txt @@ -8,7 +8,7 @@ git-request-pull - Generates a summary of pending changes SYNOPSIS -------- [verse] -'git request-pull' [-p] <start> <url> [<end>] +'git request-pull' [-p] <start> <URL> [<end>] DESCRIPTION ----------- @@ -21,7 +21,7 @@ the changes and indicates from where they can be pulled. The upstream project is expected to have the commit named by `<start>` and the output asks it to integrate the changes you made since that commit, up to the commit named by `<end>`, by visiting -the repository named by `<url>`. +the repository named by `<URL>`. OPTIONS @@ -33,14 +33,14 @@ OPTIONS Commit to start at. This names a commit that is already in the upstream history. -<url>:: +<URL>:: The repository URL to be pulled from. <end>:: Commit to end at (defaults to HEAD). This names the commit at the tip of the history you are asking to be pulled. + -When the repository named by `<url>` has the commit at a tip of a +When the repository named by `<URL>` has the commit at a tip of a ref that is different from the ref you have locally, you can use the `<local>:<remote>` syntax, to have its local name, a colon `:`, and its remote name. diff --git a/Documentation/git-restore.txt b/Documentation/git-restore.txt index 55bde91ef9..5964810caa 100644 --- a/Documentation/git-restore.txt +++ b/Documentation/git-restore.txt @@ -92,8 +92,7 @@ in linkgit:git-checkout[1] for details. The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values - are "merge" (default) and "diff3" (in addition to what is - shown by "merge" style, shows the original contents). + are "merge" (default), "diff3", and "zdiff3". --ignore-unmerged:: When restoring files on the working tree from the index, do diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt index 3db4eab4ba..41cd8cb424 100644 --- a/Documentation/git-send-email.txt +++ b/Documentation/git-send-email.txt @@ -9,7 +9,8 @@ git-send-email - Send a collection of patches as emails SYNOPSIS -------- [verse] -'git send-email' [<options>] <file|directory|rev-list options>... +'git send-email' [<options>] <file|directory>... +'git send-email' [<options>] <format-patch options> 'git send-email' --dump-aliases @@ -19,7 +20,8 @@ Takes the patches given on the command line and emails them out. Patches can be specified as files, directories (which will send all files in the directory), or directly as a revision list. In the last case, any format accepted by linkgit:git-format-patch[1] can -be passed to git send-email. +be passed to git send-email, as well as options understood by +linkgit:git-format-patch[1]. The header of the email is configurable via command-line options. If not specified on the command line, the user will be prompted with a ReadLine diff --git a/Documentation/git-shortlog.txt b/Documentation/git-shortlog.txt index c9c7f3065c..f64e77047b 100644 --- a/Documentation/git-shortlog.txt +++ b/Documentation/git-shortlog.txt @@ -8,7 +8,7 @@ git-shortlog - Summarize 'git log' output SYNOPSIS -------- [verse] -'git shortlog' [<options>] [<revision range>] [[--] <path>...] +'git shortlog' [<options>] [<revision-range>] [[--] <path>...] git log --pretty=short | 'git shortlog' [<options>] DESCRIPTION @@ -89,13 +89,13 @@ counts both authors and co-authors. If width is `0` (zero) then indent the lines of the output without wrapping them. -<revision range>:: +<revision-range>:: Show only commits in the specified revision range. When no - <revision range> is specified, it defaults to `HEAD` (i.e. the + <revision-range> is specified, it defaults to `HEAD` (i.e. the whole history leading to the current commit). `origin..HEAD` specifies all the commits reachable from the current commit (i.e. `HEAD`), but not from `origin`. For a complete list of - ways to spell <revision range>, see the "Specifying Ranges" + ways to spell <revision-range>, see the "Specifying Ranges" section of linkgit:gitrevisions[7]. [--] <path>...:: diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 42056ee9ff..9a78dd721e 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -11,7 +11,7 @@ given by a list of patterns. SYNOPSIS -------- [verse] -'git sparse-checkout <subcommand> [options]' +'git sparse-checkout <subcommand> [<options>]' DESCRIPTION diff --git a/Documentation/git-stage.txt b/Documentation/git-stage.txt index 25bcda936d..2f6aaa75b9 100644 --- a/Documentation/git-stage.txt +++ b/Documentation/git-stage.txt @@ -9,7 +9,7 @@ git-stage - Add file contents to the staging area SYNOPSIS -------- [verse] -'git stage' args... +'git stage' <arg>... DESCRIPTION diff --git a/Documentation/git-stash.txt b/Documentation/git-stash.txt index be6084ccef..6e15f47525 100644 --- a/Documentation/git-stash.txt +++ b/Documentation/git-stash.txt @@ -13,7 +13,7 @@ SYNOPSIS 'git stash' drop [-q|--quiet] [<stash>] 'git stash' ( pop | apply ) [--index] [-q|--quiet] [<stash>] 'git stash' branch <branchname> [<stash>] -'git stash' [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet] +'git stash' [push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet] [-u|--include-untracked] [-a|--all] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]] @@ -47,7 +47,7 @@ stash index (e.g. the integer `n` is equivalent to `stash@{n}`). COMMANDS -------- -push [-p|--patch] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]:: +push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]:: Save your local modifications to a new 'stash entry' and roll them back to HEAD (in the working tree and in the index). @@ -60,7 +60,7 @@ subcommand from making an unwanted stash entry. The two exceptions to this are `stash -p` which acts as alias for `stash push -p` and pathspec elements, which are allowed after a double hyphen `--` for disambiguation. -save [-p|--patch] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [<message>]:: +save [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [<message>]:: This option is deprecated in favour of 'git stash push'. It differs from "stash push" in that it cannot take pathspec. @@ -205,6 +205,16 @@ to learn how to operate the `--patch` mode. The `--patch` option implies `--keep-index`. You can use `--no-keep-index` to override this. +-S:: +--staged:: + This option is only valid for `push` and `save` commands. ++ +Stash only the changes that are currently staged. This is similar to +basic `git commit` except the state is committed to the stash instead +of current branch. ++ +The `--patch` option has priority over this one. + --pathspec-from-file=<file>:: This option is only valid for `push` command. + @@ -341,6 +351,24 @@ $ edit/build/test remaining parts $ git commit foo -m 'Remaining parts' ---------------------------------------------------------------- +Saving unrelated changes for future use:: + +When you are in the middle of massive changes and you find some +unrelated issue that you don't want to forget to fix, you can do the +change(s), stage them, and use `git stash push --staged` to stash them +out for future use. This is similar to committing the staged changes, +only the commit ends-up being in the stash and not on the current branch. ++ +---------------------------------------------------------------- +# ... hack hack hack ... +$ git add --patch foo # add unrelated changes to the index +$ git stash push --staged # save these changes to the stash +# ... hack hack hack, finish curent changes ... +$ git commit -m 'Massive' # commit fully tested changes +$ git switch fixup-branch # switch to another branch +$ git stash pop # to finish work on the saved changes +---------------------------------------------------------------- + Recovering stash entries that were cleared/dropped erroneously:: If you mistakenly drop or clear stash entries, they cannot be recovered diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 4a2c3e0408..54a4b29b47 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -314,6 +314,14 @@ Line Notes ------------------------------------------------------------ .... +Stash Information +^^^^^^^^^^^^^^^^^ + +If `--show-stash` is given, one line is printed showing the number of stash +entries if non-zero: + + # stash <N> + Changed Tracked Entries ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Documentation/git-svn.txt b/Documentation/git-svn.txt index 222b556d7a..4e92308e85 100644 --- a/Documentation/git-svn.txt +++ b/Documentation/git-svn.txt @@ -575,7 +575,7 @@ OPTIONS ------- --shared[=(false|true|umask|group|all|world|everybody)]:: ---template=<template_directory>:: +--template=<template-directory>:: Only used with the 'init' command. These are passed directly to 'git init'. diff --git a/Documentation/git-switch.txt b/Documentation/git-switch.txt index 5c438cd505..5c90f76fbe 100644 --- a/Documentation/git-switch.txt +++ b/Documentation/git-switch.txt @@ -137,8 +137,7 @@ should result in deletion of the path). The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values are - "merge" (default) and "diff3" (in addition to what is shown by - "merge" style, shows the original contents). + "merge" (default), "diff3", and "zdiff3". -q:: --quiet:: diff --git a/Documentation/git-var.txt b/Documentation/git-var.txt index 6072f936ab..387cc1b914 100644 --- a/Documentation/git-var.txt +++ b/Documentation/git-var.txt @@ -59,6 +59,9 @@ ifdef::git-default-pager[] The build you are using chose '{git-default-pager}' as the default. endif::git-default-pager[] +GIT_DEFAULT_BRANCH:: + The name of the first branch created in newly initialized repositories. + SEE ALSO -------- linkgit:git-commit-tree[1] diff --git a/Documentation/git-web--browse.txt b/Documentation/git-web--browse.txt index 8d162b56c5..f2f996cbe1 100644 --- a/Documentation/git-web--browse.txt +++ b/Documentation/git-web--browse.txt @@ -8,7 +8,7 @@ git-web--browse - Git helper script to launch a web browser SYNOPSIS -------- [verse] -'git web{litdd}browse' [<options>] <url|file>... +'git web{litdd}browse' [<options>] (<URL>|<file>)... DESCRIPTION ----------- diff --git a/Documentation/git-worktree.txt b/Documentation/git-worktree.txt index 8a7cbdd19c..9e862fbcf7 100644 --- a/Documentation/git-worktree.txt +++ b/Documentation/git-worktree.txt @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] 'git worktree add' [-f] [--detach] [--checkout] [--lock [--reason <string>]] [-b <new-branch>] <path> [<commit-ish>] -'git worktree list' [--porcelain] +'git worktree list' [-v | --porcelain] 'git worktree lock' [--reason <string>] <worktree> 'git worktree move' <worktree> <new-path> 'git worktree prune' [-n] [-v] [--expire <expire>] diff --git a/Documentation/git.txt b/Documentation/git.txt index 281c5f8cae..13f83a2a3a 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -832,8 +832,9 @@ for full details. `GIT_TRACE_REDACT`:: By default, when tracing is activated, Git redacts the values of - cookies, the "Authorization:" header, and the "Proxy-Authorization:" - header. Set this variable to `0` to prevent this redaction. + cookies, the "Authorization:" header, the "Proxy-Authorization:" + header and packfile URIs. Set this variable to `0` to prevent this + redaction. `GIT_LITERAL_PATHSPECS`:: Setting this variable to `1` will cause Git to treat all diff --git a/Documentation/gitcredentials.txt b/Documentation/gitcredentials.txt index 758bf39ba3..80517b4eb2 100644 --- a/Documentation/gitcredentials.txt +++ b/Documentation/gitcredentials.txt @@ -132,7 +132,7 @@ because the hostnames differ. Nor would it match `foo.example.com`; Git compares hostnames exactly, without considering whether two hosts are part of the same domain. Likewise, a config entry for `http://example.com` would not match: Git compares the protocols exactly. However, you may use wildcards in -the domain name and other pattern matching techniques as with the `http.<url>.*` +the domain name and other pattern matching techniques as with the `http.<URL>.*` options. If the "pattern" URL does include a path component, then this too must match @@ -147,7 +147,7 @@ CONFIGURATION OPTIONS Options for a credential context can be configured either in `credential.*` (which applies to all credentials), or -`credential.<url>.*`, where <url> matches the context as described +`credential.<URL>.*`, where <URL> matches the context as described above. The following options are available in either location: diff --git a/Documentation/gitsubmodules.txt b/Documentation/gitsubmodules.txt index 891c8da4fd..941858a6ec 100644 --- a/Documentation/gitsubmodules.txt +++ b/Documentation/gitsubmodules.txt @@ -226,7 +226,7 @@ Workflow for a third party library ---------------------------------- # Add a submodule - git submodule add <url> <path> + git submodule add <URL> <path> # Occasionally update the submodule to a new version: git -C <path> checkout <new version> diff --git a/Documentation/gitworkflows.txt b/Documentation/gitworkflows.txt index 47cf97f9be..59305265c5 100644 --- a/Documentation/gitworkflows.txt +++ b/Documentation/gitworkflows.txt @@ -394,7 +394,7 @@ request to do so by mail. Such a request looks like ------------------------------------- Please pull from - <url> <branch> + <URL> <branch> ------------------------------------- In that case, 'git pull' can do the fetch and merge in one go, as @@ -403,7 +403,7 @@ follows. .Push/pull: Merging remote topics [caption="Recipe: "] ===================================== -`git pull <url> <branch>` +`git pull <URL> <branch>` ===================================== Occasionally, the maintainer may get merge conflicts when they try to @@ -440,7 +440,7 @@ merge because you cannot format-patch merges): .format-patch/am: Keeping topics up to date [caption="Recipe: "] ===================================== -`git pull --rebase <url> <branch>` +`git pull --rebase <URL> <branch>` ===================================== You can then fix the conflicts during the rebase. Presumably you have diff --git a/Documentation/pretty-formats.txt b/Documentation/pretty-formats.txt index ef6bd420ae..0b4c1c8d98 100644 --- a/Documentation/pretty-formats.txt +++ b/Documentation/pretty-formats.txt @@ -20,7 +20,7 @@ built-in formats: * 'oneline' - <hash> <title line> + <hash> <title-line> + This is designed to be as compact as possible. @@ -29,17 +29,17 @@ This is designed to be as compact as possible. commit <hash> Author: <author> - <title line> + <title-line> * 'medium' commit <hash> Author: <author> - Date: <author date> + Date: <author-date> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'full' @@ -47,25 +47,25 @@ This is designed to be as compact as possible. Author: <author> Commit: <committer> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'fuller' commit <hash> Author: <author> - AuthorDate: <author date> + AuthorDate: <author-date> Commit: <committer> - CommitDate: <committer date> + CommitDate: <committer-date> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'reference' - <abbrev hash> (<title line>, <short author date>) + <abbrev-hash> (<title-line>, <short-author-date>) + This format is used to refer to another commit in a commit message and is the same as `--pretty='format:%C(auto)%h (%s, %ad)'`. By default, @@ -78,10 +78,10 @@ placeholders, its output is not affected by other options like From <hash> <date> From: <author> - Date: <author date> - Subject: [PATCH] <title line> + Date: <author-date> + Subject: [PATCH] <title-line> - <full commit message> + <full-commit-message> * 'mboxrd' + @@ -101,9 +101,9 @@ commits are displayed, but not the way the diff is shown e.g. with `git log --raw`. To get full object names in a raw diff format, use `--no-abbrev`. -* 'format:<string>' +* 'format:<format-string>' + -The 'format:<string>' format allows you to specify which information +The 'format:<format-string>' format allows you to specify which information you want to show. It works a little bit like printf format, with the notable exception that you get a newline with '%n' instead of '\n'. @@ -220,6 +220,12 @@ The placeholders are: inconsistent when tags are added or removed at the same time. + +** 'tags[=<bool-value>]': Instead of only considering annotated tags, + consider lightweight tags as well. +** 'abbrev=<number>': Instead of using the default number of hexadecimal digits + (which will vary according to the number of objects in the repository with a + default of 7) of the abbreviated object name, use <number> digits, or as many + digits as needed to form a unique object name. ** 'match=<pattern>': Only consider tags matching the given `glob(7)` pattern, excluding the "refs/tags/" prefix. ** 'exclude=<pattern>': Do not consider tags matching the given @@ -273,12 +279,7 @@ endif::git-rev-list[] If any option is provided multiple times the last occurrence wins. + -The boolean options accept an optional value `[=<BOOL>]`. The values -`true`, `false`, `on`, `off` etc. are all accepted. See the "boolean" -sub-section in "EXAMPLES" in linkgit:git-config[1]. If a boolean -option is given with no value, it's enabled. -+ -** 'key=<K>': only show trailers with specified key. Matching is done +** 'key=<key>': only show trailers with specified <key>. Matching is done case-insensitively and trailing colon is optional. If option is given multiple times trailer lines matching any of the keys are shown. This option automatically enables the `only` option so that @@ -286,25 +287,25 @@ option is given with no value, it's enabled. desired it can be disabled with `only=false`. E.g., `%(trailers:key=Reviewed-by)` shows trailer lines with key `Reviewed-by`. -** 'only[=<BOOL>]': select whether non-trailer lines from the trailer +** 'only[=<bool>]': select whether non-trailer lines from the trailer block should be included. -** 'separator=<SEP>': specify a separator inserted between trailer +** 'separator=<sep>': specify a separator inserted between trailer lines. When this option is not given each trailer line is - terminated with a line feed character. The string SEP may contain + terminated with a line feed character. The string <sep> may contain the literal formatting codes described above. To use comma as separator one must use `%x2C` as it would otherwise be parsed as next option. E.g., `%(trailers:key=Ticket,separator=%x2C )` shows all trailer lines whose key is "Ticket" separated by a comma and a space. -** 'unfold[=<BOOL>]': make it behave as if interpret-trailer's `--unfold` +** 'unfold[=<bool>]': make it behave as if interpret-trailer's `--unfold` option was given. E.g., `%(trailers:only,unfold=true)` unfolds and shows all trailer lines. -** 'keyonly[=<BOOL>]': only show the key part of the trailer. -** 'valueonly[=<BOOL>]': only show the value part of the trailer. -** 'key_value_separator=<SEP>': specify a separator inserted between +** 'keyonly[=<bool>]': only show the key part of the trailer. +** 'valueonly[=<bool>]': only show the value part of the trailer. +** 'key_value_separator=<sep>': specify a separator inserted between trailer lines. When this option is not given each trailer key-value pair is separated by ": ". Otherwise it shares the same semantics - as 'separator=<SEP>' above. + as 'separator=<sep>' above. NOTE: Some placeholders may depend on other options given to the revision traversal engine. For example, the `%g*` reflog options will @@ -313,6 +314,11 @@ insert an empty string unless we are traversing reflog entries (e.g., by decoration format if `--decorate` was not already provided on the command line. +The boolean options accept an optional value `[=<bool-value>]`. The values +`true`, `false`, `on`, `off` etc. are all accepted. See the "boolean" +sub-section in "EXAMPLES" in linkgit:git-config[1]. If a boolean +option is given with no value, it's enabled. + If you add a `+` (plus sign) after '%' of a placeholder, a line-feed is inserted immediately before the expansion if and only if the placeholder expands to a non-empty string. diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt index 24569b06d1..43a86fa562 100644 --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@ -1047,7 +1047,7 @@ omitted. has no effect. `--date=format:...` feeds the format `...` to your system `strftime`, -except for %z and %Z, which are handled internally. +except for %s, %z, and %Z, which are handled internally. Use `--date=format:%c` to show the date in your system locale's preferred format. See the `strftime` manual for a complete list of format placeholders. When using `-local`, the correct syntax is diff --git a/Documentation/technical/multi-pack-index.txt b/Documentation/technical/multi-pack-index.txt index 86f40f2490..b39c69da8c 100644 --- a/Documentation/technical/multi-pack-index.txt +++ b/Documentation/technical/multi-pack-index.txt @@ -17,12 +17,12 @@ is not feasible due to storage space or excessive repack times. The multi-pack-index (MIDX for short) stores a list of objects and their offsets into multiple packfiles. It contains: -- A list of packfile names. -- A sorted list of object IDs. -- A list of metadata for the ith object ID including: - - A value j referring to the jth packfile. - - An offset within the jth packfile for the object. -- If large offsets are required, we use another list of large +* A list of packfile names. +* A sorted list of object IDs. +* A list of metadata for the ith object ID including: +** A value j referring to the jth packfile. +** An offset within the jth packfile for the object. +* If large offsets are required, we use another list of large offsets similar to version 2 pack-indexes. Thus, we can provide O(log N) lookup time for any number @@ -87,11 +87,6 @@ Future Work helpful to organize packfiles by object type (commit, tree, blob, etc.) and use this metadata to help that maintenance. -- The partial clone feature records special "promisor" packs that - may point to objects that are not stored locally, but available - on request to a server. The multi-pack-index does not currently - track these promisor packs. - Related Links ------------- [0] https://bugs.chromium.org/p/git/issues/detail?id=6 diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 21e8258ccf..8a877d27e2 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -125,11 +125,11 @@ command can be requested at a time. empty-request = flush-pkt command-request = command capability-list - [command-args] + delim-pkt + command-args flush-pkt command = PKT-LINE("command=" key LF) - command-args = delim-pkt - *command-specific-arg + command-args = *command-specific-arg command-specific-args are packet line framed arguments defined by each individual command. diff --git a/Documentation/technical/rerere.txt b/Documentation/technical/rerere.txt index af5f9fc24f..35d4541433 100644 --- a/Documentation/technical/rerere.txt +++ b/Documentation/technical/rerere.txt @@ -14,9 +14,9 @@ conflicts before writing them to the rerere database. Different conflict styles and branch names are normalized by stripping the labels from the conflict markers, and removing the common ancestor -version from the `diff3` conflict style. Branches that are merged -in different order are normalized by sorting the conflict hunks. More -on each of those steps in the following sections. +version from the `diff3` or `zdiff3` conflict styles. Branches that +are merged in different order are normalized by sorting the conflict +hunks. More on each of those steps in the following sections. Once these two normalization operations are applied, a conflict ID is calculated based on the normalized conflict, which is later used by @@ -42,8 +42,8 @@ get a conflict like the following: >>>>>>> AC Doing the analogous with AC2 (forking a branch ABAC2 off of branch AB -and then merging branch AC2 into it), using the diff3 conflict style, -we get a conflict like the following: +and then merging branch AC2 into it), using the diff3 or zdiff3 +conflict style, we get a conflict like the following: <<<<<<< HEAD B diff --git a/Documentation/urls-remotes.txt b/Documentation/urls-remotes.txt index bd184cd653..86d0008f94 100644 --- a/Documentation/urls-remotes.txt +++ b/Documentation/urls-remotes.txt @@ -26,14 +26,14 @@ config file would appear like this: ------------ [remote "<name>"] - url = <url> + url = <URL> pushurl = <pushurl> push = <refspec> fetch = <refspec> ------------ The `<pushurl>` is used for pushes only. It is optional and defaults -to `<url>`. +to `<URL>`. Named file in `$GIT_DIR/remotes` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -67,10 +67,10 @@ This file should have the following format: ------------ - <url>#<head> + <URL>#<head> ------------ -`<url>` is required; `#<head>` is optional. +`<URL>` is required; `#<head>` is optional. Depending on the operation, git will use one of the following refspecs, if you don't provide one on the command line. diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 991308dfaf..8d2297c33b 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.34.1 +DEF_VER=v2.34.GIT LF=' ' @@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) # @@ -305,9 +307,6 @@ all:: # # Define NO_TCLTK if you do not want Tcl/Tk GUI. # -# Define SANE_TEXT_GREP to "-a" if you use recent versions of GNU grep -# and egrep that are pickier when their input contains non-ASCII data. -# # The TCL_PATH variable governs the location of the Tcl interpreter # used to optimize git-gui for your system. Only used if NO_TCLTK # is not set. Defaults to the bare 'tclsh'. @@ -735,6 +734,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o @@ -813,6 +813,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h @@ -1192,7 +1194,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION) @@ -1215,7 +1217,7 @@ ARFLAGS = rcs PTHREAD_CFLAGS = # For the 'sparse' target -SPARSE_FLAGS ?= +SPARSE_FLAGS ?= -std=gnu99 SP_EXTRA_FLAGS = -Wno-universal-initializer # For informing GIT-BUILD-OPTIONS of the SANITIZE=leak target @@ -1723,6 +1725,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif @@ -2252,33 +2259,30 @@ command-list.h: $(wildcard Documentation/git*.txt) hook-list.h: generate-hooklist.sh Documentation/githooks.txt $(QUIET_GEN)$(SHELL_PATH) ./generate-hooklist.sh >$@ -SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):$(GIT_VERSION):\ - $(localedir_SQ):$(NO_CURL):$(USE_GETTEXT_SCHEME):$(SANE_TOOL_PATH_SQ):\ - $(gitwebdir_SQ):$(PERL_PATH_SQ):$(SANE_TEXT_GREP):$(PAGER_ENV):\ +SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):\ + $(localedir_SQ):$(USE_GETTEXT_SCHEME):$(SANE_TOOL_PATH_SQ):\ + $(gitwebdir_SQ):$(PERL_PATH_SQ):$(PAGER_ENV):\ $(perllibdir_SQ) +GIT-SCRIPT-DEFINES: FORCE + @FLAGS='$(SCRIPT_DEFINES)'; \ + if test x"$$FLAGS" != x"`cat $@ 2>/dev/null`" ; then \ + echo >&2 " * new script parameters"; \ + echo "$$FLAGS" >$@; \ + fi + define cmd_munge_script sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ -e 's|@SHELL_PATH@|$(SHELL_PATH_SQ)|' \ -e 's|@@DIFF@@|$(DIFF_SQ)|' \ -e 's|@@LOCALEDIR@@|$(localedir_SQ)|g' \ - -e 's/@@NO_CURL@@/$(NO_CURL)/g' \ -e 's/@@USE_GETTEXT_SCHEME@@/$(USE_GETTEXT_SCHEME)/g' \ -e $(BROKEN_PATH_FIX) \ -e 's|@@GITWEBDIR@@|$(gitwebdir_SQ)|g' \ -e 's|@@PERL@@|$(PERL_PATH_SQ)|g' \ - -e 's|@@SANE_TEXT_GREP@@|$(SANE_TEXT_GREP)|g' \ -e 's|@@PAGER_ENV@@|$(PAGER_ENV_SQ)|g' \ $@.sh >$@+ endef -GIT-SCRIPT-DEFINES: FORCE - @FLAGS='$(SCRIPT_DEFINES)'; \ - if test x"$$FLAGS" != x"`cat $@ 2>/dev/null`" ; then \ - echo >&2 " * new script parameters"; \ - echo "$$FLAGS" >$@; \ - fi - - $(SCRIPT_SH_GEN) : % : %.sh GIT-SCRIPT-DEFINES $(QUIET_GEN)$(cmd_munge_script) && \ chmod +x $@+ && \ @@ -2437,7 +2441,36 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o + +REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o +REFTABLE_TEST_OBJS += reftable/merged_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o +REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/tree_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS) @@ -2453,6 +2486,8 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif @@ -2595,6 +2630,12 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE @@ -2893,7 +2934,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X @@ -3231,7 +3272,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS) @@ -1,4 +1,4 @@ -[](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) +[](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) Git - fast, scalable, distributed revision control system ========================================================= @@ -1 +1 @@ -Documentation/RelNotes/2.34.1.txt
\ No newline at end of file +Documentation/RelNotes/2.35.0.txt
\ No newline at end of file diff --git a/add-patch.c b/add-patch.c index 8c41cdfe39..573eef0cc4 100644 --- a/add-patch.c +++ b/add-patch.c @@ -413,7 +413,7 @@ static int parse_diff(struct add_p_state *s, const struct pathspec *ps) strvec_push(&args, ps->items[i].original); setup_child_process(s, &cp, NULL); - cp.argv = args.v; + strvec_pushv(&cp.args, args.v); res = capture_command(&cp, plain, 0); if (res) { strvec_clear(&args); @@ -431,7 +431,7 @@ static int parse_diff(struct add_p_state *s, const struct pathspec *ps) setup_child_process(s, &colored_cp, NULL); xsnprintf((char *)args.v[color_arg_index], 8, "--color"); - colored_cp.argv = args.v; + strvec_pushv(&colored_cp.args, args.v); colored = &s->colored; res = capture_command(&colored_cp, colored, 0); strvec_clear(&args); diff --git a/archive-tar.c b/archive-tar.c index 05d2455870..3c74db1746 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -430,7 +430,6 @@ static int write_tar_filter_archive(const struct archiver *ar, { struct strbuf cmd = STRBUF_INIT; struct child_process filter = CHILD_PROCESS_INIT; - const char *argv[2]; int r; if (!ar->data) @@ -440,14 +439,12 @@ static int write_tar_filter_archive(const struct archiver *ar, if (args->compression_level >= 0) strbuf_addf(&cmd, " -%d", args->compression_level); - argv[0] = cmd.buf; - argv[1] = NULL; - filter.argv = argv; + strvec_push(&filter.args, cmd.buf); filter.use_shell = 1; filter.in = -1; if (start_command(&filter) < 0) - die_errno(_("unable to start '%s' filter"), argv[0]); + die_errno(_("unable to start '%s' filter"), cmd.buf); close(1); if (dup2(filter.in, 1) < 0) die_errno(_("unable to redirect descriptor")); @@ -457,7 +454,7 @@ static int write_tar_filter_archive(const struct archiver *ar, close(1); if (finish_command(&filter) != 0) - die(_("'%s' filter reported error"), argv[0]); + die(_("'%s' filter reported error"), cmd.buf); strbuf_release(&cmd); return r; diff --git a/builtin/add.c b/builtin/add.c index ef6b619c45..a010b2c325 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -302,15 +302,11 @@ int interactive_add(const char **argv, const char *prefix, int patch) static int edit_patch(int argc, const char **argv, const char *prefix) { char *file = git_pathdup("ADD_EDIT.patch"); - const char *apply_argv[] = { "apply", "--recount", "--cached", - NULL, NULL }; struct child_process child = CHILD_PROCESS_INIT; struct rev_info rev; int out; struct stat st; - apply_argv[3] = file; - git_config(git_diff_basic_config, NULL); /* no "diff" UI options */ if (read_cache() < 0) @@ -338,7 +334,8 @@ static int edit_patch(int argc, const char **argv, const char *prefix) die(_("Empty patch. Aborted.")); child.git_cmd = 1; - child.argv = apply_argv; + strvec_pushl(&child.args, "apply", "--recount", "--cached", file, + NULL); if (run_command(&child)) die(_("Could not apply '%s'"), file); diff --git a/builtin/branch.c b/builtin/branch.c index 7a1d1eeb07..81b5c111cb 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -77,12 +77,11 @@ define_list_config_array(color_branch_slots); static int git_branch_config(const char *var, const char *value, void *cb) { const char *slot_name; - struct ref_sorting **sorting_tail = (struct ref_sorting **)cb; if (!strcmp(var, "branch.sort")) { if (!value) return config_error_nonbool(var); - parse_ref_sorting(sorting_tail, value); + string_list_append(cb, value); return 0; } @@ -625,7 +624,8 @@ int cmd_branch(int argc, const char **argv, const char *prefix) enum branch_track track; struct ref_filter filter; int icase = 0; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + static struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct ref_format format = REF_FORMAT_INIT; struct option options[] = { @@ -666,7 +666,7 @@ int cmd_branch(int argc, const char **argv, const char *prefix) OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), OPT_COLUMN(0, "column", &colopts, N_("list branches in columns")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), N_("print only branches of the object"), parse_opt_object_name), OPT_BOOL('i', "ignore-case", &icase, N_("sorting and filtering are case insensitive")), @@ -683,7 +683,7 @@ int cmd_branch(int argc, const char **argv, const char *prefix) if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(builtin_branch_usage, options); - git_config(git_branch_config, sorting_tail); + git_config(git_branch_config, &sorting_options); track = git_branch_track; @@ -749,8 +749,7 @@ int cmd_branch(int argc, const char **argv, const char *prefix) * local branches 'refs/heads/...' and finally remote-tracking * branches 'refs/remotes/...'. */ - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); ref_sorting_set_sort_flags_all( sorting, REF_SORTING_DETACHED_HEAD_FIRST, 1); diff --git a/builtin/checkout.c b/builtin/checkout.c index cbf73b8c9f..72beeb49aa 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -91,8 +91,8 @@ struct checkout_opts { }; struct branch_info { - const char *name; /* The short name used */ - const char *path; /* The full name of a real branch */ + char *name; /* The short name used */ + char *path; /* The full name of a real branch */ struct commit *commit; /* The named commit */ char *refname; /* The full name of the ref being checked out. */ struct object_id oid; /* The object ID of the commit being checked out. */ @@ -103,6 +103,14 @@ struct branch_info { char *checkout; }; +static void branch_info_release(struct branch_info *info) +{ + free(info->name); + free(info->path); + free(info->refname); + free(info->checkout); +} + static int post_checkout_hook(struct commit *old_commit, struct commit *new_commit, int changed) { @@ -688,9 +696,12 @@ static void setup_branch_path(struct branch_info *branch) repo_get_oid_committish(the_repository, branch->name, &branch->oid); strbuf_branchname(&buf, branch->name, INTERPRET_BRANCH_LOCAL); - if (strcmp(buf.buf, branch->name)) + if (strcmp(buf.buf, branch->name)) { + free(branch->name); branch->name = xstrdup(buf.buf); + } strbuf_splice(&buf, 0, 0, "refs/heads/", 11); + free(branch->path); branch->path = strbuf_detach(&buf, NULL); } @@ -874,7 +885,7 @@ static void update_refs_for_switch(const struct checkout_opts *opts, int ret; struct strbuf err = STRBUF_INIT; - ret = safe_create_reflog(refname, 1, &err); + ret = safe_create_reflog(refname, &err); if (ret) { fprintf(stderr, _("Can not do reflog for '%s': %s\n"), opts->new_orphan_branch, err.buf); @@ -894,7 +905,9 @@ static void update_refs_for_switch(const struct checkout_opts *opts, opts->new_branch_log, opts->quiet, opts->track); - new_branch_info->name = opts->new_branch; + free(new_branch_info->name); + free(new_branch_info->refname); + new_branch_info->name = xstrdup(opts->new_branch); setup_branch_path(new_branch_info); } @@ -1062,8 +1075,7 @@ static int switch_branches(const struct checkout_opts *opts, struct branch_info *new_branch_info) { int ret = 0; - struct branch_info old_branch_info; - void *path_to_free; + struct branch_info old_branch_info = { 0 }; struct object_id rev; int flag, writeout_error = 0; int do_merge = 1; @@ -1071,25 +1083,32 @@ static int switch_branches(const struct checkout_opts *opts, trace2_cmd_mode("branch"); memset(&old_branch_info, 0, sizeof(old_branch_info)); - old_branch_info.path = path_to_free = resolve_refdup("HEAD", 0, &rev, &flag); + old_branch_info.path = resolve_refdup("HEAD", 0, &rev, &flag); if (old_branch_info.path) old_branch_info.commit = lookup_commit_reference_gently(the_repository, &rev, 1); if (!(flag & REF_ISSYMREF)) - old_branch_info.path = NULL; + FREE_AND_NULL(old_branch_info.path); - if (old_branch_info.path) - skip_prefix(old_branch_info.path, "refs/heads/", &old_branch_info.name); + if (old_branch_info.path) { + const char *const prefix = "refs/heads/"; + const char *p; + if (skip_prefix(old_branch_info.path, prefix, &p)) + old_branch_info.name = xstrdup(p); + else + BUG("should be able to skip past '%s' in '%s'!", + prefix, old_branch_info.path); + } if (opts->new_orphan_branch && opts->orphan_from_empty_tree) { if (new_branch_info->name) BUG("'switch --orphan' should never accept a commit as starting point"); new_branch_info->commit = NULL; - new_branch_info->name = "(empty)"; + new_branch_info->name = xstrdup("(empty)"); do_merge = 1; } if (!new_branch_info->name) { - new_branch_info->name = "HEAD"; + new_branch_info->name = xstrdup("HEAD"); new_branch_info->commit = old_branch_info.commit; if (!new_branch_info->commit) die(_("You are on a branch yet to be born")); @@ -1102,7 +1121,7 @@ static int switch_branches(const struct checkout_opts *opts, if (do_merge) { ret = merge_working_tree(opts, &old_branch_info, new_branch_info, &writeout_error); if (ret) { - free(path_to_free); + branch_info_release(&old_branch_info); return ret; } } @@ -1113,7 +1132,8 @@ static int switch_branches(const struct checkout_opts *opts, update_refs_for_switch(opts, &old_branch_info, new_branch_info); ret = post_checkout_hook(old_branch_info.commit, new_branch_info->commit, 1); - free(path_to_free); + branch_info_release(&old_branch_info); + return ret || writeout_error; } @@ -1145,16 +1165,15 @@ static void setup_new_branch_info_and_source_tree( struct tree **source_tree = &opts->source_tree; struct object_id branch_rev; - new_branch_info->name = arg; + new_branch_info->name = xstrdup(arg); setup_branch_path(new_branch_info); if (!check_refname_format(new_branch_info->path, 0) && !read_ref(new_branch_info->path, &branch_rev)) oidcpy(rev, &branch_rev); - else { - free((char *)new_branch_info->path); - new_branch_info->path = NULL; /* not an existing branch */ - } + else + /* not an existing branch */ + FREE_AND_NULL(new_branch_info->path); new_branch_info->commit = lookup_commit_reference_gently(the_repository, rev, 1); if (!new_branch_info->commit) { @@ -1517,7 +1536,7 @@ static struct option *add_common_options(struct checkout_opts *opts, OPT_BOOL(0, "progress", &opts->show_progress, N_("force progress reporting")), OPT_BOOL('m', "merge", &opts->merge, N_("perform a 3-way merge with the new branch")), OPT_STRING(0, "conflict", &opts->conflict_style, N_("style"), - N_("conflict style (merge or diff3)")), + N_("conflict style (merge, diff3, or zdiff3)")), OPT_END() }; struct option *newopts = parse_options_concat(prevopts, options); @@ -1574,12 +1593,11 @@ static char cb_option = 'b'; static int checkout_main(int argc, const char **argv, const char *prefix, struct checkout_opts *opts, struct option *options, - const char * const usagestr[]) + const char * const usagestr[], + struct branch_info *new_branch_info) { - struct branch_info new_branch_info; int parseopt_flags = 0; - memset(&new_branch_info, 0, sizeof(new_branch_info)); opts->overwrite_ignore = 1; opts->prefix = prefix; opts->show_progress = -1; @@ -1688,7 +1706,7 @@ static int checkout_main(int argc, const char **argv, const char *prefix, opts->track == BRANCH_TRACK_UNSPECIFIED && !opts->new_branch; int n = parse_branchname_arg(argc, argv, dwim_ok, - &new_branch_info, opts, &rev); + new_branch_info, opts, &rev); argv += n; argc -= n; } else if (!opts->accept_ref && opts->from_treeish) { @@ -1697,7 +1715,7 @@ static int checkout_main(int argc, const char **argv, const char *prefix, if (get_oid_mb(opts->from_treeish, &rev)) die(_("could not resolve %s"), opts->from_treeish); - setup_new_branch_info_and_source_tree(&new_branch_info, + setup_new_branch_info_and_source_tree(new_branch_info, opts, &rev, opts->from_treeish); @@ -1717,7 +1735,7 @@ static int checkout_main(int argc, const char **argv, const char *prefix, * Try to give more helpful suggestion. * new_branch && argc > 1 will be caught later. */ - if (opts->new_branch && argc == 1 && !new_branch_info.commit) + if (opts->new_branch && argc == 1 && !new_branch_info->commit) die(_("'%s' is not a commit and a branch '%s' cannot be created from it"), argv[0], opts->new_branch); @@ -1766,11 +1784,10 @@ static int checkout_main(int argc, const char **argv, const char *prefix, strbuf_release(&buf); } - UNLEAK(opts); if (opts->patch_mode || opts->pathspec.nr) - return checkout_paths(opts, &new_branch_info); + return checkout_paths(opts, new_branch_info); else - return checkout_branch(opts, &new_branch_info); + return checkout_branch(opts, new_branch_info); } int cmd_checkout(int argc, const char **argv, const char *prefix) @@ -1789,6 +1806,7 @@ int cmd_checkout(int argc, const char **argv, const char *prefix) OPT_END() }; int ret; + struct branch_info new_branch_info = { 0 }; memset(&opts, 0, sizeof(opts)); opts.dwim_new_local_branch = 1; @@ -1819,7 +1837,9 @@ int cmd_checkout(int argc, const char **argv, const char *prefix) options = add_checkout_path_options(&opts, options); ret = checkout_main(argc, argv, prefix, &opts, - options, checkout_usage); + options, checkout_usage, &new_branch_info); + branch_info_release(&new_branch_info); + clear_pathspec(&opts.pathspec); FREE_AND_NULL(options); return ret; } @@ -1840,6 +1860,7 @@ int cmd_switch(int argc, const char **argv, const char *prefix) OPT_END() }; int ret; + struct branch_info new_branch_info = { 0 }; memset(&opts, 0, sizeof(opts)); opts.dwim_new_local_branch = 1; @@ -1859,7 +1880,8 @@ int cmd_switch(int argc, const char **argv, const char *prefix) cb_option = 'c'; ret = checkout_main(argc, argv, prefix, &opts, - options, switch_branch_usage); + options, switch_branch_usage, &new_branch_info); + branch_info_release(&new_branch_info); FREE_AND_NULL(options); return ret; } @@ -1881,6 +1903,7 @@ int cmd_restore(int argc, const char **argv, const char *prefix) OPT_END() }; int ret; + struct branch_info new_branch_info = { 0 }; memset(&opts, 0, sizeof(opts)); opts.accept_ref = 0; @@ -1896,7 +1919,8 @@ int cmd_restore(int argc, const char **argv, const char *prefix) options = add_checkout_path_options(&opts, options); ret = checkout_main(argc, argv, prefix, &opts, - options, restore_usage); + options, restore_usage, &new_branch_info); + branch_info_release(&new_branch_info); FREE_AND_NULL(options); return ret; } diff --git a/builtin/credential.c b/builtin/credential.c index d75dcdc64a..d7b304fa08 100644 --- a/builtin/credential.c +++ b/builtin/credential.c @@ -4,7 +4,7 @@ #include "config.h" static const char usage_msg[] = - "git credential [fill|approve|reject]"; + "git credential (fill|approve|reject)"; int cmd_credential(int argc, const char **argv, const char *prefix) { diff --git a/builtin/difftool.c b/builtin/difftool.c index 4931c10845..4ee40fe3a0 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -202,15 +202,10 @@ static void changed_files(struct hashmap *result, const char *index_path, { struct child_process update_index = CHILD_PROCESS_INIT; struct child_process diff_files = CHILD_PROCESS_INIT; - struct strbuf index_env = STRBUF_INIT, buf = STRBUF_INIT; - const char *git_dir = absolute_path(get_git_dir()), *env[] = { - NULL, NULL - }; + struct strbuf buf = STRBUF_INIT; + const char *git_dir = absolute_path(get_git_dir()); FILE *fp; - strbuf_addf(&index_env, "GIT_INDEX_FILE=%s", index_path); - env[0] = index_env.buf; - strvec_pushl(&update_index.args, "--git-dir", git_dir, "--work-tree", workdir, "update-index", "--really-refresh", "-q", @@ -222,7 +217,7 @@ static void changed_files(struct hashmap *result, const char *index_path, update_index.use_shell = 0; update_index.clean_on_exit = 1; update_index.dir = workdir; - update_index.env = env; + strvec_pushf(&update_index.env_array, "GIT_INDEX_FILE=%s", index_path); /* Ignore any errors of update-index */ run_command(&update_index); @@ -235,7 +230,7 @@ static void changed_files(struct hashmap *result, const char *index_path, diff_files.clean_on_exit = 1; diff_files.out = -1; diff_files.dir = workdir; - diff_files.env = env; + strvec_pushf(&diff_files.env_array, "GIT_INDEX_FILE=%s", index_path); if (start_command(&diff_files)) die("could not obtain raw diff"); fp = xfdopen(diff_files.out, "r"); @@ -248,7 +243,6 @@ static void changed_files(struct hashmap *result, const char *index_path, fclose(fp); if (finish_command(&diff_files)) die("diff-files did not exit properly"); - strbuf_release(&index_env); strbuf_release(&buf); } diff --git a/builtin/for-each-ref.c b/builtin/for-each-ref.c index 16a2c7d57c..6f62f40d12 100644 --- a/builtin/for-each-ref.c +++ b/builtin/for-each-ref.c @@ -17,7 +17,8 @@ static char const * const for_each_ref_usage[] = { int cmd_for_each_ref(int argc, const char **argv, const char *prefix) { int i; - struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; int maxcount = 0, icase = 0; struct ref_array array; struct ref_filter filter; @@ -39,7 +40,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) OPT_INTEGER( 0 , "count", &maxcount, N_("show only <n> matched refs")), OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")), OPT__COLOR(&format.use_color, N_("respect format colors")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), N_("print only refs which points at the given object"), parse_opt_object_name), @@ -70,8 +71,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) if (verify_ref_format(&format)) usage_with_options(for_each_ref_usage, opts); - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); filter.ignore_case = icase; diff --git a/builtin/fsck.c b/builtin/fsck.c index 27b9e78094..9e54892311 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -944,15 +944,13 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) if (the_repository->settings.core_commit_graph) { struct child_process commit_graph_verify = CHILD_PROCESS_INIT; - const char *verify_argv[] = { "commit-graph", "verify", NULL, NULL, NULL }; prepare_alt_odb(the_repository); for (odb = the_repository->objects->odb; odb; odb = odb->next) { child_process_init(&commit_graph_verify); - commit_graph_verify.argv = verify_argv; commit_graph_verify.git_cmd = 1; - verify_argv[2] = "--object-dir"; - verify_argv[3] = odb->path; + strvec_pushl(&commit_graph_verify.args, "commit-graph", + "verify", "--object-dir", odb->path, NULL); if (run_command(&commit_graph_verify)) errors_found |= ERROR_COMMIT_GRAPH; } @@ -960,15 +958,13 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) if (the_repository->settings.core_multi_pack_index) { struct child_process midx_verify = CHILD_PROCESS_INIT; - const char *midx_argv[] = { "multi-pack-index", "verify", NULL, NULL, NULL }; prepare_alt_odb(the_repository); for (odb = the_repository->objects->odb; odb; odb = odb->next) { child_process_init(&midx_verify); - midx_verify.argv = midx_argv; midx_verify.git_cmd = 1; - midx_argv[2] = "--object-dir"; - midx_argv[3] = odb->path; + strvec_pushl(&midx_verify.args, "multi-pack-index", + "verify", "--object-dir", odb->path, NULL); if (run_command(&midx_verify)) errors_found |= ERROR_MULTI_PACK_INDEX; } diff --git a/builtin/help.c b/builtin/help.c index 75cd2fb407..d387131dd8 100644 --- a/builtin/help.c +++ b/builtin/help.c @@ -212,11 +212,10 @@ static int check_emacsclient_version(void) { struct strbuf buffer = STRBUF_INIT; struct child_process ec_process = CHILD_PROCESS_INIT; - const char *argv_ec[] = { "emacsclient", "--version", NULL }; int version; /* emacsclient prints its version number on stderr */ - ec_process.argv = argv_ec; + strvec_pushl(&ec_process.args, "emacsclient", "--version", NULL); ec_process.err = -1; ec_process.stdout_to_stderr = 1; if (start_command(&ec_process)) diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index 318949c3d7..44448fa61d 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -54,7 +54,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) struct transport *transport; const struct ref *ref; struct ref_array ref_array; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct option options[] = { OPT__QUIET(&quiet, N_("do not print remote URL")), @@ -68,7 +68,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) OPT_BIT(0, "refs", &flags, N_("do not show peeled tags"), REF_NORMAL), OPT_BOOL(0, "get-url", &get_url, N_("take url.<base>.insteadOf into account")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_SET_INT_F(0, "exit-code", &status, N_("exit with exit code 2 if no matching refs are found"), 2, PARSE_OPT_NOCOMPLETE), @@ -86,8 +86,6 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) packet_trace_identity("ls-remote"); - UNLEAK(sorting); - if (argc > 1) { int i; CALLOC_ARRAY(pattern, argc); @@ -139,8 +137,13 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) item->symref = xstrdup_or_null(ref->symref); } - if (sorting) + if (sorting_options.nr) { + struct ref_sorting *sorting; + + sorting = ref_sorting_options(&sorting_options); ref_array_sort(sorting, &ref_array); + ref_sorting_release(sorting); + } for (i = 0; i < ref_array.nr; i++) { const struct ref_array_item *ref = ref_array.items[i]; diff --git a/builtin/merge-file.c b/builtin/merge-file.c index 06a2f90c48..e695867ee5 100644 --- a/builtin/merge-file.c +++ b/builtin/merge-file.c @@ -34,6 +34,8 @@ int cmd_merge_file(int argc, const char **argv, const char *prefix) struct option options[] = { OPT_BOOL('p', "stdout", &to_stdout, N_("send results to standard output")), OPT_SET_INT(0, "diff3", &xmp.style, N_("use a diff3 based merge"), XDL_MERGE_DIFF3), + OPT_SET_INT(0, "zdiff3", &xmp.style, N_("use a zealous diff3 based merge"), + XDL_MERGE_ZEALOUS_DIFF3), OPT_SET_INT(0, "ours", &xmp.favor, N_("for conflicts, use our version"), XDL_MERGE_FAVOR_OURS), OPT_SET_INT(0, "theirs", &xmp.favor, N_("for conflicts, use their version"), diff --git a/builtin/merge.c b/builtin/merge.c index ea3112e0c0..5f0476b0b7 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -310,10 +310,9 @@ static int save_state(struct object_id *stash) int len; struct child_process cp = CHILD_PROCESS_INIT; struct strbuf buffer = STRBUF_INIT; - const char *argv[] = {"stash", "create", NULL}; int rc = -1; - cp.argv = argv; + strvec_pushl(&cp.args, "stash", "create", NULL); cp.out = -1; cp.git_cmd = 1; diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 075d15d706..4480ba3982 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -167,6 +167,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv) usage_with_options(builtin_multi_pack_index_verify_usage, options); + FREE_AND_NULL(options); + return verify_midx_file(the_repository, opts.object_dir, opts.flags); } @@ -191,6 +193,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv) usage_with_options(builtin_multi_pack_index_expire_usage, options); + FREE_AND_NULL(options); + return expire_midx_packs(the_repository, opts.object_dir, opts.flags); } diff --git a/builtin/notes.c b/builtin/notes.c index 71c59583a1..85d1abad88 100644 --- a/builtin/notes.c +++ b/builtin/notes.c @@ -134,14 +134,13 @@ static void copy_obj_to_fd(int fd, const struct object_id *oid) static void write_commented_object(int fd, const struct object_id *object) { - const char *show_args[5] = - {"show", "--stat", "--no-notes", oid_to_hex(object), NULL}; struct child_process show = CHILD_PROCESS_INIT; struct strbuf buf = STRBUF_INIT; struct strbuf cbuf = STRBUF_INIT; /* Invoke "git show --stat --no-notes $object" */ - show.argv = show_args; + strvec_pushl(&show.args, "show", "--stat", "--no-notes", + oid_to_hex(object), NULL); show.no_stdin = 1; show.out = -1; show.err = 0; diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1a3dd445f8..857be7826f 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -4148,11 +4148,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) read_packs_list_from_stdin(); if (rev_list_unpacked) add_unreachable_loose_objects(); - } else if (!use_internal_rev_list) + } else if (!use_internal_rev_list) { read_object_list_from_stdin(); - else { + } else { get_object_list(rp.nr, rp.v); - strvec_clear(&rp); } cleanup_preferred_base(); if (include_tag && nr_result) @@ -4162,7 +4161,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) the_repository); if (non_empty && !nr_result) - return 0; + goto cleanup; if (nr_result) { trace2_region_enter("pack-objects", "prepare-pack", the_repository); @@ -4183,5 +4182,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) " pack-reused %"PRIu32), written, written_delta, reused, reused_delta, reuse_packfile_objects); + +cleanup: + strvec_clear(&rp); + return 0; } diff --git a/builtin/pull.c b/builtin/pull.c index 1cfaf9f343..c8457619d8 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -970,7 +970,7 @@ static void show_advice_pull_non_ff(void) "You can do so by running one of the following commands sometime before\n" "your next pull:\n" "\n" - " git config pull.rebase false # merge (the default strategy)\n" + " git config pull.rebase false # merge\n" " git config pull.rebase true # rebase\n" " git config pull.ff only # fast-forward only\n" "\n" diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 49b846d960..313b372a11 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -812,16 +812,13 @@ static int run_and_feed_hook(const char *hook_name, feed_fn feed, { struct child_process proc = CHILD_PROCESS_INIT; struct async muxer; - const char *argv[2]; int code; + const char *hook_path = find_hook(hook_name); - argv[0] = find_hook(hook_name); - if (!argv[0]) + if (!hook_path) return 0; - argv[1] = NULL; - - proc.argv = argv; + strvec_push(&proc.args, hook_path); proc.in = -1; proc.stdout_to_stderr = 1; proc.trace2_hook_name = hook_name; @@ -943,23 +940,21 @@ static int run_receive_hook(struct command *commands, static int run_update_hook(struct command *cmd) { - const char *argv[5]; struct child_process proc = CHILD_PROCESS_INIT; int code; + const char *hook_path = find_hook("update"); - argv[0] = find_hook("update"); - if (!argv[0]) + if (!hook_path) return 0; - argv[1] = cmd->ref_name; - argv[2] = oid_to_hex(&cmd->old_oid); - argv[3] = oid_to_hex(&cmd->new_oid); - argv[4] = NULL; + strvec_push(&proc.args, hook_path); + strvec_push(&proc.args, cmd->ref_name); + strvec_push(&proc.args, oid_to_hex(&cmd->old_oid)); + strvec_push(&proc.args, oid_to_hex(&cmd->new_oid)); proc.no_stdin = 1; proc.stdout_to_stderr = 1; proc.err = use_sideband ? -1 : 0; - proc.argv = argv; proc.trace2_hook_name = "update"; code = start_command(&proc); @@ -1117,22 +1112,20 @@ static int run_proc_receive_hook(struct command *commands, struct child_process proc = CHILD_PROCESS_INIT; struct async muxer; struct command *cmd; - const char *argv[2]; struct packet_reader reader; struct strbuf cap = STRBUF_INIT; struct strbuf errmsg = STRBUF_INIT; int hook_use_push_options = 0; int version = 0; int code; + const char *hook_path = find_hook("proc-receive"); - argv[0] = find_hook("proc-receive"); - if (!argv[0]) { + if (!hook_path) { rp_error("cannot find hook 'proc-receive'"); return -1; } - argv[1] = NULL; - proc.argv = argv; + strvec_push(&proc.args, hook_path); proc.in = -1; proc.out = -1; proc.trace2_hook_name = "proc-receive"; @@ -1370,23 +1363,11 @@ static const char *push_to_deploy(unsigned char *sha1, struct strvec *env, const char *work_tree) { - const char *update_refresh[] = { - "update-index", "-q", "--ignore-submodules", "--refresh", NULL - }; - const char *diff_files[] = { - "diff-files", "--quiet", "--ignore-submodules", "--", NULL - }; - const char *diff_index[] = { - "diff-index", "--quiet", "--cached", "--ignore-submodules", - NULL, "--", NULL - }; - const char *read_tree[] = { - "read-tree", "-u", "-m", NULL, NULL - }; struct child_process child = CHILD_PROCESS_INIT; - child.argv = update_refresh; - child.env = env->v; + strvec_pushl(&child.args, "update-index", "-q", "--ignore-submodules", + "--refresh", NULL); + strvec_pushv(&child.env_array, env->v); child.dir = work_tree; child.no_stdin = 1; child.stdout_to_stderr = 1; @@ -1396,8 +1377,9 @@ static const char *push_to_deploy(unsigned char *sha1, /* run_command() does not clean up completely; reinitialize */ child_process_init(&child); - child.argv = diff_files; - child.env = env->v; + strvec_pushl(&child.args, "diff-files", "--quiet", + "--ignore-submodules", "--", NULL); + strvec_pushv(&child.env_array, env->v); child.dir = work_tree; child.no_stdin = 1; child.stdout_to_stderr = 1; @@ -1405,12 +1387,13 @@ static const char *push_to_deploy(unsigned char *sha1, if (run_command(&child)) return "Working directory has unstaged changes"; - /* diff-index with either HEAD or an empty tree */ - diff_index[4] = head_has_history() ? "HEAD" : empty_tree_oid_hex(); - child_process_init(&child); - child.argv = diff_index; - child.env = env->v; + strvec_pushl(&child.args, "diff-index", "--quiet", "--cached", + "--ignore-submodules", + /* diff-index with either HEAD or an empty tree */ + head_has_history() ? "HEAD" : empty_tree_oid_hex(), + "--", NULL); + strvec_pushv(&child.env_array, env->v); child.no_stdin = 1; child.no_stdout = 1; child.stdout_to_stderr = 0; @@ -1418,10 +1401,10 @@ static const char *push_to_deploy(unsigned char *sha1, if (run_command(&child)) return "Working directory has staged changes"; - read_tree[3] = hash_to_hex(sha1); child_process_init(&child); - child.argv = read_tree; - child.env = env->v; + strvec_pushl(&child.args, "read-tree", "-u", "-m", hash_to_hex(sha1), + NULL); + strvec_pushv(&child.env_array, env->v); child.dir = work_tree; child.no_stdin = 1; child.no_stdout = 1; @@ -2219,7 +2202,8 @@ static const char *unpack(int err_fd, struct shallow_info *si) close(err_fd); return "unable to create temporary object directory"; } - child.env = tmp_objdir_env(tmp_objdir); + if (tmp_objdir) + strvec_pushv(&child.env_array, tmp_objdir_env(tmp_objdir)); /* * Normally we just pass the tmp_objdir environment to the child @@ -2566,25 +2550,25 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) &push_options); if (pack_lockfile) unlink_or_warn(pack_lockfile); + sigchain_push(SIGPIPE, SIG_IGN); if (report_status_v2) report_v2(commands, unpack_status); else if (report_status) report(commands, unpack_status); + sigchain_pop(SIGPIPE); run_receive_hook(commands, "post-receive", 1, &push_options); run_update_post_hook(commands); string_list_clear(&push_options, 0); if (auto_gc) { - const char *argv_gc_auto[] = { - "gc", "--auto", "--quiet", NULL, - }; struct child_process proc = CHILD_PROCESS_INIT; proc.no_stdin = 1; proc.stdout_to_stderr = 1; proc.err = use_sideband ? -1 : 0; proc.git_cmd = proc.close_object_store = 1; - proc.argv = argv_gc_auto; + strvec_pushl(&proc.args, "gc", "--auto", "--quiet", + NULL); if (!start_command(&proc)) { if (use_sideband) diff --git a/builtin/repack.c b/builtin/repack.c index 0b2d1e5d82..9b0be6a6ab 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -258,9 +258,11 @@ static void repack_promisor_objects(const struct pack_objects_args *args, for_each_packed_object(write_oid, &cmd, FOR_EACH_OBJECT_PROMISOR_ONLY); - if (cmd.in == -1) + if (cmd.in == -1) { /* No packed objects; cmd was never started */ + child_process_clear(&cmd); return; + } close(cmd.in); @@ -842,7 +844,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) fname_old = mkpathdup("%s-%s%s", packtmp, item->string, exts[ext].name); - if (((uintptr_t)item->util) & (1 << ext)) { + if (((uintptr_t)item->util) & ((uintptr_t)1 << ext)) { struct stat statbuffer; if (!stat(fname_old, &statbuffer)) { statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); diff --git a/builtin/replace.c b/builtin/replace.c index 946938d011..6ff1734d58 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -258,11 +258,10 @@ static int import_object(struct object_id *oid, enum object_type type, return error_errno(_("unable to open %s for reading"), filename); if (!raw && type == OBJ_TREE) { - const char *argv[] = { "mktree", NULL }; struct child_process cmd = CHILD_PROCESS_INIT; struct strbuf result = STRBUF_INIT; - cmd.argv = argv; + strvec_push(&cmd.args, "mktree"); cmd.git_cmd = 1; cmd.in = fd; cmd.out = -1; diff --git a/builtin/reset.c b/builtin/reset.c index 7393595349..b1ff699b43 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -25,6 +25,7 @@ #include "cache-tree.h" #include "submodule.h" #include "submodule-config.h" +#include "dir.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -136,21 +137,36 @@ static void update_index_from_diff(struct diff_queue_struct *q, int intent_to_add = *(int *)data; for (i = 0; i < q->nr; i++) { + int pos; struct diff_filespec *one = q->queue[i]->one; - int is_missing = !(one->mode && !is_null_oid(&one->oid)); + int is_in_reset_tree = one->mode && !is_null_oid(&one->oid); struct cache_entry *ce; - if (is_missing && !intent_to_add) { + if (!is_in_reset_tree && !intent_to_add) { remove_file_from_cache(one->path); continue; } ce = make_cache_entry(&the_index, one->mode, &one->oid, one->path, 0, 0); + + /* + * If the file 1) corresponds to an existing index entry with + * skip-worktree set, or 2) does not exist in the index but is + * outside the sparse checkout definition, add a skip-worktree bit + * to the new index entry. Note that a sparse index will be expanded + * if this entry is outside the sparse cone - this is necessary + * to properly construct the reset sparse directory. + */ + pos = cache_name_pos(one->path, strlen(one->path)); + if ((pos >= 0 && ce_skip_worktree(active_cache[pos])) || + (pos < 0 && !path_in_sparse_checkout(one->path, &the_index))) + ce->ce_flags |= CE_SKIP_WORKTREE; + if (!ce) die(_("make_cache_entry failed for path '%s'"), one->path); - if (is_missing) { + if (!is_in_reset_tree) { ce->ce_flags |= CE_INTENT_TO_ADD; set_object_name_for_intent_to_add_entry(ce); } @@ -158,6 +174,82 @@ static void update_index_from_diff(struct diff_queue_struct *q, } } +static int pathspec_needs_expanded_index(const struct pathspec *pathspec) +{ + unsigned int i, pos; + int res = 0; + char *skip_worktree_seen = NULL; + + /* + * When using a magic pathspec, assume for the sake of simplicity that + * the index needs to be expanded to match all matchable files. + */ + if (pathspec->magic) + return 1; + + for (i = 0; i < pathspec->nr; i++) { + struct pathspec_item item = pathspec->items[i]; + + /* + * If the pathspec item has a wildcard, the index should be expanded + * if the pathspec has the possibility of matching a subset of entries inside + * of a sparse directory (but not the entire directory). + * + * If the pathspec item is a literal path, the index only needs to be expanded + * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't + * expand for in-cone files) and b) it doesn't match any sparse directories + * (since we can reset whole sparse directories without expanding them). + */ + if (item.nowildcard_len < item.len) { + /* + * Special case: if the pattern is a path inside the cone + * followed by only wildcards, the pattern cannot match + * partial sparse directories, so we don't expand the index. + */ + if (path_in_cone_mode_sparse_checkout(item.original, &the_index) && + strspn(item.original + item.nowildcard_len, "*") == item.len - item.nowildcard_len) + continue; + + for (pos = 0; pos < active_nr; pos++) { + struct cache_entry *ce = active_cache[pos]; + + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + + /* + * If the pre-wildcard length is longer than the sparse + * directory name and the sparse directory is the first + * component of the pathspec, need to expand the index. + */ + if (item.nowildcard_len > ce_namelen(ce) && + !strncmp(item.original, ce->name, ce_namelen(ce))) { + res = 1; + break; + } + + /* + * If the pre-wildcard length is shorter than the sparse + * directory and the pathspec does not match the whole + * directory, need to expand the index. + */ + if (!strncmp(item.original, ce->name, item.nowildcard_len) && + wildmatch(item.original, ce->name, 0)) { + res = 1; + break; + } + } + } else if (!path_in_cone_mode_sparse_checkout(item.original, &the_index) && + !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) + res = 1; + + if (res > 0) + break; + } + + free(skip_worktree_seen); + return res; +} + static int read_from_tree(const struct pathspec *pathspec, struct object_id *tree_oid, int intent_to_add) @@ -170,7 +262,13 @@ static int read_from_tree(const struct pathspec *pathspec, opt.format_callback = update_index_from_diff; opt.format_callback_data = &intent_to_add; opt.flags.override_submodule_config = 1; + opt.flags.recursive = 1; opt.repo = the_repository; + opt.change = diff_change; + opt.add_remove = diff_addremove; + + if (pathspec->nr && the_index.sparse_index && pathspec_needs_expanded_index(pathspec)) + ensure_full_index(&the_index); if (do_diff_cache(tree_oid, &opt)) return 1; @@ -249,9 +347,6 @@ static void parse_args(struct pathspec *pathspec, } *rev_ret = rev; - if (read_cache() < 0) - die(_("index file corrupt")); - parse_pathspec(pathspec, 0, PATHSPEC_PREFER_FULL | (patch_mode ? PATHSPEC_PREFIX_ORIGIN : 0), @@ -397,6 +492,12 @@ int cmd_reset(int argc, const char **argv, const char *prefix) if (intent_to_add && reset_type != MIXED) die(_("-N can only be used with --mixed")); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + + if (read_cache() < 0) + die(_("index file corrupt")); + /* Soft reset does not touch the index file nor the working tree * at all, but requires them in a good order. Other resets reset * the index file to the tree object we are switching to. */ diff --git a/builtin/show-branch.c b/builtin/show-branch.c index 082449293b..f1e8318592 100644 --- a/builtin/show-branch.c +++ b/builtin/show-branch.c @@ -761,6 +761,7 @@ int cmd_show_branch(int ac, const char **av, const char *prefix) char *logmsg; char *nth_desc; const char *msg; + char *end; timestamp_t timestamp; int tz; @@ -770,11 +771,12 @@ int cmd_show_branch(int ac, const char **av, const char *prefix) reflog = i; break; } - msg = strchr(logmsg, '\t'); - if (!msg) - msg = "(none)"; - else - msg++; + + end = strchr(logmsg, '\n'); + if (end) + *end = '\0'; + + msg = (*logmsg == '\0') ? "(none)" : logmsg; reflog_msg[i] = xstrfmt("(%s) %s", show_date(timestamp, tz, DATE_MODE(RELATIVE)), diff --git a/builtin/stash.c b/builtin/stash.c index a0ccc8654d..18c812bbe0 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -27,11 +27,11 @@ static const char * const git_stash_usage[] = { N_("git stash ( pop | apply ) [--index] [-q|--quiet] [<stash>]"), N_("git stash branch <branchname> [<stash>]"), "git stash clear", - N_("git stash [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" + N_("git stash [push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [-m|--message <message>]\n" " [--pathspec-from-file=<file> [--pathspec-file-nul]]\n" " [--] [<pathspec>...]]"), - N_("git stash save [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" + N_("git stash save [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [<message>]"), NULL }; @@ -1132,6 +1132,38 @@ done: return ret; } +static int stash_staged(struct stash_info *info, struct strbuf *out_patch, + int quiet) +{ + int ret = 0; + struct child_process cp_diff_tree = CHILD_PROCESS_INIT; + struct index_state istate = { NULL }; + + if (write_index_as_tree(&info->w_tree, &istate, the_repository->index_file, + 0, NULL)) { + ret = -1; + goto done; + } + + cp_diff_tree.git_cmd = 1; + strvec_pushl(&cp_diff_tree.args, "diff-tree", "-p", "-U1", "HEAD", + oid_to_hex(&info->w_tree), "--", NULL); + if (pipe_command(&cp_diff_tree, NULL, 0, out_patch, 0, NULL, 0)) { + ret = -1; + goto done; + } + + if (!out_patch->len) { + if (!quiet) + fprintf_ln(stderr, _("No staged changes")); + ret = 1; + } + +done: + discard_index(&istate); + return ret; +} + static int stash_patch(struct stash_info *info, const struct pathspec *ps, struct strbuf *out_patch, int quiet) { @@ -1258,7 +1290,7 @@ done: } static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_buf, - int include_untracked, int patch_mode, + int include_untracked, int patch_mode, int only_staged, struct stash_info *info, struct strbuf *patch, int quiet) { @@ -1337,6 +1369,16 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b } else if (ret > 0) { goto done; } + } else if (only_staged) { + ret = stash_staged(info, patch, quiet); + if (ret < 0) { + if (!quiet) + fprintf_ln(stderr, _("Cannot save the current " + "staged state")); + goto done; + } else if (ret > 0) { + goto done; + } } else { if (stash_working_tree(info, ps)) { if (!quiet) @@ -1395,7 +1437,7 @@ static int create_stash(int argc, const char **argv, const char *prefix) if (!check_changes_tracked_files(&ps)) return 0; - ret = do_create_stash(&ps, &stash_msg_buf, 0, 0, &info, + ret = do_create_stash(&ps, &stash_msg_buf, 0, 0, 0, &info, NULL, 0); if (!ret) printf_ln("%s", oid_to_hex(&info.w_commit)); @@ -1405,7 +1447,7 @@ static int create_stash(int argc, const char **argv, const char *prefix) } static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int quiet, - int keep_index, int patch_mode, int include_untracked) + int keep_index, int patch_mode, int include_untracked, int only_staged) { int ret = 0; struct stash_info info; @@ -1423,6 +1465,17 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q goto done; } + /* --patch overrides --staged */ + if (patch_mode) + only_staged = 0; + + if (only_staged && include_untracked) { + fprintf_ln(stderr, _("Can't use --staged and --include-untracked" + " or --all at the same time")); + ret = -1; + goto done; + } + read_cache_preload(NULL); if (!include_untracked && ps->nr) { int i; @@ -1463,7 +1516,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q if (stash_msg) strbuf_addstr(&stash_msg_buf, stash_msg); - if (do_create_stash(ps, &stash_msg_buf, include_untracked, patch_mode, + if (do_create_stash(ps, &stash_msg_buf, include_untracked, patch_mode, only_staged, &info, &patch, quiet)) { ret = -1; goto done; @@ -1480,7 +1533,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q printf_ln(_("Saved working directory and index state %s"), stash_msg_buf.buf); - if (!patch_mode) { + if (!(patch_mode || only_staged)) { if (include_untracked && !ps->nr) { struct child_process cp = CHILD_PROCESS_INIT; @@ -1598,6 +1651,7 @@ static int push_stash(int argc, const char **argv, const char *prefix, { int force_assume = 0; int keep_index = -1; + int only_staged = 0; int patch_mode = 0; int include_untracked = 0; int quiet = 0; @@ -1608,6 +1662,8 @@ static int push_stash(int argc, const char **argv, const char *prefix, struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), + OPT_BOOL('S', "staged", &only_staged, + N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), OPT__QUIET(&quiet, N_("quiet mode")), @@ -1646,6 +1702,9 @@ static int push_stash(int argc, const char **argv, const char *prefix, if (patch_mode) die(_("--pathspec-from-file is incompatible with --patch")); + if (only_staged) + die(_("--pathspec-from-file is incompatible with --staged")); + if (ps.nr) die(_("--pathspec-from-file is incompatible with pathspec arguments")); @@ -1657,12 +1716,13 @@ static int push_stash(int argc, const char **argv, const char *prefix, } return do_push_stash(&ps, stash_msg, quiet, keep_index, patch_mode, - include_untracked); + include_untracked, only_staged); } static int save_stash(int argc, const char **argv, const char *prefix) { int keep_index = -1; + int only_staged = 0; int patch_mode = 0; int include_untracked = 0; int quiet = 0; @@ -1673,6 +1733,8 @@ static int save_stash(int argc, const char **argv, const char *prefix) struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), + OPT_BOOL('S', "staged", &only_staged, + N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), OPT__QUIET(&quiet, N_("quiet mode")), @@ -1694,7 +1756,7 @@ static int save_stash(int argc, const char **argv, const char *prefix) memset(&ps, 0, sizeof(ps)); ret = do_push_stash(&ps, stash_msg, quiet, keep_index, - patch_mode, include_untracked); + patch_mode, include_untracked, only_staged); strbuf_release(&stash_msg_buf); return ret; diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index e630f0c730..9b25a508e6 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1503,16 +1503,17 @@ static void deinit_submodule(const char *path, const char *prefix, struct strbuf sb_rm = STRBUF_INIT; const char *format; - /* - * protect submodules containing a .git directory - * NEEDSWORK: instead of dying, automatically call - * absorbgitdirs and (possibly) warn. - */ - if (is_directory(sub_git_dir)) - die(_("Submodule work tree '%s' contains a .git " - "directory (use 'rm -rf' if you really want " - "to remove it including all of its history)"), - displaypath); + if (is_directory(sub_git_dir)) { + if (!(flags & OPT_QUIET)) + warning(_("Submodule work tree '%s' contains a .git " + "directory. This will be replaced with a " + ".git file by using absorbgitdirs."), + displaypath); + + absorb_git_dir_into_superproject(path, + ABSORB_GITDIR_RECURSE_SUBMODULES); + + } if (!(flags & OPT_FORCE)) { struct child_process cp_rm = CHILD_PROCESS_INIT; diff --git a/builtin/tag.c b/builtin/tag.c index 6fe646710d..41863c5ab7 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -178,7 +178,6 @@ static const char tag_template_nocleanup[] = static int git_tag_config(const char *var, const char *value, void *cb) { int status; - struct ref_sorting **sorting_tail = (struct ref_sorting **)cb; if (!strcmp(var, "tag.gpgsign")) { config_sign_tag = git_config_bool(var, value); @@ -188,7 +187,7 @@ static int git_tag_config(const char *var, const char *value, void *cb) if (!strcmp(var, "tag.sort")) { if (!value) return config_error_nonbool(var); - parse_ref_sorting(sorting_tail, value); + string_list_append(cb, value); return 0; } @@ -436,7 +435,8 @@ int cmd_tag(int argc, const char **argv, const char *prefix) struct ref_transaction *transaction; struct strbuf err = STRBUF_INIT; struct ref_filter filter; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct ref_format format = REF_FORMAT_INIT; int icase = 0; int edit_flag = 0; @@ -470,7 +470,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix) OPT_WITHOUT(&filter.no_commit, N_("print only tags that don't contain the commit")), OPT_MERGED(&filter, N_("print only tags that are merged")), OPT_NO_MERGED(&filter, N_("print only tags that are not merged")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), { OPTION_CALLBACK, 0, "points-at", &filter.points_at, N_("object"), N_("print only tags of the object"), PARSE_OPT_LASTARG_DEFAULT, @@ -486,7 +486,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix) setup_ref_filter_porcelain_msg(); - git_config(git_tag_config, sorting_tail); + git_config(git_tag_config, &sorting_options); memset(&opt, 0, sizeof(opt)); memset(&filter, 0, sizeof(filter)); @@ -525,8 +525,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix) die(_("--column and -n are incompatible")); colopts = 0; } - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); filter.ignore_case = icase; if (cmdmode == 'l') { diff --git a/builtin/upload-archive.c b/builtin/upload-archive.c index 24654b4c9b..98d028dae6 100644 --- a/builtin/upload-archive.c +++ b/builtin/upload-archive.c @@ -77,7 +77,7 @@ static ssize_t process_input(int child_fd, int band) int cmd_upload_archive(int argc, const char **argv, const char *prefix) { - struct child_process writer = { argv }; + struct child_process writer = CHILD_PROCESS_INIT; if (argc == 2 && !strcmp(argv[1], "-h")) usage(upload_archive_usage); @@ -89,9 +89,10 @@ int cmd_upload_archive(int argc, const char **argv, const char *prefix) * multiplexed out to our fd#1. If the child dies, we tell the other * end over channel #3. */ - argv[0] = "upload-archive--writer"; writer.out = writer.err = -1; writer.git_cmd = 1; + strvec_push(&writer.args, "upload-archive--writer"); + strvec_pushv(&writer.args, argv + 1); if (start_command(&writer)) { int err = errno; packet_write_fmt(1, "NACK unable to spawn subprocess\n"); diff --git a/builtin/var.c b/builtin/var.c index 6c6f46b4ae..491db27429 100644 --- a/builtin/var.c +++ b/builtin/var.c @@ -5,6 +5,7 @@ */ #include "builtin.h" #include "config.h" +#include "refs.h" static const char var_usage[] = "git var (-l | <variable>)"; @@ -27,6 +28,11 @@ static const char *pager(int flag) return pgm; } +static const char *default_branch(int flag) +{ + return git_default_branch_name(1); +} + struct git_var { const char *name; const char *(*read)(int); @@ -36,6 +42,7 @@ static struct git_var git_vars[] = { { "GIT_AUTHOR_IDENT", git_author_info }, { "GIT_EDITOR", editor }, { "GIT_PAGER", pager }, + { "GIT_DEFAULT_BRANCH", default_branch }, { "", NULL }, }; diff --git a/builtin/worktree.c b/builtin/worktree.c index d22ece93e1..a396cfdc64 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -72,7 +72,7 @@ static void delete_worktrees_dir_if_empty(void) static void prune_worktree(const char *id, const char *reason) { if (show_only || verbose) - printf_ln(_("Removing %s/%s: %s"), "worktrees", id, reason); + fprintf_ln(stderr, _("Removing %s/%s: %s"), "worktrees", id, reason); if (!show_only) delete_git_dir(id); } @@ -349,18 +349,18 @@ static int add_worktree(const char *path, const char *refname, strvec_push(&cp.args, "--quiet"); } - cp.env = child_env.v; + strvec_pushv(&cp.env_array, child_env.v); ret = run_command(&cp); if (ret) goto done; if (opts->checkout) { - cp.argv = NULL; - strvec_clear(&cp.args); + struct child_process cp = CHILD_PROCESS_INIT; + cp.git_cmd = 1; strvec_pushl(&cp.args, "reset", "--hard", "--no-recurse-submodules", NULL); if (opts->quiet) strvec_push(&cp.args, "--quiet"); - cp.env = child_env.v; + strvec_pushv(&cp.env_array, child_env.v); ret = run_command(&cp); if (ret) goto done; @@ -385,12 +385,11 @@ done: const char *hook = find_hook("post-checkout"); if (hook) { const char *env[] = { "GIT_DIR", "GIT_WORK_TREE", NULL }; - cp.git_cmd = 0; + struct child_process cp = CHILD_PROCESS_INIT; cp.no_stdin = 1; cp.stdout_to_stderr = 1; cp.dir = path; - cp.env = env; - cp.argv = NULL; + strvec_pushv(&cp.env_array, env); cp.trace2_hook_name = "post-checkout"; strvec_pushl(&cp.args, absolute_path(hook), oid_to_hex(null_oid()), @@ -418,24 +417,24 @@ static void print_preparing_worktree_line(int detach, if (force_new_branch) { struct commit *commit = lookup_commit_reference_by_name(new_branch); if (!commit) - printf_ln(_("Preparing worktree (new branch '%s')"), new_branch); + fprintf_ln(stderr, _("Preparing worktree (new branch '%s')"), new_branch); else - printf_ln(_("Preparing worktree (resetting branch '%s'; was at %s)"), + fprintf_ln(stderr, _("Preparing worktree (resetting branch '%s'; was at %s)"), new_branch, find_unique_abbrev(&commit->object.oid, DEFAULT_ABBREV)); } else if (new_branch) { - printf_ln(_("Preparing worktree (new branch '%s')"), new_branch); + fprintf_ln(stderr, _("Preparing worktree (new branch '%s')"), new_branch); } else { struct strbuf s = STRBUF_INIT; if (!detach && !strbuf_check_branch_ref(&s, branch) && ref_exists(s.buf)) - printf_ln(_("Preparing worktree (checking out '%s')"), + fprintf_ln(stderr, _("Preparing worktree (checking out '%s')"), branch); else { struct commit *commit = lookup_commit_reference_by_name(branch); if (!commit) die(_("invalid reference: %s"), branch); - printf_ln(_("Preparing worktree (detached HEAD %s)"), + fprintf_ln(stderr, _("Preparing worktree (detached HEAD %s)"), find_unique_abbrev(&commit->object.oid, DEFAULT_ABBREV)); } strbuf_release(&s); @@ -1006,7 +1005,7 @@ static int remove_worktree(int ac, const char **av, const char *prefix) static void report_repair(int iserr, const char *path, const char *msg, void *cb_data) { if (!iserr) { - printf_ln(_("repair: %s: %s"), msg, path); + fprintf_ln(stderr, _("repair: %s: %s"), msg, path); } else { int *exit_status = (int *)cb_data; fprintf_ln(stderr, _("error: %s: %s"), msg, path); diff --git a/cache-tree.c b/cache-tree.c index 79d168192d..65ca993361 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -741,15 +741,26 @@ out: return ret; } +static void prime_cache_tree_sparse_dir(struct cache_tree *it, + struct tree *tree) +{ + + oidcpy(&it->oid, &tree->object.oid); + it->entry_count = 1; +} + static void prime_cache_tree_rec(struct repository *r, struct cache_tree *it, - struct tree *tree) + struct tree *tree, + struct strbuf *tree_path) { struct tree_desc desc; struct name_entry entry; int cnt; + int base_path_len = tree_path->len; oidcpy(&it->oid, &tree->object.oid); + init_tree_desc(&desc, tree->buffer, tree->size); cnt = 0; while (tree_entry(&desc, &entry)) { @@ -758,14 +769,40 @@ static void prime_cache_tree_rec(struct repository *r, else { struct cache_tree_sub *sub; struct tree *subtree = lookup_tree(r, &entry.oid); + if (!subtree->object.parsed) parse_tree(subtree); sub = cache_tree_sub(it, entry.path); sub->cache_tree = cache_tree(); - prime_cache_tree_rec(r, sub->cache_tree, subtree); + + /* + * Recursively-constructed subtree path is only needed when working + * in a sparse index (where it's used to determine whether the + * subtree is a sparse directory in the index). + */ + if (r->index->sparse_index) { + strbuf_setlen(tree_path, base_path_len); + strbuf_grow(tree_path, base_path_len + entry.pathlen + 1); + strbuf_add(tree_path, entry.path, entry.pathlen); + strbuf_addch(tree_path, '/'); + } + + /* + * If a sparse index is in use, the directory being processed may be + * sparse. To confirm that, we can check whether an entry with that + * exact name exists in the index. If it does, the created subtree + * should be sparse. Otherwise, cache tree expansion should continue + * as normal. + */ + if (r->index->sparse_index && + index_entry_exists(r->index, tree_path->buf, tree_path->len)) + prime_cache_tree_sparse_dir(sub->cache_tree, subtree); + else + prime_cache_tree_rec(r, sub->cache_tree, subtree, tree_path); cnt += sub->cache_tree->entry_count; } } + it->entry_count = cnt; } @@ -773,11 +810,14 @@ void prime_cache_tree(struct repository *r, struct index_state *istate, struct tree *tree) { + struct strbuf tree_path = STRBUF_INIT; + trace2_region_enter("cache-tree", "prime_cache_tree", the_repository); cache_tree_free(&istate->cache_tree); istate->cache_tree = cache_tree(); - prime_cache_tree_rec(r, istate->cache_tree, tree); + prime_cache_tree_rec(r, istate->cache_tree, tree, &tree_path); + strbuf_release(&tree_path); istate->cache_changed |= CACHE_TREE_CHANGED; trace2_region_leave("cache-tree", "prime_cache_tree", the_repository); } @@ -817,6 +817,16 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na int index_name_pos(struct index_state *, const char *name, int namelen); /* + * Determines whether an entry with the given name exists within the + * given index. The return value is 1 if an exact match is found, otherwise + * it is 0. Note that, unlike index_name_pos, this function does not expand + * the index if it is sparse. If an item exists within the full index but it + * is contained within a sparse directory (and not in the sparse index), 0 is + * returned. + */ +int index_entry_exists(struct index_state *, const char *name, int namelen); + +/* * Some functions return the negative complement of an insert position when a * precise match was not found but a position was found where the entry would * need to be inserted. This helper protects that logic from any integer @@ -1588,6 +1598,7 @@ timestamp_t approxidate_careful(const char *, int *); timestamp_t approxidate_relative(const char *date); void parse_date_format(const char *format, struct date_mode *mode); int date_overflows(timestamp_t date); +time_t tm_to_time_t(const struct tm *tm); #define IDENT_STRICT 1 #define IDENT_NO_DATE 2 diff --git a/ci/check-directional-formatting.bash b/ci/check-directional-formatting.bash new file mode 100755 index 0000000000..e6211b141a --- /dev/null +++ b/ci/check-directional-formatting.bash @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script verifies that the non-binary files tracked in the Git index do +# not contain any Unicode directional formatting: such formatting could be used +# to deceive reviewers into interpreting code differently from the compiler. +# This is intended to run on an Ubuntu agent in a GitHub workflow. +# +# To allow translated messages to introduce such directional formatting in the +# future, we exclude the `.po` files from this validation. +# +# Neither GNU grep nor `git grep` (not even with `-P`) handle `\u` as a way to +# specify UTF-8. +# +# To work around that, we use `printf` to produce the pattern as a byte +# sequence, and then feed that to `git grep` as a byte sequence (setting +# `LC_CTYPE` to make sure that the arguments are interpreted as intended). +# +# Note: we need to use Bash here because its `printf` interprets `\uNNNN` as +# UTF-8 code points, as desired. Running this script through Ubuntu's `dash`, +# for example, would use a `printf` that does not understand that syntax. + +# U+202a..U+2a2e: LRE, RLE, PDF, LRO and RLO +# U+2066..U+2069: LRI, RLI, FSI and PDI +regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)' + +! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \ + -- ':(exclude,attr:binary)' ':(exclude)*.po' diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 1d0e48f451..dbcebad2fb 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -11,18 +11,11 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" -case "$jobname" in -linux-clang|linux-gcc|linux-leaks) - sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" +case "$runs_on_pool" in +ubuntu-latest) sudo apt-get -q update sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ - $UBUNTU_COMMON_PKGS - case "$jobname" in - linux-gcc) - sudo apt-get -q -y install gcc-8 - ;; - esac - + $UBUNTU_COMMON_PKGS $CC_PACKAGE mkdir --parents "$P4_PATH" pushd "$P4_PATH" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" @@ -37,7 +30,7 @@ linux-clang|linux-gcc|linux-leaks) cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . popd ;; -osx-clang|osx-gcc) +macos-latest) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 # Uncomment this if you want to run perf tests: # brew install gnu-time @@ -51,15 +44,17 @@ osx-clang|osx-gcc) brew install --cask --no-quarantine perforce } || brew install homebrew/cask/perforce - case "$jobname" in - osx-gcc) - brew install gcc@9 - # Just in case the image is updated to contain gcc@9 - # pre-installed but not linked. - brew link gcc@9 - ;; - esac + + if test -n "$CC_PACKAGE" + then + BREW_PACKAGE=${CC_PACKAGE/-/@} + brew install "$BREW_PACKAGE" + brew link "$BREW_PACKAGE" + fi ;; +esac + +case "$jobname" in StaticAnalysis) sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ @@ -77,7 +72,7 @@ Documentation) test -n "$ALREADY_HAVE_ASCIIDOCTOR" || sudo gem install --version 1.5.8 asciidoctor ;; -linux-gcc-default|linux-gcc-4.8) +linux-gcc-default) sudo apt-get -q update sudo apt-get -q -y install $UBUNTU_COMMON_PKGS ;; diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 07a8c6b199..78b7e326da 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -4,7 +4,7 @@ # case "$jobname" in -Linux32) +linux32) linux32 --32bit i386 sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev \ @@ -34,7 +34,7 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if test "$TRAVIS_DEBUG_MODE" = true || test true = "$GITHUB_ACTIONS" + if test true = "$GITHUB_ACTIONS" then return fi @@ -60,7 +60,7 @@ skip_good_tree () { cat <<-EOF $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. - The log of that build job is available at $(url_for_job_id $prev_good_job_id) + The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id To force a re-build delete the branch's cache and then hit 'Restart job'. EOF fi @@ -91,29 +91,7 @@ export MAKEFLAGS= # and installing dependencies. set -ex -if test true = "$TRAVIS" -then - CI_TYPE=travis - # When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not - # what we want here. We want the source branch instead. - CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" - CI_COMMIT="$TRAVIS_COMMIT" - CI_JOB_ID="$TRAVIS_JOB_ID" - CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" - CI_OS_NAME="$TRAVIS_OS_NAME" - CI_REPO_SLUG="$TRAVIS_REPO_SLUG" - - cache_dir="$HOME/travis-cache" - - url_for_job_id () { - echo "https://travis-ci.org/$CI_REPO_SLUG/jobs/$1" - } - - BREW_INSTALL_PACKAGES="git-lfs gettext" - export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --immediate" - MAKEFLAGS="$MAKEFLAGS --jobs=2" -elif test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" +if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" then CI_TYPE=azure-pipelines # We are running in Azure Pipelines @@ -130,10 +108,6 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - url_for_job_id () { - echo "$SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$1" - } - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" MAKEFLAGS="$MAKEFLAGS --jobs=10" @@ -182,11 +156,15 @@ export DEFAULT_TEST_TARGET=prove export GIT_TEST_CLONE_2GB=true export SKIP_DASHED_BUILT_INS=YesPlease -case "$jobname" in -linux-clang|linux-gcc|linux-leaks) +case "$runs_on_pool" in +ubuntu-latest) + if test "$jobname" = "linux-gcc-default" + then + break + fi + if [ "$jobname" = linux-gcc ] then - export CC=gcc-8 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3" else MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2" @@ -206,24 +184,20 @@ linux-clang|linux-gcc|linux-leaks) GIT_LFS_PATH="$HOME/custom/git-lfs" export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" ;; -osx-clang|osx-gcc) +macos-latest) if [ "$jobname" = osx-gcc ] then - export CC=gcc-9 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" else MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" fi - - # t9810 occasionally fails on Travis CI OS X - # t9816 occasionally fails with "TAP out of sequence errors" on - # Travis CI OS X - export GIT_SKIP_TESTS="t9810 t9816" - ;; -linux-gcc-default) ;; -Linux32) +esac + +case "$jobname" in +linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc @@ -231,9 +205,6 @@ linux-musl) MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" ;; -esac - -case "$jobname" in linux-leaks) export SANITIZE=leak export GIT_TEST_PASSING_SANITIZE_LEAK=true diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index c70d6cdbf2..57277eefcd 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -39,8 +39,6 @@ do test_name="${test_name##*/}" trash_dir="trash directory.$test_name" case "$CI_TYPE" in - travis) - ;; azure-pipelines) mkdir -p failed-test-artifacts mv "$trash_dir" failed-test-artifacts @@ -88,11 +86,3 @@ do fi fi done - -if [ $combined_trash_size -gt 0 ] -then - echo "------------------------------------------------------------------------" - echo "Trash directories embedded in this log can be extracted by running:" - echo - echo " curl https://api.travis-ci.org/v3/job/$TRAVIS_JOB_ID/log.txt |./ci/util/extract-trash-dirs.sh" -fi diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index cc62616d80..280dda7d28 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -10,16 +10,13 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -if test "$jobname" = "pedantic" -then - export DEVOPTS=pedantic -fi +export MAKE_TARGETS="all test" -make case "$jobname" in linux-gcc) export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main - make test + ;; +linux-TEST-vars) export GIT_TEST_SPLIT_INDEX=yes export GIT_TEST_MERGE_ALGORITHM=recursive export GIT_TEST_FULL_IN_PACK_ARRAY=true @@ -33,23 +30,25 @@ linux-gcc) export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master export GIT_TEST_WRITE_REV_INDEX=1 export GIT_TEST_CHECKOUT_WORKERS=2 - make test ;; linux-clang) export GIT_TEST_DEFAULT_HASH=sha1 - make test + ;; +linux-sha256) export GIT_TEST_DEFAULT_HASH=sha256 - make test ;; -linux-gcc-4.8|pedantic) +pedantic) # Don't run the tests; we only care about whether Git can be - # built with GCC 4.8 or with pedantic - ;; -*) - make test + # built. + export DEVOPTS=pedantic + export MAKE_TARGETS=all ;; esac +# Any new "test" targets should not go after this "make", but should +# adjust $MAKE_TARGETS. Otherwise compilation-only targets above will +# start running tests. +make $MAKE_TARGETS check_unignored_build_artifacts save_good_tree diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index 8d47a5fda3..6cd832efb9 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -15,7 +15,7 @@ then fi case "$jobname" in -Linux32) +linux32) switch_cmd="linux32 --32bit i386" ;; linux-musl) @@ -47,15 +47,6 @@ else else useradd -u $HOST_UID $CI_USER fi - - # Due to a bug the test suite was run as root in the past, so - # a prove state file created back then is only accessible by - # root. Now that bug is fixed, the test suite is run as a - # regular user, but the prove state file coming from Travis - # CI's cache might still be owned by root. - # Make sure that this user has rights to any cached files, - # including an existing prove state file. - test -n "$cache_dir" && chown -R $HOST_UID:$HOST_UID "$cache_dir" fi # Build and test diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 37fa372052..af89d1624a 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -6,7 +6,7 @@ . ${0%/*}/lib.sh case "$jobname" in -Linux32) +linux32) CI_CONTAINER="daald/ubuntu32:xenial" ;; linux-musl) @@ -25,7 +25,7 @@ docker pull "$CI_CONTAINER" # root@container:/# export jobname=<jobname> # root@container:/# /usr/src/git/ci/run-docker-build.sh <host-user-id> -container_cache_dir=/tmp/travis-cache +container_cache_dir=/tmp/container-cache docker run \ --interactive \ diff --git a/command-list.txt b/command-list.txt index eb9cee8dee..675c28f0bd 100644 --- a/command-list.txt +++ b/command-list.txt @@ -43,7 +43,7 @@ # specified here, which can only have "guide" attribute and nothing # else. # -### command list (do not change this line, also do not change alignment) +### command list (do not change this line) # command name category [category] [category] git-add mainporcelain worktree git-am mainporcelain @@ -60,9 +60,9 @@ git-cat-file plumbinginterrogators git-check-attr purehelpers git-check-ignore purehelpers git-check-mailmap purehelpers +git-check-ref-format purehelpers git-checkout mainporcelain git-checkout-index plumbingmanipulators -git-check-ref-format purehelpers git-cherry plumbinginterrogators complete git-cherry-pick mainporcelain git-citool mainporcelain @@ -111,7 +111,6 @@ git-index-pack plumbingmanipulators git-init mainporcelain init git-instaweb ancillaryinterrogators complete git-interpret-trailers purehelpers -gitk mainporcelain git-log mainporcelain info git-ls-files plumbinginterrogators git-ls-remote plumbinginterrogators @@ -124,11 +123,11 @@ git-merge-base plumbinginterrogators git-merge-file plumbingmanipulators git-merge-index plumbingmanipulators git-merge-one-file purehelpers -git-mergetool ancillarymanipulators complete git-merge-tree ancillaryinterrogators -git-multi-pack-index plumbingmanipulators +git-mergetool ancillarymanipulators complete git-mktag plumbingmanipulators git-mktree plumbingmanipulators +git-multi-pack-index plumbingmanipulators git-mv mainporcelain worktree git-name-rev plumbinginterrogators git-notes mainporcelain @@ -154,23 +153,23 @@ git-request-pull foreignscminterface complete git-rerere ancillaryinterrogators git-reset mainporcelain history git-restore mainporcelain worktree -git-revert mainporcelain git-rev-list plumbinginterrogators git-rev-parse plumbinginterrogators +git-revert mainporcelain git-rm mainporcelain worktree git-send-email foreignscminterface complete git-send-pack synchingrepositories +git-sh-i18n purehelpers +git-sh-setup purehelpers git-shell synchelpers git-shortlog mainporcelain git-show mainporcelain info git-show-branch ancillaryinterrogators complete git-show-index plumbinginterrogators git-show-ref plumbinginterrogators -git-sh-i18n purehelpers -git-sh-setup purehelpers git-sparse-checkout mainporcelain -git-stash mainporcelain git-stage complete +git-stash mainporcelain git-status mainporcelain info git-stripspace purehelpers git-submodule mainporcelain @@ -189,7 +188,6 @@ git-var plumbinginterrogators git-verify-commit ancillaryinterrogators git-verify-pack plumbinginterrogators git-verify-tag ancillaryinterrogators -gitweb ancillaryinterrogators git-whatchanged ancillaryinterrogators complete git-worktree mainporcelain git-write-tree plumbingmanipulators @@ -204,6 +202,7 @@ gitfaq guide gitglossary guide githooks guide gitignore guide +gitk mainporcelain gitmailmap guide gitmodules guide gitnamespaces guide @@ -211,6 +210,7 @@ gitremote-helpers guide gitrepository-layout guide gitrevisions guide gitsubmodules guide -gittutorial-2 guide gittutorial guide +gittutorial-2 guide +gitweb ancillaryinterrogators gitworkflows guide diff --git a/compat/.gitattributes b/compat/.gitattributes new file mode 100644 index 0000000000..40dbfb170d --- /dev/null +++ b/compat/.gitattributes @@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space diff --git a/compat/mingw.c b/compat/mingw.c index 9e0cd1e097..e14f2d5f77 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1083,6 +1083,7 @@ int pipe(int filedes[2]) return 0; } +#ifndef __MINGW64__ struct tm *gmtime_r(const time_t *timep, struct tm *result) { if (gmtime_s(result, timep) == 0) @@ -1096,6 +1097,7 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) return result; return NULL; } +#endif char *mingw_getcwd(char *pointer, int len) { diff --git a/compat/unsetenv.c b/compat/unsetenv.c index bf5fd7063b..b9d34af613 100644 --- a/compat/unsetenv.c +++ b/compat/unsetenv.c @@ -1,6 +1,6 @@ #include "../git-compat-util.h" -void gitunsetenv (const char *name) +int gitunsetenv(const char *name) { #if !defined(__MINGW32__) extern char **environ; @@ -24,4 +24,6 @@ void gitunsetenv (const char *name) ++dst; } environ[dst] = NULL; + + return 0; } diff --git a/compat/zlib-uncompress2.c b/compat/zlib-uncompress2.c new file mode 100644 index 0000000000..722610b971 --- /dev/null +++ b/compat/zlib-uncompress2.c @@ -0,0 +1,95 @@ +/* taken from zlib's uncompr.c + + commit cacf7f1d4e3d44d871b605da3b647f07d718623f + Author: Mark Adler <madler@alumni.caltech.edu> + Date: Sun Jan 15 09:18:46 2017 -0800 + + zlib 1.2.11 + +*/ + +#include "../reftable/system.h" +#define z_const + +/* + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include <zlib.h> + +/* clang-format off */ + +/* =========================================================================== + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. +*/ +int ZEXPORT uncompress2 ( + Bytef *dest, + uLongf *destLen, + const Bytef *source, + uLong *sourceLen) { + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = inflateInit(&stream); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} diff --git a/config.mak.uname b/config.mak.uname index 3236a4918a..a3a779327f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -58,7 +58,6 @@ ifeq ($(uname_S),Linux) # -lrt is needed for clock_gettime on glibc <= 2.16 NEEDS_LIBRT = YesPlease HAVE_GETDELIM = YesPlease - SANE_TEXT_GREP=-a FREAD_READS_DIRECTORIES = UnfortunatelyYes BASIC_CFLAGS += -DHAVE_SYSINFO PROCFS_EXECUTABLE_PATH = /proc/self/exe @@ -262,6 +261,10 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease @@ -517,6 +520,7 @@ ifeq ($(uname_S),Interix) endif endif ifeq ($(uname_S),Minix) + NO_UNCOMPRESS2 = YesPlease NO_IPV6 = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease diff --git a/configure.ac b/configure.ac index 031e8d3fee..d60d494ee4 100644 --- a/configure.ac +++ b/configure.ac @@ -507,14 +507,6 @@ if test -n "$ASCIIDOC"; then esac fi -if grep -a ascii configure.ac >/dev/null; then - AC_MSG_RESULT([Using 'grep -a' for sane_grep]) - SANE_TEXT_GREP=-a -else - SANE_TEXT_GREP= -fi -GIT_CONF_SUBST([SANE_TEXT_GREP]) - ## Checks for libraries. AC_MSG_NOTICE([CHECKS for libraries]) # @@ -672,9 +664,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS, diff --git a/connected.c b/connected.c index 35bd4a2638..ed3025e7a2 100644 --- a/connected.c +++ b/connected.c @@ -109,7 +109,8 @@ no_promisor_pack_found: _("Checking connectivity")); rev_list.git_cmd = 1; - rev_list.env = opt->env; + if (opt->env) + strvec_pushv(&rev_list.env_array, opt->env); rev_list.in = -1; rev_list.no_stdout = 1; if (opt->err_fd) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index fd1399c440..5100f56bb3 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -208,7 +208,7 @@ endif() if(CMAKE_C_COMPILER_ID STREQUAL "MSVC") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}) - add_compile_options(/MP) + add_compile_options(/MP /std:c11) endif() #default behaviour @@ -647,6 +647,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res @@ -669,7 +675,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif() @@ -781,7 +787,6 @@ foreach(script ${git_shell_scripts}) string(REPLACE "@@USE_GETTEXT_SCHEME@@" "" content "${content}") string(REPLACE "# @@BROKEN_PATH_FIX@@" "" content "${content}") string(REPLACE "@@PERL@@" "${PERL_PATH}" content "${content}") - string(REPLACE "@@SANE_TEXT_GREP@@" "-a" content "${content}") string(REPLACE "@@PAGER_ENV@@" "LESS=FRX LV=-c" content "${content}") file(WRITE ${CMAKE_BINARY_DIR}/${script} ${content}) endforeach() @@ -909,11 +914,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index d2584450ba..1a25789d28 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g; @@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM"; @@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj"> diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 7c3a75373a..377d6c5494 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1566,7 +1566,7 @@ _git_checkout () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --*) __gitcomp_builtin checkout @@ -2001,7 +2001,7 @@ __git_log_shortlog_options=" " __git_log_pretty_formats="oneline short medium full fuller reference email raw format: tformat: mboxrd" -__git_log_date_formats="relative iso8601 iso8601-strict rfc2822 short local default raw unix format:" +__git_log_date_formats="relative iso8601 iso8601-strict rfc2822 short local default human raw unix auto: format:" _git_log () { @@ -2359,16 +2359,7 @@ _git_send_email () return ;; --*) - __gitcomp_builtin send-email "--annotate --bcc --cc --cc-cmd --chain-reply-to - --compose --confirm= --dry-run --envelope-sender - --from --identity - --in-reply-to --no-chain-reply-to --no-signed-off-by-cc - --no-suppress-from --no-thread --quiet --reply-to - --signed-off-by-cc --smtp-pass --smtp-server - --smtp-server-port --smtp-encryption= --smtp-user - --subject --suppress-cc= --suppress-from --thread --to - --validate --no-validate - $__git_format_patch_extra_options" + __gitcomp_builtin send-email "$__git_format_patch_extra_options" return ;; esac @@ -2446,7 +2437,7 @@ _git_switch () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --*) __gitcomp_builtin switch @@ -2886,7 +2877,7 @@ _git_restore () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --source=*) __git_complete_refs --cur="${cur##--source=}" diff --git a/contrib/git-jump/README b/contrib/git-jump/README index 2f618a7f97..8bcace29d2 100644 --- a/contrib/git-jump/README +++ b/contrib/git-jump/README @@ -65,6 +65,9 @@ git jump diff --cached # jump to merge conflicts git jump merge +# documentation conflicts are hard; skip past them for now +git jump merge :^Documentation + # jump to all instances of foo_bar git jump grep foo_bar diff --git a/contrib/git-jump/git-jump b/contrib/git-jump/git-jump index 931b0fe3a9..92dbd4cde1 100755 --- a/contrib/git-jump/git-jump +++ b/contrib/git-jump/git-jump @@ -39,7 +39,7 @@ mode_diff() { } mode_merge() { - git ls-files -u | + git ls-files -u "$@" | perl -pe 's/^.*?\t//' | sort -u | while IFS= read fn; do @@ -613,7 +613,7 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf, struct filter_params { const char *src; - unsigned long size; + size_t size; int fd; const char *cmd; const char *path; @@ -326,22 +326,18 @@ static int run_access_hook(struct daemon_service *service, const char *dir, { struct child_process child = CHILD_PROCESS_INIT; struct strbuf buf = STRBUF_INIT; - const char *argv[8]; - const char **arg = argv; char *eol; int seen_errors = 0; - *arg++ = access_hook; - *arg++ = service->name; - *arg++ = path; - *arg++ = hi->hostname.buf; - *arg++ = get_canon_hostname(hi); - *arg++ = get_ip_address(hi); - *arg++ = hi->tcp_port.buf; - *arg = NULL; + strvec_push(&child.args, access_hook); + strvec_push(&child.args, service->name); + strvec_push(&child.args, path); + strvec_push(&child.args, hi->hostname.buf); + strvec_push(&child.args, get_canon_hostname(hi)); + strvec_push(&child.args, get_ip_address(hi)); + strvec_push(&child.args, hi->tcp_port.buf); child.use_shell = 1; - child.argv = argv; child.no_stdin = 1; child.no_stderr = 1; child.out = -1; @@ -922,7 +918,7 @@ static void handle(int incoming, struct sockaddr *addr, socklen_t addrlen) #endif } - cld.argv = cld_argv.v; + strvec_pushv(&cld.args, cld_argv.v); cld.in = incoming; cld.out = dup(incoming); @@ -9,7 +9,7 @@ /* * This is like mktime, but without normalization of tm_wday and tm_yday. */ -static time_t tm_to_time_t(const struct tm *tm) +time_t tm_to_time_t(const struct tm *tm) { static const int mdays[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 @@ -90,15 +90,15 @@ static inline unsigned long get_delta_hdr_size(const unsigned char **datap, const unsigned char *top) { const unsigned char *data = *datap; - unsigned long cmd, size = 0; + size_t cmd, size = 0; int i = 0; do { cmd = *data++; - size |= (cmd & 0x7f) << i; + size |= st_left_shift(cmd & 0x7f, i); i += 7; } while (cmd & 0x80 && data < top); *datap = data; - return size; + return cast_size_t_to_ulong(size); } #endif @@ -6921,19 +6921,15 @@ static char *run_textconv(struct repository *r, size_t *outsize) { struct diff_tempfile *temp; - const char *argv[3]; - const char **arg = argv; struct child_process child = CHILD_PROCESS_INIT; struct strbuf buf = STRBUF_INIT; int err = 0; temp = prepare_temp_file(r, spec->path, spec); - *arg++ = pgm; - *arg++ = temp->name; - *arg = NULL; + strvec_push(&child.args, pgm); + strvec_push(&child.args, temp->name); child.use_shell = 1; - child.argv = argv; child.out = -1; if (start_command(&child)) { remove_tempfile(); diff --git a/diffcore-delta.c b/diffcore-delta.c index 5668ace60d..18d8f766d7 100644 --- a/diffcore-delta.c +++ b/diffcore-delta.c @@ -133,10 +133,10 @@ static struct spanhash_top *hash_chars(struct repository *r, i = INITIAL_HASH_SIZE; hash = xmalloc(st_add(sizeof(*hash), - st_mult(sizeof(struct spanhash), 1<<i))); + st_mult(sizeof(struct spanhash), (size_t)1 << i))); hash->alloc_log2 = i; hash->free = INITIAL_FREE(i); - memset(hash->data, 0, sizeof(struct spanhash) * (1<<i)); + memset(hash->data, 0, sizeof(struct spanhash) * ((size_t)1 << i)); n = 0; accum1 = accum2 = 0; @@ -159,7 +159,7 @@ static struct spanhash_top *hash_chars(struct repository *r, n = 0; accum1 = accum2 = 0; } - QSORT(hash->data, 1ul << hash->alloc_log2, spanhash_cmp); + QSORT(hash->data, (size_t)1ul << hash->alloc_log2, spanhash_cmp); return hash; } @@ -1,6 +1,7 @@ #include "cache.h" #include "config.h" #include "strbuf.h" +#include "strvec.h" #include "run-command.h" #include "sigchain.h" @@ -55,7 +56,6 @@ static int launch_specified_editor(const char *editor, const char *path, if (strcmp(editor, ":")) { struct strbuf realpath = STRBUF_INIT; - const char *args[] = { editor, NULL, NULL }; struct child_process p = CHILD_PROCESS_INIT; int ret, sig; int print_waiting_for_editor = advice_enabled(ADVICE_WAITING_FOR_EDITOR) && isatty(2); @@ -77,10 +77,10 @@ static int launch_specified_editor(const char *editor, const char *path, } strbuf_realpath(&realpath, path, 1); - args[1] = realpath.buf; - p.argv = args; - p.env = env; + strvec_pushl(&p.args, editor, realpath.buf, NULL); + if (env) + strvec_pushv(&p.env_array, (const char **)env); p.use_shell = 1; p.trace2_child_class = "editor"; if (start_command(&p) < 0) { @@ -82,11 +82,13 @@ static int create_file(const char *path, unsigned int mode) return open(path, O_WRONLY | O_CREAT | O_EXCL, mode); } -void *read_blob_entry(const struct cache_entry *ce, unsigned long *size) +void *read_blob_entry(const struct cache_entry *ce, size_t *size) { enum object_type type; - void *blob_data = read_object_file(&ce->oid, &type, size); + unsigned long ul; + void *blob_data = read_object_file(&ce->oid, &type, &ul); + *size = ul; if (blob_data) { if (type == OBJ_BLOB) return blob_data; @@ -271,7 +273,7 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca int fd, ret, fstat_done = 0; char *new_blob; struct strbuf buf = STRBUF_INIT; - unsigned long size; + size_t size; ssize_t wrote; size_t newsize = 0; struct stat st; @@ -52,7 +52,7 @@ int finish_delayed_checkout(struct checkout *state, int *nr_checkouts, */ void unlink_entry(const struct cache_entry *ce); -void *read_blob_entry(const struct cache_entry *ce, unsigned long *size); +void *read_blob_entry(const struct cache_entry *ce, size_t *size); int fstat_checkout_output(int fd, const struct checkout *state, struct stat *st); void update_ce_after_write(const struct checkout *state, struct cache_entry *ce, struct stat *st); diff --git a/fetch-pack.c b/fetch-pack.c index a9604f35a3..34987a2c30 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -25,6 +25,7 @@ #include "shallow.h" #include "commit-reach.h" #include "commit-graph.h" +#include "sigchain.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -956,6 +957,8 @@ static int get_pack(struct fetch_pack_args *args, strvec_push(index_pack_args, cmd.args.v[i]); } + sigchain_push(SIGPIPE, SIG_IGN); + cmd.in = demux.out; cmd.git_cmd = 1; if (start_command(&cmd)) @@ -986,6 +989,8 @@ static int get_pack(struct fetch_pack_args *args, if (use_sideband && finish_async(&demux)) die(_("error in sideband demultiplexer")); + sigchain_pop(SIGPIPE); + /* * Now that index-pack has succeeded, write the promisor file using the * obtained .keep filename if necessary @@ -1653,8 +1658,13 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, receive_wanted_refs(&reader, sought, nr_sought); /* get the pack(s) */ + if (git_env_bool("GIT_TRACE_REDACT", 1)) + reader.options |= PACKET_READ_REDACT_URI_PATH; if (process_section_header(&reader, "packfile-uris", 1)) receive_packfile_uris(&reader, &packfile_uris); + /* We don't expect more URIs. Reset to avoid expensive URI check. */ + reader.options &= ~PACKET_READ_REDACT_URI_PATH; + process_section_header(&reader, "packfile", 0); /* diff --git a/generate-cmdlist.sh b/generate-cmdlist.sh index 9dbbb08e70..205541e0f7 100755 --- a/generate-cmdlist.sh +++ b/generate-cmdlist.sh @@ -6,37 +6,38 @@ die () { } command_list () { - eval "grep -ve '^#' $exclude_programs" <"$1" -} - -get_categories () { - tr ' ' '\012'| - grep -v '^$' | - sort | - uniq + while read cmd rest + do + case "$cmd" in + "#"* | '') + # Ignore comments and allow empty lines + continue + ;; + *) + case "$exclude_programs" in + *":$cmd:"*) + ;; + *) + echo "$cmd $rest" + ;; + esac + esac + done <"$1" } category_list () { - command_list "$1" | - cut -c 40- | - get_categories -} - -get_synopsis () { - sed -n ' - /^NAME/,/'"$1"'/H - ${ - x - s/.*'"$1"' - \(.*\)/N_("\1")/ - p - }' "Documentation/$1.txt" + echo "$1" | + cut -d' ' -f2- | + tr ' ' '\012' | + grep -v '^$' | + LC_ALL=C sort -u } define_categories () { echo echo "/* Command categories */" bit=0 - category_list "$1" | + echo "$1" | while read cat do echo "#define CAT_$cat (1UL << $bit)" @@ -50,7 +51,7 @@ define_category_names () { echo "/* Category names */" echo "static const char *category_names[] = {" bit=0 - category_list "$1" | + echo "$1" | while read cat do echo " \"$cat\", /* (1UL << $bit) */" @@ -63,27 +64,38 @@ define_category_names () { print_command_list () { echo "static struct cmdname_help command_list[] = {" - command_list "$1" | + echo "$1" | while read cmd rest do - printf " { \"$cmd\", $(get_synopsis $cmd), 0" - for cat in $(echo "$rest" | get_categories) + synopsis= + while read line do - printf " | CAT_$cat" - done + case "$line" in + "$cmd - "*) + synopsis=${line#$cmd - } + break + ;; + esac + done <"Documentation/$cmd.txt" + + printf '\t{ "%s", N_("%s"), 0' "$cmd" "$synopsis" + printf " | CAT_%s" $rest echo " }," done echo "};" } -exclude_programs= +exclude_programs=: while test "--exclude-program" = "$1" do shift - exclude_programs="$exclude_programs -e \"^$1 \"" + exclude_programs="$exclude_programs$1:" shift done +commands="$(command_list "$1")" +categories="$(category_list "$commands")" + echo "/* Automatically generated by generate-cmdlist.sh */ struct cmdname_help { const char *name; @@ -91,8 +103,8 @@ struct cmdname_help { uint32_t category; }; " -define_categories "$1" +define_categories "$categories" echo -define_category_names "$1" +define_category_names "$categories" echo -print_command_list "$1" +print_command_list "$commands" diff --git a/git-add--interactive.perl b/git-add--interactive.perl index bc3a1e8eff..95887fd8e5 100755 --- a/git-add--interactive.perl +++ b/git-add--interactive.perl @@ -1175,15 +1175,17 @@ sub prompt_single_character { ReadMode 'cbreak'; my $key = ReadKey 0; ReadMode 'restore'; - if ($use_termcap and $key eq "\e") { - while (!defined $term_escapes{$key}) { - my $next = ReadKey 0.5; - last if (!defined $next); - $key .= $next; + if (defined $key) { + if ($use_termcap and $key eq "\e") { + while (!defined $term_escapes{$key}) { + my $next = ReadKey 0.5; + last if (!defined $next); + $key .= $next; + } + $key =~ s/\e/^[/; } - $key =~ s/\e/^[/; + print "$key"; } - print "$key" if defined $key; print "\n"; return $key; } else { diff --git a/git-compat-util.h b/git-compat-util.h index d70ce14286..5fa54a7afe 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,19 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#if __STDC_VERSION__ - 0 < 199901L +/* + * Git is in a testing period for mandatory C99 support in the compiler. If + * your compiler is reasonably recent, you can try to enable C99 support (or, + * for MSVC, C11 support). If you encounter a problem and can't enable C99 + * support with your compiler (such as with "-std=gnu99") and don't have access + * to one with this support, such as GCC or Clang, you can remove this #if + * directive, but please report the details of your system to + * git@vger.kernel.org. + */ +#error "Required C99 support is in a test phase. Please see git-compat-util.h for more details." +#endif + #ifdef USE_MSVC_CRTDBG /* * For these to work they must appear very early in each @@ -113,6 +126,14 @@ #define unsigned_mult_overflows(a, b) \ ((a) && (b) > maximum_unsigned_value_of_type(a) / (a)) +/* + * Returns true if the left shift of "a" by "shift" bits will + * overflow. The type of "a" must be unsigned. + */ +#define unsigned_left_shift_overflows(a, shift) \ + ((shift) < bitsizeof(a) && \ + (a) > maximum_unsigned_value_of_type(a) >> (shift)) + #ifdef __GNUC__ #define TYPEOF(x) (__typeof__(x)) #else @@ -127,7 +148,9 @@ /* Approximation of the length of the decimal representation of this type. */ #define decimal_length(x) ((int)(sizeof(x) * 2.56 + 0.5) + 1) -#if defined(__sun__) +#ifdef __MINGW64__ +#define _POSIX_C_SOURCE 1 +#elif defined(__sun__) /* * On Solaris, when _XOPEN_EXTENDED is set, its header file * forces the programs to be XPG4v2, defeating any _XOPEN_SOURCE @@ -729,7 +752,7 @@ char *gitmkdtemp(char *); #ifdef NO_UNSETENV #define unsetenv gitunsetenv -void gitunsetenv(const char *); +int gitunsetenv(const char *); #endif #ifdef NO_STRCASESTR @@ -862,6 +885,23 @@ static inline size_t st_sub(size_t a, size_t b) return a - b; } +static inline size_t st_left_shift(size_t a, unsigned shift) +{ + if (unsigned_left_shift_overflows(a, shift)) + die("size_t overflow: %"PRIuMAX" << %u", + (uintmax_t)a, shift); + return a << shift; +} + +static inline unsigned long cast_size_t_to_ulong(size_t a) +{ + if (a != (unsigned long)a) + die("object too large to read on this platform: %" + PRIuMAX" is cut off to %lu", + (uintmax_t)a, (unsigned long)a); + return (unsigned long)a; +} + #ifdef HAVE_ALLOCA_H # include <alloca.h> # define xalloca(size) (alloca(size)) diff --git a/git-filter-branch.sh b/git-filter-branch.sh index cb89372813..3a51d4507c 100755 --- a/git-filter-branch.sh +++ b/git-filter-branch.sh @@ -579,7 +579,7 @@ if [ "$filter_tag_name" ]; then git hash-object -t tag -w --stdin) || die "Could not create new tag object for $ref" if git cat-file tag "$ref" | \ - sane_grep '^-----BEGIN PGP SIGNATURE-----' >/dev/null 2>&1 + grep '^-----BEGIN PGP SIGNATURE-----' >/dev/null 2>&1 then warn "gpg signature stripped from tag object $sha1t" fi diff --git a/git-instaweb.sh b/git-instaweb.sh index 7c55229773..4349566c89 100755 --- a/git-instaweb.sh +++ b/git-instaweb.sh @@ -49,7 +49,7 @@ resolve_full_httpd () { *apache2*|*lighttpd*|*httpd*) # yes, *httpd* covers *lighttpd* above, but it is there for clarity # ensure that the apache2/lighttpd command ends with "-f" - if ! echo "$httpd" | sane_grep -- '-f *$' >/dev/null 2>&1 + if ! echo "$httpd" | grep -- '-f *$' >/dev/null 2>&1 then httpd="$httpd -f" fi @@ -380,10 +380,7 @@ TypesConfig "$fqgitdir/mime.types" DirectoryIndex gitweb.cgi EOF - # check to see if Dennis Stosberg's mod_perl compatibility patch - # (<20060621130708.Gcbc6e5c@leonov.stosberg.net>) has been applied - if test -f "$module_path/mod_perl.so" && - sane_grep 'MOD_PERL' "$root/gitweb.cgi" >/dev/null + if test -f "$module_path/mod_perl.so" then # favor mod_perl if available cat >> "$conf" <<EOF @@ -402,7 +399,7 @@ EOF # plain-old CGI resolve_full_httpd list_mods=$(echo "$full_httpd" | sed 's/-f$/-l/') - $list_mods | sane_grep 'mod_cgi\.c' >/dev/null 2>&1 || \ + $list_mods | grep 'mod_cgi\.c' >/dev/null 2>&1 || \ if test -f "$module_path/mod_cgi.so" then echo "LoadModule cgi_module $module_path/mod_cgi.so" >> "$conf" diff --git a/git-send-email.perl b/git-send-email.perl index 5262d88ee3..04087221aa 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -40,7 +40,8 @@ package main; sub usage { print <<EOT; -git send-email [options] <file | directory | rev-list options > +git send-email' [<options>] <file|directory> +git send-email' [<options>] <format-patch options> git send-email --dump-aliases Composing: @@ -113,9 +114,38 @@ EOT exit(1); } +sub uniq { + my %seen; + grep !$seen{$_}++, @_; +} + sub completion_helper { - print Git::command('format-patch', '--git-completion-helper'); - exit(0); + my ($original_opts) = @_; + my %not_for_completion = ( + "git-completion-helper" => undef, + "h" => undef, + ); + my @send_email_opts = (); + + foreach my $key (keys %$original_opts) { + unless (exists $not_for_completion{$key}) { + $key =~ s/!$//; + + if ($key =~ /[:=][si]$/) { + $key =~ s/[:=][si]$//; + push (@send_email_opts, "--$_=") foreach (split (/\|/, $key)); + } else { + push (@send_email_opts, "--$_") foreach (split (/\|/, $key)); + } + } + } + + my @format_patch_opts = split(/ /, Git::command('format-patch', '--git-completion-helper')); + my @opts = (@send_email_opts, @format_patch_opts); + @opts = uniq (grep !/^$/, @opts); + # There's an implicit '\n' here already, no need to add an explicit one. + print "@opts"; + exit(0); } # most mail servers generate the Date: header, but not all... @@ -425,10 +455,11 @@ my %known_config_keys; my $key = "sendemail.identity"; $identity = Git::config(@repo, $key) if exists $known_config_keys{$key}; } -my $rc = GetOptions( +my %identity_options = ( "identity=s" => \$identity, "no-identity" => \$no_identity, ); +my $rc = GetOptions(%identity_options); usage() unless $rc; undef $identity if $no_identity; @@ -444,14 +475,17 @@ undef $identity if $no_identity; my $help; my $git_completion_helper; -$rc = GetOptions("h" => \$help, - "dump-aliases" => \$dump_aliases); +my %dump_aliases_options = ( + "h" => \$help, + "dump-aliases" => \$dump_aliases, +); +$rc = GetOptions(%dump_aliases_options); usage() unless $rc; die __("--dump-aliases incompatible with other options\n") if !$help and $dump_aliases and @ARGV; -$rc = GetOptions( +my %options = ( "sender|from=s" => \$sender, - "in-reply-to=s" => \$initial_in_reply_to, + "in-reply-to=s" => \$initial_in_reply_to, "reply-to=s" => \$reply_to, "subject=s" => \$initial_subject, "to=s" => \@getopt_to, @@ -508,7 +542,8 @@ $rc = GetOptions( "batch-size=i" => \$batch_size, "relogin-delay=i" => \$relogin_delay, "git-completion-helper" => \$git_completion_helper, - ); +); +$rc = GetOptions(%options); # Munge any "either config or getopt, not both" variables my @initial_to = @getopt_to ? @getopt_to : ($no_to ? () : @config_to); @@ -516,7 +551,8 @@ my @initial_cc = @getopt_cc ? @getopt_cc : ($no_cc ? () : @config_cc); my @initial_bcc = @getopt_bcc ? @getopt_bcc : ($no_bcc ? () : @config_bcc); usage() if $help; -completion_helper() if $git_completion_helper; +my %all_options = (%options, %dump_aliases_options, %identity_options); +completion_helper(\%all_options) if $git_completion_helper; unless ($rc) { usage(); } diff --git a/git-sh-setup.sh b/git-sh-setup.sh index 960982f9d5..b93f39288c 100644 --- a/git-sh-setup.sh +++ b/git-sh-setup.sh @@ -173,14 +173,6 @@ git_pager() { eval "$GIT_PAGER" '"$@"' } -sane_grep () { - GREP_OPTIONS= LC_ALL=C grep @@SANE_TEXT_GREP@@ "$@" -} - -sane_egrep () { - GREP_OPTIONS= LC_ALL=C egrep @@SANE_TEXT_GREP@@ "$@" -} - is_bare_repository () { git rev-parse --is-bare-repository } @@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s /* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 + /* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20 #define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) /* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 + /* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32 #define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ) diff --git a/http-backend.c b/http-backend.c index 3d6e2ff17f..4dd4d939f8 100644 --- a/http-backend.c +++ b/http-backend.c @@ -480,7 +480,7 @@ static void run_service(const char **argv, int buffer_input) strvec_pushf(&cld.env_array, "GIT_COMMITTER_EMAIL=%s@http.%s", user, host); - cld.argv = argv; + strvec_pushv(&cld.args, argv); if (buffer_input || gzipped_request || req_len >= 0) cld.in = -1; cld.git_cmd = 1; diff --git a/http-fetch.c b/http-fetch.c index fa642462a9..c7c7d391ac 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -4,6 +4,7 @@ #include "http.h" #include "walker.h" #include "strvec.h" +#include "urlmatch.h" static const char http_fetch_usage[] = "git http-fetch " "[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin | --packfile=hash | commit-id] url"; @@ -63,8 +64,17 @@ static void fetch_single_packfile(struct object_id *packfile_hash, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); if (results.curl_result != CURLE_OK) { - die("Unable to get pack file %s\n%s", preq->url, - curl_errorstr); + struct url_info url; + char *nurl = url_normalize(preq->url, &url); + if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) { + die("unable to get pack file '%s'\n%s", preq->url, + curl_errorstr); + } else { + die("failed to get '%.*s' url from '%.*s' " + "(full URL redacted due to GIT_TRACE_REDACT setting)\n%s", + (int)url.scheme_len, url.url, + (int)url.host_len, &url.url[url.host_off], curl_errorstr); + } } } else { die("Unable to start request"); @@ -2126,8 +2126,9 @@ int finish_http_pack_request(struct http_pack_request *preq) ip.git_cmd = 1; ip.in = tmpfile_fd; - ip.argv = preq->index_pack_args ? preq->index_pack_args - : default_index_pack_args; + strvec_pushv(&ip.args, preq->index_pack_args ? + preq->index_pack_args : + default_index_pack_args); if (preq->preserve_index_pack_stdout) ip.out = 0; diff --git a/mergesort.c b/mergesort.c index 6216835566..bd9c6ef8ee 100644 --- a/mergesort.c +++ b/mergesort.c @@ -63,7 +63,7 @@ void *llist_mergesort(void *list, void *next = get_next_fn(list); if (next) set_next_fn(list, NULL); - for (i = 0; n & (1 << i); i++) + for (i = 0; n & ((size_t)1 << i); i++) list = llist_merge(ranks[i], list, get_next_fn, set_next_fn, compare_fn); n++; @@ -57,15 +57,15 @@ const unsigned char *get_midx_checksum(struct multi_pack_index *m) return m->data + m->data_len - the_hash_algo->rawsz; } -char *get_midx_filename(const char *object_dir) +void get_midx_filename(struct strbuf *out, const char *object_dir) { - return xstrfmt("%s/pack/multi-pack-index", object_dir); + strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); } -char *get_midx_rev_filename(struct multi_pack_index *m) +void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m) { - return xstrfmt("%s/pack/multi-pack-index-%s.rev", - m->object_dir, hash_to_hex(get_midx_checksum(m))); + get_midx_filename(out, m->object_dir); + strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m))); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -89,28 +89,30 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local size_t midx_size; void *midx_map = NULL; uint32_t hash_version; - char *midx_name = get_midx_filename(object_dir); + struct strbuf midx_name = STRBUF_INIT; uint32_t i; const char *cur_pack_name; struct chunkfile *cf = NULL; - fd = git_open(midx_name); + get_midx_filename(&midx_name, object_dir); + + fd = git_open(midx_name.buf); if (fd < 0) goto cleanup_fail; if (fstat(fd, &st)) { - error_errno(_("failed to read %s"), midx_name); + error_errno(_("failed to read %s"), midx_name.buf); goto cleanup_fail; } midx_size = xsize_t(st.st_size); if (midx_size < MIDX_MIN_SIZE) { - error(_("multi-pack-index file %s is too small"), midx_name); + error(_("multi-pack-index file %s is too small"), midx_name.buf); goto cleanup_fail; } - FREE_AND_NULL(midx_name); + strbuf_release(&midx_name); midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); @@ -179,12 +181,13 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs); trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects); + free_chunkfile(cf); return m; cleanup_fail: free(m); - free(midx_name); - free(cf); + strbuf_release(&midx_name); + free_chunkfile(cf); if (midx_map) munmap(midx_map, midx_size); if (0 <= fd) @@ -1130,7 +1133,7 @@ static int write_midx_internal(const char *object_dir, const char *refs_snapshot, unsigned flags) { - char *midx_name; + struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t i; struct hashfile *f = NULL; @@ -1141,10 +1144,10 @@ static int write_midx_internal(const char *object_dir, int result = 0; struct chunkfile *cf; - midx_name = get_midx_filename(object_dir); - if (safe_create_leading_directories(midx_name)) + get_midx_filename(&midx_name, object_dir); + if (safe_create_leading_directories(midx_name.buf)) die_errno(_("unable to create leading directories of %s"), - midx_name); + midx_name.buf); if (!packs_to_include) { /* @@ -1373,7 +1376,7 @@ static int write_midx_internal(const char *object_dir, pack_name_concat_len += MIDX_CHUNK_ALIGNMENT - (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT); - hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR); f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk)); if (ctx.nr - dropped_packs == 0) { @@ -1410,9 +1413,9 @@ static int write_midx_internal(const char *object_dir, ctx.pack_order = midx_pack_order(&ctx); if (flags & MIDX_WRITE_REV_INDEX) - write_midx_reverse_index(midx_name, midx_hash, &ctx); + write_midx_reverse_index(midx_name.buf, midx_hash, &ctx); if (flags & MIDX_WRITE_BITMAP) { - if (write_midx_bitmap(midx_name, midx_hash, &ctx, + if (write_midx_bitmap(midx_name.buf, midx_hash, &ctx, refs_snapshot, flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; @@ -1442,7 +1445,7 @@ cleanup: free(ctx.entries); free(ctx.pack_perm); free(ctx.pack_order); - free(midx_name); + strbuf_release(&midx_name); return result; } @@ -1506,20 +1509,22 @@ static void clear_midx_files_ext(const char *object_dir, const char *ext, void clear_midx_file(struct repository *r) { - char *midx = get_midx_filename(r->objects->odb->path); + struct strbuf midx = STRBUF_INIT; + + get_midx_filename(&midx, r->objects->odb->path); if (r->objects && r->objects->multi_pack_index) { close_midx(r->objects->multi_pack_index); r->objects->multi_pack_index = NULL; } - if (remove_path(midx)) - die(_("failed to clear multi-pack-index at %s"), midx); + if (remove_path(midx.buf)) + die(_("failed to clear multi-pack-index at %s"), midx.buf); clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL); clear_midx_files_ext(r->objects->odb->path, ".rev", NULL); - free(midx); + strbuf_release(&midx); } static int verify_midx_error; @@ -1572,12 +1577,15 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag if (!m) { int result = 0; struct stat sb; - char *filename = get_midx_filename(object_dir); - if (!stat(filename, &sb)) { + struct strbuf filename = STRBUF_INIT; + + get_midx_filename(&filename, object_dir); + + if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); result = 1; } - free(filename); + strbuf_release(&filename); return result; } @@ -1610,7 +1618,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag * Remaining tests assume that we have objects, so we can * return here. */ - return verify_midx_error; + goto cleanup; } if (flags & MIDX_PROGRESS) @@ -1688,7 +1696,9 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag } stop_progress(&progress); +cleanup: free(pairs); + close_midx(m); return verify_midx_error; } @@ -48,8 +48,8 @@ struct multi_pack_index { #define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3) const unsigned char *get_midx_checksum(struct multi_pack_index *m); -char *get_midx_filename(const char *object_dir); -char *get_midx_rev_filename(struct multi_pack_index *m); +void get_midx_filename(struct strbuf *out, const char *object_dir); +void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m); struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local); int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); diff --git a/object-file.c b/object-file.c index c3d866a287..eb1426f98c 100644 --- a/object-file.c +++ b/object-file.c @@ -165,7 +165,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx) BUG("trying to finalize unknown hash"); } - const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { { NULL, @@ -184,8 +183,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, { "sha1", - /* "sha1", big-endian */ - 0x73686131, + GIT_SHA1_FORMAT_ID, GIT_SHA1_RAWSZ, GIT_SHA1_HEXSZ, GIT_SHA1_BLKSZ, @@ -200,8 +198,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, { "sha256", - /* "s256", big-endian */ - 0x73323536, + GIT_SHA256_FORMAT_ID, GIT_SHA256_RAWSZ, GIT_SHA256_HEXSZ, GIT_SHA256_BLKSZ, @@ -797,7 +794,7 @@ static void fill_alternate_refs_command(struct child_process *cmd, } } - cmd->env = local_repo_env; + strvec_pushv(&cmd->env_array, (const char **)local_repo_env); cmd->out = -1; } @@ -1306,7 +1303,7 @@ static void *unpack_loose_rest(git_zstream *stream, int parse_loose_header(const char *hdr, struct object_info *oi) { const char *type_buf = hdr; - unsigned long size; + size_t size; int type, type_len = 0; /* @@ -1341,12 +1338,12 @@ int parse_loose_header(const char *hdr, struct object_info *oi) if (c > 9) break; hdr++; - size = size * 10 + c; + size = st_add(st_mult(size, 10), c); } } if (oi->sizep) - *oi->sizep = size; + *oi->sizep = cast_size_t_to_ulong(size); /* * The length must be followed by a zero byte @@ -2425,7 +2422,7 @@ struct oidtree *odb_loose_cache(struct object_directory *odb, struct strbuf buf = STRBUF_INIT; size_t word_bits = bitsizeof(odb->loose_objects_subdir_seen[0]); size_t word_index = subdir_nr / word_bits; - size_t mask = 1u << (subdir_nr % word_bits); + size_t mask = (size_t)1u << (subdir_nr % word_bits); uint32_t *bitmap; if (subdir_nr < 0 || diff --git a/pack-bitmap.c b/pack-bitmap.c index f47a0a7db4..f772d3cb7f 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -292,9 +292,12 @@ static int load_bitmap_entries_v1(struct bitmap_index *index) char *midx_bitmap_filename(struct multi_pack_index *midx) { - return xstrfmt("%s-%s.bitmap", - get_midx_filename(midx->object_dir), - hash_to_hex(get_midx_checksum(midx))); + struct strbuf buf = STRBUF_INIT; + + get_midx_filename(&buf, midx->object_dir); + strbuf_addf(&buf, "-%s.bitmap", hash_to_hex(get_midx_checksum(midx))); + + return strbuf_detach(&buf, NULL); } char *pack_bitmap_filename(struct packed_git *p) @@ -324,10 +327,12 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } if (bitmap_git->pack || bitmap_git->midx) { + struct strbuf buf = STRBUF_INIT; + get_midx_filename(&buf, midx->object_dir); /* ignore extra bitmap file; we can only handle one */ - warning("ignoring extra bitmap file: %s", - get_midx_filename(midx->object_dir)); + warning("ignoring extra bitmap file: %s", buf.buf); close(fd); + strbuf_release(&buf); return -1; } @@ -1721,6 +1726,12 @@ void test_bitmap_walk(struct rev_info *revs) else die("mismatch in bitmap results"); + bitmap_free(result); + bitmap_free(tdata.base); + bitmap_free(tdata.commits); + bitmap_free(tdata.trees); + bitmap_free(tdata.blobs); + bitmap_free(tdata.tags); free_bitmap_index(bitmap_git); } @@ -1748,7 +1759,7 @@ int test_bitmap_hashes(struct repository *r) struct object_id oid; uint32_t i, index_pos; - if (!bitmap_git->hashes) + if (!bitmap_git || !bitmap_git->hashes) goto cleanup; for (i = 0; i < bitmap_num_objects(bitmap_git); i++) { @@ -1848,9 +1859,17 @@ void free_bitmap_index(struct bitmap_index *b) ewah_pool_free(b->trees); ewah_pool_free(b->blobs); ewah_pool_free(b->tags); + if (b->bitmaps) { + struct stored_bitmap *sb; + kh_foreach_value(b->bitmaps, sb, { + ewah_pool_free(sb->root); + free(sb); + }); + } kh_destroy_oid_map(b->bitmaps); free(b->ext_index.objects); free(b->ext_index.hashes); + kh_destroy_oid_pos(b->ext_index.positions); bitmap_free(b->result); bitmap_free(b->haves); if (bitmap_is_midx(b)) { diff --git a/pack-revindex.c b/pack-revindex.c index 0e4a31d9db..70d0fbafcb 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -296,14 +296,14 @@ int load_pack_revindex(struct packed_git *p) int load_midx_revindex(struct multi_pack_index *m) { - char *revindex_name; + struct strbuf revindex_name = STRBUF_INIT; int ret; if (m->revindex_data) return 0; - revindex_name = get_midx_rev_filename(m); + get_midx_rev_filename(&revindex_name, m); - ret = load_revindex_from_disk(revindex_name, + ret = load_revindex_from_disk(revindex_name.buf, m->num_objects, &m->revindex_map, &m->revindex_len); @@ -313,7 +313,7 @@ int load_midx_revindex(struct multi_pack_index *m) m->revindex_data = (const uint32_t *)((const char *)m->revindex_map + RIDX_HEADER_SIZE); cleanup: - free(revindex_name); + strbuf_release(&revindex_name); return ret; } diff --git a/packfile.c b/packfile.c index 89402cfc69..11bb262482 100644 --- a/packfile.c +++ b/packfile.c @@ -324,7 +324,8 @@ void close_pack_index(struct packed_git *p) } } -void close_pack_revindex(struct packed_git *p) { +static void close_pack_revindex(struct packed_git *p) +{ if (!p->revindex_map) return; @@ -1060,7 +1061,7 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) { unsigned shift; - unsigned long size, c; + size_t size, c; unsigned long used = 0; c = buf[used++]; @@ -1068,16 +1069,16 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf, size = c & 15; shift = 4; while (c & 0x80) { - if (len <= used || bitsizeof(long) <= shift) { + if (len <= used || (bitsizeof(long) - 7) <= shift) { error("bad object header"); size = used = 0; break; } c = buf[used++]; - size += (c & 0x7f) << shift; + size = st_add(size, st_left_shift(c & 0x7f, shift)); shift += 7; } - *sizep = size; + *sizep = cast_size_t_to_ulong(size); return used; } diff --git a/packfile.h b/packfile.h index 186146779d..a3f6723857 100644 --- a/packfile.h +++ b/packfile.h @@ -90,7 +90,6 @@ uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); void close_pack_windows(struct packed_git *); -void close_pack_revindex(struct packed_git *); void close_pack(struct packed_git *); void close_object_store(struct raw_object_store *o); void unuse_pack(struct pack_window **); @@ -8,7 +8,7 @@ #define DEFAULT_PAGER "less" #endif -static struct child_process pager_process = CHILD_PROCESS_INIT; +static struct child_process pager_process; static const char *pager_program; /* Is the value coming back from term_columns() just a guess? */ @@ -124,6 +124,8 @@ void setup_pager(void) setenv("GIT_PAGER_IN_USE", "true", 1); + child_process_init(&pager_process); + /* spawn the pager */ prepare_pager_args(&pager_process, pager); pager_process.in = -1; diff --git a/parallel-checkout.c b/parallel-checkout.c index ed9c999520..8dd7e7bad4 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -261,7 +261,7 @@ static int write_pc_item_to_fd(struct parallel_checkout_item *pc_item, int fd, struct stream_filter *filter; struct strbuf buf = STRBUF_INIT; char *blob; - unsigned long size; + size_t size; ssize_t wrote; /* Sanity check */ diff --git a/pkt-line.c b/pkt-line.c index 2dc8ac274b..8e43c2def4 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -370,6 +370,32 @@ int packet_length(const char lenbuf_hex[4]) return (val < 0) ? val : (val << 8) | hex2chr(lenbuf_hex + 2); } +static char *find_packfile_uri_path(const char *buffer) +{ + const char *URI_MARK = "://"; + char *path; + int len; + + /* First char is sideband mark */ + buffer += 1; + + len = strspn(buffer, "0123456789abcdefABCDEF"); + /* size of SHA1 and SHA256 hash */ + if (!(len == 40 || len == 64) || buffer[len] != ' ') + return NULL; /* required "<hash>SP" not seen */ + + path = strstr(buffer + len + 1, URI_MARK); + if (!path) + return NULL; + + path = strchr(path + strlen(URI_MARK), '/'); + if (!path || !*(path + 1)) + return NULL; + + /* position after '/' */ + return ++path; +} + enum packet_read_status packet_read_with_status(int fd, char **src_buffer, size_t *src_len, char *buffer, unsigned size, int *pktlen, @@ -377,6 +403,7 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, { int len; char linelen[4]; + char *uri_path_start; if (get_packet_data(fd, src_buffer, src_len, linelen, 4, options) < 0) { *pktlen = -1; @@ -427,7 +454,18 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, len--; buffer[len] = 0; - packet_trace(buffer, len, 0); + if (options & PACKET_READ_REDACT_URI_PATH && + (uri_path_start = find_packfile_uri_path(buffer))) { + const char *redacted = "<redacted>"; + struct strbuf tracebuf = STRBUF_INIT; + strbuf_insert(&tracebuf, 0, buffer, len); + strbuf_splice(&tracebuf, uri_path_start - buffer, + strlen(uri_path_start), redacted, strlen(redacted)); + packet_trace(tracebuf.buf, tracebuf.len, 0); + strbuf_release(&tracebuf); + } else { + packet_trace(buffer, len, 0); + } if ((options & PACKET_READ_DIE_ON_ERR_PACKET) && starts_with(buffer, "ERR ")) diff --git a/pkt-line.h b/pkt-line.h index 467ae01357..6d2a63db23 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -87,6 +87,7 @@ void packet_fflush(FILE *f); #define PACKET_READ_CHOMP_NEWLINE (1u<<1) #define PACKET_READ_DIE_ON_ERR_PACKET (1u<<2) #define PACKET_READ_GENTLE_ON_READ_ERROR (1u<<3) +#define PACKET_READ_REDACT_URI_PATH (1u<<4) int packet_read(int fd, char *buffer, unsigned size, int options); /* @@ -1275,28 +1275,66 @@ int format_set_trailers_options(struct process_trailer_options *opts, static size_t parse_describe_args(const char *start, struct strvec *args) { - const char *options[] = { "match", "exclude" }; + struct { + char *name; + enum { + DESCRIBE_ARG_BOOL, + DESCRIBE_ARG_INTEGER, + DESCRIBE_ARG_STRING, + } type; + } option[] = { + { "tags", DESCRIBE_ARG_BOOL}, + { "abbrev", DESCRIBE_ARG_INTEGER }, + { "exclude", DESCRIBE_ARG_STRING }, + { "match", DESCRIBE_ARG_STRING }, + }; const char *arg = start; for (;;) { - const char *matched = NULL; + int found = 0; const char *argval; size_t arglen = 0; + int optval = 0; int i; - for (i = 0; i < ARRAY_SIZE(options); i++) { - if (match_placeholder_arg_value(arg, options[i], &arg, - &argval, &arglen)) { - matched = options[i]; + for (i = 0; !found && i < ARRAY_SIZE(option); i++) { + switch (option[i].type) { + case DESCRIBE_ARG_BOOL: + if (match_placeholder_bool_arg(arg, option[i].name, &arg, &optval)) { + if (optval) + strvec_pushf(args, "--%s", option[i].name); + else + strvec_pushf(args, "--no-%s", option[i].name); + found = 1; + } + break; + case DESCRIBE_ARG_INTEGER: + if (match_placeholder_arg_value(arg, option[i].name, &arg, + &argval, &arglen)) { + char *endptr; + if (!arglen) + return 0; + strtol(argval, &endptr, 10); + if (endptr - argval != arglen) + return 0; + strvec_pushf(args, "--%s=%.*s", option[i].name, (int)arglen, argval); + found = 1; + } + break; + case DESCRIBE_ARG_STRING: + if (match_placeholder_arg_value(arg, option[i].name, &arg, + &argval, &arglen)) { + if (!arglen) + return 0; + strvec_pushf(args, "--%s=%.*s", option[i].name, (int)arglen, argval); + found = 1; + } break; } } - if (!matched) + if (!found) break; - if (!arglen) - return 0; - strvec_pushf(args, "--%s=%.*s", matched, (int)arglen, argval); } return arg - start; } @@ -8,15 +8,12 @@ static char *do_askpass(const char *cmd, const char *prompt) { struct child_process pass = CHILD_PROCESS_INIT; - const char *args[3]; static struct strbuf buffer = STRBUF_INIT; int err = 0; - args[0] = cmd; - args[1] = prompt; - args[2] = NULL; + strvec_push(&pass.args, cmd); + strvec_push(&pass.args, prompt); - pass.argv = args; pass.out = -1; if (start_command(&pass)) diff --git a/read-cache.c b/read-cache.c index f398659662..cbe73f14e5 100644 --- a/read-cache.c +++ b/read-cache.c @@ -68,6 +68,11 @@ */ #define CACHE_ENTRY_PATH_LENGTH 80 +enum index_search_mode { + NO_EXPAND_SPARSE = 0, + EXPAND_SPARSE = 1 +}; + static inline struct cache_entry *mem_pool__ce_alloc(struct mem_pool *mem_pool, size_t len) { struct cache_entry *ce; @@ -551,7 +556,10 @@ int cache_name_stage_compare(const char *name1, int len1, int stage1, const char return 0; } -static int index_name_stage_pos(struct index_state *istate, const char *name, int namelen, int stage) +static int index_name_stage_pos(struct index_state *istate, + const char *name, int namelen, + int stage, + enum index_search_mode search_mode) { int first, last; @@ -570,7 +578,7 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in first = next+1; } - if (istate->sparse_index && + if (search_mode == EXPAND_SPARSE && istate->sparse_index && first > 0) { /* Note: first <= istate->cache_nr */ struct cache_entry *ce = istate->cache[first - 1]; @@ -586,7 +594,7 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in ce_namelen(ce) < namelen && !strncmp(name, ce->name, ce_namelen(ce))) { ensure_full_index(istate); - return index_name_stage_pos(istate, name, namelen, stage); + return index_name_stage_pos(istate, name, namelen, stage, search_mode); } } @@ -595,7 +603,12 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in int index_name_pos(struct index_state *istate, const char *name, int namelen) { - return index_name_stage_pos(istate, name, namelen, 0); + return index_name_stage_pos(istate, name, namelen, 0, EXPAND_SPARSE); +} + +int index_entry_exists(struct index_state *istate, const char *name, int namelen) +{ + return index_name_stage_pos(istate, name, namelen, 0, NO_EXPAND_SPARSE) >= 0; } int remove_index_entry_at(struct index_state *istate, int pos) @@ -1237,7 +1250,7 @@ static int has_dir_name(struct index_state *istate, */ } - pos = index_name_stage_pos(istate, name, len, stage); + pos = index_name_stage_pos(istate, name, len, stage, EXPAND_SPARSE); if (pos >= 0) { /* * Found one, but not so fast. This could @@ -1337,7 +1350,7 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e strcmp(ce->name, istate->cache[istate->cache_nr - 1]->name) > 0) pos = index_pos_to_insert_pos(istate->cache_nr); else - pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); + pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce), EXPAND_SPARSE); /* existing match? Just replace it. */ if (pos >= 0) { @@ -1372,7 +1385,7 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e if (!ok_to_replace) return error(_("'%s' appears as both a file and as a directory"), ce->name); - pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); + pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce), EXPAND_SPARSE); pos = -pos-1; } return pos + 1; @@ -2352,9 +2365,17 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) if (!istate->repo) istate->repo = the_repository; + + /* + * If the command explicitly requires a full index, force it + * to be full. Otherwise, correct the sparsity based on repository + * settings and other properties of the index (if necessary). + */ prepare_repo_settings(istate->repo); if (istate->repo->settings.command_requires_full_index) ensure_full_index(istate); + else + ensure_correct_sparsity(istate); return istate->cache_nr; diff --git a/ref-filter.c b/ref-filter.c index 08a3f839c9..7260fce31d 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -2470,6 +2470,12 @@ static int memcasecmp(const void *vs1, const void *vs2, size_t n) return 0; } +struct ref_sorting { + struct ref_sorting *next; + int atom; /* index into used_atom array (internal) */ + enum ref_sorting_order sort_flags; +}; + static int cmp_ref_sorting(struct ref_sorting *s, struct ref_array_item *a, struct ref_array_item *b) { struct atom_value *va, *vb; @@ -2663,7 +2669,7 @@ static int parse_sorting_atom(const char *atom) } /* If no sorting option is given, use refname to sort as default */ -struct ref_sorting *ref_default_sorting(void) +static struct ref_sorting *ref_default_sorting(void) { static const char cstr_name[] = "refname"; @@ -2674,7 +2680,7 @@ struct ref_sorting *ref_default_sorting(void) return sorting; } -void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) +static void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) { struct ref_sorting *s; @@ -2692,17 +2698,25 @@ void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) s->atom = parse_sorting_atom(arg); } -int parse_opt_ref_sorting(const struct option *opt, const char *arg, int unset) +struct ref_sorting *ref_sorting_options(struct string_list *options) { + struct string_list_item *item; + struct ref_sorting *sorting = NULL, **tail = &sorting; + + if (!options->nr) { + sorting = ref_default_sorting(); + } else { + for_each_string_list_item(item, options) + parse_ref_sorting(tail, item->string); + } + /* - * NEEDSWORK: We should probably clear the list in this case, but we've - * already munged the global used_atoms list, which would need to be - * undone. + * From here on, the ref_sorting list should be used to talk + * about the sort order used for the output. The caller + * should not touch the string form anymore. */ - BUG_ON_OPT_NEG(unset); - - parse_ref_sorting(opt->value, arg); - return 0; + string_list_clear(options, 0); + return sorting; } void ref_sorting_release(struct ref_sorting *sorting) diff --git a/ref-filter.h b/ref-filter.h index 6228458d30..aa0eea4ecf 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -23,16 +23,13 @@ #define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD) struct atom_value; +struct ref_sorting; -struct ref_sorting { - struct ref_sorting *next; - int atom; /* index into used_atom array (internal) */ - enum { - REF_SORTING_REVERSE = 1<<0, - REF_SORTING_ICASE = 1<<1, - REF_SORTING_VERSION = 1<<2, - REF_SORTING_DETACHED_HEAD_FIRST = 1<<3, - } sort_flags; +enum ref_sorting_order { + REF_SORTING_REVERSE = 1<<0, + REF_SORTING_ICASE = 1<<1, + REF_SORTING_VERSION = 1<<2, + REF_SORTING_DETACHED_HEAD_FIRST = 1<<3, }; struct ref_array_item { @@ -97,9 +94,8 @@ struct ref_format { #define OPT_NO_MERGED(f, h) _OPT_MERGED_NO_MERGED("no-merged", f, h) #define OPT_REF_SORT(var) \ - OPT_CALLBACK_F(0, "sort", (var), \ - N_("key"), N_("field name to sort on"), \ - PARSE_OPT_NONEG, parse_opt_ref_sorting) + OPT_STRING_LIST(0, "sort", (var), \ + N_("key"), N_("field name to sort on")) /* * API for filtering a set of refs. Based on the type of refs the user @@ -121,14 +117,10 @@ int format_ref_array_item(struct ref_array_item *info, struct ref_format *format, struct strbuf *final_buf, struct strbuf *error_buf); -/* Parse a single sort specifier and add it to the list */ -void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *atom); -/* Callback function for parsing the sort option */ -int parse_opt_ref_sorting(const struct option *opt, const char *arg, int unset); -/* Default sort option based on refname */ -struct ref_sorting *ref_default_sorting(void); /* Release a "struct ref_sorting" */ void ref_sorting_release(struct ref_sorting *); +/* Convert list of sort options into ref_sorting */ +struct ref_sorting *ref_sorting_options(struct string_list *); /* Function to parse --merged and --no-merged options */ int parse_opt_merge_filter(const struct option *opt, const char *arg, int unset); /* Get the current HEAD's description */ @@ -269,9 +269,10 @@ char *refs_resolve_refdup(struct ref_store *refs, struct object_id *oid, int *flags) { const char *result; + int ignore_errno; result = refs_resolve_ref_unsafe(refs, refname, resolve_flags, - oid, flags); + oid, flags, &ignore_errno); return xstrdup_or_null(result); } @@ -291,20 +292,17 @@ struct ref_filter { void *cb_data; }; -int refs_read_ref_full(struct ref_store *refs, const char *refname, - int resolve_flags, struct object_id *oid, int *flags) +int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags) { - if (refs_resolve_ref_unsafe(refs, refname, resolve_flags, oid, flags)) + int ignore_errno; + struct ref_store *refs = get_main_ref_store(the_repository); + + if (refs_resolve_ref_unsafe(refs, refname, resolve_flags, + oid, flags, &ignore_errno)) return 0; return -1; } -int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags) -{ - return refs_read_ref_full(get_main_ref_store(the_repository), refname, - resolve_flags, oid, flags); -} - int read_ref(const char *refname, struct object_id *oid) { return read_ref_full(refname, RESOLVE_REF_READING, oid, NULL); @@ -312,7 +310,9 @@ int read_ref(const char *refname, struct object_id *oid) int refs_ref_exists(struct ref_store *refs, const char *refname) { - return !!refs_resolve_ref_unsafe(refs, refname, RESOLVE_REF_READING, NULL, NULL); + int ignore_errno; + return !!refs_resolve_ref_unsafe(refs, refname, RESOLVE_REF_READING, + NULL, NULL, &ignore_errno); } int ref_exists(const char *refname) @@ -655,13 +655,16 @@ int expand_ref(struct repository *repo, const char *str, int len, struct object_id oid_from_ref; struct object_id *this_result; int flag; + struct ref_store *refs = get_main_ref_store(repo); + int ignore_errno; this_result = refs_found ? &oid_from_ref : oid; strbuf_reset(&fullref); strbuf_addf(&fullref, *p, len, str); - r = refs_resolve_ref_unsafe(get_main_ref_store(repo), - fullref.buf, RESOLVE_REF_READING, - this_result, &flag); + r = refs_resolve_ref_unsafe(refs, fullref.buf, + RESOLVE_REF_READING, + this_result, &flag, + &ignore_errno); if (r) { if (!refs_found++) *ref = xstrdup(r); @@ -690,12 +693,14 @@ int repo_dwim_log(struct repository *r, const char *str, int len, for (p = ref_rev_parse_rules; *p; p++) { struct object_id hash; const char *ref, *it; + int ignore_errno; strbuf_reset(&path); strbuf_addf(&path, *p, len, str); ref = refs_resolve_ref_unsafe(refs, path.buf, RESOLVE_REF_READING, - oid ? &hash : NULL, NULL); + oid ? &hash : NULL, NULL, + &ignore_errno); if (!ref) continue; if (refs_reflog_exists(refs, path.buf)) @@ -1089,6 +1094,13 @@ int ref_transaction_update(struct ref_transaction *transaction, if (flags & ~REF_TRANSACTION_UPDATE_ALLOWED_FLAGS) BUG("illegal flags 0x%x passed to ref_transaction_update()", flags); + /* + * Clear flags outside the allowed set; this should be a noop because + * of the BUG() check above, but it works around a -Wnonnull warning + * with some versions of "gcc -O3". + */ + flags &= REF_TRANSACTION_UPDATE_ALLOWED_FLAGS; + flags |= (new_oid ? REF_HAVE_NEW : 0) | (old_oid ? REF_HAVE_OLD : 0); ref_transaction_add_update(transaction, refname, flags, @@ -1373,32 +1385,14 @@ const char *find_descendant_ref(const char *dirname, return NULL; } -int refs_rename_ref_available(struct ref_store *refs, - const char *old_refname, - const char *new_refname) -{ - struct string_list skip = STRING_LIST_INIT_NODUP; - struct strbuf err = STRBUF_INIT; - int ok; - - string_list_insert(&skip, old_refname); - ok = !refs_verify_refname_available(refs, new_refname, - NULL, &skip, &err); - if (!ok) - error("%s", err.buf); - - string_list_clear(&skip, 0); - strbuf_release(&err); - return ok; -} - int refs_head_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data) { struct object_id oid; int flag; + int ignore_errno; - if (!refs_read_ref_full(refs, "HEAD", RESOLVE_REF_READING, - &oid, &flag)) + if (refs_resolve_ref_unsafe(refs, "HEAD", RESOLVE_REF_READING, + &oid, &flag, &ignore_errno)) return fn("HEAD", &oid, flag, cb_data); return 0; @@ -1649,7 +1643,8 @@ int for_each_fullref_in_prefixes(const char *namespace, static int refs_read_special_head(struct ref_store *ref_store, const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type) + struct strbuf *referent, unsigned int *type, + int *failure_errno) { struct strbuf full_path = STRBUF_INIT; struct strbuf content = STRBUF_INIT; @@ -1659,7 +1654,8 @@ static int refs_read_special_head(struct ref_store *ref_store, if (strbuf_read_file(&content, full_path.buf, 0) < 0) goto done; - result = parse_loose_ref_contents(content.buf, oid, referent, type); + result = parse_loose_ref_contents(content.buf, oid, referent, type, + failure_errno); done: strbuf_release(&full_path); @@ -1667,30 +1663,33 @@ done: return result; } -int refs_read_raw_ref(struct ref_store *ref_store, - const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type) +int refs_read_raw_ref(struct ref_store *ref_store, const char *refname, + struct object_id *oid, struct strbuf *referent, + unsigned int *type, int *failure_errno) { + assert(failure_errno); if (!strcmp(refname, "FETCH_HEAD") || !strcmp(refname, "MERGE_HEAD")) { return refs_read_special_head(ref_store, refname, oid, referent, - type); + type, failure_errno); } return ref_store->be->read_raw_ref(ref_store, refname, oid, referent, - type, &errno); + type, failure_errno); } -/* This function needs to return a meaningful errno on failure */ const char *refs_resolve_ref_unsafe(struct ref_store *refs, const char *refname, int resolve_flags, - struct object_id *oid, int *flags) + struct object_id *oid, + int *flags, int *failure_errno) { static struct strbuf sb_refname = STRBUF_INIT; struct object_id unused_oid; int unused_flags; int symref_count; + assert(failure_errno); + if (!oid) oid = &unused_oid; if (!flags) @@ -1701,7 +1700,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { if (!(resolve_flags & RESOLVE_REF_ALLOW_BAD_NAME) || !refname_is_safe(refname)) { - errno = EINVAL; + *failure_errno = EINVAL; return NULL; } @@ -1719,9 +1718,11 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, for (symref_count = 0; symref_count < SYMREF_MAXDEPTH; symref_count++) { unsigned int read_flags = 0; - if (refs_read_raw_ref(refs, refname, - oid, &sb_refname, &read_flags)) { + if (refs_read_raw_ref(refs, refname, oid, &sb_refname, + &read_flags, failure_errno)) { *flags |= read_flags; + if (errno) + *failure_errno = errno; /* In reading mode, refs must eventually resolve */ if (resolve_flags & RESOLVE_REF_READING) @@ -1732,9 +1733,9 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, * may show errors besides ENOENT if there are * similarly-named refs. */ - if (errno != ENOENT && - errno != EISDIR && - errno != ENOTDIR) + if (*failure_errno != ENOENT && + *failure_errno != EISDIR && + *failure_errno != ENOTDIR) return NULL; oidclr(oid); @@ -1761,7 +1762,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { if (!(resolve_flags & RESOLVE_REF_ALLOW_BAD_NAME) || !refname_is_safe(refname)) { - errno = EINVAL; + *failure_errno = EINVAL; return NULL; } @@ -1769,7 +1770,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, } } - errno = ELOOP; + *failure_errno = ELOOP; return NULL; } @@ -1784,8 +1785,10 @@ int refs_init_db(struct strbuf *err) const char *resolve_ref_unsafe(const char *refname, int resolve_flags, struct object_id *oid, int *flags) { + int ignore_errno; + return refs_resolve_ref_unsafe(get_main_ref_store(the_repository), refname, - resolve_flags, oid, flags); + resolve_flags, oid, flags, &ignore_errno); } int resolve_gitlink_ref(const char *submodule, const char *refname, @@ -1793,14 +1796,15 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, { struct ref_store *refs; int flags; + int ignore_errno; refs = get_submodule_ref_store(submodule); if (!refs) return -1; - if (!refs_resolve_ref_unsafe(refs, refname, 0, oid, &flags) || - is_null_oid(oid)) + if (!refs_resolve_ref_unsafe(refs, refname, 0, oid, &flags, + &ignore_errno) || is_null_oid(oid)) return -1; return 0; } @@ -2102,8 +2106,11 @@ static int run_transaction_hook(struct ref_transaction *transaction, update->refname); if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - if (errno != EPIPE) + if (errno != EPIPE) { + /* Don't leak errno outside this API */ + errno = 0; ret = -1; + } break; } } @@ -2238,6 +2245,13 @@ int refs_verify_refname_available(struct ref_store *refs, strbuf_grow(&dirname, strlen(refname) + 1); for (slash = strchr(refname, '/'); slash; slash = strchr(slash + 1, '/')) { + /* + * Just saying "Is a directory" when we e.g. can't + * lock some multi-level ref isn't very informative, + * the user won't be told *what* is a directory, so + * let's not use strerror() below. + */ + int ignore_errno; /* Expand dirname to the new prefix, not including the trailing slash: */ strbuf_add(&dirname, refname + dirname.len, slash - refname - dirname.len); @@ -2249,7 +2263,8 @@ int refs_verify_refname_available(struct ref_store *refs, if (skip && string_list_has_string(skip, dirname.buf)) continue; - if (!refs_read_raw_ref(refs, dirname.buf, &oid, &referent, &type)) { + if (!refs_read_raw_ref(refs, dirname.buf, &oid, &referent, + &type, &ignore_errno)) { strbuf_addf(err, _("'%s' exists; cannot create '%s'"), dirname.buf, refname); goto cleanup; @@ -2358,16 +2373,15 @@ int reflog_exists(const char *refname) } int refs_create_reflog(struct ref_store *refs, const char *refname, - int force_create, struct strbuf *err) + struct strbuf *err) { - return refs->be->create_reflog(refs, refname, force_create, err); + return refs->be->create_reflog(refs, refname, err); } -int safe_create_reflog(const char *refname, int force_create, - struct strbuf *err) +int safe_create_reflog(const char *refname, struct strbuf *err) { return refs_create_reflog(get_main_ref_store(the_repository), refname, - force_create, err); + err); } int refs_delete_reflog(struct ref_store *refs, const char *refname) @@ -58,6 +58,11 @@ struct worktree; * resolved. The function returns NULL for such ref names. * Caps and underscores refers to the special refs, such as HEAD, * FETCH_HEAD and friends, that all live outside of the refs/ directory. + * + * Callers should not inspect "errno" on failure, but rather pass in a + * "failure_errno" parameter, on failure the "errno" will indicate the + * type of failure encountered, but not necessarily one that came from + * a syscall. We might have faked it up. */ #define RESOLVE_REF_READING 0x01 #define RESOLVE_REF_NO_RECURSE 0x02 @@ -67,7 +72,8 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, const char *refname, int resolve_flags, struct object_id *oid, - int *flags); + int *flags, int *failure_errno); + const char *resolve_ref_unsafe(const char *refname, int resolve_flags, struct object_id *oid, int *flags); @@ -77,8 +83,6 @@ char *refs_resolve_refdup(struct ref_store *refs, char *resolve_refdup(const char *refname, int resolve_flags, struct object_id *oid, int *flags); -int refs_read_ref_full(struct ref_store *refs, const char *refname, - int resolve_flags, struct object_id *oid, int *flags); int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags); int read_ref(const char *refname, struct object_id *oid); @@ -413,8 +417,8 @@ int refs_pack_refs(struct ref_store *refs, unsigned int flags); * Setup reflog before using. Fill in err and return -1 on failure. */ int refs_create_reflog(struct ref_store *refs, const char *refname, - int force_create, struct strbuf *err); -int safe_create_reflog(const char *refname, int force_create, struct strbuf *err); + struct strbuf *err); +int safe_create_reflog(const char *refname, struct strbuf *err); /** Reads log for the value of ref during at_time. **/ int read_ref_at(struct ref_store *refs, diff --git a/refs/debug.c b/refs/debug.c index 8667c64023..791423c6a7 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -284,6 +284,7 @@ static int debug_print_reflog_ent(struct object_id *old_oid, int ret; char o[GIT_MAX_HEXSZ + 1] = "null"; char n[GIT_MAX_HEXSZ + 1] = "null"; + char *msgend = strchrnul(msg, '\n'); if (old_oid) oid_to_hex_r(o, old_oid); if (new_oid) @@ -291,8 +292,10 @@ static int debug_print_reflog_ent(struct object_id *old_oid, ret = dbg->fn(old_oid, new_oid, committer, timestamp, tz, msg, dbg->cb_data); - trace_printf_key(&trace_refs, "reflog_ent %s (ret %d): %s -> %s, %s %ld \"%s\"\n", - dbg->refname, ret, o, n, committer, (long int)timestamp, msg); + trace_printf_key(&trace_refs, + "reflog_ent %s (ret %d): %s -> %s, %s %ld \"%.*s\"\n", + dbg->refname, ret, o, n, committer, + (long int)timestamp, (int)(msgend - msg), msg); return ret; } @@ -339,11 +342,10 @@ static int debug_reflog_exists(struct ref_store *ref_store, const char *refname) } static int debug_create_reflog(struct ref_store *ref_store, const char *refname, - int force_create, struct strbuf *err) + struct strbuf *err) { struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; - int res = drefs->refs->be->create_reflog(drefs->refs, refname, - force_create, err); + int res = drefs->refs->be->create_reflog(drefs->refs, refname, err); trace_printf_key(&trace_refs, "create_reflog: %s: %d\n", refname, res); return res; } diff --git a/refs/files-backend.c b/refs/files-backend.c index 151b0056fe..237a2afb5d 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -282,10 +282,11 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, create_dir_entry(dir->cache, refname.buf, refname.len)); } else { + int ignore_errno; if (!refs_resolve_ref_unsafe(&refs->base, refname.buf, RESOLVE_REF_READING, - &oid, &flag)) { + &oid, &flag, &ignore_errno)) { oidclr(&oid); flag |= REF_ISBROKEN; } else if (is_null_oid(&oid)) { @@ -357,6 +358,7 @@ static int files_read_raw_ref(struct ref_store *ref_store, const char *refname, int fd; int ret = -1; int remaining_retries = 3; + int myerr = 0; *type = 0; strbuf_reset(&sb_path); @@ -383,11 +385,14 @@ stat_ref: goto out; if (lstat(path, &st) < 0) { - if (errno != ENOENT) + int ignore_errno; + myerr = errno; + errno = 0; + if (myerr != ENOENT) goto out; - if (refs_read_raw_ref(refs->packed_ref_store, refname, - oid, referent, type)) { - errno = ENOENT; + if (refs_read_raw_ref(refs->packed_ref_store, refname, oid, + referent, type, &ignore_errno)) { + myerr = ENOENT; goto out; } ret = 0; @@ -398,7 +403,9 @@ stat_ref: if (S_ISLNK(st.st_mode)) { strbuf_reset(&sb_contents); if (strbuf_readlink(&sb_contents, path, st.st_size) < 0) { - if (errno == ENOENT || errno == EINVAL) + myerr = errno; + errno = 0; + if (myerr == ENOENT || myerr == EINVAL) /* inconsistent with lstat; retry */ goto stat_ref; else @@ -420,14 +427,15 @@ stat_ref: /* Is it a directory? */ if (S_ISDIR(st.st_mode)) { + int ignore_errno; /* * Even though there is a directory where the loose * ref is supposed to be, there could still be a * packed ref: */ - if (refs_read_raw_ref(refs->packed_ref_store, refname, - oid, referent, type)) { - errno = EISDIR; + if (refs_read_raw_ref(refs->packed_ref_store, refname, oid, + referent, type, &ignore_errno)) { + myerr = EISDIR; goto out; } ret = 0; @@ -440,7 +448,8 @@ stat_ref: */ fd = open(path, O_RDONLY); if (fd < 0) { - if (errno == ENOENT && !S_ISLNK(st.st_mode)) + myerr = errno; + if (myerr == ENOENT && !S_ISLNK(st.st_mode)) /* inconsistent with lstat; retry */ goto stat_ref; else @@ -448,26 +457,29 @@ stat_ref: } strbuf_reset(&sb_contents); if (strbuf_read(&sb_contents, fd, 256) < 0) { - int save_errno = errno; + myerr = errno; close(fd); - errno = save_errno; goto out; } close(fd); strbuf_rtrim(&sb_contents); buf = sb_contents.buf; - ret = parse_loose_ref_contents(buf, oid, referent, type); + ret = parse_loose_ref_contents(buf, oid, referent, type, &myerr); out: - *failure_errno = errno; + if (ret && !myerr) + BUG("returning non-zero %d, should have set myerr!", ret); + *failure_errno = myerr; + strbuf_release(&sb_path); strbuf_release(&sb_contents); return ret; } int parse_loose_ref_contents(const char *buf, struct object_id *oid, - struct strbuf *referent, unsigned int *type) + struct strbuf *referent, unsigned int *type, + int *failure_errno) { const char *p; if (skip_prefix(buf, "ref:", &buf)) { @@ -486,7 +498,7 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid, if (parse_oid_hex(buf, oid, &p) || (*p != '\0' && !isspace(*p))) { *type |= REF_ISBROKEN; - errno = EINVAL; + *failure_errno = EINVAL; return -1; } return 0; @@ -995,11 +1007,12 @@ static int create_reflock(const char *path, void *cb) * Locks a ref returning the lock on success and NULL on failure. */ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, - const char *refname, int *type, + const char *refname, struct strbuf *err) { struct strbuf ref_file = STRBUF_INIT; struct ref_lock *lock; + int ignore_errno; files_assert_main_repository(refs, "lock_ref_oid_basic"); assert(err); @@ -1007,16 +1020,6 @@ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, CALLOC_ARRAY(lock, 1); files_ref_path(refs, &ref_file, refname); - if (!refs_resolve_ref_unsafe(&refs->base, refname, - RESOLVE_REF_NO_RECURSE, - &lock->old_oid, type)) { - if (!refs_verify_refname_available(&refs->base, refname, - NULL, NULL, err)) - strbuf_addf(err, "unable to resolve reference '%s': %s", - refname, strerror(errno)); - - goto error_return; - } /* * If the ref did not exist and we are creating it, make sure @@ -1036,9 +1039,8 @@ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, goto error_return; } - if (refs_read_ref_full(&refs->base, lock->ref_name, - 0, - &lock->old_oid, NULL)) + if (!refs_resolve_ref_unsafe(&refs->base, lock->ref_name, 0, + &lock->old_oid, NULL, &ignore_errno)) oidclr(&lock->old_oid); goto out; @@ -1358,6 +1360,35 @@ static int commit_ref_update(struct files_ref_store *refs, const struct object_id *oid, const char *logmsg, struct strbuf *err); +/* + * Emit a better error message than lockfile.c's + * unable_to_lock_message() would in case there is a D/F conflict with + * another existing reference. If there would be a conflict, emit an error + * message and return false; otherwise, return true. + * + * Note that this function is not safe against all races with other + * processes, and that's not its job. We'll emit a more verbose error on D/f + * conflicts if we get past it into lock_ref_oid_basic(). + */ +static int refs_rename_ref_available(struct ref_store *refs, + const char *old_refname, + const char *new_refname) +{ + struct string_list skip = STRING_LIST_INIT_NODUP; + struct strbuf err = STRBUF_INIT; + int ok; + + string_list_insert(&skip, old_refname); + ok = !refs_verify_refname_available(refs, new_refname, + NULL, &skip, &err); + if (!ok) + error("%s", err.buf); + + string_list_clear(&skip, 0); + strbuf_release(&err); + return ok; +} + static int files_copy_or_rename_ref(struct ref_store *ref_store, const char *oldrefname, const char *newrefname, const char *logmsg, int copy) @@ -1373,6 +1404,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, struct strbuf tmp_renamed_log = STRBUF_INIT; int log, ret; struct strbuf err = STRBUF_INIT; + int ignore_errno; files_reflog_path(refs, &sb_oldref, oldrefname); files_reflog_path(refs, &sb_newref, newrefname); @@ -1386,7 +1418,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, if (!refs_resolve_ref_unsafe(&refs->base, oldrefname, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, - &orig_oid, &flag)) { + &orig_oid, &flag, &ignore_errno)) { ret = error("refname %s not found", oldrefname); goto out; } @@ -1430,9 +1462,9 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, * the safety anyway; we want to delete the reference whatever * its current value. */ - if (!copy && !refs_read_ref_full(&refs->base, newrefname, - RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, - NULL, NULL) && + if (!copy && refs_resolve_ref_unsafe(&refs->base, newrefname, + RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, + NULL, NULL, &ignore_errno) && refs_delete_ref(&refs->base, NULL, newrefname, NULL, REF_NO_DEREF)) { if (errno == EISDIR) { @@ -1458,7 +1490,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, logmoved = log; - lock = lock_ref_oid_basic(refs, newrefname, NULL, &err); + lock = lock_ref_oid_basic(refs, newrefname, &err); if (!lock) { if (copy) error("unable to copy '%s' to '%s': %s", oldrefname, newrefname, err.buf); @@ -1480,7 +1512,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, goto out; rollback: - lock = lock_ref_oid_basic(refs, oldrefname, NULL, &err); + lock = lock_ref_oid_basic(refs, oldrefname, &err); if (!lock) { error("unable to lock %s for rollback: %s", oldrefname, err.buf); strbuf_release(&err); @@ -1639,15 +1671,14 @@ error: return -1; } -static int files_create_reflog(struct ref_store *ref_store, - const char *refname, int force_create, +static int files_create_reflog(struct ref_store *ref_store, const char *refname, struct strbuf *err) { struct files_ref_store *refs = files_downcast(ref_store, REF_STORE_WRITE, "create_reflog"); int fd; - if (log_ref_setup(refs, refname, force_create, &fd, err)) + if (log_ref_setup(refs, refname, 1, &fd, err)) return -1; if (fd >= 0) @@ -1797,10 +1828,12 @@ static int commit_ref_update(struct files_ref_store *refs, */ int head_flag; const char *head_ref; + int ignore_errno; head_ref = refs_resolve_ref_unsafe(&refs->base, "HEAD", RESOLVE_REF_READING, - NULL, &head_flag); + NULL, &head_flag, + &ignore_errno); if (head_ref && (head_flag & REF_ISSYMREF) && !strcmp(head_ref, lock->ref_name)) { struct strbuf log_err = STRBUF_INIT; @@ -1844,9 +1877,12 @@ static void update_symref_reflog(struct files_ref_store *refs, { struct strbuf err = STRBUF_INIT; struct object_id new_oid; + int ignore_errno; + if (logmsg && - !refs_read_ref_full(&refs->base, target, - RESOLVE_REF_READING, &new_oid, NULL) && + refs_resolve_ref_unsafe(&refs->base, target, + RESOLVE_REF_READING, &new_oid, NULL, + &ignore_errno) && files_log_ref_write(refs, refname, &lock->old_oid, &new_oid, logmsg, 0, &err)) { error("%s", err.buf); @@ -1887,7 +1923,7 @@ static int files_create_symref(struct ref_store *ref_store, struct ref_lock *lock; int ret; - lock = lock_ref_oid_basic(refs, refname, NULL, &err); + lock = lock_ref_oid_basic(refs, refname, &err); if (!lock) { error("%s", err.buf); strbuf_release(&err); @@ -2120,6 +2156,7 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator) (struct files_reflog_iterator *)ref_iterator; struct dir_iterator *diter = iter->dir_iterator; int ok; + int ignore_errno; while ((ok = dir_iterator_advance(diter)) == ITER_OK) { int flags; @@ -2131,9 +2168,10 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator) if (ends_with(diter->basename, ".lock")) continue; - if (refs_read_ref_full(iter->ref_store, - diter->relative_path, 0, - &iter->oid, &flags)) { + if (!refs_resolve_ref_unsafe(iter->ref_store, + diter->relative_path, 0, + &iter->oid, &flags, + &ignore_errno)) { error("bad ref for %s", diter->path.buf); continue; } @@ -2477,9 +2515,11 @@ static int lock_ref_for_update(struct files_ref_store *refs, * the transaction, so we have to read it here * to record and possibly check old_oid: */ - if (refs_read_ref_full(&refs->base, - referent.buf, 0, - &lock->old_oid, NULL)) { + int ignore_errno; + if (!refs_resolve_ref_unsafe(&refs->base, + referent.buf, 0, + &lock->old_oid, NULL, + &ignore_errno)) { if (update->flags & REF_HAVE_OLD) { strbuf_addf(err, "cannot lock ref '%s': " "error reading reference", @@ -3091,7 +3131,6 @@ static int files_reflog_expire(struct ref_store *ref_store, struct strbuf log_file_sb = STRBUF_INIT; char *log_file; int status = 0; - int type; struct strbuf err = STRBUF_INIT; const struct object_id *oid; @@ -3105,7 +3144,7 @@ static int files_reflog_expire(struct ref_store *ref_store, * reference itself, plus we might need to update the * reference if --updateref was specified: */ - lock = lock_ref_oid_basic(refs, refname, &type, &err); + lock = lock_ref_oid_basic(refs, refname, &err); if (!lock) { error("cannot lock ref '%s': %s", refname, err.buf); strbuf_release(&err); @@ -3167,9 +3206,20 @@ static int files_reflog_expire(struct ref_store *ref_store, * a reference if there are no remaining reflog * entries. */ - int update = (flags & EXPIRE_REFLOGS_UPDATE_REF) && - !(type & REF_ISSYMREF) && - !is_null_oid(&cb.last_kept_oid); + int update = 0; + + if ((flags & EXPIRE_REFLOGS_UPDATE_REF) && + !is_null_oid(&cb.last_kept_oid)) { + int ignore_errno; + int type; + const char *ref; + + ref = refs_resolve_ref_unsafe(&refs->base, refname, + RESOLVE_REF_NO_RECURSE, + NULL, &type, + &ignore_errno); + update = !!(ref && !(type & REF_ISSYMREF)); + } if (close_lock_file_gently(&reflog_lock)) { status |= error("couldn't write %s: %s", log_file, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 1c5211b03e..67152c664e 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -1354,6 +1354,7 @@ int is_packed_transaction_needed(struct ref_store *ref_store, ret = 0; for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; + int failure_errno; unsigned int type; struct object_id oid; @@ -1364,9 +1365,9 @@ int is_packed_transaction_needed(struct ref_store *ref_store, */ continue; - if (!refs_read_raw_ref(ref_store, update->refname, - &oid, &referent, &type) || - errno != ENOENT) { + if (!refs_read_raw_ref(ref_store, update->refname, &oid, + &referent, &type, &failure_errno) || + failure_errno != ENOENT) { /* * We have to actually delete that reference * -> this transaction is needed. @@ -1628,8 +1629,7 @@ static int packed_reflog_exists(struct ref_store *ref_store, } static int packed_create_reflog(struct ref_store *ref_store, - const char *refname, int force_create, - struct strbuf *err) + const char *refname, struct strbuf *err) { BUG("packed reference store does not support reflogs"); } diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 12224742ed..46a839539e 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -150,9 +150,9 @@ struct ref_update { const char refname[FLEX_ARRAY]; }; -int refs_read_raw_ref(struct ref_store *ref_store, - const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type); +int refs_read_raw_ref(struct ref_store *ref_store, const char *refname, + struct object_id *oid, struct strbuf *referent, + unsigned int *type, int *failure_errno); /* * Write an error to `err` and return a nonzero value iff the same @@ -229,20 +229,6 @@ const char *find_descendant_ref(const char *dirname, const struct string_list *extras, const struct string_list *skip); -/* - * Check whether an attempt to rename old_refname to new_refname would - * cause a D/F conflict with any existing reference (other than - * possibly old_refname). If there would be a conflict, emit an error - * message and return false; otherwise, return true. - * - * Note that this function is not safe against all races with other - * processes (though rename_ref() catches some races that might get by - * this check). - */ -int refs_rename_ref_available(struct ref_store *refs, - const char *old_refname, - const char *new_refname); - /* We allow "recursive" symbolic refs. Only within reason, though */ #define SYMREF_MAXDEPTH 5 @@ -606,7 +592,7 @@ typedef int for_each_reflog_ent_reverse_fn(struct ref_store *ref_store, void *cb_data); typedef int reflog_exists_fn(struct ref_store *ref_store, const char *refname); typedef int create_reflog_fn(struct ref_store *ref_store, const char *refname, - int force_create, struct strbuf *err); + struct strbuf *err); typedef int delete_reflog_fn(struct ref_store *ref_store, const char *refname); typedef int reflog_expire_fn(struct ref_store *ref_store, const char *refname, @@ -713,10 +699,12 @@ struct ref_store { }; /* - * Parse contents of a loose ref file. + * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for + * invalid contents. */ int parse_loose_ref_contents(const char *buf, struct object_id *oid, - struct strbuf *referent, unsigned int *type); + struct strbuf *referent, unsigned int *type, + int *failure_errno); /* * Fill in the generic part of refs and add it to our collection of diff --git a/reftable/LICENSE b/reftable/LICENSE new file mode 100644 index 0000000000..402e0f9356 --- /dev/null +++ b/reftable/LICENSE @@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/reftable/basics.c b/reftable/basics.c new file mode 100644 index 0000000000..f761e48028 --- /dev/null +++ b/reftable/basics.c @@ -0,0 +1,128 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" + +void put_be24(uint8_t *out, uint32_t i) +{ + out[0] = (uint8_t)((i >> 16) & 0xff); + out[1] = (uint8_t)((i >> 8) & 0xff); + out[2] = (uint8_t)(i & 0xff); +} + +uint32_t get_be24(uint8_t *in) +{ + return (uint32_t)(in[0]) << 16 | (uint32_t)(in[1]) << 8 | + (uint32_t)(in[2]); +} + +void put_be16(uint8_t *out, uint16_t i) +{ + out[0] = (uint8_t)((i >> 8) & 0xff); + out[1] = (uint8_t)(i & 0xff); +} + +int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args) +{ + size_t lo = 0; + size_t hi = sz; + + /* Invariants: + * + * (hi == sz) || f(hi) == true + * (lo == 0 && f(0) == true) || fi(lo) == false + */ + while (hi - lo > 1) { + size_t mid = lo + (hi - lo) / 2; + + if (f(mid, args)) + hi = mid; + else + lo = mid; + } + + if (lo) + return hi; + + return f(0, args) ? 0 : 1; +} + +void free_names(char **a) +{ + char **p; + if (!a) { + return; + } + for (p = a; *p; p++) { + reftable_free(*p); + } + reftable_free(a); +} + +int names_length(char **names) +{ + char **p = names; + for (; *p; p++) { + /* empty */ + } + return p - names; +} + +void parse_names(char *buf, int size, char ***namesp) +{ + char **names = NULL; + size_t names_cap = 0; + size_t names_len = 0; + + char *p = buf; + char *end = buf + size; + while (p < end) { + char *next = strchr(p, '\n'); + if (next && next < end) { + *next = 0; + } else { + next = end; + } + if (p < next) { + if (names_len == names_cap) { + names_cap = 2 * names_cap + 1; + names = reftable_realloc( + names, names_cap * sizeof(*names)); + } + names[names_len++] = xstrdup(p); + } + p = next + 1; + } + + names = reftable_realloc(names, (names_len + 1) * sizeof(*names)); + names[names_len] = NULL; + *namesp = names; +} + +int names_equal(char **a, char **b) +{ + int i = 0; + for (; a[i] && b[i]; i++) { + if (strcmp(a[i], b[i])) { + return 0; + } + } + + return a[i] == b[i]; +} + +int common_prefix_size(struct strbuf *a, struct strbuf *b) +{ + int p = 0; + for (; p < a->len && p < b->len; p++) { + if (a->buf[p] != b->buf[p]) + break; + } + + return p; +} diff --git a/reftable/basics.h b/reftable/basics.h new file mode 100644 index 0000000000..096b36862b --- /dev/null +++ b/reftable/basics.h @@ -0,0 +1,60 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BASICS_H +#define BASICS_H + +/* + * miscellaneous utilities that are not provided by Git. + */ + +#include "system.h" + +/* Bigendian en/decoding of integers */ + +void put_be24(uint8_t *out, uint32_t i); +uint32_t get_be24(uint8_t *in); +void put_be16(uint8_t *out, uint16_t i); + +/* + * find smallest index i in [0, sz) at which f(i) is true, assuming + * that f is ascending. Return sz if f(i) is false for all indices. + * + * Contrary to bsearch(3), this returns something useful if the argument is not + * found. + */ +int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args); + +/* + * Frees a NULL terminated array of malloced strings. The array itself is also + * freed. + */ +void free_names(char **a); + +/* parse a newline separated list of names. `size` is the length of the buffer, + * without terminating '\0'. Empty names are discarded. */ +void parse_names(char *buf, int size, char ***namesp); + +/* compares two NULL-terminated arrays of strings. */ +int names_equal(char **a, char **b); + +/* returns the array size of a NULL-terminated array of strings. */ +int names_length(char **names); + +/* Allocation routines; they invoke the functions set through + * reftable_set_alloc() */ +void *reftable_malloc(size_t sz); +void *reftable_realloc(void *p, size_t sz); +void reftable_free(void *p); +void *reftable_calloc(size_t sz); + +/* Find the longest shared prefix size of `a` and `b` */ +struct strbuf; +int common_prefix_size(struct strbuf *a, struct strbuf *b); + +#endif diff --git a/reftable/basics_test.c b/reftable/basics_test.c new file mode 100644 index 0000000000..1fcd229725 --- /dev/null +++ b/reftable/basics_test.c @@ -0,0 +1,98 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "test_framework.h" +#include "reftable-tests.h" + +struct binsearch_args { + int key; + int *arr; +}; + +static int binsearch_func(size_t i, void *void_args) +{ + struct binsearch_args *args = void_args; + + return args->key < args->arr[i]; +} + +static void test_binsearch(void) +{ + int arr[] = { 2, 4, 6, 8, 10 }; + size_t sz = ARRAY_SIZE(arr); + struct binsearch_args args = { + .arr = arr, + }; + + int i = 0; + for (i = 1; i < 11; i++) { + int res; + args.key = i; + res = binsearch(sz, &binsearch_func, &args); + + if (res < sz) { + EXPECT(args.key < arr[res]); + if (res > 0) { + EXPECT(args.key >= arr[res - 1]); + } + } else { + EXPECT(args.key == 10 || args.key == 11); + } + } +} + +static void test_names_length(void) +{ + char *a[] = { "a", "b", NULL }; + EXPECT(names_length(a) == 2); +} + +static void test_parse_names_normal(void) +{ + char in[] = "a\nb\n"; + char **out = NULL; + parse_names(in, strlen(in), &out); + EXPECT(!strcmp(out[0], "a")); + EXPECT(!strcmp(out[1], "b")); + EXPECT(!out[2]); + free_names(out); +} + +static void test_parse_names_drop_empty(void) +{ + char in[] = "a\n\n"; + char **out = NULL; + parse_names(in, strlen(in), &out); + EXPECT(!strcmp(out[0], "a")); + EXPECT(!out[1]); + free_names(out); +} + +static void test_common_prefix(void) +{ + struct strbuf s1 = STRBUF_INIT; + struct strbuf s2 = STRBUF_INIT; + strbuf_addstr(&s1, "abcdef"); + strbuf_addstr(&s2, "abc"); + EXPECT(common_prefix_size(&s1, &s2) == 3); + strbuf_release(&s1); + strbuf_release(&s2); +} + +int basics_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_common_prefix); + RUN_TEST(test_parse_names_normal); + RUN_TEST(test_parse_names_drop_empty); + RUN_TEST(test_binsearch); + RUN_TEST(test_names_length); + return 0; +} diff --git a/reftable/block.c b/reftable/block.c new file mode 100644 index 0000000000..855e3f5c94 --- /dev/null +++ b/reftable/block.c @@ -0,0 +1,437 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "block.h" + +#include "blocksource.h" +#include "constants.h" +#include "record.h" +#include "reftable-error.h" +#include "system.h" +#include <zlib.h> + +int header_size(int version) +{ + switch (version) { + case 1: + return 24; + case 2: + return 28; + } + abort(); +} + +int footer_size(int version) +{ + switch (version) { + case 1: + return 68; + case 2: + return 72; + } + abort(); +} + +static int block_writer_register_restart(struct block_writer *w, int n, + int is_restart, struct strbuf *key) +{ + int rlen = w->restart_len; + if (rlen >= MAX_RESTARTS) { + is_restart = 0; + } + + if (is_restart) { + rlen++; + } + if (2 + 3 * rlen + n > w->block_size - w->next) + return -1; + if (is_restart) { + if (w->restart_len == w->restart_cap) { + w->restart_cap = w->restart_cap * 2 + 1; + w->restarts = reftable_realloc( + w->restarts, sizeof(uint32_t) * w->restart_cap); + } + + w->restarts[w->restart_len++] = w->next; + } + + w->next += n; + + strbuf_reset(&w->last_key); + strbuf_addbuf(&w->last_key, key); + w->entries++; + return 0; +} + +void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, + uint32_t block_size, uint32_t header_off, int hash_size) +{ + bw->buf = buf; + bw->hash_size = hash_size; + bw->block_size = block_size; + bw->header_off = header_off; + bw->buf[header_off] = typ; + bw->next = header_off + 4; + bw->restart_interval = 16; + bw->entries = 0; + bw->restart_len = 0; + bw->last_key.len = 0; +} + +uint8_t block_writer_type(struct block_writer *bw) +{ + return bw->buf[bw->header_off]; +} + +/* adds the reftable_record to the block. Returns -1 if it does not fit, 0 on + success */ +int block_writer_add(struct block_writer *w, struct reftable_record *rec) +{ + struct strbuf empty = STRBUF_INIT; + struct strbuf last = + w->entries % w->restart_interval == 0 ? empty : w->last_key; + struct string_view out = { + .buf = w->buf + w->next, + .len = w->block_size - w->next, + }; + + struct string_view start = out; + + int is_restart = 0; + struct strbuf key = STRBUF_INIT; + int n = 0; + + reftable_record_key(rec, &key); + n = reftable_encode_key(&is_restart, out, last, key, + reftable_record_val_type(rec)); + if (n < 0) + goto done; + string_view_consume(&out, n); + + n = reftable_record_encode(rec, out, w->hash_size); + if (n < 0) + goto done; + string_view_consume(&out, n); + + if (block_writer_register_restart(w, start.len - out.len, is_restart, + &key) < 0) + goto done; + + strbuf_release(&key); + return 0; + +done: + strbuf_release(&key); + return -1; +} + +int block_writer_finish(struct block_writer *w) +{ + int i; + for (i = 0; i < w->restart_len; i++) { + put_be24(w->buf + w->next, w->restarts[i]); + w->next += 3; + } + + put_be16(w->buf + w->next, w->restart_len); + w->next += 2; + put_be24(w->buf + 1 + w->header_off, w->next); + + if (block_writer_type(w) == BLOCK_TYPE_LOG) { + int block_header_skip = 4 + w->header_off; + uLongf src_len = w->next - block_header_skip; + uLongf dest_cap = src_len * 1.001 + 12; + + uint8_t *compressed = reftable_malloc(dest_cap); + while (1) { + uLongf out_dest_len = dest_cap; + int zresult = compress2(compressed, &out_dest_len, + w->buf + block_header_skip, + src_len, 9); + if (zresult == Z_BUF_ERROR && dest_cap < LONG_MAX) { + dest_cap *= 2; + compressed = + reftable_realloc(compressed, dest_cap); + if (compressed) + continue; + } + + if (Z_OK != zresult) { + reftable_free(compressed); + return REFTABLE_ZLIB_ERROR; + } + + memcpy(w->buf + block_header_skip, compressed, + out_dest_len); + w->next = out_dest_len + block_header_skip; + reftable_free(compressed); + break; + } + } + return w->next; +} + +uint8_t block_reader_type(struct block_reader *r) +{ + return r->block.data[r->header_off]; +} + +int block_reader_init(struct block_reader *br, struct reftable_block *block, + uint32_t header_off, uint32_t table_block_size, + int hash_size) +{ + uint32_t full_block_size = table_block_size; + uint8_t typ = block->data[header_off]; + uint32_t sz = get_be24(block->data + header_off + 1); + + uint16_t restart_count = 0; + uint32_t restart_start = 0; + uint8_t *restart_bytes = NULL; + + if (!reftable_is_block_type(typ)) + return REFTABLE_FORMAT_ERROR; + + if (typ == BLOCK_TYPE_LOG) { + int block_header_skip = 4 + header_off; + uLongf dst_len = sz - block_header_skip; /* total size of dest + buffer. */ + uLongf src_len = block->len - block_header_skip; + /* Log blocks specify the *uncompressed* size in their header. + */ + uint8_t *uncompressed = reftable_malloc(sz); + + /* Copy over the block header verbatim. It's not compressed. */ + memcpy(uncompressed, block->data, block_header_skip); + + /* Uncompress */ + if (Z_OK != + uncompress2(uncompressed + block_header_skip, &dst_len, + block->data + block_header_skip, &src_len)) { + reftable_free(uncompressed); + return REFTABLE_ZLIB_ERROR; + } + + if (dst_len + block_header_skip != sz) + return REFTABLE_FORMAT_ERROR; + + /* We're done with the input data. */ + reftable_block_done(block); + block->data = uncompressed; + block->len = sz; + block->source = malloc_block_source(); + full_block_size = src_len + block_header_skip; + } else if (full_block_size == 0) { + full_block_size = sz; + } else if (sz < full_block_size && sz < block->len && + block->data[sz] != 0) { + /* If the block is smaller than the full block size, it is + padded (data followed by '\0') or the next block is + unaligned. */ + full_block_size = sz; + } + + restart_count = get_be16(block->data + sz - 2); + restart_start = sz - 2 - 3 * restart_count; + restart_bytes = block->data + restart_start; + + /* transfer ownership. */ + br->block = *block; + block->data = NULL; + block->len = 0; + + br->hash_size = hash_size; + br->block_len = restart_start; + br->full_block_size = full_block_size; + br->header_off = header_off; + br->restart_count = restart_count; + br->restart_bytes = restart_bytes; + + return 0; +} + +static uint32_t block_reader_restart_offset(struct block_reader *br, int i) +{ + return get_be24(br->restart_bytes + 3 * i); +} + +void block_reader_start(struct block_reader *br, struct block_iter *it) +{ + it->br = br; + strbuf_reset(&it->last_key); + it->next_off = br->header_off + 4; +} + +struct restart_find_args { + int error; + struct strbuf key; + struct block_reader *r; +}; + +static int restart_key_less(size_t idx, void *args) +{ + struct restart_find_args *a = args; + uint32_t off = block_reader_restart_offset(a->r, idx); + struct string_view in = { + .buf = a->r->block.data + off, + .len = a->r->block_len - off, + }; + + /* the restart key is verbatim in the block, so this could avoid the + alloc for decoding the key */ + struct strbuf rkey = STRBUF_INIT; + struct strbuf last_key = STRBUF_INIT; + uint8_t unused_extra; + int n = reftable_decode_key(&rkey, &unused_extra, last_key, in); + int result; + if (n < 0) { + a->error = 1; + return -1; + } + + result = strbuf_cmp(&a->key, &rkey); + strbuf_release(&rkey); + return result; +} + +void block_iter_copy_from(struct block_iter *dest, struct block_iter *src) +{ + dest->br = src->br; + dest->next_off = src->next_off; + strbuf_reset(&dest->last_key); + strbuf_addbuf(&dest->last_key, &src->last_key); +} + +int block_iter_next(struct block_iter *it, struct reftable_record *rec) +{ + struct string_view in = { + .buf = it->br->block.data + it->next_off, + .len = it->br->block_len - it->next_off, + }; + struct string_view start = in; + struct strbuf key = STRBUF_INIT; + uint8_t extra = 0; + int n = 0; + + if (it->next_off >= it->br->block_len) + return 1; + + n = reftable_decode_key(&key, &extra, it->last_key, in); + if (n < 0) + return -1; + + string_view_consume(&in, n); + n = reftable_record_decode(rec, key, extra, in, it->br->hash_size); + if (n < 0) + return -1; + string_view_consume(&in, n); + + strbuf_reset(&it->last_key); + strbuf_addbuf(&it->last_key, &key); + it->next_off += start.len - in.len; + strbuf_release(&key); + return 0; +} + +int block_reader_first_key(struct block_reader *br, struct strbuf *key) +{ + struct strbuf empty = STRBUF_INIT; + int off = br->header_off + 4; + struct string_view in = { + .buf = br->block.data + off, + .len = br->block_len - off, + }; + + uint8_t extra = 0; + int n = reftable_decode_key(key, &extra, empty, in); + if (n < 0) + return n; + + return 0; +} + +int block_iter_seek(struct block_iter *it, struct strbuf *want) +{ + return block_reader_seek(it->br, it, want); +} + +void block_iter_close(struct block_iter *it) +{ + strbuf_release(&it->last_key); +} + +int block_reader_seek(struct block_reader *br, struct block_iter *it, + struct strbuf *want) +{ + struct restart_find_args args = { + .key = *want, + .r = br, + }; + struct reftable_record rec = reftable_new_record(block_reader_type(br)); + struct strbuf key = STRBUF_INIT; + int err = 0; + struct block_iter next = { + .last_key = STRBUF_INIT, + }; + + int i = binsearch(br->restart_count, &restart_key_less, &args); + if (args.error) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + it->br = br; + if (i > 0) { + i--; + it->next_off = block_reader_restart_offset(br, i); + } else { + it->next_off = br->header_off + 4; + } + + /* We're looking for the last entry less/equal than the wanted key, so + we have to go one entry too far and then back up. + */ + while (1) { + block_iter_copy_from(&next, it); + err = block_iter_next(&next, &rec); + if (err < 0) + goto done; + + reftable_record_key(&rec, &key); + if (err > 0 || strbuf_cmp(&key, want) >= 0) { + err = 0; + goto done; + } + + block_iter_copy_from(it, &next); + } + +done: + strbuf_release(&key); + strbuf_release(&next.last_key); + reftable_record_destroy(&rec); + + return err; +} + +void block_writer_release(struct block_writer *bw) +{ + FREE_AND_NULL(bw->restarts); + strbuf_release(&bw->last_key); + /* the block is not owned. */ +} + +void reftable_block_done(struct reftable_block *blockp) +{ + struct reftable_block_source source = blockp->source; + if (blockp && source.ops) + source.ops->return_block(source.arg, blockp); + blockp->data = NULL; + blockp->len = 0; + blockp->source.ops = NULL; + blockp->source.arg = NULL; +} diff --git a/reftable/block.h b/reftable/block.h new file mode 100644 index 0000000000..e207706a64 --- /dev/null +++ b/reftable/block.h @@ -0,0 +1,127 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCK_H +#define BLOCK_H + +#include "basics.h" +#include "record.h" +#include "reftable-blocksource.h" + +/* + * Writes reftable blocks. The block_writer is reused across blocks to minimize + * allocation overhead. + */ +struct block_writer { + uint8_t *buf; + uint32_t block_size; + + /* Offset ofof the global header. Nonzero in the first block only. */ + uint32_t header_off; + + /* How often to restart keys. */ + int restart_interval; + int hash_size; + + /* Offset of next uint8_t to write. */ + uint32_t next; + uint32_t *restarts; + uint32_t restart_len; + uint32_t restart_cap; + + struct strbuf last_key; + int entries; +}; + +/* + * initializes the blockwriter to write `typ` entries, using `buf` as temporary + * storage. `buf` is not owned by the block_writer. */ +void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, + uint32_t block_size, uint32_t header_off, int hash_size); + +/* returns the block type (eg. 'r' for ref records. */ +uint8_t block_writer_type(struct block_writer *bw); + +/* appends the record, or -1 if it doesn't fit. */ +int block_writer_add(struct block_writer *w, struct reftable_record *rec); + +/* appends the key restarts, and compress the block if necessary. */ +int block_writer_finish(struct block_writer *w); + +/* clears out internally allocated block_writer members. */ +void block_writer_release(struct block_writer *bw); + +/* Read a block. */ +struct block_reader { + /* offset of the block header; nonzero for the first block in a + * reftable. */ + uint32_t header_off; + + /* the memory block */ + struct reftable_block block; + int hash_size; + + /* size of the data, excluding restart data. */ + uint32_t block_len; + uint8_t *restart_bytes; + uint16_t restart_count; + + /* size of the data in the file. For log blocks, this is the compressed + * size. */ + uint32_t full_block_size; +}; + +/* Iterate over entries in a block */ +struct block_iter { + /* offset within the block of the next entry to read. */ + uint32_t next_off; + struct block_reader *br; + + /* key for last entry we read. */ + struct strbuf last_key; +}; + +/* initializes a block reader. */ +int block_reader_init(struct block_reader *br, struct reftable_block *bl, + uint32_t header_off, uint32_t table_block_size, + int hash_size); + +/* Position `it` at start of the block */ +void block_reader_start(struct block_reader *br, struct block_iter *it); + +/* Position `it` to the `want` key in the block */ +int block_reader_seek(struct block_reader *br, struct block_iter *it, + struct strbuf *want); + +/* Returns the block type (eg. 'r' for refs) */ +uint8_t block_reader_type(struct block_reader *r); + +/* Decodes the first key in the block */ +int block_reader_first_key(struct block_reader *br, struct strbuf *key); + +void block_iter_copy_from(struct block_iter *dest, struct block_iter *src); + +/* return < 0 for error, 0 for OK, > 0 for EOF. */ +int block_iter_next(struct block_iter *it, struct reftable_record *rec); + +/* Seek to `want` with in the block pointed to by `it` */ +int block_iter_seek(struct block_iter *it, struct strbuf *want); + +/* deallocate memory for `it`. The block reader and its block is left intact. */ +void block_iter_close(struct block_iter *it); + +/* size of file header, depending on format version */ +int header_size(int version); + +/* size of file footer, depending on format version */ +int footer_size(int version); + +/* returns a block to its source. */ +void reftable_block_done(struct reftable_block *ret); + +#endif diff --git a/reftable/block_test.c b/reftable/block_test.c new file mode 100644 index 0000000000..4b3ea262dc --- /dev/null +++ b/reftable/block_test.c @@ -0,0 +1,120 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "block.h" + +#include "system.h" +#include "blocksource.h" +#include "basics.h" +#include "constants.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static void test_block_read_write(void) +{ + const int header_off = 21; /* random */ + char *names[30]; + const int N = ARRAY_SIZE(names); + const int block_size = 1024; + struct reftable_block block = { NULL }; + struct block_writer bw = { + .last_key = STRBUF_INIT, + }; + struct reftable_ref_record ref = { NULL }; + struct reftable_record rec = { NULL }; + int i = 0; + int n; + struct block_reader br = { 0 }; + struct block_iter it = { .last_key = STRBUF_INIT }; + int j = 0; + struct strbuf want = STRBUF_INIT; + + block.data = reftable_calloc(block_size); + block.len = block_size; + block.source = malloc_block_source(); + block_writer_init(&bw, BLOCK_TYPE_REF, block.data, block_size, + header_off, hash_size(GIT_SHA1_FORMAT_ID)); + reftable_record_from_ref(&rec, &ref); + + for (i = 0; i < N; i++) { + char name[100]; + uint8_t hash[GIT_SHA1_RAWSZ]; + snprintf(name, sizeof(name), "branch%02d", i); + memset(hash, i, sizeof(hash)); + + ref.refname = name; + ref.value_type = REFTABLE_REF_VAL1; + ref.value.val1 = hash; + + names[i] = xstrdup(name); + n = block_writer_add(&bw, &rec); + ref.refname = NULL; + ref.value_type = REFTABLE_REF_DELETION; + EXPECT(n == 0); + } + + n = block_writer_finish(&bw); + EXPECT(n > 0); + + block_writer_release(&bw); + + block_reader_init(&br, &block, header_off, block_size, GIT_SHA1_RAWSZ); + + block_reader_start(&br, &it); + + while (1) { + int r = block_iter_next(&it, &rec); + EXPECT(r >= 0); + if (r > 0) { + break; + } + EXPECT_STREQ(names[j], ref.refname); + j++; + } + + reftable_record_release(&rec); + block_iter_close(&it); + + for (i = 0; i < N; i++) { + struct block_iter it = { .last_key = STRBUF_INIT }; + strbuf_reset(&want); + strbuf_addstr(&want, names[i]); + + n = block_reader_seek(&br, &it, &want); + EXPECT(n == 0); + + n = block_iter_next(&it, &rec); + EXPECT(n == 0); + + EXPECT_STREQ(names[i], ref.refname); + + want.len--; + n = block_reader_seek(&br, &it, &want); + EXPECT(n == 0); + + n = block_iter_next(&it, &rec); + EXPECT(n == 0); + EXPECT_STREQ(names[10 * (i / 10)], ref.refname); + + block_iter_close(&it); + } + + reftable_record_release(&rec); + reftable_block_done(&br.block); + strbuf_release(&want); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } +} + +int block_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_block_read_write); + return 0; +} diff --git a/reftable/blocksource.c b/reftable/blocksource.c new file mode 100644 index 0000000000..0044eecd9a --- /dev/null +++ b/reftable/blocksource.c @@ -0,0 +1,148 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "blocksource.h" +#include "reftable-blocksource.h" +#include "reftable-error.h" + +static void strbuf_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static void strbuf_close(void *b) +{ +} + +static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + struct strbuf *b = v; + assert(off + size <= b->len); + dest->data = reftable_calloc(size); + memcpy(dest->data, b->buf + off, size); + dest->len = size; + return size; +} + +static uint64_t strbuf_size(void *b) +{ + return ((struct strbuf *)b)->len; +} + +static struct reftable_block_source_vtable strbuf_vtable = { + .size = &strbuf_size, + .read_block = &strbuf_read_block, + .return_block = &strbuf_return_block, + .close = &strbuf_close, +}; + +void block_source_from_strbuf(struct reftable_block_source *bs, + struct strbuf *buf) +{ + assert(!bs->ops); + bs->ops = &strbuf_vtable; + bs->arg = buf; +} + +static void malloc_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static struct reftable_block_source_vtable malloc_vtable = { + .return_block = &malloc_return_block, +}; + +static struct reftable_block_source malloc_block_source_instance = { + .ops = &malloc_vtable, +}; + +struct reftable_block_source malloc_block_source(void) +{ + return malloc_block_source_instance; +} + +struct file_block_source { + int fd; + uint64_t size; +}; + +static uint64_t file_size(void *b) +{ + return ((struct file_block_source *)b)->size; +} + +static void file_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static void file_close(void *b) +{ + int fd = ((struct file_block_source *)b)->fd; + if (fd > 0) { + close(fd); + ((struct file_block_source *)b)->fd = 0; + } + + reftable_free(b); +} + +static int file_read_block(void *v, struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + struct file_block_source *b = v; + assert(off + size <= b->size); + dest->data = reftable_malloc(size); + if (pread(b->fd, dest->data, size, off) != size) + return -1; + dest->len = size; + return size; +} + +static struct reftable_block_source_vtable file_vtable = { + .size = &file_size, + .read_block = &file_read_block, + .return_block = &file_return_block, + .close = &file_close, +}; + +int reftable_block_source_from_file(struct reftable_block_source *bs, + const char *name) +{ + struct stat st = { 0 }; + int err = 0; + int fd = open(name, O_RDONLY); + struct file_block_source *p = NULL; + if (fd < 0) { + if (errno == ENOENT) { + return REFTABLE_NOT_EXIST_ERROR; + } + return -1; + } + + err = fstat(fd, &st); + if (err < 0) + return -1; + + p = reftable_calloc(sizeof(struct file_block_source)); + p->size = st.st_size; + p->fd = fd; + + assert(!bs->ops); + bs->ops = &file_vtable; + bs->arg = p; + return 0; +} diff --git a/reftable/blocksource.h b/reftable/blocksource.h new file mode 100644 index 0000000000..072e2727ad --- /dev/null +++ b/reftable/blocksource.h @@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf(struct reftable_block_source *bs, + struct strbuf *buf); + +struct reftable_block_source malloc_block_source(void); + +#endif diff --git a/reftable/constants.h b/reftable/constants.h new file mode 100644 index 0000000000..5eee72c4c1 --- /dev/null +++ b/reftable/constants.h @@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif diff --git a/reftable/dump.c b/reftable/dump.c new file mode 100644 index 0000000000..155953d1b8 --- /dev/null +++ b/reftable/dump.c @@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack(const char *stackdir) +{ + struct reftable_stack *stack = NULL; + struct reftable_write_options cfg = { 0 }; + + int err = reftable_new_stack(&stack, stackdir, cfg); + if (err < 0) + goto done; + + err = reftable_stack_compact_all(stack, NULL); + if (err < 0) + goto done; +done: + if (stack) { + reftable_stack_destroy(stack); + } + return err; +} + +static void print_help(void) +{ + printf("usage: dump [-cst] arg\n\n" + "options: \n" + " -c compact\n" + " -t dump table\n" + " -s dump stack\n" + " -6 sha256 hash format\n" + " -h this help\n" + "\n"); +} + +int reftable_dump_main(int argc, char *const *argv) +{ + int err = 0; + int opt_dump_table = 0; + int opt_dump_stack = 0; + int opt_compact = 0; + uint32_t opt_hash_id = GIT_SHA1_FORMAT_ID; + const char *arg = NULL, *argv0 = argv[0]; + + for (; argc > 1; argv++, argc--) + if (*argv[1] != '-') + break; + else if (!strcmp("-t", argv[1])) + opt_dump_table = 1; + else if (!strcmp("-6", argv[1])) + opt_hash_id = GIT_SHA256_FORMAT_ID; + else if (!strcmp("-s", argv[1])) + opt_dump_stack = 1; + else if (!strcmp("-c", argv[1])) + opt_compact = 1; + else if (!strcmp("-?", argv[1]) || !strcmp("-h", argv[1])) { + print_help(); + return 2; + } + + if (argc != 2) { + fprintf(stderr, "need argument\n"); + print_help(); + return 2; + } + + arg = argv[1]; + + if (opt_dump_table) { + err = reftable_reader_print_file(arg); + } else if (opt_dump_stack) { + err = reftable_stack_print_directory(arg, opt_hash_id); + } else if (opt_compact) { + err = compact_stack(arg); + } + + if (err < 0) { + fprintf(stderr, "%s: %s: %s\n", argv0, arg, + reftable_error_str(err)); + return 1; + } + return 0; +} diff --git a/reftable/error.c b/reftable/error.c new file mode 100644 index 0000000000..f6f16def92 --- /dev/null +++ b/reftable/error.c @@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str(int err) +{ + static char buf[250]; + switch (err) { + case REFTABLE_IO_ERROR: + return "I/O error"; + case REFTABLE_FORMAT_ERROR: + return "corrupt reftable file"; + case REFTABLE_NOT_EXIST_ERROR: + return "file does not exist"; + case REFTABLE_LOCK_ERROR: + return "data is outdated"; + case REFTABLE_API_ERROR: + return "misuse of the reftable API"; + case REFTABLE_ZLIB_ERROR: + return "zlib failure"; + case REFTABLE_NAME_CONFLICT: + return "file/directory conflict"; + case REFTABLE_EMPTY_TABLE_ERROR: + return "wrote empty table"; + case REFTABLE_REFNAME_ERROR: + return "invalid refname"; + case -1: + return "general error"; + default: + snprintf(buf, sizeof(buf), "unknown error code %d", err); + return buf; + } +} diff --git a/reftable/generic.c b/reftable/generic.c new file mode 100644 index 0000000000..7a8a738d86 --- /dev/null +++ b/reftable/generic.c @@ -0,0 +1,169 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "record.h" +#include "generic.h" +#include "reftable-iterator.h" +#include "reftable-generic.h" + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_seek_log(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = ~((uint64_t)0), + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref) +{ + struct reftable_iterator it = { NULL }; + int err = reftable_table_seek_ref(tab, &it, name); + if (err) + goto done; + + err = reftable_iterator_next_ref(&it, ref); + if (err) + goto done; + + if (strcmp(ref->refname, name) || + reftable_ref_record_is_deletion(ref)) { + reftable_ref_record_release(ref); + err = 1; + goto done; + } + +done: + reftable_iterator_destroy(&it); + return err; +} + +int reftable_table_print(struct reftable_table *tab) { + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + struct reftable_log_record log = { NULL }; + uint32_t hash_id = reftable_table_hash_id(tab); + int err = reftable_table_seek_ref(tab, &it, ""); + if (err < 0) { + return err; + } + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) { + return err; + } + reftable_ref_record_print(&ref, hash_id); + } + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); + + err = reftable_table_seek_log(tab, &it, ""); + if (err < 0) { + return err; + } + while (1) { + err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + if (err < 0) { + return err; + } + reftable_log_record_print(&log, hash_id); + } + reftable_iterator_destroy(&it); + reftable_log_record_release(&log); + return 0; +} + +uint64_t reftable_table_max_update_index(struct reftable_table *tab) +{ + return tab->ops->max_update_index(tab->table_arg); +} + +uint64_t reftable_table_min_update_index(struct reftable_table *tab) +{ + return tab->ops->min_update_index(tab->table_arg); +} + +uint32_t reftable_table_hash_id(struct reftable_table *tab) +{ + return tab->ops->hash_id(tab->table_arg); +} + +void reftable_iterator_destroy(struct reftable_iterator *it) +{ + if (!it->ops) { + return; + } + it->ops->close(it->iter_arg); + it->ops = NULL; + FREE_AND_NULL(it->iter_arg); +} + +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, ref); + return iterator_next(it, &rec); +} + +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, log); + return iterator_next(it, &rec); +} + +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) +{ + return it->ops->next(it->iter_arg, rec); +} + +static int empty_iterator_next(void *arg, struct reftable_record *rec) +{ + return 1; +} + +static void empty_iterator_close(void *arg) +{ +} + +static struct reftable_iterator_vtable empty_vtable = { + .next = &empty_iterator_next, + .close = &empty_iterator_close, +}; + +void iterator_set_empty(struct reftable_iterator *it) +{ + assert(!it->ops); + it->iter_arg = NULL; + it->ops = &empty_vtable; +} diff --git a/reftable/generic.h b/reftable/generic.h new file mode 100644 index 0000000000..98886a0640 --- /dev/null +++ b/reftable/generic.h @@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { + int (*seek_record)(void *tab, struct reftable_iterator *it, + struct reftable_record *); + uint32_t (*hash_id)(void *tab); + uint64_t (*min_update_index)(void *tab); + uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { + int (*next)(void *iter_arg, struct reftable_record *rec); + void (*close)(void *iter_arg); +}; + +void iterator_set_empty(struct reftable_iterator *it); +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec); + +#endif diff --git a/reftable/iter.c b/reftable/iter.c new file mode 100644 index 0000000000..93d04f735b --- /dev/null +++ b/reftable/iter.c @@ -0,0 +1,194 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "iter.h" + +#include "system.h" + +#include "block.h" +#include "generic.h" +#include "constants.h" +#include "reader.h" +#include "reftable-error.h" + +int iterator_is_null(struct reftable_iterator *it) +{ + return !it->ops; +} + +static void filtering_ref_iterator_close(void *iter_arg) +{ + struct filtering_ref_iterator *fri = iter_arg; + strbuf_release(&fri->oid); + reftable_iterator_destroy(&fri->it); +} + +static int filtering_ref_iterator_next(void *iter_arg, + struct reftable_record *rec) +{ + struct filtering_ref_iterator *fri = iter_arg; + struct reftable_ref_record *ref = rec->data; + int err = 0; + while (1) { + err = reftable_iterator_next_ref(&fri->it, ref); + if (err != 0) { + break; + } + + if (fri->double_check) { + struct reftable_iterator it = { NULL }; + + err = reftable_table_seek_ref(&fri->tab, &it, + ref->refname); + if (err == 0) { + err = reftable_iterator_next_ref(&it, ref); + } + + reftable_iterator_destroy(&it); + + if (err < 0) { + break; + } + + if (err > 0) { + continue; + } + } + + if (ref->value_type == REFTABLE_REF_VAL2 && + (!memcmp(fri->oid.buf, ref->value.val2.target_value, + fri->oid.len) || + !memcmp(fri->oid.buf, ref->value.val2.value, + fri->oid.len))) + return 0; + + if (ref->value_type == REFTABLE_REF_VAL1 && + !memcmp(fri->oid.buf, ref->value.val1, fri->oid.len)) { + return 0; + } + } + + reftable_ref_record_release(ref); + return err; +} + +static struct reftable_iterator_vtable filtering_ref_iterator_vtable = { + .next = &filtering_ref_iterator_next, + .close = &filtering_ref_iterator_close, +}; + +void iterator_from_filtering_ref_iterator(struct reftable_iterator *it, + struct filtering_ref_iterator *fri) +{ + assert(!it->ops); + it->iter_arg = fri; + it->ops = &filtering_ref_iterator_vtable; +} + +static void indexed_table_ref_iter_close(void *p) +{ + struct indexed_table_ref_iter *it = p; + block_iter_close(&it->cur); + reftable_block_done(&it->block_reader.block); + reftable_free(it->offsets); + strbuf_release(&it->oid); +} + +static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) +{ + uint64_t off; + int err = 0; + if (it->offset_idx == it->offset_len) { + it->is_finished = 1; + return 1; + } + + reftable_block_done(&it->block_reader.block); + + off = it->offsets[it->offset_idx++]; + err = reader_init_block_reader(it->r, &it->block_reader, off, + BLOCK_TYPE_REF); + if (err < 0) { + return err; + } + if (err > 0) { + /* indexed block does not exist. */ + return REFTABLE_FORMAT_ERROR; + } + block_reader_start(&it->block_reader, &it->cur); + return 0; +} + +static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec) +{ + struct indexed_table_ref_iter *it = p; + struct reftable_ref_record *ref = rec->data; + + while (1) { + int err = block_iter_next(&it->cur, rec); + if (err < 0) { + return err; + } + + if (err > 0) { + err = indexed_table_ref_iter_next_block(it); + if (err < 0) { + return err; + } + + if (it->is_finished) { + return 1; + } + continue; + } + /* BUG */ + if (!memcmp(it->oid.buf, ref->value.val2.target_value, + it->oid.len) || + !memcmp(it->oid.buf, ref->value.val2.value, it->oid.len)) { + return 0; + } + } +} + +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, + struct reftable_reader *r, uint8_t *oid, + int oid_len, uint64_t *offsets, int offset_len) +{ + struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; + struct indexed_table_ref_iter *itr = + reftable_calloc(sizeof(struct indexed_table_ref_iter)); + int err = 0; + + *itr = empty; + itr->r = r; + strbuf_add(&itr->oid, oid, oid_len); + + itr->offsets = offsets; + itr->offset_len = offset_len; + + err = indexed_table_ref_iter_next_block(itr); + if (err < 0) { + reftable_free(itr); + } else { + *dest = itr; + } + return err; +} + +static struct reftable_iterator_vtable indexed_table_ref_iter_vtable = { + .next = &indexed_table_ref_iter_next, + .close = &indexed_table_ref_iter_close, +}; + +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, + struct indexed_table_ref_iter *itr) +{ + assert(!it->ops); + it->iter_arg = itr; + it->ops = &indexed_table_ref_iter_vtable; +} diff --git a/reftable/iter.h b/reftable/iter.h new file mode 100644 index 0000000000..09eb0cbfa5 --- /dev/null +++ b/reftable/iter.h @@ -0,0 +1,69 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef ITER_H +#define ITER_H + +#include "system.h" +#include "block.h" +#include "record.h" + +#include "reftable-iterator.h" +#include "reftable-generic.h" + +/* Returns true for a zeroed out iterator, such as the one returned from + * iterator_destroy. */ +int iterator_is_null(struct reftable_iterator *it); + +/* iterator that produces only ref records that point to `oid` */ +struct filtering_ref_iterator { + int double_check; + struct reftable_table tab; + struct strbuf oid; + struct reftable_iterator it; +}; +#define FILTERING_REF_ITERATOR_INIT \ + { \ + .oid = STRBUF_INIT \ + } + +void iterator_from_filtering_ref_iterator(struct reftable_iterator *, + struct filtering_ref_iterator *); + +/* iterator that produces only ref records that point to `oid`, + * but using the object index. + */ +struct indexed_table_ref_iter { + struct reftable_reader *r; + struct strbuf oid; + + /* mutable */ + uint64_t *offsets; + + /* Points to the next offset to read. */ + int offset_idx; + int offset_len; + struct block_reader block_reader; + struct block_iter cur; + int is_finished; +}; + +#define INDEXED_TABLE_REF_ITER_INIT \ + { \ + .cur = { .last_key = STRBUF_INIT }, .oid = STRBUF_INIT, \ + } + +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, + struct indexed_table_ref_iter *itr); + +/* Takes ownership of `offsets` */ +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, + struct reftable_reader *r, uint8_t *oid, + int oid_len, uint64_t *offsets, int offset_len); + +#endif diff --git a/reftable/merged.c b/reftable/merged.c new file mode 100644 index 0000000000..e5b53da6db --- /dev/null +++ b/reftable/merged.c @@ -0,0 +1,362 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "merged.h" + +#include "constants.h" +#include "iter.h" +#include "pq.h" +#include "reader.h" +#include "record.h" +#include "generic.h" +#include "reftable-merged.h" +#include "reftable-error.h" +#include "system.h" + +static int merged_iter_init(struct merged_iter *mi) +{ + int i = 0; + for (i = 0; i < mi->stack_len; i++) { + struct reftable_record rec = reftable_new_record(mi->typ); + int err = iterator_next(&mi->stack[i], &rec); + if (err < 0) { + return err; + } + + if (err > 0) { + reftable_iterator_destroy(&mi->stack[i]); + reftable_record_destroy(&rec); + } else { + struct pq_entry e = { + .rec = rec, + .index = i, + }; + merged_iter_pqueue_add(&mi->pq, e); + } + } + + return 0; +} + +static void merged_iter_close(void *p) +{ + struct merged_iter *mi = p; + int i = 0; + merged_iter_pqueue_release(&mi->pq); + for (i = 0; i < mi->stack_len; i++) { + reftable_iterator_destroy(&mi->stack[i]); + } + reftable_free(mi->stack); +} + +static int merged_iter_advance_nonnull_subiter(struct merged_iter *mi, + size_t idx) +{ + struct reftable_record rec = reftable_new_record(mi->typ); + struct pq_entry e = { + .rec = rec, + .index = idx, + }; + int err = iterator_next(&mi->stack[idx], &rec); + if (err < 0) + return err; + + if (err > 0) { + reftable_iterator_destroy(&mi->stack[idx]); + reftable_record_destroy(&rec); + return 0; + } + + merged_iter_pqueue_add(&mi->pq, e); + return 0; +} + +static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx) +{ + if (iterator_is_null(&mi->stack[idx])) + return 0; + return merged_iter_advance_nonnull_subiter(mi, idx); +} + +static int merged_iter_next_entry(struct merged_iter *mi, + struct reftable_record *rec) +{ + struct strbuf entry_key = STRBUF_INIT; + struct pq_entry entry = { 0 }; + int err = 0; + + if (merged_iter_pqueue_is_empty(mi->pq)) + return 1; + + entry = merged_iter_pqueue_remove(&mi->pq); + err = merged_iter_advance_subiter(mi, entry.index); + if (err < 0) + return err; + + /* + One can also use reftable as datacenter-local storage, where the ref + database is maintained in globally consistent database (eg. + CockroachDB or Spanner). In this scenario, replication delays together + with compaction may cause newer tables to contain older entries. In + such a deployment, the loop below must be changed to collect all + entries for the same key, and return new the newest one. + */ + reftable_record_key(&entry.rec, &entry_key); + while (!merged_iter_pqueue_is_empty(mi->pq)) { + struct pq_entry top = merged_iter_pqueue_top(mi->pq); + struct strbuf k = STRBUF_INIT; + int err = 0, cmp = 0; + + reftable_record_key(&top.rec, &k); + + cmp = strbuf_cmp(&k, &entry_key); + strbuf_release(&k); + + if (cmp > 0) { + break; + } + + merged_iter_pqueue_remove(&mi->pq); + err = merged_iter_advance_subiter(mi, top.index); + if (err < 0) { + return err; + } + reftable_record_destroy(&top.rec); + } + + reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id)); + reftable_record_destroy(&entry.rec); + strbuf_release(&entry_key); + return 0; +} + +static int merged_iter_next(struct merged_iter *mi, struct reftable_record *rec) +{ + while (1) { + int err = merged_iter_next_entry(mi, rec); + if (err == 0 && mi->suppress_deletions && + reftable_record_is_deletion(rec)) { + continue; + } + + return err; + } +} + +static int merged_iter_next_void(void *p, struct reftable_record *rec) +{ + struct merged_iter *mi = p; + if (merged_iter_pqueue_is_empty(mi->pq)) + return 1; + + return merged_iter_next(mi, rec); +} + +static struct reftable_iterator_vtable merged_iter_vtable = { + .next = &merged_iter_next_void, + .close = &merged_iter_close, +}; + +static void iterator_from_merged_iter(struct reftable_iterator *it, + struct merged_iter *mi) +{ + assert(!it->ops); + it->iter_arg = mi; + it->ops = &merged_iter_vtable; +} + +int reftable_new_merged_table(struct reftable_merged_table **dest, + struct reftable_table *stack, int n, + uint32_t hash_id) +{ + struct reftable_merged_table *m = NULL; + uint64_t last_max = 0; + uint64_t first_min = 0; + int i = 0; + for (i = 0; i < n; i++) { + uint64_t min = reftable_table_min_update_index(&stack[i]); + uint64_t max = reftable_table_max_update_index(&stack[i]); + + if (reftable_table_hash_id(&stack[i]) != hash_id) { + return REFTABLE_FORMAT_ERROR; + } + if (i == 0 || min < first_min) { + first_min = min; + } + if (i == 0 || max > last_max) { + last_max = max; + } + } + + m = reftable_calloc(sizeof(struct reftable_merged_table)); + m->stack = stack; + m->stack_len = n; + m->min = first_min; + m->max = last_max; + m->hash_id = hash_id; + *dest = m; + return 0; +} + +/* clears the list of subtable, without affecting the readers themselves. */ +void merged_table_release(struct reftable_merged_table *mt) +{ + FREE_AND_NULL(mt->stack); + mt->stack_len = 0; +} + +void reftable_merged_table_free(struct reftable_merged_table *mt) +{ + if (!mt) { + return; + } + merged_table_release(mt); + reftable_free(mt); +} + +uint64_t +reftable_merged_table_max_update_index(struct reftable_merged_table *mt) +{ + return mt->max; +} + +uint64_t +reftable_merged_table_min_update_index(struct reftable_merged_table *mt) +{ + return mt->min; +} + +static int reftable_table_seek_record(struct reftable_table *tab, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + return tab->ops->seek_record(tab->table_arg, it, rec); +} + +static int merged_table_seek_record(struct reftable_merged_table *mt, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_iterator *iters = reftable_calloc( + sizeof(struct reftable_iterator) * mt->stack_len); + struct merged_iter merged = { + .stack = iters, + .typ = reftable_record_type(rec), + .hash_id = mt->hash_id, + .suppress_deletions = mt->suppress_deletions, + }; + int n = 0; + int err = 0; + int i = 0; + for (i = 0; i < mt->stack_len && err == 0; i++) { + int e = reftable_table_seek_record(&mt->stack[i], &iters[n], + rec); + if (e < 0) { + err = e; + } + if (e == 0) { + n++; + } + } + if (err < 0) { + int i = 0; + for (i = 0; i < n; i++) { + reftable_iterator_destroy(&iters[i]); + } + reftable_free(iters); + return err; + } + + merged.stack_len = n; + err = merged_iter_init(&merged); + if (err < 0) { + merged_iter_close(&merged); + return err; + } else { + struct merged_iter *p = + reftable_malloc(sizeof(struct merged_iter)); + *p = merged; + iterator_from_merged_iter(it, p); + } + return 0; +} + +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return merged_table_seek_record(mt, it, &rec); +} + +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name, uint64_t update_index) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = update_index, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return merged_table_seek_record(mt, it, &rec); +} + +int reftable_merged_table_seek_log(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name) +{ + uint64_t max = ~((uint64_t)0); + return reftable_merged_table_seek_log_at(mt, it, name, max); +} + +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt) +{ + return mt->hash_id; +} + +static int reftable_merged_table_seek_void(void *tab, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + return merged_table_seek_record(tab, it, rec); +} + +static uint32_t reftable_merged_table_hash_id_void(void *tab) +{ + return reftable_merged_table_hash_id(tab); +} + +static uint64_t reftable_merged_table_min_update_index_void(void *tab) +{ + return reftable_merged_table_min_update_index(tab); +} + +static uint64_t reftable_merged_table_max_update_index_void(void *tab) +{ + return reftable_merged_table_max_update_index(tab); +} + +static struct reftable_table_vtable merged_table_vtable = { + .seek_record = reftable_merged_table_seek_void, + .hash_id = reftable_merged_table_hash_id_void, + .min_update_index = reftable_merged_table_min_update_index_void, + .max_update_index = reftable_merged_table_max_update_index_void, +}; + +void reftable_table_from_merged_table(struct reftable_table *tab, + struct reftable_merged_table *merged) +{ + assert(!tab->ops); + tab->ops = &merged_table_vtable; + tab->table_arg = merged; +} diff --git a/reftable/merged.h b/reftable/merged.h new file mode 100644 index 0000000000..7d9f95d27e --- /dev/null +++ b/reftable/merged.h @@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { + struct reftable_table *stack; + size_t stack_len; + uint32_t hash_id; + + /* If unset, produce deletions. This is useful for compaction. For the + * full stack, deletions should be produced. */ + int suppress_deletions; + + uint64_t min; + uint64_t max; +}; + +struct merged_iter { + struct reftable_iterator *stack; + uint32_t hash_id; + size_t stack_len; + uint8_t typ; + int suppress_deletions; + struct merged_iter_pqueue pq; +}; + +void merged_table_release(struct reftable_merged_table *mt); + +#endif diff --git a/reftable/merged_test.c b/reftable/merged_test.c new file mode 100644 index 0000000000..24461e8a80 --- /dev/null +++ b/reftable/merged_test.c @@ -0,0 +1,468 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "merged.h" + +#include "system.h" + +#include "basics.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-merged.h" +#include "reftable-tests.h" +#include "reftable-generic.h" +#include "reftable-writer.h" + +static void write_test_table(struct strbuf *buf, + struct reftable_ref_record refs[], int n) +{ + int min = 0xffffffff; + int max = 0; + int i = 0; + int err; + + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_writer *w = NULL; + for (i = 0; i < n; i++) { + uint64_t ui = refs[i].update_index; + if (ui > max) { + max = ui; + } + if (ui < min) { + min = ui; + } + } + + w = reftable_new_writer(&strbuf_add_void, buf, &opts); + reftable_writer_set_limits(w, min, max); + + for (i = 0; i < n; i++) { + uint64_t before = refs[i].update_index; + int n = reftable_writer_add_ref(w, &refs[i]); + EXPECT(n == 0); + EXPECT(before == refs[i].update_index); + } + + err = reftable_writer_close(w); + EXPECT_ERR(err); + + reftable_writer_free(w); +} + +static void write_test_log_table(struct strbuf *buf, + struct reftable_log_record logs[], int n, + uint64_t update_index) +{ + int i = 0; + int err; + + struct reftable_write_options opts = { + .block_size = 256, + .exact_log_message = 1, + }; + struct reftable_writer *w = NULL; + w = reftable_new_writer(&strbuf_add_void, buf, &opts); + reftable_writer_set_limits(w, update_index, update_index); + + for (i = 0; i < n; i++) { + int err = reftable_writer_add_log(w, &logs[i]); + EXPECT_ERR(err); + } + + err = reftable_writer_close(w); + EXPECT_ERR(err); + + reftable_writer_free(w); +} + +static struct reftable_merged_table * +merged_table_from_records(struct reftable_ref_record **refs, + struct reftable_block_source **source, + struct reftable_reader ***readers, int *sizes, + struct strbuf *buf, int n) +{ + int i = 0; + struct reftable_merged_table *mt = NULL; + int err; + struct reftable_table *tabs = + reftable_calloc(n * sizeof(struct reftable_table)); + *readers = reftable_calloc(n * sizeof(struct reftable_reader *)); + *source = reftable_calloc(n * sizeof(**source)); + for (i = 0; i < n; i++) { + write_test_table(&buf[i], refs[i], sizes[i]); + block_source_from_strbuf(&(*source)[i], &buf[i]); + + err = reftable_new_reader(&(*readers)[i], &(*source)[i], + "name"); + EXPECT_ERR(err); + reftable_table_from_reader(&tabs[i], (*readers)[i]); + } + + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + return mt; +} + +static void readers_destroy(struct reftable_reader **readers, size_t n) +{ + int i = 0; + for (; i < n; i++) + reftable_reader_free(readers[i]); + reftable_free(readers); +} + +static void test_merged_between(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 0 }; + + struct reftable_ref_record r1[] = { { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + } }; + struct reftable_ref_record r2[] = { { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_REF_DELETION, + } }; + + struct reftable_ref_record *refs[] = { r1, r2 }; + int sizes[] = { 1, 1 }; + struct strbuf bufs[2] = { STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 2); + int i; + struct reftable_ref_record ref = { NULL }; + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_ref(mt, &it, "a"); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + EXPECT(ref.update_index == 2); + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + readers_destroy(readers, 2); + reftable_merged_table_free(mt); + for (i = 0; i < ARRAY_SIZE(bufs); i++) { + strbuf_release(&bufs[i]); + } + reftable_free(bs); +} + +static void test_merged(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; + struct reftable_ref_record r1[] = { + { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + { + .refname = "c", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + } + }; + struct reftable_ref_record r2[] = { { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_REF_DELETION, + } }; + struct reftable_ref_record r3[] = { + { + .refname = "c", + .update_index = 3, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash2, + }, + { + .refname = "d", + .update_index = 3, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + }; + + struct reftable_ref_record want[] = { + r2[0], + r1[1], + r3[0], + r3[1], + }; + + struct reftable_ref_record *refs[] = { r1, r2, r3 }; + int sizes[3] = { 3, 1, 2 }; + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); + + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_ref(mt, &it, "a"); + struct reftable_ref_record *out = NULL; + size_t len = 0; + size_t cap = 0; + int i = 0; + + EXPECT_ERR(err); + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); + EXPECT(reftable_merged_table_min_update_index(mt) == 1); + + while (len < 100) { /* cap loops/recursion. */ + struct reftable_ref_record ref = { NULL }; + int err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (len == cap) { + cap = 2 * cap + 1; + out = reftable_realloc( + out, sizeof(struct reftable_ref_record) * cap); + } + out[len++] = ref; + } + reftable_iterator_destroy(&it); + + EXPECT(ARRAY_SIZE(want) == len); + for (i = 0; i < len; i++) { + EXPECT(reftable_ref_record_equal(&want[i], &out[i], + GIT_SHA1_RAWSZ)); + } + for (i = 0; i < len; i++) { + reftable_ref_record_release(&out[i]); + } + reftable_free(out); + + for (i = 0; i < 3; i++) { + strbuf_release(&bufs[i]); + } + readers_destroy(readers, 3); + reftable_merged_table_free(mt); + reftable_free(bs); +} + +static struct reftable_merged_table * +merged_table_from_log_records(struct reftable_log_record **logs, + struct reftable_block_source **source, + struct reftable_reader ***readers, int *sizes, + struct strbuf *buf, int n) +{ + int i = 0; + struct reftable_merged_table *mt = NULL; + int err; + struct reftable_table *tabs = + reftable_calloc(n * sizeof(struct reftable_table)); + *readers = reftable_calloc(n * sizeof(struct reftable_reader *)); + *source = reftable_calloc(n * sizeof(**source)); + for (i = 0; i < n; i++) { + write_test_log_table(&buf[i], logs[i], sizes[i], i + 1); + block_source_from_strbuf(&(*source)[i], &buf[i]); + + err = reftable_new_reader(&(*readers)[i], &(*source)[i], + "name"); + EXPECT_ERR(err); + reftable_table_from_reader(&tabs[i], (*readers)[i]); + } + + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + return mt; +} + +static void test_merged_logs(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; + uint8_t hash3[GIT_SHA1_RAWSZ] = { 3 }; + struct reftable_log_record r1[] = { + { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash2, + /* deletion */ + .name = "jane doe", + .email = "jane@invalid", + .message = "message2", + } + }, + { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash1, + .new_hash = hash2, + .name = "jane doe", + .email = "jane@invalid", + .message = "message1", + } + }, + }; + struct reftable_log_record r2[] = { + { + .refname = "a", + .update_index = 3, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .new_hash = hash3, + .name = "jane doe", + .email = "jane@invalid", + .message = "message3", + } + }, + }; + struct reftable_log_record r3[] = { + { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_LOG_DELETION, + }, + }; + struct reftable_log_record want[] = { + r2[0], + r3[0], + r1[1], + }; + + struct reftable_log_record *logs[] = { r1, r2, r3 }; + int sizes[3] = { 2, 1, 1 }; + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = merged_table_from_log_records( + logs, &bs, &readers, sizes, bufs, 3); + + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_log(mt, &it, "a"); + struct reftable_log_record *out = NULL; + size_t len = 0; + size_t cap = 0; + int i = 0; + + EXPECT_ERR(err); + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); + EXPECT(reftable_merged_table_min_update_index(mt) == 1); + + while (len < 100) { /* cap loops/recursion. */ + struct reftable_log_record log = { NULL }; + int err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + if (len == cap) { + cap = 2 * cap + 1; + out = reftable_realloc( + out, sizeof(struct reftable_log_record) * cap); + } + out[len++] = log; + } + reftable_iterator_destroy(&it); + + EXPECT(ARRAY_SIZE(want) == len); + for (i = 0; i < len; i++) { + EXPECT(reftable_log_record_equal(&want[i], &out[i], + GIT_SHA1_RAWSZ)); + } + + err = reftable_merged_table_seek_log_at(mt, &it, "a", 2); + EXPECT_ERR(err); + reftable_log_record_release(&out[0]); + err = reftable_iterator_next_log(&it, &out[0]); + EXPECT_ERR(err); + EXPECT(reftable_log_record_equal(&out[0], &r3[0], GIT_SHA1_RAWSZ)); + reftable_iterator_destroy(&it); + + for (i = 0; i < len; i++) { + reftable_log_record_release(&out[i]); + } + reftable_free(out); + + for (i = 0; i < 3; i++) { + strbuf_release(&bufs[i]); + } + readers_destroy(readers, 3); + reftable_merged_table_free(mt); + reftable_free(bs); +} + +static void test_default_write_opts(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + struct reftable_ref_record rec = { + .refname = "master", + .update_index = 1, + }; + int err; + struct reftable_block_source source = { NULL }; + struct reftable_table *tab = reftable_calloc(sizeof(*tab) * 1); + uint32_t hash_id; + struct reftable_reader *rd = NULL; + struct reftable_merged_table *merged = NULL; + + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_add_ref(w, &rec); + EXPECT_ERR(err); + + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + + block_source_from_strbuf(&source, &buf); + + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + hash_id = reftable_reader_hash_id(rd); + EXPECT(hash_id == GIT_SHA1_FORMAT_ID); + + reftable_table_from_reader(&tab[0], rd); + err = reftable_new_merged_table(&merged, tab, 1, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + + reftable_reader_free(rd); + reftable_merged_table_free(merged); + strbuf_release(&buf); +} + +/* XXX test refs_for(oid) */ + +int merged_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_merged_logs); + RUN_TEST(test_merged_between); + RUN_TEST(test_merged); + RUN_TEST(test_default_write_opts); + return 0; +} diff --git a/reftable/pq.c b/reftable/pq.c new file mode 100644 index 0000000000..efc474017a --- /dev/null +++ b/reftable/pq.c @@ -0,0 +1,105 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "pq.h" + +#include "reftable-record.h" +#include "system.h" +#include "basics.h" + +int pq_less(struct pq_entry *a, struct pq_entry *b) +{ + struct strbuf ak = STRBUF_INIT; + struct strbuf bk = STRBUF_INIT; + int cmp = 0; + reftable_record_key(&a->rec, &ak); + reftable_record_key(&b->rec, &bk); + + cmp = strbuf_cmp(&ak, &bk); + + strbuf_release(&ak); + strbuf_release(&bk); + + if (cmp == 0) + return a->index > b->index; + + return cmp < 0; +} + +struct pq_entry merged_iter_pqueue_top(struct merged_iter_pqueue pq) +{ + return pq.heap[0]; +} + +int merged_iter_pqueue_is_empty(struct merged_iter_pqueue pq) +{ + return pq.len == 0; +} + +struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq) +{ + int i = 0; + struct pq_entry e = pq->heap[0]; + pq->heap[0] = pq->heap[pq->len - 1]; + pq->len--; + + i = 0; + while (i < pq->len) { + int min = i; + int j = 2 * i + 1; + int k = 2 * i + 2; + if (j < pq->len && pq_less(&pq->heap[j], &pq->heap[i])) { + min = j; + } + if (k < pq->len && pq_less(&pq->heap[k], &pq->heap[min])) { + min = k; + } + + if (min == i) { + break; + } + + SWAP(pq->heap[i], pq->heap[min]); + i = min; + } + + return e; +} + +void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, struct pq_entry e) +{ + int i = 0; + if (pq->len == pq->cap) { + pq->cap = 2 * pq->cap + 1; + pq->heap = reftable_realloc(pq->heap, + pq->cap * sizeof(struct pq_entry)); + } + + pq->heap[pq->len++] = e; + i = pq->len - 1; + while (i > 0) { + int j = (i - 1) / 2; + if (pq_less(&pq->heap[j], &pq->heap[i])) { + break; + } + + SWAP(pq->heap[j], pq->heap[i]); + + i = j; + } +} + +void merged_iter_pqueue_release(struct merged_iter_pqueue *pq) +{ + int i = 0; + for (i = 0; i < pq->len; i++) { + reftable_record_destroy(&pq->heap[i].rec); + } + FREE_AND_NULL(pq->heap); + pq->len = pq->cap = 0; +} diff --git a/reftable/pq.h b/reftable/pq.h new file mode 100644 index 0000000000..56fc1b6d87 --- /dev/null +++ b/reftable/pq.h @@ -0,0 +1,33 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { + int index; + struct reftable_record rec; +}; + +struct merged_iter_pqueue { + struct pq_entry *heap; + size_t len; + size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top(struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty(struct merged_iter_pqueue pq); +void merged_iter_pqueue_check(struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, struct pq_entry e); +void merged_iter_pqueue_release(struct merged_iter_pqueue *pq); +int pq_less(struct pq_entry *a, struct pq_entry *b); + +#endif diff --git a/reftable/pq_test.c b/reftable/pq_test.c new file mode 100644 index 0000000000..c9bb05e37b --- /dev/null +++ b/reftable/pq_test.c @@ -0,0 +1,82 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "constants.h" +#include "pq.h" +#include "record.h" +#include "reftable-tests.h" +#include "test_framework.h" + +void merged_iter_pqueue_check(struct merged_iter_pqueue pq) +{ + int i; + for (i = 1; i < pq.len; i++) { + int parent = (i - 1) / 2; + + EXPECT(pq_less(&pq.heap[parent], &pq.heap[i])); + } +} + +static void test_pq(void) +{ + char *names[54] = { NULL }; + int N = ARRAY_SIZE(names) - 1; + + struct merged_iter_pqueue pq = { NULL }; + const char *last = NULL; + + int i = 0; + for (i = 0; i < N; i++) { + char name[100]; + snprintf(name, sizeof(name), "%02d", i); + names[i] = xstrdup(name); + } + + i = 1; + do { + struct reftable_record rec = + reftable_new_record(BLOCK_TYPE_REF); + struct pq_entry e = { 0 }; + + reftable_record_as_ref(&rec)->refname = names[i]; + e.rec = rec; + merged_iter_pqueue_add(&pq, e); + merged_iter_pqueue_check(pq); + i = (i * 7) % N; + } while (i != 1); + + while (!merged_iter_pqueue_is_empty(pq)) { + struct pq_entry e = merged_iter_pqueue_remove(&pq); + struct reftable_ref_record *ref = + reftable_record_as_ref(&e.rec); + + merged_iter_pqueue_check(pq); + + if (last) { + EXPECT(strcmp(last, ref->refname) < 0); + } + last = ref->refname; + ref->refname = NULL; + reftable_free(ref); + } + + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + + merged_iter_pqueue_release(&pq); +} + +int pq_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_pq); + return 0; +} diff --git a/reftable/publicbasics.c b/reftable/publicbasics.c new file mode 100644 index 0000000000..0ad7d5c0ff --- /dev/null +++ b/reftable/publicbasics.c @@ -0,0 +1,65 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-malloc.h" + +#include "basics.h" +#include "system.h" + +static void *(*reftable_malloc_ptr)(size_t sz); +static void *(*reftable_realloc_ptr)(void *, size_t); +static void (*reftable_free_ptr)(void *); + +void *reftable_malloc(size_t sz) +{ + if (reftable_malloc_ptr) + return (*reftable_malloc_ptr)(sz); + return malloc(sz); +} + +void *reftable_realloc(void *p, size_t sz) +{ + if (reftable_realloc_ptr) + return (*reftable_realloc_ptr)(p, sz); + return realloc(p, sz); +} + +void reftable_free(void *p) +{ + if (reftable_free_ptr) + reftable_free_ptr(p); + else + free(p); +} + +void *reftable_calloc(size_t sz) +{ + void *p = reftable_malloc(sz); + memset(p, 0, sz); + return p; +} + +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)) +{ + reftable_malloc_ptr = malloc; + reftable_realloc_ptr = realloc; + reftable_free_ptr = free; +} + +int hash_size(uint32_t id) +{ + switch (id) { + case 0: + case GIT_SHA1_FORMAT_ID: + return GIT_SHA1_RAWSZ; + case GIT_SHA256_FORMAT_ID: + return GIT_SHA256_RAWSZ; + } + abort(); +} diff --git a/reftable/reader.c b/reftable/reader.c new file mode 100644 index 0000000000..006709a645 --- /dev/null +++ b/reftable/reader.c @@ -0,0 +1,801 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reader.h" + +#include "system.h" +#include "block.h" +#include "constants.h" +#include "generic.h" +#include "iter.h" +#include "record.h" +#include "reftable-error.h" +#include "reftable-generic.h" +#include "tree.h" + +uint64_t block_source_size(struct reftable_block_source *source) +{ + return source->ops->size(source->arg); +} + +int block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + int result = source->ops->read_block(source->arg, dest, off, size); + dest->source = *source; + return result; +} + +void block_source_close(struct reftable_block_source *source) +{ + if (!source->ops) { + return; + } + + source->ops->close(source->arg); + source->ops = NULL; +} + +static struct reftable_reader_offsets * +reader_offsets_for(struct reftable_reader *r, uint8_t typ) +{ + switch (typ) { + case BLOCK_TYPE_REF: + return &r->ref_offsets; + case BLOCK_TYPE_LOG: + return &r->log_offsets; + case BLOCK_TYPE_OBJ: + return &r->obj_offsets; + } + abort(); +} + +static int reader_get_block(struct reftable_reader *r, + struct reftable_block *dest, uint64_t off, + uint32_t sz) +{ + if (off >= r->size) + return 0; + + if (off + sz > r->size) { + sz = r->size - off; + } + + return block_source_read_block(&r->source, dest, off, sz); +} + +uint32_t reftable_reader_hash_id(struct reftable_reader *r) +{ + return r->hash_id; +} + +const char *reader_name(struct reftable_reader *r) +{ + return r->name; +} + +static int parse_footer(struct reftable_reader *r, uint8_t *footer, + uint8_t *header) +{ + uint8_t *f = footer; + uint8_t first_block_typ; + int err = 0; + uint32_t computed_crc; + uint32_t file_crc; + + if (memcmp(f, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + f += 4; + + if (memcmp(footer, header, header_size(r->version))) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + f++; + r->block_size = get_be24(f); + + f += 3; + r->min_update_index = get_be64(f); + f += 8; + r->max_update_index = get_be64(f); + f += 8; + + if (r->version == 1) { + r->hash_id = GIT_SHA1_FORMAT_ID; + } else { + r->hash_id = get_be32(f); + switch (r->hash_id) { + case GIT_SHA1_FORMAT_ID: + break; + case GIT_SHA256_FORMAT_ID: + break; + default: + err = REFTABLE_FORMAT_ERROR; + goto done; + } + f += 4; + } + + r->ref_offsets.index_offset = get_be64(f); + f += 8; + + r->obj_offsets.offset = get_be64(f); + f += 8; + + r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1); + r->obj_offsets.offset >>= 5; + + r->obj_offsets.index_offset = get_be64(f); + f += 8; + r->log_offsets.offset = get_be64(f); + f += 8; + r->log_offsets.index_offset = get_be64(f); + f += 8; + + computed_crc = crc32(0, footer, f - footer); + file_crc = get_be32(f); + f += 4; + if (computed_crc != file_crc) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + first_block_typ = header[header_size(r->version)]; + r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); + r->ref_offsets.offset = 0; + r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || + r->log_offsets.offset > 0); + r->obj_offsets.is_present = r->obj_offsets.offset > 0; + err = 0; +done: + return err; +} + +int init_reader(struct reftable_reader *r, struct reftable_block_source *source, + const char *name) +{ + struct reftable_block footer = { NULL }; + struct reftable_block header = { NULL }; + int err = 0; + uint64_t file_size = block_source_size(source); + + /* Need +1 to read type of first block. */ + uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */ + memset(r, 0, sizeof(struct reftable_reader)); + + if (read_size > file_size) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + err = block_source_read_block(source, &header, 0, read_size); + if (err != read_size) { + err = REFTABLE_IO_ERROR; + goto done; + } + + if (memcmp(header.data, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + r->version = header.data[4]; + if (r->version != 1 && r->version != 2) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + r->size = file_size - footer_size(r->version); + r->source = *source; + r->name = xstrdup(name); + r->hash_id = 0; + + err = block_source_read_block(source, &footer, r->size, + footer_size(r->version)); + if (err != footer_size(r->version)) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = parse_footer(r, footer.data, header.data); +done: + reftable_block_done(&footer); + reftable_block_done(&header); + return err; +} + +struct table_iter { + struct reftable_reader *r; + uint8_t typ; + uint64_t block_off; + struct block_iter bi; + int is_finished; +}; +#define TABLE_ITER_INIT \ + { \ + .bi = {.last_key = STRBUF_INIT } \ + } + +static void table_iter_copy_from(struct table_iter *dest, + struct table_iter *src) +{ + dest->r = src->r; + dest->typ = src->typ; + dest->block_off = src->block_off; + dest->is_finished = src->is_finished; + block_iter_copy_from(&dest->bi, &src->bi); +} + +static int table_iter_next_in_block(struct table_iter *ti, + struct reftable_record *rec) +{ + int res = block_iter_next(&ti->bi, rec); + if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { + ((struct reftable_ref_record *)rec->data)->update_index += + ti->r->min_update_index; + } + + return res; +} + +static void table_iter_block_done(struct table_iter *ti) +{ + if (!ti->bi.br) { + return; + } + reftable_block_done(&ti->bi.br->block); + FREE_AND_NULL(ti->bi.br); + + ti->bi.last_key.len = 0; + ti->bi.next_off = 0; +} + +static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off, + int version) +{ + int32_t result = 0; + + if (off == 0) { + data += header_size(version); + } + + *typ = data[0]; + if (reftable_is_block_type(*typ)) { + result = get_be24(data + 1); + } + return result; +} + +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, + uint64_t next_off, uint8_t want_typ) +{ + int32_t guess_block_size = r->block_size ? r->block_size : + DEFAULT_BLOCK_SIZE; + struct reftable_block block = { NULL }; + uint8_t block_typ = 0; + int err = 0; + uint32_t header_off = next_off ? 0 : header_size(r->version); + int32_t block_size = 0; + + if (next_off >= r->size) + return 1; + + err = reader_get_block(r, &block, next_off, guess_block_size); + if (err < 0) + return err; + + block_size = extract_block_size(block.data, &block_typ, next_off, + r->version); + if (block_size < 0) + return block_size; + + if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) { + reftable_block_done(&block); + return 1; + } + + if (block_size > guess_block_size) { + reftable_block_done(&block); + err = reader_get_block(r, &block, next_off, block_size); + if (err < 0) { + return err; + } + } + + return block_reader_init(br, &block, header_off, r->block_size, + hash_size(r->hash_id)); +} + +static int table_iter_next_block(struct table_iter *dest, + struct table_iter *src) +{ + uint64_t next_block_off = src->block_off + src->bi.br->full_block_size; + struct block_reader br = { 0 }; + int err = 0; + + dest->r = src->r; + dest->typ = src->typ; + dest->block_off = next_block_off; + + err = reader_init_block_reader(src->r, &br, next_block_off, src->typ); + if (err > 0) { + dest->is_finished = 1; + return 1; + } + if (err != 0) + return err; + else { + struct block_reader *brp = + reftable_malloc(sizeof(struct block_reader)); + *brp = br; + + dest->is_finished = 0; + block_reader_start(brp, &dest->bi); + } + return 0; +} + +static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) +{ + if (reftable_record_type(rec) != ti->typ) + return REFTABLE_API_ERROR; + + while (1) { + struct table_iter next = TABLE_ITER_INIT; + int err = 0; + if (ti->is_finished) { + return 1; + } + + err = table_iter_next_in_block(ti, rec); + if (err <= 0) { + return err; + } + + err = table_iter_next_block(&next, ti); + if (err != 0) { + ti->is_finished = 1; + } + table_iter_block_done(ti); + if (err != 0) { + return err; + } + table_iter_copy_from(ti, &next); + block_iter_close(&next.bi); + } +} + +static int table_iter_next_void(void *ti, struct reftable_record *rec) +{ + return table_iter_next(ti, rec); +} + +static void table_iter_close(void *p) +{ + struct table_iter *ti = p; + table_iter_block_done(ti); + block_iter_close(&ti->bi); +} + +static struct reftable_iterator_vtable table_iter_vtable = { + .next = &table_iter_next_void, + .close = &table_iter_close, +}; + +static void iterator_from_table_iter(struct reftable_iterator *it, + struct table_iter *ti) +{ + assert(!it->ops); + it->iter_arg = ti; + it->ops = &table_iter_vtable; +} + +static int reader_table_iter_at(struct reftable_reader *r, + struct table_iter *ti, uint64_t off, + uint8_t typ) +{ + struct block_reader br = { 0 }; + struct block_reader *brp = NULL; + + int err = reader_init_block_reader(r, &br, off, typ); + if (err != 0) + return err; + + brp = reftable_malloc(sizeof(struct block_reader)); + *brp = br; + ti->r = r; + ti->typ = block_reader_type(brp); + ti->block_off = off; + block_reader_start(brp, &ti->bi); + return 0; +} + +static int reader_start(struct reftable_reader *r, struct table_iter *ti, + uint8_t typ, int index) +{ + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); + uint64_t off = offs->offset; + if (index) { + off = offs->index_offset; + if (off == 0) { + return 1; + } + typ = BLOCK_TYPE_INDEX; + } + + return reader_table_iter_at(r, ti, off, typ); +} + +static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti, + struct reftable_record *want) +{ + struct reftable_record rec = + reftable_new_record(reftable_record_type(want)); + struct strbuf want_key = STRBUF_INIT; + struct strbuf got_key = STRBUF_INIT; + struct table_iter next = TABLE_ITER_INIT; + int err = -1; + + reftable_record_key(want, &want_key); + + while (1) { + err = table_iter_next_block(&next, ti); + if (err < 0) + goto done; + + if (err > 0) { + break; + } + + err = block_reader_first_key(next.bi.br, &got_key); + if (err < 0) + goto done; + + if (strbuf_cmp(&got_key, &want_key) > 0) { + table_iter_block_done(&next); + break; + } + + table_iter_block_done(ti); + table_iter_copy_from(ti, &next); + } + + err = block_iter_seek(&ti->bi, &want_key); + if (err < 0) + goto done; + err = 0; + +done: + block_iter_close(&next.bi); + reftable_record_destroy(&rec); + strbuf_release(&want_key); + strbuf_release(&got_key); + return err; +} + +static int reader_seek_indexed(struct reftable_reader *r, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_index_record want_index = { .last_key = STRBUF_INIT }; + struct reftable_record want_index_rec = { NULL }; + struct reftable_index_record index_result = { .last_key = STRBUF_INIT }; + struct reftable_record index_result_rec = { NULL }; + struct table_iter index_iter = TABLE_ITER_INIT; + struct table_iter next = TABLE_ITER_INIT; + int err = 0; + + reftable_record_key(rec, &want_index.last_key); + reftable_record_from_index(&want_index_rec, &want_index); + reftable_record_from_index(&index_result_rec, &index_result); + + err = reader_start(r, &index_iter, reftable_record_type(rec), 1); + if (err < 0) + goto done; + + err = reader_seek_linear(r, &index_iter, &want_index_rec); + while (1) { + err = table_iter_next(&index_iter, &index_result_rec); + table_iter_block_done(&index_iter); + if (err != 0) + goto done; + + err = reader_table_iter_at(r, &next, index_result.offset, 0); + if (err != 0) + goto done; + + err = block_iter_seek(&next.bi, &want_index.last_key); + if (err < 0) + goto done; + + if (next.typ == reftable_record_type(rec)) { + err = 0; + break; + } + + if (next.typ != BLOCK_TYPE_INDEX) { + err = REFTABLE_FORMAT_ERROR; + break; + } + + table_iter_copy_from(&index_iter, &next); + } + + if (err == 0) { + struct table_iter empty = TABLE_ITER_INIT; + struct table_iter *malloced = + reftable_calloc(sizeof(struct table_iter)); + *malloced = empty; + table_iter_copy_from(malloced, &next); + iterator_from_table_iter(it, malloced); + } +done: + block_iter_close(&next.bi); + table_iter_close(&index_iter); + reftable_record_release(&want_index_rec); + reftable_record_release(&index_result_rec); + return err; +} + +static int reader_seek_internal(struct reftable_reader *r, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_reader_offsets *offs = + reader_offsets_for(r, reftable_record_type(rec)); + uint64_t idx = offs->index_offset; + struct table_iter ti = TABLE_ITER_INIT; + int err = 0; + if (idx > 0) + return reader_seek_indexed(r, it, rec); + + err = reader_start(r, &ti, reftable_record_type(rec), 0); + if (err < 0) + return err; + err = reader_seek_linear(r, &ti, rec); + if (err < 0) + return err; + else { + struct table_iter *p = + reftable_malloc(sizeof(struct table_iter)); + *p = ti; + iterator_from_table_iter(it, p); + } + + return 0; +} + +static int reader_seek(struct reftable_reader *r, struct reftable_iterator *it, + struct reftable_record *rec) +{ + uint8_t typ = reftable_record_type(rec); + + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); + if (!offs->is_present) { + iterator_set_empty(it); + return 0; + } + + return reader_seek_internal(r, it, rec); +} + +int reftable_reader_seek_ref(struct reftable_reader *r, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return reader_seek(r, it, &rec); +} + +int reftable_reader_seek_log_at(struct reftable_reader *r, + struct reftable_iterator *it, const char *name, + uint64_t update_index) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = update_index, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return reader_seek(r, it, &rec); +} + +int reftable_reader_seek_log(struct reftable_reader *r, + struct reftable_iterator *it, const char *name) +{ + uint64_t max = ~((uint64_t)0); + return reftable_reader_seek_log_at(r, it, name, max); +} + +void reader_close(struct reftable_reader *r) +{ + block_source_close(&r->source); + FREE_AND_NULL(r->name); +} + +int reftable_new_reader(struct reftable_reader **p, + struct reftable_block_source *src, char const *name) +{ + struct reftable_reader *rd = + reftable_calloc(sizeof(struct reftable_reader)); + int err = init_reader(rd, src, name); + if (err == 0) { + *p = rd; + } else { + block_source_close(src); + reftable_free(rd); + } + return err; +} + +void reftable_reader_free(struct reftable_reader *r) +{ + reader_close(r); + reftable_free(r); +} + +static int reftable_reader_refs_for_indexed(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct reftable_obj_record want = { + .hash_prefix = oid, + .hash_prefix_len = r->object_id_len, + }; + struct reftable_record want_rec = { NULL }; + struct reftable_iterator oit = { NULL }; + struct reftable_obj_record got = { NULL }; + struct reftable_record got_rec = { NULL }; + int err = 0; + struct indexed_table_ref_iter *itr = NULL; + + /* Look through the reverse index. */ + reftable_record_from_obj(&want_rec, &want); + err = reader_seek(r, &oit, &want_rec); + if (err != 0) + goto done; + + /* read out the reftable_obj_record */ + reftable_record_from_obj(&got_rec, &got); + err = iterator_next(&oit, &got_rec); + if (err < 0) + goto done; + + if (err > 0 || + memcmp(want.hash_prefix, got.hash_prefix, r->object_id_len)) { + /* didn't find it; return empty iterator */ + iterator_set_empty(it); + err = 0; + goto done; + } + + err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id), + got.offsets, got.offset_len); + if (err < 0) + goto done; + got.offsets = NULL; + iterator_from_indexed_table_ref_iter(it, itr); + +done: + reftable_iterator_destroy(&oit); + reftable_record_release(&got_rec); + return err; +} + +static int reftable_reader_refs_for_unindexed(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct table_iter ti_empty = TABLE_ITER_INIT; + struct table_iter *ti = reftable_calloc(sizeof(struct table_iter)); + struct filtering_ref_iterator *filter = NULL; + struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; + int oid_len = hash_size(r->hash_id); + int err; + + *ti = ti_empty; + err = reader_start(r, ti, BLOCK_TYPE_REF, 0); + if (err < 0) { + reftable_free(ti); + return err; + } + + filter = reftable_malloc(sizeof(struct filtering_ref_iterator)); + *filter = empty; + + strbuf_add(&filter->oid, oid, oid_len); + reftable_table_from_reader(&filter->tab, r); + filter->double_check = 0; + iterator_from_table_iter(&filter->it, ti); + + iterator_from_filtering_ref_iterator(it, filter); + return 0; +} + +int reftable_reader_refs_for(struct reftable_reader *r, + struct reftable_iterator *it, uint8_t *oid) +{ + if (r->obj_offsets.is_present) + return reftable_reader_refs_for_indexed(r, it, oid); + return reftable_reader_refs_for_unindexed(r, it, oid); +} + +uint64_t reftable_reader_max_update_index(struct reftable_reader *r) +{ + return r->max_update_index; +} + +uint64_t reftable_reader_min_update_index(struct reftable_reader *r) +{ + return r->min_update_index; +} + +/* generic table interface. */ + +static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it, + struct reftable_record *rec) +{ + return reader_seek(tab, it, rec); +} + +static uint32_t reftable_reader_hash_id_void(void *tab) +{ + return reftable_reader_hash_id(tab); +} + +static uint64_t reftable_reader_min_update_index_void(void *tab) +{ + return reftable_reader_min_update_index(tab); +} + +static uint64_t reftable_reader_max_update_index_void(void *tab) +{ + return reftable_reader_max_update_index(tab); +} + +static struct reftable_table_vtable reader_vtable = { + .seek_record = reftable_reader_seek_void, + .hash_id = reftable_reader_hash_id_void, + .min_update_index = reftable_reader_min_update_index_void, + .max_update_index = reftable_reader_max_update_index_void, +}; + +void reftable_table_from_reader(struct reftable_table *tab, + struct reftable_reader *reader) +{ + assert(!tab->ops); + tab->ops = &reader_vtable; + tab->table_arg = reader; +} + + +int reftable_reader_print_file(const char *tablename) +{ + struct reftable_block_source src = { NULL }; + int err = reftable_block_source_from_file(&src, tablename); + struct reftable_reader *r = NULL; + struct reftable_table tab = { NULL }; + if (err < 0) + goto done; + + err = reftable_new_reader(&r, &src, tablename); + if (err < 0) + goto done; + + reftable_table_from_reader(&tab, r); + err = reftable_table_print(&tab); +done: + reftable_reader_free(r); + return err; +} diff --git a/reftable/reader.h b/reftable/reader.h new file mode 100644 index 0000000000..e869165f23 --- /dev/null +++ b/reftable/reader.h @@ -0,0 +1,64 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef READER_H +#define READER_H + +#include "block.h" +#include "record.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" + +uint64_t block_source_size(struct reftable_block_source *source); + +int block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size); +void block_source_close(struct reftable_block_source *source); + +/* metadata for a block type */ +struct reftable_reader_offsets { + int is_present; + uint64_t offset; + uint64_t index_offset; +}; + +/* The state for reading a reftable file. */ +struct reftable_reader { + /* for convience, associate a name with the instance. */ + char *name; + struct reftable_block_source source; + + /* Size of the file, excluding the footer. */ + uint64_t size; + + /* 'sha1' for SHA1, 's256' for SHA-256 */ + uint32_t hash_id; + + uint32_t block_size; + uint64_t min_update_index; + uint64_t max_update_index; + /* Length of the OID keys in the 'o' section */ + int object_id_len; + int version; + + struct reftable_reader_offsets ref_offsets; + struct reftable_reader_offsets obj_offsets; + struct reftable_reader_offsets log_offsets; +}; + +int init_reader(struct reftable_reader *r, struct reftable_block_source *source, + const char *name); +void reader_close(struct reftable_reader *r); +const char *reader_name(struct reftable_reader *r); + +/* initialize a block reader to read from `r` */ +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, + uint64_t next_off, uint8_t want_typ); + +#endif diff --git a/reftable/readwrite_test.c b/reftable/readwrite_test.c new file mode 100644 index 0000000000..5f6bcc2f77 --- /dev/null +++ b/reftable/readwrite_test.c @@ -0,0 +1,652 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "block.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" +#include "reftable-writer.h" + +static const int update_index = 5; + +static void test_buffer(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_block out = { NULL }; + int n; + uint8_t in[] = "hello"; + strbuf_add(&buf, in, sizeof(in)); + block_source_from_strbuf(&source, &buf); + EXPECT(block_source_size(&source) == 6); + n = block_source_read_block(&source, &out, 0, sizeof(in)); + EXPECT(n == sizeof(in)); + EXPECT(!memcmp(in, out.data, n)); + reftable_block_done(&out); + + n = block_source_read_block(&source, &out, 1, 2); + EXPECT(n == 2); + EXPECT(!memcmp(out.data, "el", 2)); + + reftable_block_done(&out); + block_source_close(&source); + strbuf_release(&buf); +} + +static void write_table(char ***names, struct strbuf *buf, int N, + int block_size, uint32_t hash_id) +{ + struct reftable_write_options opts = { + .block_size = block_size, + .hash_id = hash_id, + }; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, buf, &opts); + struct reftable_ref_record ref = { NULL }; + int i = 0, n; + struct reftable_log_record log = { NULL }; + const struct reftable_stats *stats = NULL; + *names = reftable_calloc(sizeof(char *) * (N + 1)); + reftable_writer_set_limits(w, update_index, update_index); + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; + char name[100]; + int n; + + set_test_hash(hash, i); + + snprintf(name, sizeof(name), "refs/heads/branch%02d", i); + + ref.refname = name; + ref.update_index = update_index; + ref.value_type = REFTABLE_REF_VAL1; + ref.value.val1 = hash; + (*names)[i] = xstrdup(name); + + n = reftable_writer_add_ref(w, &ref); + EXPECT(n == 0); + } + + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; + char name[100]; + int n; + + set_test_hash(hash, i); + + snprintf(name, sizeof(name), "refs/heads/branch%02d", i); + + log.refname = name; + log.update_index = update_index; + log.value_type = REFTABLE_LOG_UPDATE; + log.value.update.new_hash = hash; + log.value.update.message = "message"; + + n = reftable_writer_add_log(w, &log); + EXPECT(n == 0); + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + stats = writer_stats(w); + for (i = 0; i < stats->ref_stats.blocks; i++) { + int off = i * opts.block_size; + if (off == 0) { + off = header_size( + (hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1); + } + EXPECT(buf->buf[off] == 'r'); + } + + EXPECT(stats->log_stats.blocks > 0); + reftable_writer_free(w); +} + +static void test_log_buffer_size(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_write_options opts = { + .block_size = 4096, + }; + int err; + int i; + struct reftable_log_record + log = { .refname = "refs/heads/master", + .update_index = 0xa, + .value_type = REFTABLE_LOG_UPDATE, + .value = { .update = { + .name = "Han-Wen Nienhuys", + .email = "hanwen@google.com", + .tz_offset = 100, + .time = 0x5e430672, + .message = "commit: 9\n", + } } }; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + /* This tests buffer extension for log compression. Must use a random + hash, to ensure that the compressed part is larger than the original. + */ + uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ]; + for (i = 0; i < GIT_SHA1_RAWSZ; i++) { + hash1[i] = (uint8_t)(rand() % 256); + hash2[i] = (uint8_t)(rand() % 256); + } + log.value.update.old_hash = hash1; + log.value.update.new_hash = hash2; + reftable_writer_set_limits(w, update_index, update_index); + err = reftable_writer_add_log(w, &log); + EXPECT_ERR(err); + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + strbuf_release(&buf); +} + +static void test_log_write_read(void) +{ + int N = 2; + char **names = reftable_calloc(sizeof(char *) * (N + 1)); + int err; + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_ref_record ref = { NULL }; + int i = 0; + struct reftable_log_record log = { NULL }; + int n; + struct reftable_iterator it = { NULL }; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + const struct reftable_stats *stats = NULL; + reftable_writer_set_limits(w, 0, N); + for (i = 0; i < N; i++) { + char name[256]; + struct reftable_ref_record ref = { NULL }; + snprintf(name, sizeof(name), "b%02d%0*d", i, 130, 7); + names[i] = xstrdup(name); + ref.refname = name; + ref.update_index = i; + + err = reftable_writer_add_ref(w, &ref); + EXPECT_ERR(err); + } + for (i = 0; i < N; i++) { + uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ]; + struct reftable_log_record log = { NULL }; + set_test_hash(hash1, i); + set_test_hash(hash2, i + 1); + + log.refname = names[i]; + log.update_index = i; + log.value_type = REFTABLE_LOG_UPDATE; + log.value.update.old_hash = hash1; + log.value.update.new_hash = hash2; + + err = reftable_writer_add_log(w, &log); + EXPECT_ERR(err); + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + stats = writer_stats(w); + EXPECT(stats->log_stats.blocks > 0); + reftable_writer_free(w); + w = NULL; + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.log"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, names[N - 1]); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + + /* end of iteration. */ + err = reftable_iterator_next_ref(&it, &ref); + EXPECT(0 < err); + + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); + + err = reftable_reader_seek_log(&rd, &it, ""); + EXPECT_ERR(err); + + i = 0; + while (1) { + int err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + + EXPECT_ERR(err); + EXPECT_STREQ(names[i], log.refname); + EXPECT(i == log.update_index); + i++; + reftable_log_record_release(&log); + } + + EXPECT(i == N); + reftable_iterator_destroy(&it); + + /* cleanup. */ + strbuf_release(&buf); + free_names(names); + reader_close(&rd); +} + +static void test_table_read_write_sequential(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_iterator it = { NULL }; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err = 0; + int j = 0; + + write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, ""); + EXPECT_ERR(err); + + while (1) { + struct reftable_ref_record ref = { NULL }; + int r = reftable_iterator_next_ref(&it, &ref); + EXPECT(r >= 0); + if (r > 0) { + break; + } + EXPECT(0 == strcmp(names[j], ref.refname)); + EXPECT(update_index == ref.update_index); + + j++; + reftable_ref_record_release(&ref); + } + EXPECT(j == N); + reftable_iterator_destroy(&it); + strbuf_release(&buf); + free_names(names); + + reader_close(&rd); +} + +static void test_table_write_small_table(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 1; + write_table(&names, &buf, N, 4096, GIT_SHA1_FORMAT_ID); + EXPECT(buf.len < 200); + strbuf_release(&buf); + free_names(names); +} + +static void test_table_read_api(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + int err; + int i; + struct reftable_log_record log = { NULL }; + struct reftable_iterator it = { NULL }; + + write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, names[0]); + EXPECT_ERR(err); + + err = reftable_iterator_next_log(&it, &log); + EXPECT(err == REFTABLE_API_ERROR); + + strbuf_release(&buf); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + reftable_iterator_destroy(&it); + reftable_free(names); + reader_close(&rd); + strbuf_release(&buf); +} + +static void test_table_read_write_seek(int index, int hash_id) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + int err; + int i = 0; + + struct reftable_iterator it = { NULL }; + struct strbuf pastLast = STRBUF_INIT; + struct reftable_ref_record ref = { NULL }; + + write_table(&names, &buf, N, 256, hash_id); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + EXPECT(hash_id == reftable_reader_hash_id(&rd)); + + if (!index) { + rd.ref_offsets.index_offset = 0; + } else { + EXPECT(rd.ref_offsets.index_offset > 0); + } + + for (i = 1; i < N; i++) { + int err = reftable_reader_seek_ref(&rd, &it, names[i]); + EXPECT_ERR(err); + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + EXPECT(0 == strcmp(names[i], ref.refname)); + EXPECT(REFTABLE_REF_VAL1 == ref.value_type); + EXPECT(i == ref.value.val1[0]); + + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + } + + strbuf_addstr(&pastLast, names[N - 1]); + strbuf_addstr(&pastLast, "/"); + + err = reftable_reader_seek_ref(&rd, &it, pastLast.buf); + if (err == 0) { + struct reftable_ref_record ref = { NULL }; + int err = reftable_iterator_next_ref(&it, &ref); + EXPECT(err > 0); + } else { + EXPECT(err > 0); + } + + strbuf_release(&pastLast); + reftable_iterator_destroy(&it); + + strbuf_release(&buf); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + reftable_free(names); + reader_close(&rd); +} + +static void test_table_read_write_seek_linear(void) +{ + test_table_read_write_seek(0, GIT_SHA1_FORMAT_ID); +} + +static void test_table_read_write_seek_linear_sha256(void) +{ + test_table_read_write_seek(0, GIT_SHA256_FORMAT_ID); +} + +static void test_table_read_write_seek_index(void) +{ + test_table_read_write_seek(1, GIT_SHA1_FORMAT_ID); +} + +static void test_table_refs_for(int indexed) +{ + int N = 50; + char **want_names = reftable_calloc(sizeof(char *) * (N + 1)); + int want_names_len = 0; + uint8_t want_hash[GIT_SHA1_RAWSZ]; + + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_ref_record ref = { NULL }; + int i = 0; + int n; + int err; + struct reftable_reader rd; + struct reftable_block_source source = { NULL }; + + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + struct reftable_iterator it = { NULL }; + int j; + + set_test_hash(want_hash, 4); + + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA1_RAWSZ]; + char fill[51] = { 0 }; + char name[100]; + uint8_t hash1[GIT_SHA1_RAWSZ]; + uint8_t hash2[GIT_SHA1_RAWSZ]; + struct reftable_ref_record ref = { NULL }; + + memset(hash, i, sizeof(hash)); + memset(fill, 'x', 50); + /* Put the variable part in the start */ + snprintf(name, sizeof(name), "br%02d%s", i, fill); + name[40] = 0; + ref.refname = name; + + set_test_hash(hash1, i / 4); + set_test_hash(hash2, 3 + i / 4); + ref.value_type = REFTABLE_REF_VAL2; + ref.value.val2.value = hash1; + ref.value.val2.target_value = hash2; + + /* 80 bytes / entry, so 3 entries per block. Yields 17 + */ + /* blocks. */ + n = reftable_writer_add_ref(w, &ref); + EXPECT(n == 0); + + if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) || + !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) { + want_names[want_names_len++] = xstrdup(name); + } + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + reftable_writer_free(w); + w = NULL; + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + if (!indexed) { + rd.obj_offsets.is_present = 0; + } + + err = reftable_reader_seek_ref(&rd, &it, ""); + EXPECT_ERR(err); + reftable_iterator_destroy(&it); + + err = reftable_reader_refs_for(&rd, &it, want_hash); + EXPECT_ERR(err); + + j = 0; + while (1) { + int err = reftable_iterator_next_ref(&it, &ref); + EXPECT(err >= 0); + if (err > 0) { + break; + } + + EXPECT(j < want_names_len); + EXPECT(0 == strcmp(ref.refname, want_names[j])); + j++; + reftable_ref_record_release(&ref); + } + EXPECT(j == want_names_len); + + strbuf_release(&buf); + free_names(want_names); + reftable_iterator_destroy(&it); + reader_close(&rd); +} + +static void test_table_refs_for_no_index(void) +{ + test_table_refs_for(0); +} + +static void test_table_refs_for_obj_index(void) +{ + test_table_refs_for(1); +} + +static void test_write_empty_table(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_block_source source = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_ref_record rec = { NULL }; + struct reftable_iterator it = { NULL }; + int err; + + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_close(w); + EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR); + reftable_writer_free(w); + + EXPECT(buf.len == header_size(1) + footer_size(1)); + + block_source_from_strbuf(&source, &buf); + + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(rd, &it, ""); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &rec); + EXPECT(err > 0); + + reftable_iterator_destroy(&it); + reftable_reader_free(rd); + strbuf_release(&buf); +} + +static void test_write_key_order(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_ref_record refs[2] = { + { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value = { + .symref = "target", + }, + }, { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value = { + .symref = "target", + }, + } + }; + int err; + + reftable_writer_set_limits(w, 1, 1); + err = reftable_writer_add_ref(w, &refs[0]); + EXPECT_ERR(err); + err = reftable_writer_add_ref(w, &refs[1]); + printf("%d\n", err); + EXPECT(err == REFTABLE_API_ERROR); + reftable_writer_close(w); + reftable_writer_free(w); + strbuf_release(&buf); +} + +static void test_corrupt_table_empty(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err; + + block_source_from_strbuf(&source, &buf); + err = init_reader(&rd, &source, "file.log"); + EXPECT(err == REFTABLE_FORMAT_ERROR); +} + +static void test_corrupt_table(void) +{ + uint8_t zeros[1024] = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err; + strbuf_add(&buf, zeros, sizeof(zeros)); + + block_source_from_strbuf(&source, &buf); + err = init_reader(&rd, &source, "file.log"); + EXPECT(err == REFTABLE_FORMAT_ERROR); + strbuf_release(&buf); +} + +int readwrite_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_corrupt_table); + RUN_TEST(test_corrupt_table_empty); + RUN_TEST(test_log_write_read); + RUN_TEST(test_write_key_order); + RUN_TEST(test_table_read_write_seek_linear_sha256); + RUN_TEST(test_log_buffer_size); + RUN_TEST(test_table_write_small_table); + RUN_TEST(test_buffer); + RUN_TEST(test_table_read_api); + RUN_TEST(test_table_read_write_sequential); + RUN_TEST(test_table_read_write_seek_linear); + RUN_TEST(test_table_read_write_seek_index); + RUN_TEST(test_table_refs_for_no_index); + RUN_TEST(test_table_refs_for_obj_index); + RUN_TEST(test_write_empty_table); + return 0; +} diff --git a/reftable/record.c b/reftable/record.c new file mode 100644 index 0000000000..6a5dac32dc --- /dev/null +++ b/reftable/record.c @@ -0,0 +1,1212 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +/* record.c - methods for different types of records. */ + +#include "record.h" + +#include "system.h" +#include "constants.h" +#include "reftable-error.h" +#include "basics.h" + +int get_var_int(uint64_t *dest, struct string_view *in) +{ + int ptr = 0; + uint64_t val; + + if (in->len == 0) + return -1; + val = in->buf[ptr] & 0x7f; + + while (in->buf[ptr] & 0x80) { + ptr++; + if (ptr > in->len) { + return -1; + } + val = (val + 1) << 7 | (uint64_t)(in->buf[ptr] & 0x7f); + } + + *dest = val; + return ptr + 1; +} + +int put_var_int(struct string_view *dest, uint64_t val) +{ + uint8_t buf[10] = { 0 }; + int i = 9; + int n = 0; + buf[i] = (uint8_t)(val & 0x7f); + i--; + while (1) { + val >>= 7; + if (!val) { + break; + } + val--; + buf[i] = 0x80 | (uint8_t)(val & 0x7f); + i--; + } + + n = sizeof(buf) - i - 1; + if (dest->len < n) + return -1; + memcpy(dest->buf, &buf[i + 1], n); + return n; +} + +int reftable_is_block_type(uint8_t typ) +{ + switch (typ) { + case BLOCK_TYPE_REF: + case BLOCK_TYPE_LOG: + case BLOCK_TYPE_OBJ: + case BLOCK_TYPE_INDEX: + return 1; + } + return 0; +} + +uint8_t *reftable_ref_record_val1(struct reftable_ref_record *rec) +{ + switch (rec->value_type) { + case REFTABLE_REF_VAL1: + return rec->value.val1; + case REFTABLE_REF_VAL2: + return rec->value.val2.value; + default: + return NULL; + } +} + +uint8_t *reftable_ref_record_val2(struct reftable_ref_record *rec) +{ + switch (rec->value_type) { + case REFTABLE_REF_VAL2: + return rec->value.val2.target_value; + default: + return NULL; + } +} + +static int decode_string(struct strbuf *dest, struct string_view in) +{ + int start_len = in.len; + uint64_t tsize = 0; + int n = get_var_int(&tsize, &in); + if (n <= 0) + return -1; + string_view_consume(&in, n); + if (in.len < tsize) + return -1; + + strbuf_reset(dest); + strbuf_add(dest, in.buf, tsize); + string_view_consume(&in, tsize); + + return start_len - in.len; +} + +static int encode_string(char *str, struct string_view s) +{ + struct string_view start = s; + int l = strlen(str); + int n = put_var_int(&s, l); + if (n < 0) + return -1; + string_view_consume(&s, n); + if (s.len < l) + return -1; + memcpy(s.buf, str, l); + string_view_consume(&s, l); + + return start.len - s.len; +} + +int reftable_encode_key(int *restart, struct string_view dest, + struct strbuf prev_key, struct strbuf key, + uint8_t extra) +{ + struct string_view start = dest; + int prefix_len = common_prefix_size(&prev_key, &key); + uint64_t suffix_len = key.len - prefix_len; + int n = put_var_int(&dest, (uint64_t)prefix_len); + if (n < 0) + return -1; + string_view_consume(&dest, n); + + *restart = (prefix_len == 0); + + n = put_var_int(&dest, suffix_len << 3 | (uint64_t)extra); + if (n < 0) + return -1; + string_view_consume(&dest, n); + + if (dest.len < suffix_len) + return -1; + memcpy(dest.buf, key.buf + prefix_len, suffix_len); + string_view_consume(&dest, suffix_len); + + return start.len - dest.len; +} + +int reftable_decode_key(struct strbuf *key, uint8_t *extra, + struct strbuf last_key, struct string_view in) +{ + int start_len = in.len; + uint64_t prefix_len = 0; + uint64_t suffix_len = 0; + int n = get_var_int(&prefix_len, &in); + if (n < 0) + return -1; + string_view_consume(&in, n); + + if (prefix_len > last_key.len) + return -1; + + n = get_var_int(&suffix_len, &in); + if (n <= 0) + return -1; + string_view_consume(&in, n); + + *extra = (uint8_t)(suffix_len & 0x7); + suffix_len >>= 3; + + if (in.len < suffix_len) + return -1; + + strbuf_reset(key); + strbuf_add(key, last_key.buf, prefix_len); + strbuf_add(key, in.buf, suffix_len); + string_view_consume(&in, suffix_len); + + return start_len - in.len; +} + +static void reftable_ref_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_ref_record *rec = + (const struct reftable_ref_record *)r; + strbuf_reset(dest); + strbuf_addstr(dest, rec->refname); +} + +static void reftable_ref_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_ref_record *ref = rec; + const struct reftable_ref_record *src = src_rec; + assert(hash_size > 0); + + /* This is simple and correct, but we could probably reuse the hash + * fields. */ + reftable_ref_record_release(ref); + if (src->refname) { + ref->refname = xstrdup(src->refname); + } + ref->update_index = src->update_index; + ref->value_type = src->value_type; + switch (src->value_type) { + case REFTABLE_REF_DELETION: + break; + case REFTABLE_REF_VAL1: + ref->value.val1 = reftable_malloc(hash_size); + memcpy(ref->value.val1, src->value.val1, hash_size); + break; + case REFTABLE_REF_VAL2: + ref->value.val2.value = reftable_malloc(hash_size); + memcpy(ref->value.val2.value, src->value.val2.value, hash_size); + ref->value.val2.target_value = reftable_malloc(hash_size); + memcpy(ref->value.val2.target_value, + src->value.val2.target_value, hash_size); + break; + case REFTABLE_REF_SYMREF: + ref->value.symref = xstrdup(src->value.symref); + break; + } +} + +static char hexdigit(int c) +{ + if (c <= 9) + return '0' + c; + return 'a' + (c - 10); +} + +static void hex_format(char *dest, uint8_t *src, int hash_size) +{ + assert(hash_size > 0); + if (src) { + int i = 0; + for (i = 0; i < hash_size; i++) { + dest[2 * i] = hexdigit(src[i] >> 4); + dest[2 * i + 1] = hexdigit(src[i] & 0xf); + } + dest[2 * hash_size] = 0; + } +} + +void reftable_ref_record_print(struct reftable_ref_record *ref, + uint32_t hash_id) +{ + char hex[2 * GIT_SHA256_RAWSZ + 1] = { 0 }; /* BUG */ + printf("ref{%s(%" PRIu64 ") ", ref->refname, ref->update_index); + switch (ref->value_type) { + case REFTABLE_REF_SYMREF: + printf("=> %s", ref->value.symref); + break; + case REFTABLE_REF_VAL2: + hex_format(hex, ref->value.val2.value, hash_size(hash_id)); + printf("val 2 %s", hex); + hex_format(hex, ref->value.val2.target_value, + hash_size(hash_id)); + printf("(T %s)", hex); + break; + case REFTABLE_REF_VAL1: + hex_format(hex, ref->value.val1, hash_size(hash_id)); + printf("val 1 %s", hex); + break; + case REFTABLE_REF_DELETION: + printf("delete"); + break; + } + printf("}\n"); +} + +static void reftable_ref_record_release_void(void *rec) +{ + reftable_ref_record_release(rec); +} + +void reftable_ref_record_release(struct reftable_ref_record *ref) +{ + switch (ref->value_type) { + case REFTABLE_REF_SYMREF: + reftable_free(ref->value.symref); + break; + case REFTABLE_REF_VAL2: + reftable_free(ref->value.val2.target_value); + reftable_free(ref->value.val2.value); + break; + case REFTABLE_REF_VAL1: + reftable_free(ref->value.val1); + break; + case REFTABLE_REF_DELETION: + break; + default: + abort(); + } + + reftable_free(ref->refname); + memset(ref, 0, sizeof(struct reftable_ref_record)); +} + +static uint8_t reftable_ref_record_val_type(const void *rec) +{ + const struct reftable_ref_record *r = + (const struct reftable_ref_record *)rec; + return r->value_type; +} + +static int reftable_ref_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_ref_record *r = + (const struct reftable_ref_record *)rec; + struct string_view start = s; + int n = put_var_int(&s, r->update_index); + assert(hash_size > 0); + if (n < 0) + return -1; + string_view_consume(&s, n); + + switch (r->value_type) { + case REFTABLE_REF_SYMREF: + n = encode_string(r->value.symref, s); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + break; + case REFTABLE_REF_VAL2: + if (s.len < 2 * hash_size) { + return -1; + } + memcpy(s.buf, r->value.val2.value, hash_size); + string_view_consume(&s, hash_size); + memcpy(s.buf, r->value.val2.target_value, hash_size); + string_view_consume(&s, hash_size); + break; + case REFTABLE_REF_VAL1: + if (s.len < hash_size) { + return -1; + } + memcpy(s.buf, r->value.val1, hash_size); + string_view_consume(&s, hash_size); + break; + case REFTABLE_REF_DELETION: + break; + default: + abort(); + } + + return start.len - s.len; +} + +static int reftable_ref_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct reftable_ref_record *r = rec; + struct string_view start = in; + uint64_t update_index = 0; + int n = get_var_int(&update_index, &in); + if (n < 0) + return n; + string_view_consume(&in, n); + + reftable_ref_record_release(r); + + assert(hash_size > 0); + + r->refname = reftable_realloc(r->refname, key.len + 1); + memcpy(r->refname, key.buf, key.len); + r->update_index = update_index; + r->refname[key.len] = 0; + r->value_type = val_type; + switch (val_type) { + case REFTABLE_REF_VAL1: + if (in.len < hash_size) { + return -1; + } + + r->value.val1 = reftable_malloc(hash_size); + memcpy(r->value.val1, in.buf, hash_size); + string_view_consume(&in, hash_size); + break; + + case REFTABLE_REF_VAL2: + if (in.len < 2 * hash_size) { + return -1; + } + + r->value.val2.value = reftable_malloc(hash_size); + memcpy(r->value.val2.value, in.buf, hash_size); + string_view_consume(&in, hash_size); + + r->value.val2.target_value = reftable_malloc(hash_size); + memcpy(r->value.val2.target_value, in.buf, hash_size); + string_view_consume(&in, hash_size); + break; + + case REFTABLE_REF_SYMREF: { + struct strbuf dest = STRBUF_INIT; + int n = decode_string(&dest, in); + if (n < 0) { + return -1; + } + string_view_consume(&in, n); + r->value.symref = dest.buf; + } break; + + case REFTABLE_REF_DELETION: + break; + default: + abort(); + break; + } + + return start.len - in.len; +} + +static int reftable_ref_record_is_deletion_void(const void *p) +{ + return reftable_ref_record_is_deletion( + (const struct reftable_ref_record *)p); +} + +static struct reftable_record_vtable reftable_ref_record_vtable = { + .key = &reftable_ref_record_key, + .type = BLOCK_TYPE_REF, + .copy_from = &reftable_ref_record_copy_from, + .val_type = &reftable_ref_record_val_type, + .encode = &reftable_ref_record_encode, + .decode = &reftable_ref_record_decode, + .release = &reftable_ref_record_release_void, + .is_deletion = &reftable_ref_record_is_deletion_void, +}; + +static void reftable_obj_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_obj_record *rec = + (const struct reftable_obj_record *)r; + strbuf_reset(dest); + strbuf_add(dest, rec->hash_prefix, rec->hash_prefix_len); +} + +static void reftable_obj_record_release(void *rec) +{ + struct reftable_obj_record *obj = rec; + FREE_AND_NULL(obj->hash_prefix); + FREE_AND_NULL(obj->offsets); + memset(obj, 0, sizeof(struct reftable_obj_record)); +} + +static void reftable_obj_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_obj_record *obj = rec; + const struct reftable_obj_record *src = + (const struct reftable_obj_record *)src_rec; + + reftable_obj_record_release(obj); + *obj = *src; + obj->hash_prefix = reftable_malloc(obj->hash_prefix_len); + memcpy(obj->hash_prefix, src->hash_prefix, obj->hash_prefix_len); + + obj->offsets = reftable_malloc(obj->offset_len * sizeof(uint64_t)); + COPY_ARRAY(obj->offsets, src->offsets, obj->offset_len); +} + +static uint8_t reftable_obj_record_val_type(const void *rec) +{ + const struct reftable_obj_record *r = rec; + if (r->offset_len > 0 && r->offset_len < 8) + return r->offset_len; + return 0; +} + +static int reftable_obj_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_obj_record *r = rec; + struct string_view start = s; + int i = 0; + int n = 0; + uint64_t last = 0; + if (r->offset_len == 0 || r->offset_len >= 8) { + n = put_var_int(&s, r->offset_len); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + } + if (r->offset_len == 0) + return start.len - s.len; + n = put_var_int(&s, r->offsets[0]); + if (n < 0) + return -1; + string_view_consume(&s, n); + + last = r->offsets[0]; + for (i = 1; i < r->offset_len; i++) { + int n = put_var_int(&s, r->offsets[i] - last); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + last = r->offsets[i]; + } + return start.len - s.len; +} + +static int reftable_obj_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_obj_record *r = rec; + uint64_t count = val_type; + int n = 0; + uint64_t last; + int j; + r->hash_prefix = reftable_malloc(key.len); + memcpy(r->hash_prefix, key.buf, key.len); + r->hash_prefix_len = key.len; + + if (val_type == 0) { + n = get_var_int(&count, &in); + if (n < 0) { + return n; + } + + string_view_consume(&in, n); + } + + r->offsets = NULL; + r->offset_len = 0; + if (count == 0) + return start.len - in.len; + + r->offsets = reftable_malloc(count * sizeof(uint64_t)); + r->offset_len = count; + + n = get_var_int(&r->offsets[0], &in); + if (n < 0) + return n; + string_view_consume(&in, n); + + last = r->offsets[0]; + j = 1; + while (j < count) { + uint64_t delta = 0; + int n = get_var_int(&delta, &in); + if (n < 0) { + return n; + } + string_view_consume(&in, n); + + last = r->offsets[j] = (delta + last); + j++; + } + return start.len - in.len; +} + +static int not_a_deletion(const void *p) +{ + return 0; +} + +static struct reftable_record_vtable reftable_obj_record_vtable = { + .key = &reftable_obj_record_key, + .type = BLOCK_TYPE_OBJ, + .copy_from = &reftable_obj_record_copy_from, + .val_type = &reftable_obj_record_val_type, + .encode = &reftable_obj_record_encode, + .decode = &reftable_obj_record_decode, + .release = &reftable_obj_record_release, + .is_deletion = not_a_deletion, +}; + +void reftable_log_record_print(struct reftable_log_record *log, + uint32_t hash_id) +{ + char hex[GIT_SHA256_RAWSZ + 1] = { 0 }; + + switch (log->value_type) { + case REFTABLE_LOG_DELETION: + printf("log{%s(%" PRIu64 ") delete", log->refname, + log->update_index); + break; + case REFTABLE_LOG_UPDATE: + printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", + log->refname, log->update_index, log->value.update.name, + log->value.update.email, log->value.update.time, + log->value.update.tz_offset); + hex_format(hex, log->value.update.old_hash, hash_size(hash_id)); + printf("%s => ", hex); + hex_format(hex, log->value.update.new_hash, hash_size(hash_id)); + printf("%s\n\n%s\n}\n", hex, log->value.update.message); + break; + } +} + +static void reftable_log_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_log_record *rec = + (const struct reftable_log_record *)r; + int len = strlen(rec->refname); + uint8_t i64[8]; + uint64_t ts = 0; + strbuf_reset(dest); + strbuf_add(dest, (uint8_t *)rec->refname, len + 1); + + ts = (~ts) - rec->update_index; + put_be64(&i64[0], ts); + strbuf_add(dest, i64, sizeof(i64)); +} + +static void reftable_log_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_log_record *dst = rec; + const struct reftable_log_record *src = + (const struct reftable_log_record *)src_rec; + + reftable_log_record_release(dst); + *dst = *src; + if (dst->refname) { + dst->refname = xstrdup(dst->refname); + } + switch (dst->value_type) { + case REFTABLE_LOG_DELETION: + break; + case REFTABLE_LOG_UPDATE: + if (dst->value.update.email) { + dst->value.update.email = + xstrdup(dst->value.update.email); + } + if (dst->value.update.name) { + dst->value.update.name = + xstrdup(dst->value.update.name); + } + if (dst->value.update.message) { + dst->value.update.message = + xstrdup(dst->value.update.message); + } + + if (dst->value.update.new_hash) { + dst->value.update.new_hash = reftable_malloc(hash_size); + memcpy(dst->value.update.new_hash, + src->value.update.new_hash, hash_size); + } + if (dst->value.update.old_hash) { + dst->value.update.old_hash = reftable_malloc(hash_size); + memcpy(dst->value.update.old_hash, + src->value.update.old_hash, hash_size); + } + break; + } +} + +static void reftable_log_record_release_void(void *rec) +{ + struct reftable_log_record *r = rec; + reftable_log_record_release(r); +} + +void reftable_log_record_release(struct reftable_log_record *r) +{ + reftable_free(r->refname); + switch (r->value_type) { + case REFTABLE_LOG_DELETION: + break; + case REFTABLE_LOG_UPDATE: + reftable_free(r->value.update.new_hash); + reftable_free(r->value.update.old_hash); + reftable_free(r->value.update.name); + reftable_free(r->value.update.email); + reftable_free(r->value.update.message); + break; + } + memset(r, 0, sizeof(struct reftable_log_record)); +} + +static uint8_t reftable_log_record_val_type(const void *rec) +{ + const struct reftable_log_record *log = + (const struct reftable_log_record *)rec; + + return reftable_log_record_is_deletion(log) ? 0 : 1; +} + +static uint8_t zero[GIT_SHA256_RAWSZ] = { 0 }; + +static int reftable_log_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_log_record *r = rec; + struct string_view start = s; + int n = 0; + uint8_t *oldh = NULL; + uint8_t *newh = NULL; + if (reftable_log_record_is_deletion(r)) + return 0; + + oldh = r->value.update.old_hash; + newh = r->value.update.new_hash; + if (!oldh) { + oldh = zero; + } + if (!newh) { + newh = zero; + } + + if (s.len < 2 * hash_size) + return -1; + + memcpy(s.buf, oldh, hash_size); + memcpy(s.buf + hash_size, newh, hash_size); + string_view_consume(&s, 2 * hash_size); + + n = encode_string(r->value.update.name ? r->value.update.name : "", s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + n = encode_string(r->value.update.email ? r->value.update.email : "", + s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + n = put_var_int(&s, r->value.update.time); + if (n < 0) + return -1; + string_view_consume(&s, n); + + if (s.len < 2) + return -1; + + put_be16(s.buf, r->value.update.tz_offset); + string_view_consume(&s, 2); + + n = encode_string( + r->value.update.message ? r->value.update.message : "", s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + return start.len - s.len; +} + +static int reftable_log_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_log_record *r = rec; + uint64_t max = 0; + uint64_t ts = 0; + struct strbuf dest = STRBUF_INIT; + int n; + + if (key.len <= 9 || key.buf[key.len - 9] != 0) + return REFTABLE_FORMAT_ERROR; + + r->refname = reftable_realloc(r->refname, key.len - 8); + memcpy(r->refname, key.buf, key.len - 8); + ts = get_be64(key.buf + key.len - 8); + + r->update_index = (~max) - ts; + + if (val_type != r->value_type) { + switch (r->value_type) { + case REFTABLE_LOG_UPDATE: + FREE_AND_NULL(r->value.update.old_hash); + FREE_AND_NULL(r->value.update.new_hash); + FREE_AND_NULL(r->value.update.message); + FREE_AND_NULL(r->value.update.email); + FREE_AND_NULL(r->value.update.name); + break; + case REFTABLE_LOG_DELETION: + break; + } + } + + r->value_type = val_type; + if (val_type == REFTABLE_LOG_DELETION) + return 0; + + if (in.len < 2 * hash_size) + return REFTABLE_FORMAT_ERROR; + + r->value.update.old_hash = + reftable_realloc(r->value.update.old_hash, hash_size); + r->value.update.new_hash = + reftable_realloc(r->value.update.new_hash, hash_size); + + memcpy(r->value.update.old_hash, in.buf, hash_size); + memcpy(r->value.update.new_hash, in.buf + hash_size, hash_size); + + string_view_consume(&in, 2 * hash_size); + + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.name = + reftable_realloc(r->value.update.name, dest.len + 1); + memcpy(r->value.update.name, dest.buf, dest.len); + r->value.update.name[dest.len] = 0; + + strbuf_reset(&dest); + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.email = + reftable_realloc(r->value.update.email, dest.len + 1); + memcpy(r->value.update.email, dest.buf, dest.len); + r->value.update.email[dest.len] = 0; + + ts = 0; + n = get_var_int(&ts, &in); + if (n < 0) + goto done; + string_view_consume(&in, n); + r->value.update.time = ts; + if (in.len < 2) + goto done; + + r->value.update.tz_offset = get_be16(in.buf); + string_view_consume(&in, 2); + + strbuf_reset(&dest); + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.message = + reftable_realloc(r->value.update.message, dest.len + 1); + memcpy(r->value.update.message, dest.buf, dest.len); + r->value.update.message[dest.len] = 0; + + strbuf_release(&dest); + return start.len - in.len; + +done: + strbuf_release(&dest); + return REFTABLE_FORMAT_ERROR; +} + +static int null_streq(char *a, char *b) +{ + char *empty = ""; + if (!a) + a = empty; + + if (!b) + b = empty; + + return 0 == strcmp(a, b); +} + +static int zero_hash_eq(uint8_t *a, uint8_t *b, int sz) +{ + if (!a) + a = zero; + + if (!b) + b = zero; + + return !memcmp(a, b, sz); +} + +int reftable_log_record_equal(struct reftable_log_record *a, + struct reftable_log_record *b, int hash_size) +{ + if (!(null_streq(a->refname, b->refname) && + a->update_index == b->update_index && + a->value_type == b->value_type)) + return 0; + + switch (a->value_type) { + case REFTABLE_LOG_DELETION: + return 1; + case REFTABLE_LOG_UPDATE: + return null_streq(a->value.update.name, b->value.update.name) && + a->value.update.time == b->value.update.time && + a->value.update.tz_offset == b->value.update.tz_offset && + null_streq(a->value.update.email, + b->value.update.email) && + null_streq(a->value.update.message, + b->value.update.message) && + zero_hash_eq(a->value.update.old_hash, + b->value.update.old_hash, hash_size) && + zero_hash_eq(a->value.update.new_hash, + b->value.update.new_hash, hash_size); + } + + abort(); +} + +static int reftable_log_record_is_deletion_void(const void *p) +{ + return reftable_log_record_is_deletion( + (const struct reftable_log_record *)p); +} + +static struct reftable_record_vtable reftable_log_record_vtable = { + .key = &reftable_log_record_key, + .type = BLOCK_TYPE_LOG, + .copy_from = &reftable_log_record_copy_from, + .val_type = &reftable_log_record_val_type, + .encode = &reftable_log_record_encode, + .decode = &reftable_log_record_decode, + .release = &reftable_log_record_release_void, + .is_deletion = &reftable_log_record_is_deletion_void, +}; + +struct reftable_record reftable_new_record(uint8_t typ) +{ + struct reftable_record rec = { NULL }; + switch (typ) { + case BLOCK_TYPE_REF: { + struct reftable_ref_record *r = + reftable_calloc(sizeof(struct reftable_ref_record)); + reftable_record_from_ref(&rec, r); + return rec; + } + + case BLOCK_TYPE_OBJ: { + struct reftable_obj_record *r = + reftable_calloc(sizeof(struct reftable_obj_record)); + reftable_record_from_obj(&rec, r); + return rec; + } + case BLOCK_TYPE_LOG: { + struct reftable_log_record *r = + reftable_calloc(sizeof(struct reftable_log_record)); + reftable_record_from_log(&rec, r); + return rec; + } + case BLOCK_TYPE_INDEX: { + struct reftable_index_record empty = { .last_key = + STRBUF_INIT }; + struct reftable_index_record *r = + reftable_calloc(sizeof(struct reftable_index_record)); + *r = empty; + reftable_record_from_index(&rec, r); + return rec; + } + } + abort(); + return rec; +} + +/* clear out the record, yielding the reftable_record data that was + * encapsulated. */ +static void *reftable_record_yield(struct reftable_record *rec) +{ + void *p = rec->data; + rec->data = NULL; + return p; +} + +void reftable_record_destroy(struct reftable_record *rec) +{ + reftable_record_release(rec); + reftable_free(reftable_record_yield(rec)); +} + +static void reftable_index_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_index_record *rec = r; + strbuf_reset(dest); + strbuf_addbuf(dest, &rec->last_key); +} + +static void reftable_index_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_index_record *dst = rec; + const struct reftable_index_record *src = src_rec; + + strbuf_reset(&dst->last_key); + strbuf_addbuf(&dst->last_key, &src->last_key); + dst->offset = src->offset; +} + +static void reftable_index_record_release(void *rec) +{ + struct reftable_index_record *idx = rec; + strbuf_release(&idx->last_key); +} + +static uint8_t reftable_index_record_val_type(const void *rec) +{ + return 0; +} + +static int reftable_index_record_encode(const void *rec, struct string_view out, + int hash_size) +{ + const struct reftable_index_record *r = + (const struct reftable_index_record *)rec; + struct string_view start = out; + + int n = put_var_int(&out, r->offset); + if (n < 0) + return n; + + string_view_consume(&out, n); + + return start.len - out.len; +} + +static int reftable_index_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_index_record *r = rec; + int n = 0; + + strbuf_reset(&r->last_key); + strbuf_addbuf(&r->last_key, &key); + + n = get_var_int(&r->offset, &in); + if (n < 0) + return n; + + string_view_consume(&in, n); + return start.len - in.len; +} + +static struct reftable_record_vtable reftable_index_record_vtable = { + .key = &reftable_index_record_key, + .type = BLOCK_TYPE_INDEX, + .copy_from = &reftable_index_record_copy_from, + .val_type = &reftable_index_record_val_type, + .encode = &reftable_index_record_encode, + .decode = &reftable_index_record_decode, + .release = &reftable_index_record_release, + .is_deletion = ¬_a_deletion, +}; + +void reftable_record_key(struct reftable_record *rec, struct strbuf *dest) +{ + rec->ops->key(rec->data, dest); +} + +uint8_t reftable_record_type(struct reftable_record *rec) +{ + return rec->ops->type; +} + +int reftable_record_encode(struct reftable_record *rec, struct string_view dest, + int hash_size) +{ + return rec->ops->encode(rec->data, dest, hash_size); +} + +void reftable_record_copy_from(struct reftable_record *rec, + struct reftable_record *src, int hash_size) +{ + assert(src->ops->type == rec->ops->type); + + rec->ops->copy_from(rec->data, src->data, hash_size); +} + +uint8_t reftable_record_val_type(struct reftable_record *rec) +{ + return rec->ops->val_type(rec->data); +} + +int reftable_record_decode(struct reftable_record *rec, struct strbuf key, + uint8_t extra, struct string_view src, int hash_size) +{ + return rec->ops->decode(rec->data, key, extra, src, hash_size); +} + +void reftable_record_release(struct reftable_record *rec) +{ + rec->ops->release(rec->data); +} + +int reftable_record_is_deletion(struct reftable_record *rec) +{ + return rec->ops->is_deletion(rec->data); +} + +void reftable_record_from_ref(struct reftable_record *rec, + struct reftable_ref_record *ref_rec) +{ + assert(!rec->ops); + rec->data = ref_rec; + rec->ops = &reftable_ref_record_vtable; +} + +void reftable_record_from_obj(struct reftable_record *rec, + struct reftable_obj_record *obj_rec) +{ + assert(!rec->ops); + rec->data = obj_rec; + rec->ops = &reftable_obj_record_vtable; +} + +void reftable_record_from_index(struct reftable_record *rec, + struct reftable_index_record *index_rec) +{ + assert(!rec->ops); + rec->data = index_rec; + rec->ops = &reftable_index_record_vtable; +} + +void reftable_record_from_log(struct reftable_record *rec, + struct reftable_log_record *log_rec) +{ + assert(!rec->ops); + rec->data = log_rec; + rec->ops = &reftable_log_record_vtable; +} + +struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *rec) +{ + assert(reftable_record_type(rec) == BLOCK_TYPE_REF); + return rec->data; +} + +struct reftable_log_record *reftable_record_as_log(struct reftable_record *rec) +{ + assert(reftable_record_type(rec) == BLOCK_TYPE_LOG); + return rec->data; +} + +static int hash_equal(uint8_t *a, uint8_t *b, int hash_size) +{ + if (a && b) + return !memcmp(a, b, hash_size); + + return a == b; +} + +int reftable_ref_record_equal(struct reftable_ref_record *a, + struct reftable_ref_record *b, int hash_size) +{ + assert(hash_size > 0); + if (!(0 == strcmp(a->refname, b->refname) && + a->update_index == b->update_index && + a->value_type == b->value_type)) + return 0; + + switch (a->value_type) { + case REFTABLE_REF_SYMREF: + return !strcmp(a->value.symref, b->value.symref); + case REFTABLE_REF_VAL2: + return hash_equal(a->value.val2.value, b->value.val2.value, + hash_size) && + hash_equal(a->value.val2.target_value, + b->value.val2.target_value, hash_size); + case REFTABLE_REF_VAL1: + return hash_equal(a->value.val1, b->value.val1, hash_size); + case REFTABLE_REF_DELETION: + return 1; + default: + abort(); + } +} + +int reftable_ref_record_compare_name(const void *a, const void *b) +{ + return strcmp(((struct reftable_ref_record *)a)->refname, + ((struct reftable_ref_record *)b)->refname); +} + +int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref) +{ + return ref->value_type == REFTABLE_REF_DELETION; +} + +int reftable_log_record_compare_key(const void *a, const void *b) +{ + const struct reftable_log_record *la = a; + const struct reftable_log_record *lb = b; + + int cmp = strcmp(la->refname, lb->refname); + if (cmp) + return cmp; + if (la->update_index > lb->update_index) + return -1; + return (la->update_index < lb->update_index) ? 1 : 0; +} + +int reftable_log_record_is_deletion(const struct reftable_log_record *log) +{ + return (log->value_type == REFTABLE_LOG_DELETION); +} + +void string_view_consume(struct string_view *s, int n) +{ + s->buf += n; + s->len -= n; +} diff --git a/reftable/record.h b/reftable/record.h new file mode 100644 index 0000000000..498e8c50bf --- /dev/null +++ b/reftable/record.h @@ -0,0 +1,139 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef RECORD_H +#define RECORD_H + +#include "system.h" + +#include <stdint.h> + +#include "reftable-record.h" + +/* + * A substring of existing string data. This structure takes no responsibility + * for the lifetime of the data it points to. + */ +struct string_view { + uint8_t *buf; + size_t len; +}; + +/* Advance `s.buf` by `n`, and decrease length. */ +void string_view_consume(struct string_view *s, int n); + +/* utilities for de/encoding varints */ + +int get_var_int(uint64_t *dest, struct string_view *in); +int put_var_int(struct string_view *dest, uint64_t val); + +/* Methods for records. */ +struct reftable_record_vtable { + /* encode the key of to a uint8_t strbuf. */ + void (*key)(const void *rec, struct strbuf *dest); + + /* The record type of ('r' for ref). */ + uint8_t type; + + void (*copy_from)(void *dest, const void *src, int hash_size); + + /* a value of [0..7], indicating record subvariants (eg. ref vs. symref + * vs ref deletion) */ + uint8_t (*val_type)(const void *rec); + + /* encodes rec into dest, returning how much space was used. */ + int (*encode)(const void *rec, struct string_view dest, int hash_size); + + /* decode data from `src` into the record. */ + int (*decode)(void *rec, struct strbuf key, uint8_t extra, + struct string_view src, int hash_size); + + /* deallocate and null the record. */ + void (*release)(void *rec); + + /* is this a tombstone? */ + int (*is_deletion)(const void *rec); +}; + +/* record is a generic wrapper for different types of records. */ +struct reftable_record { + void *data; + struct reftable_record_vtable *ops; +}; + +/* returns true for recognized block types. Block start with the block type. */ +int reftable_is_block_type(uint8_t typ); + +/* creates a malloced record of the given type. Dispose with record_destroy */ +struct reftable_record reftable_new_record(uint8_t typ); + +/* Encode `key` into `dest`. Sets `is_restart` to indicate a restart. Returns + * number of bytes written. */ +int reftable_encode_key(int *is_restart, struct string_view dest, + struct strbuf prev_key, struct strbuf key, + uint8_t extra); + +/* Decode into `key` and `extra` from `in` */ +int reftable_decode_key(struct strbuf *key, uint8_t *extra, + struct strbuf last_key, struct string_view in); + +/* reftable_index_record are used internally to speed up lookups. */ +struct reftable_index_record { + uint64_t offset; /* Offset of block */ + struct strbuf last_key; /* Last key of the block. */ +}; + +/* reftable_obj_record stores an object ID => ref mapping. */ +struct reftable_obj_record { + uint8_t *hash_prefix; /* leading bytes of the object ID */ + int hash_prefix_len; /* number of leading bytes. Constant + * across a single table. */ + uint64_t *offsets; /* a vector of file offsets. */ + int offset_len; +}; + +/* see struct record_vtable */ + +void reftable_record_key(struct reftable_record *rec, struct strbuf *dest); +uint8_t reftable_record_type(struct reftable_record *rec); +void reftable_record_copy_from(struct reftable_record *rec, + struct reftable_record *src, int hash_size); +uint8_t reftable_record_val_type(struct reftable_record *rec); +int reftable_record_encode(struct reftable_record *rec, struct string_view dest, + int hash_size); +int reftable_record_decode(struct reftable_record *rec, struct strbuf key, + uint8_t extra, struct string_view src, + int hash_size); +int reftable_record_is_deletion(struct reftable_record *rec); + +/* zeroes out the embedded record */ +void reftable_record_release(struct reftable_record *rec); + +/* clear and deallocate embedded record, and zero `rec`. */ +void reftable_record_destroy(struct reftable_record *rec); + +/* initialize generic records from concrete records. The generic record should + * be zeroed out. */ +void reftable_record_from_obj(struct reftable_record *rec, + struct reftable_obj_record *objrec); +void reftable_record_from_index(struct reftable_record *rec, + struct reftable_index_record *idxrec); +void reftable_record_from_ref(struct reftable_record *rec, + struct reftable_ref_record *refrec); +void reftable_record_from_log(struct reftable_record *rec, + struct reftable_log_record *logrec); +struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *ref); +struct reftable_log_record *reftable_record_as_log(struct reftable_record *ref); + +/* for qsort. */ +int reftable_ref_record_compare_name(const void *a, const void *b); + +/* for qsort. */ +int reftable_log_record_compare_key(const void *a, const void *b); + +#endif diff --git a/reftable/record_test.c b/reftable/record_test.c new file mode 100644 index 0000000000..f4ad7cace4 --- /dev/null +++ b/reftable/record_test.c @@ -0,0 +1,412 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#include "record.h" + +#include "system.h" +#include "basics.h" +#include "constants.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static void test_copy(struct reftable_record *rec) +{ + struct reftable_record copy = + reftable_new_record(reftable_record_type(rec)); + reftable_record_copy_from(©, rec, GIT_SHA1_RAWSZ); + /* do it twice to catch memory leaks */ + reftable_record_copy_from(©, rec, GIT_SHA1_RAWSZ); + switch (reftable_record_type(©)) { + case BLOCK_TYPE_REF: + EXPECT(reftable_ref_record_equal(reftable_record_as_ref(©), + reftable_record_as_ref(rec), + GIT_SHA1_RAWSZ)); + break; + case BLOCK_TYPE_LOG: + EXPECT(reftable_log_record_equal(reftable_record_as_log(©), + reftable_record_as_log(rec), + GIT_SHA1_RAWSZ)); + break; + } + reftable_record_destroy(©); +} + +static void test_varint_roundtrip(void) +{ + uint64_t inputs[] = { 0, + 1, + 27, + 127, + 128, + 257, + 4096, + ((uint64_t)1 << 63), + ((uint64_t)1 << 63) + ((uint64_t)1 << 63) - 1 }; + int i = 0; + for (i = 0; i < ARRAY_SIZE(inputs); i++) { + uint8_t dest[10]; + + struct string_view out = { + .buf = dest, + .len = sizeof(dest), + }; + uint64_t in = inputs[i]; + int n = put_var_int(&out, in); + uint64_t got = 0; + + EXPECT(n > 0); + out.len = n; + n = get_var_int(&got, &out); + EXPECT(n > 0); + + EXPECT(got == in); + } +} + +static void test_common_prefix(void) +{ + struct { + const char *a, *b; + int want; + } cases[] = { + { "abc", "ab", 2 }, + { "", "abc", 0 }, + { "abc", "abd", 2 }, + { "abc", "pqr", 0 }, + }; + + int i = 0; + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct strbuf a = STRBUF_INIT; + struct strbuf b = STRBUF_INIT; + strbuf_addstr(&a, cases[i].a); + strbuf_addstr(&b, cases[i].b); + EXPECT(common_prefix_size(&a, &b) == cases[i].want); + + strbuf_release(&a); + strbuf_release(&b); + } +} + +static void set_hash(uint8_t *h, int j) +{ + int i = 0; + for (i = 0; i < hash_size(GIT_SHA1_FORMAT_ID); i++) { + h[i] = (j >> i) & 0xff; + } +} + +static void test_reftable_ref_record_roundtrip(void) +{ + int i = 0; + + for (i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) { + struct reftable_ref_record in = { NULL }; + struct reftable_ref_record out = { NULL }; + struct reftable_record rec_out = { NULL }; + struct strbuf key = STRBUF_INIT; + struct reftable_record rec = { NULL }; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + + int n, m; + + in.value_type = i; + switch (i) { + case REFTABLE_REF_DELETION: + break; + case REFTABLE_REF_VAL1: + in.value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val1, 1); + break; + case REFTABLE_REF_VAL2: + in.value.val2.value = reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val2.value, 1); + in.value.val2.target_value = + reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val2.target_value, 2); + break; + case REFTABLE_REF_SYMREF: + in.value.symref = xstrdup("target"); + break; + } + in.refname = xstrdup("refs/heads/master"); + + reftable_record_from_ref(&rec, &in); + test_copy(&rec); + + EXPECT(reftable_record_val_type(&rec) == i); + + reftable_record_key(&rec, &key); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + + /* decode into a non-zero reftable_record to test for leaks. */ + + reftable_record_from_ref(&rec_out, &out); + m = reftable_record_decode(&rec_out, key, i, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(reftable_ref_record_equal(&in, &out, GIT_SHA1_RAWSZ)); + reftable_record_release(&rec_out); + + strbuf_release(&key); + reftable_ref_record_release(&in); + } +} + +static void test_reftable_log_record_equal(void) +{ + struct reftable_log_record in[2] = { + { + .refname = xstrdup("refs/heads/master"), + .update_index = 42, + }, + { + .refname = xstrdup("refs/heads/master"), + .update_index = 22, + } + }; + + EXPECT(!reftable_log_record_equal(&in[0], &in[1], GIT_SHA1_RAWSZ)); + in[1].update_index = in[0].update_index; + EXPECT(reftable_log_record_equal(&in[0], &in[1], GIT_SHA1_RAWSZ)); + reftable_log_record_release(&in[0]); + reftable_log_record_release(&in[1]); +} + +static void test_reftable_log_record_roundtrip(void) +{ + int i; + struct reftable_log_record in[2] = { + { + .refname = xstrdup("refs/heads/master"), + .update_index = 42, + .value_type = REFTABLE_LOG_UPDATE, + .value = { + .update = { + .old_hash = reftable_malloc(GIT_SHA1_RAWSZ), + .new_hash = reftable_malloc(GIT_SHA1_RAWSZ), + .name = xstrdup("han-wen"), + .email = xstrdup("hanwen@google.com"), + .message = xstrdup("test"), + .time = 1577123507, + .tz_offset = 100, + }, + } + }, + { + .refname = xstrdup("refs/heads/master"), + .update_index = 22, + .value_type = REFTABLE_LOG_DELETION, + } + }; + set_test_hash(in[0].value.update.new_hash, 1); + set_test_hash(in[0].value.update.old_hash, 2); + for (i = 0; i < ARRAY_SIZE(in); i++) { + struct reftable_record rec = { NULL }; + struct strbuf key = STRBUF_INIT; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + /* populate out, to check for leaks. */ + struct reftable_log_record out = { + .refname = xstrdup("old name"), + .value_type = REFTABLE_LOG_UPDATE, + .value = { + .update = { + .new_hash = reftable_calloc(GIT_SHA1_RAWSZ), + .old_hash = reftable_calloc(GIT_SHA1_RAWSZ), + .name = xstrdup("old name"), + .email = xstrdup("old@email"), + .message = xstrdup("old message"), + }, + }, + }; + struct reftable_record rec_out = { NULL }; + int n, m, valtype; + + reftable_record_from_log(&rec, &in[i]); + + test_copy(&rec); + + reftable_record_key(&rec, &key); + + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n >= 0); + reftable_record_from_log(&rec_out, &out); + valtype = reftable_record_val_type(&rec); + m = reftable_record_decode(&rec_out, key, valtype, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(reftable_log_record_equal(&in[i], &out, GIT_SHA1_RAWSZ)); + reftable_log_record_release(&in[i]); + strbuf_release(&key); + reftable_record_release(&rec_out); + } +} + +static void test_u24_roundtrip(void) +{ + uint32_t in = 0x112233; + uint8_t dest[3]; + uint32_t out; + put_be24(dest, in); + out = get_be24(dest); + EXPECT(in == out); +} + +static void test_key_roundtrip(void) +{ + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct strbuf last_key = STRBUF_INIT; + struct strbuf key = STRBUF_INIT; + struct strbuf roundtrip = STRBUF_INIT; + int restart; + uint8_t extra; + int n, m; + uint8_t rt_extra; + + strbuf_addstr(&last_key, "refs/heads/master"); + strbuf_addstr(&key, "refs/tags/bla"); + extra = 6; + n = reftable_encode_key(&restart, dest, last_key, key, extra); + EXPECT(!restart); + EXPECT(n > 0); + + m = reftable_decode_key(&roundtrip, &rt_extra, last_key, dest); + EXPECT(n == m); + EXPECT(0 == strbuf_cmp(&key, &roundtrip)); + EXPECT(rt_extra == extra); + + strbuf_release(&last_key); + strbuf_release(&key); + strbuf_release(&roundtrip); +} + +static void test_reftable_obj_record_roundtrip(void) +{ + uint8_t testHash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 4, 0 }; + uint64_t till9[] = { 1, 2, 3, 4, 500, 600, 700, 800, 9000 }; + struct reftable_obj_record recs[3] = { { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + .offsets = till9, + .offset_len = 3, + }, + { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + .offsets = till9, + .offset_len = 9, + }, + { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + } }; + int i = 0; + for (i = 0; i < ARRAY_SIZE(recs); i++) { + struct reftable_obj_record in = recs[i]; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct reftable_record rec = { NULL }; + struct strbuf key = STRBUF_INIT; + struct reftable_obj_record out = { NULL }; + struct reftable_record rec_out = { NULL }; + int n, m; + uint8_t extra; + + reftable_record_from_obj(&rec, &in); + test_copy(&rec); + reftable_record_key(&rec, &key); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + extra = reftable_record_val_type(&rec); + reftable_record_from_obj(&rec_out, &out); + m = reftable_record_decode(&rec_out, key, extra, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(in.hash_prefix_len == out.hash_prefix_len); + EXPECT(in.offset_len == out.offset_len); + + EXPECT(!memcmp(in.hash_prefix, out.hash_prefix, + in.hash_prefix_len)); + EXPECT(0 == memcmp(in.offsets, out.offsets, + sizeof(uint64_t) * in.offset_len)); + strbuf_release(&key); + reftable_record_release(&rec_out); + } +} + +static void test_reftable_index_record_roundtrip(void) +{ + struct reftable_index_record in = { + .offset = 42, + .last_key = STRBUF_INIT, + }; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct strbuf key = STRBUF_INIT; + struct reftable_record rec = { NULL }; + struct reftable_index_record out = { .last_key = STRBUF_INIT }; + struct reftable_record out_rec = { NULL }; + int n, m; + uint8_t extra; + + strbuf_addstr(&in.last_key, "refs/heads/master"); + reftable_record_from_index(&rec, &in); + reftable_record_key(&rec, &key); + test_copy(&rec); + + EXPECT(0 == strbuf_cmp(&key, &in.last_key)); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + + extra = reftable_record_val_type(&rec); + reftable_record_from_index(&out_rec, &out); + m = reftable_record_decode(&out_rec, key, extra, dest, GIT_SHA1_RAWSZ); + EXPECT(m == n); + + EXPECT(in.offset == out.offset); + + reftable_record_release(&out_rec); + strbuf_release(&key); + strbuf_release(&in.last_key); +} + +int record_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_reftable_log_record_equal); + RUN_TEST(test_reftable_log_record_roundtrip); + RUN_TEST(test_reftable_ref_record_roundtrip); + RUN_TEST(test_varint_roundtrip); + RUN_TEST(test_key_roundtrip); + RUN_TEST(test_common_prefix); + RUN_TEST(test_reftable_obj_record_roundtrip); + RUN_TEST(test_reftable_index_record_roundtrip); + RUN_TEST(test_u24_roundtrip); + return 0; +} diff --git a/reftable/refname.c b/reftable/refname.c new file mode 100644 index 0000000000..9573496932 --- /dev/null +++ b/reftable/refname.c @@ -0,0 +1,209 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "reftable-error.h" +#include "basics.h" +#include "refname.h" +#include "reftable-iterator.h" + +struct find_arg { + char **names; + const char *want; +}; + +static int find_name(size_t k, void *arg) +{ + struct find_arg *f_arg = arg; + return strcmp(f_arg->names[k], f_arg->want) >= 0; +} + +static int modification_has_ref(struct modification *mod, const char *name) +{ + struct reftable_ref_record ref = { NULL }; + int err = 0; + + if (mod->add_len > 0) { + struct find_arg arg = { + .names = mod->add, + .want = name, + }; + int idx = binsearch(mod->add_len, find_name, &arg); + if (idx < mod->add_len && !strcmp(mod->add[idx], name)) { + return 0; + } + } + + if (mod->del_len > 0) { + struct find_arg arg = { + .names = mod->del, + .want = name, + }; + int idx = binsearch(mod->del_len, find_name, &arg); + if (idx < mod->del_len && !strcmp(mod->del[idx], name)) { + return 1; + } + } + + err = reftable_table_read_ref(&mod->tab, name, &ref); + reftable_ref_record_release(&ref); + return err; +} + +static void modification_release(struct modification *mod) +{ + /* don't delete the strings themselves; they're owned by ref records. + */ + FREE_AND_NULL(mod->add); + FREE_AND_NULL(mod->del); + mod->add_len = 0; + mod->del_len = 0; +} + +static int modification_has_ref_with_prefix(struct modification *mod, + const char *prefix) +{ + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + int err = 0; + + if (mod->add_len > 0) { + struct find_arg arg = { + .names = mod->add, + .want = prefix, + }; + int idx = binsearch(mod->add_len, find_name, &arg); + if (idx < mod->add_len && + !strncmp(prefix, mod->add[idx], strlen(prefix))) + goto done; + } + err = reftable_table_seek_ref(&mod->tab, &it, prefix); + if (err) + goto done; + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err) + goto done; + + if (mod->del_len > 0) { + struct find_arg arg = { + .names = mod->del, + .want = ref.refname, + }; + int idx = binsearch(mod->del_len, find_name, &arg); + if (idx < mod->del_len && + !strcmp(ref.refname, mod->del[idx])) { + continue; + } + } + + if (strncmp(ref.refname, prefix, strlen(prefix))) { + err = 1; + goto done; + } + err = 0; + goto done; + } + +done: + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + return err; +} + +static int validate_refname(const char *name) +{ + while (1) { + char *next = strchr(name, '/'); + if (!*name) { + return REFTABLE_REFNAME_ERROR; + } + if (!next) { + return 0; + } + if (next - name == 0 || (next - name == 1 && *name == '.') || + (next - name == 2 && name[0] == '.' && name[1] == '.')) + return REFTABLE_REFNAME_ERROR; + name = next + 1; + } + return 0; +} + +int validate_ref_record_addition(struct reftable_table tab, + struct reftable_ref_record *recs, size_t sz) +{ + struct modification mod = { + .tab = tab, + .add = reftable_calloc(sizeof(char *) * sz), + .del = reftable_calloc(sizeof(char *) * sz), + }; + int i = 0; + int err = 0; + for (; i < sz; i++) { + if (reftable_ref_record_is_deletion(&recs[i])) { + mod.del[mod.del_len++] = recs[i].refname; + } else { + mod.add[mod.add_len++] = recs[i].refname; + } + } + + err = modification_validate(&mod); + modification_release(&mod); + return err; +} + +static void strbuf_trim_component(struct strbuf *sl) +{ + while (sl->len > 0) { + int is_slash = (sl->buf[sl->len - 1] == '/'); + strbuf_setlen(sl, sl->len - 1); + if (is_slash) + break; + } +} + +int modification_validate(struct modification *mod) +{ + struct strbuf slashed = STRBUF_INIT; + int err = 0; + int i = 0; + for (; i < mod->add_len; i++) { + err = validate_refname(mod->add[i]); + if (err) + goto done; + strbuf_reset(&slashed); + strbuf_addstr(&slashed, mod->add[i]); + strbuf_addstr(&slashed, "/"); + + err = modification_has_ref_with_prefix(mod, slashed.buf); + if (err == 0) { + err = REFTABLE_NAME_CONFLICT; + goto done; + } + if (err < 0) + goto done; + + strbuf_reset(&slashed); + strbuf_addstr(&slashed, mod->add[i]); + while (slashed.len) { + strbuf_trim_component(&slashed); + err = modification_has_ref(mod, slashed.buf); + if (err == 0) { + err = REFTABLE_NAME_CONFLICT; + goto done; + } + if (err < 0) + goto done; + } + } + err = 0; +done: + strbuf_release(&slashed); + return err; +} diff --git a/reftable/refname.h b/reftable/refname.h new file mode 100644 index 0000000000..a24b40fcb4 --- /dev/null +++ b/reftable/refname.h @@ -0,0 +1,29 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { + struct reftable_table tab; + + char **add; + size_t add_len; + + char **del; + size_t del_len; +}; + +int validate_ref_record_addition(struct reftable_table tab, + struct reftable_ref_record *recs, size_t sz); + +int modification_validate(struct modification *mod); + +#endif diff --git a/reftable/refname_test.c b/reftable/refname_test.c new file mode 100644 index 0000000000..8645cd93bb --- /dev/null +++ b/reftable/refname_test.c @@ -0,0 +1,102 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "block.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "refname.h" +#include "reftable-error.h" +#include "reftable-writer.h" +#include "system.h" + +#include "test_framework.h" +#include "reftable-tests.h" + +struct testcase { + char *add; + char *del; + int error_code; +}; + +static void test_conflict(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_ref_record rec = { + .refname = "a/b", + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "destination", /* make sure it's not a symref. + */ + .update_index = 1, + }; + int err; + int i; + struct reftable_block_source source = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_table tab = { NULL }; + struct testcase cases[] = { + { "a/b/c", NULL, REFTABLE_NAME_CONFLICT }, + { "b", NULL, 0 }, + { "a", NULL, REFTABLE_NAME_CONFLICT }, + { "a", "a/b", 0 }, + + { "p/", NULL, REFTABLE_REFNAME_ERROR }, + { "p//q", NULL, REFTABLE_REFNAME_ERROR }, + { "p/./q", NULL, REFTABLE_REFNAME_ERROR }, + { "p/../q", NULL, REFTABLE_REFNAME_ERROR }, + + { "a/b/c", "a/b", 0 }, + { NULL, "a//b", 0 }, + }; + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_add_ref(w, &rec); + EXPECT_ERR(err); + + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + + block_source_from_strbuf(&source, &buf); + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + reftable_table_from_reader(&tab, rd); + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct modification mod = { + .tab = tab, + }; + + if (cases[i].add) { + mod.add = &cases[i].add; + mod.add_len = 1; + } + if (cases[i].del) { + mod.del = &cases[i].del; + mod.del_len = 1; + } + + err = modification_validate(&mod); + EXPECT(err == cases[i].error_code); + } + + reftable_reader_free(rd); + strbuf_release(&buf); +} + +int refname_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_conflict); + return 0; +} diff --git a/reftable/reftable-blocksource.h b/reftable/reftable-blocksource.h new file mode 100644 index 0000000000..5aa3990a57 --- /dev/null +++ b/reftable/reftable-blocksource.h @@ -0,0 +1,49 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_BLOCKSOURCE_H +#define REFTABLE_BLOCKSOURCE_H + +#include <stdint.h> + +/* block_source is a generic wrapper for a seekable readable file. + */ +struct reftable_block_source { + struct reftable_block_source_vtable *ops; + void *arg; +}; + +/* a contiguous segment of bytes. It keeps track of its generating block_source + * so it can return itself into the pool. */ +struct reftable_block { + uint8_t *data; + int len; + struct reftable_block_source source; +}; + +/* block_source_vtable are the operations that make up block_source */ +struct reftable_block_source_vtable { + /* returns the size of a block source */ + uint64_t (*size)(void *source); + + /* reads a segment from the block source. It is an error to read + beyond the end of the block */ + int (*read_block)(void *source, struct reftable_block *dest, + uint64_t off, uint32_t size); + /* mark the block as read; may return the data back to malloc */ + void (*return_block)(void *source, struct reftable_block *blockp); + + /* release all resources associated with the block source */ + void (*close)(void *source); +}; + +/* opens a file on the file system as a block_source */ +int reftable_block_source_from_file(struct reftable_block_source *block_src, + const char *name); + +#endif diff --git a/reftable/reftable-error.h b/reftable/reftable-error.h new file mode 100644 index 0000000000..6f89bedf1a --- /dev/null +++ b/reftable/reftable-error.h @@ -0,0 +1,62 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ERROR_H +#define REFTABLE_ERROR_H + +/* + * Errors in reftable calls are signaled with negative integer return values. 0 + * means success. + */ +enum reftable_error { + /* Unexpected file system behavior */ + REFTABLE_IO_ERROR = -2, + + /* Format inconsistency on reading data */ + REFTABLE_FORMAT_ERROR = -3, + + /* File does not exist. Returned from block_source_from_file(), because + * it needs special handling in stack. + */ + REFTABLE_NOT_EXIST_ERROR = -4, + + /* Trying to write out-of-date data. */ + REFTABLE_LOCK_ERROR = -5, + + /* Misuse of the API: + * - on writing a record with NULL refname. + * - on writing a reftable_ref_record outside the table limits + * - on writing a ref or log record before the stack's + * next_update_inde*x + * - on writing a log record with multiline message with + * exact_log_message unset + * - on reading a reftable_ref_record from log iterator, or vice versa. + * + * When a call misuses the API, the internal state of the library is + * kept unchanged. + */ + REFTABLE_API_ERROR = -6, + + /* Decompression error */ + REFTABLE_ZLIB_ERROR = -7, + + /* Wrote a table without blocks. */ + REFTABLE_EMPTY_TABLE_ERROR = -8, + + /* Dir/file conflict. */ + REFTABLE_NAME_CONFLICT = -9, + + /* Invalid ref name. */ + REFTABLE_REFNAME_ERROR = -10, +}; + +/* convert the numeric error code to a string. The string should not be + * deallocated. */ +const char *reftable_error_str(int err); + +#endif diff --git a/reftable/reftable-generic.h b/reftable/reftable-generic.h new file mode 100644 index 0000000000..d239751a77 --- /dev/null +++ b/reftable/reftable-generic.h @@ -0,0 +1,47 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_GENERIC_H +#define REFTABLE_GENERIC_H + +#include "reftable-iterator.h" + +struct reftable_table_vtable; + +/* + * Provides a unified API for reading tables, either merged tables, or single + * readers. */ +struct reftable_table { + struct reftable_table_vtable *ops; + void *table_arg; +}; + +int reftable_table_seek_log(struct reftable_table *tab, + struct reftable_iterator *it, const char *name); + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name); + +/* returns the hash ID from a generic reftable_table */ +uint32_t reftable_table_hash_id(struct reftable_table *tab); + +/* returns the max update_index covered by this table. */ +uint64_t reftable_table_max_update_index(struct reftable_table *tab); + +/* returns the min update_index covered by this table. */ +uint64_t reftable_table_min_update_index(struct reftable_table *tab); + +/* convenience function to read a single ref. Returns < 0 for error, 0 + for success, and 1 if ref not found. */ +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref); + +/* dump table contents onto stdout for debugging */ +int reftable_table_print(struct reftable_table *tab); + +#endif diff --git a/reftable/reftable-iterator.h b/reftable/reftable-iterator.h new file mode 100644 index 0000000000..d3eee7af35 --- /dev/null +++ b/reftable/reftable-iterator.h @@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable; + +/* iterator is the generic interface for walking over data stored in a + * reftable. + */ +struct reftable_iterator { + struct reftable_iterator_vtable *ops; + void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: + * end of iteration. + */ +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: + * end of iteration. + */ +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy(struct reftable_iterator *it); + +#endif diff --git a/reftable/reftable-malloc.h b/reftable/reftable-malloc.h new file mode 100644 index 0000000000..5f2185f1f3 --- /dev/null +++ b/reftable/reftable-malloc.h @@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)); + +#endif diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h new file mode 100644 index 0000000000..1a6d16915a --- /dev/null +++ b/reftable/reftable-merged.h @@ -0,0 +1,72 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_MERGED_H +#define REFTABLE_MERGED_H + +#include "reftable-iterator.h" + +/* + * Merged tables + * + * A ref database kept in a sequence of table files. The merged_table presents a + * unified view to reading (seeking, iterating) a sequence of immutable tables. + * + * The merged tables are on purpose kept disconnected from their actual storage + * (eg. files on disk), because it is useful to merge tables aren't files. For + * example, the per-workspace and global ref namespace can be implemented as a + * merged table of two stacks of file-backed reftables. + */ + +/* A merged table is implements seeking/iterating over a stack of tables. */ +struct reftable_merged_table; + +/* A generic reftable; see below. */ +struct reftable_table; + +/* reftable_new_merged_table creates a new merged table. It takes ownership of + the stack array. +*/ +int reftable_new_merged_table(struct reftable_merged_table **dest, + struct reftable_table *stack, int n, + uint32_t hash_id); + +/* returns an iterator positioned just before 'name' */ +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name); + +/* returns an iterator for log entry, at given update_index */ +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name, uint64_t update_index); + +/* like reftable_merged_table_seek_log_at but look for the newest entry. */ +int reftable_merged_table_seek_log(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name); + +/* returns the max update_index covered by this merged table. */ +uint64_t +reftable_merged_table_max_update_index(struct reftable_merged_table *mt); + +/* returns the min update_index covered by this merged table. */ +uint64_t +reftable_merged_table_min_update_index(struct reftable_merged_table *mt); + +/* releases memory for the merged_table */ +void reftable_merged_table_free(struct reftable_merged_table *m); + +/* return the hash ID of the merged table. */ +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *m); + +/* create a generic table from reftable_merged_table */ +void reftable_table_from_merged_table(struct reftable_table *tab, + struct reftable_merged_table *table); + +#endif diff --git a/reftable/reftable-reader.h b/reftable/reftable-reader.h new file mode 100644 index 0000000000..4a4bc2fdf8 --- /dev/null +++ b/reftable/reftable-reader.h @@ -0,0 +1,101 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_READER_H +#define REFTABLE_READER_H + +#include "reftable-iterator.h" +#include "reftable-blocksource.h" + +/* + * Reading single tables + * + * The follow routines are for reading single files. For an + * application-level interface, skip ahead to struct + * reftable_merged_table and struct reftable_stack. + */ + +/* The reader struct is a handle to an open reftable file. */ +struct reftable_reader; + +/* Generic table. */ +struct reftable_table; + +/* reftable_new_reader opens a reftable for reading. If successful, + * returns 0 code and sets pp. The name is used for creating a + * stack. Typically, it is the basename of the file. The block source + * `src` is owned by the reader, and is closed on calling + * reftable_reader_destroy(). On error, the block source `src` is + * closed as well. + */ +int reftable_new_reader(struct reftable_reader **pp, + struct reftable_block_source *src, const char *name); + +/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted + in the table. To seek to the start of the table, use name = "". + + example: + + struct reftable_reader *r = NULL; + int err = reftable_new_reader(&r, &src, "filename"); + if (err < 0) { ... } + struct reftable_iterator it = {0}; + err = reftable_reader_seek_ref(r, &it, "refs/heads/master"); + if (err < 0) { ... } + struct reftable_ref_record ref = {0}; + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) { + ..error handling.. + } + ..found.. + } + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); +*/ +int reftable_reader_seek_ref(struct reftable_reader *r, + struct reftable_iterator *it, const char *name); + +/* returns the hash ID used in this table. */ +uint32_t reftable_reader_hash_id(struct reftable_reader *r); + +/* seek to logs for the given name, older than update_index. To seek to the + start of the table, use name = "". +*/ +int reftable_reader_seek_log_at(struct reftable_reader *r, + struct reftable_iterator *it, const char *name, + uint64_t update_index); + +/* seek to newest log entry for given name. */ +int reftable_reader_seek_log(struct reftable_reader *r, + struct reftable_iterator *it, const char *name); + +/* closes and deallocates a reader. */ +void reftable_reader_free(struct reftable_reader *); + +/* return an iterator for the refs pointing to `oid`. */ +int reftable_reader_refs_for(struct reftable_reader *r, + struct reftable_iterator *it, uint8_t *oid); + +/* return the max_update_index for a table */ +uint64_t reftable_reader_max_update_index(struct reftable_reader *r); + +/* return the min_update_index for a table */ +uint64_t reftable_reader_min_update_index(struct reftable_reader *r); + +/* creates a generic table from a file reader. */ +void reftable_table_from_reader(struct reftable_table *tab, + struct reftable_reader *reader); + +/* print table onto stdout for debugging. */ +int reftable_reader_print_file(const char *tablename); + +#endif diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h new file mode 100644 index 0000000000..5370d2288c --- /dev/null +++ b/reftable/reftable-record.h @@ -0,0 +1,114 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_RECORD_H +#define REFTABLE_RECORD_H + +#include <stdint.h> + +/* + * Basic data types + * + * Reftables store the state of each ref in struct reftable_ref_record, and they + * store a sequence of reflog updates in struct reftable_log_record. + */ + +/* reftable_ref_record holds a ref database entry target_value */ +struct reftable_ref_record { + char *refname; /* Name of the ref, malloced. */ + uint64_t update_index; /* Logical timestamp at which this value is + * written */ + + enum { + /* tombstone to hide deletions from earlier tables */ + REFTABLE_REF_DELETION = 0x0, + + /* a simple ref */ + REFTABLE_REF_VAL1 = 0x1, + /* a tag, plus its peeled hash */ + REFTABLE_REF_VAL2 = 0x2, + + /* a symbolic reference */ + REFTABLE_REF_SYMREF = 0x3, +#define REFTABLE_NR_REF_VALUETYPES 4 + } value_type; + union { + uint8_t *val1; /* malloced hash. */ + struct { + uint8_t *value; /* first value, malloced hash */ + uint8_t *target_value; /* second value, malloced hash */ + } val2; + char *symref; /* referent, malloced 0-terminated string */ + } value; +}; + +/* Returns the first hash, or NULL if `rec` is not of type + * REFTABLE_REF_VAL1 or REFTABLE_REF_VAL2. */ +uint8_t *reftable_ref_record_val1(struct reftable_ref_record *rec); + +/* Returns the second hash, or NULL if `rec` is not of type + * REFTABLE_REF_VAL2. */ +uint8_t *reftable_ref_record_val2(struct reftable_ref_record *rec); + +/* returns whether 'ref' represents a deletion */ +int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref); + +/* prints a reftable_ref_record onto stdout. Useful for debugging. */ +void reftable_ref_record_print(struct reftable_ref_record *ref, + uint32_t hash_id); + +/* frees and nulls all pointer values inside `ref`. */ +void reftable_ref_record_release(struct reftable_ref_record *ref); + +/* returns whether two reftable_ref_records are the same. Useful for testing. */ +int reftable_ref_record_equal(struct reftable_ref_record *a, + struct reftable_ref_record *b, int hash_size); + +/* reftable_log_record holds a reflog entry */ +struct reftable_log_record { + char *refname; + uint64_t update_index; /* logical timestamp of a transactional update. + */ + + enum { + /* tombstone to hide deletions from earlier tables */ + REFTABLE_LOG_DELETION = 0x0, + + /* a simple update */ + REFTABLE_LOG_UPDATE = 0x1, +#define REFTABLE_NR_LOG_VALUETYPES 2 + } value_type; + + union { + struct { + uint8_t *new_hash; + uint8_t *old_hash; + char *name; + char *email; + uint64_t time; + int16_t tz_offset; + char *message; + } update; + } value; +}; + +/* returns whether 'ref' represents the deletion of a log record. */ +int reftable_log_record_is_deletion(const struct reftable_log_record *log); + +/* frees and nulls all pointer values. */ +void reftable_log_record_release(struct reftable_log_record *log); + +/* returns whether two records are equal. Useful for testing. */ +int reftable_log_record_equal(struct reftable_log_record *a, + struct reftable_log_record *b, int hash_size); + +/* dumps a reftable_log_record on stdout, for debugging/testing. */ +void reftable_log_record_print(struct reftable_log_record *log, + uint32_t hash_id); + +#endif diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h new file mode 100644 index 0000000000..1b602dda58 --- /dev/null +++ b/reftable/reftable-stack.h @@ -0,0 +1,128 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_STACK_H +#define REFTABLE_STACK_H + +#include "reftable-writer.h" + +/* + * The stack presents an interface to a mutable sequence of reftables. + + * A stack can be mutated by pushing a table to the top of the stack. + + * The reftable_stack automatically compacts files on disk to ensure good + * amortized performance. + * + * For windows and other platforms that cannot have open files as rename + * destinations, concurrent access from multiple processes needs the rand() + * random seed to be randomized. + */ +struct reftable_stack; + +/* open a new reftable stack. The tables along with the table list will be + * stored in 'dir'. Typically, this should be .git/reftables. + */ +int reftable_new_stack(struct reftable_stack **dest, const char *dir, + struct reftable_write_options config); + +/* returns the update_index at which a next table should be written. */ +uint64_t reftable_stack_next_update_index(struct reftable_stack *st); + +/* holds a transaction to add tables at the top of a stack. */ +struct reftable_addition; + +/* + * returns a new transaction to add reftables to the given stack. As a side + * effect, the ref database is locked. + */ +int reftable_stack_new_addition(struct reftable_addition **dest, + struct reftable_stack *st); + +/* Adds a reftable to transaction. */ +int reftable_addition_add(struct reftable_addition *add, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg); + +/* Commits the transaction, releasing the lock. After calling this, + * reftable_addition_destroy should still be called. + */ +int reftable_addition_commit(struct reftable_addition *add); + +/* Release all non-committed data from the transaction, and deallocate the + * transaction. Releases the lock if held. */ +void reftable_addition_destroy(struct reftable_addition *add); + +/* add a new table to the stack. The write_table function must call + * reftable_writer_set_limits, add refs and return an error value. */ +int reftable_stack_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *write_arg), + void *write_arg); + +/* returns the merged_table for seeking. This table is valid until the + * next write or reload, and should not be closed or deleted. + */ +struct reftable_merged_table * +reftable_stack_merged_table(struct reftable_stack *st); + +/* frees all resources associated with the stack. */ +void reftable_stack_destroy(struct reftable_stack *st); + +/* Reloads the stack if necessary. This is very cheap to run if the stack was up + * to date */ +int reftable_stack_reload(struct reftable_stack *st); + +/* Policy for expiring reflog entries. */ +struct reftable_log_expiry_config { + /* Drop entries older than this timestamp */ + uint64_t time; + + /* Drop older entries */ + uint64_t min_update_index; +}; + +/* compacts all reftables into a giant table. Expire reflog entries if config is + * non-NULL */ +int reftable_stack_compact_all(struct reftable_stack *st, + struct reftable_log_expiry_config *config); + +/* heuristically compact unbalanced table stack. */ +int reftable_stack_auto_compact(struct reftable_stack *st); + +/* delete stale .ref tables. */ +int reftable_stack_clean(struct reftable_stack *st); + +/* convenience function to read a single ref. Returns < 0 for error, 0 for + * success, and 1 if ref not found. */ +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, + struct reftable_ref_record *ref); + +/* convenience function to read a single log. Returns < 0 for error, 0 for + * success, and 1 if ref not found. */ +int reftable_stack_read_log(struct reftable_stack *st, const char *refname, + struct reftable_log_record *log); + +/* statistics on past compactions. */ +struct reftable_compaction_stats { + uint64_t bytes; /* total number of bytes written */ + uint64_t entries_written; /* total number of entries written, including + failures. */ + int attempts; /* how often we tried to compact */ + int failures; /* failures happen on concurrent updates */ +}; + +/* return statistics for compaction up till now. */ +struct reftable_compaction_stats * +reftable_stack_compaction_stats(struct reftable_stack *st); + +/* print the entire stack represented by the directory */ +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id); + +#endif diff --git a/reftable/reftable-tests.h b/reftable/reftable-tests.h new file mode 100644 index 0000000000..0019cbcfa4 --- /dev/null +++ b/reftable/reftable-tests.h @@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main(int argc, const char **argv); +int block_test_main(int argc, const char **argv); +int merged_test_main(int argc, const char **argv); +int pq_test_main(int argc, const char **argv); +int record_test_main(int argc, const char **argv); +int refname_test_main(int argc, const char **argv); +int readwrite_test_main(int argc, const char **argv); +int stack_test_main(int argc, const char **argv); +int tree_test_main(int argc, const char **argv); +int reftable_dump_main(int argc, char *const *argv); + +#endif diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h new file mode 100644 index 0000000000..af36462ced --- /dev/null +++ b/reftable/reftable-writer.h @@ -0,0 +1,148 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_WRITER_H +#define REFTABLE_WRITER_H + +#include "reftable-record.h" + +#include <stdint.h> +#include <unistd.h> /* ssize_t */ + +/* Writing single reftables */ + +/* reftable_write_options sets options for writing a single reftable. */ +struct reftable_write_options { + /* boolean: do not pad out blocks to block size. */ + unsigned unpadded : 1; + + /* the blocksize. Should be less than 2^24. */ + uint32_t block_size; + + /* boolean: do not generate a SHA1 => ref index. */ + unsigned skip_index_objects : 1; + + /* how often to write complete keys in each block. */ + int restart_interval; + + /* 4-byte identifier ("sha1", "s256") of the hash. + * Defaults to SHA1 if unset + */ + uint32_t hash_id; + + /* boolean: do not check ref names for validity or dir/file conflicts. + */ + unsigned skip_name_check : 1; + + /* boolean: copy log messages exactly. If unset, check that the message + * is a single line, and add '\n' if missing. + */ + unsigned exact_log_message : 1; +}; + +/* reftable_block_stats holds statistics for a single block type */ +struct reftable_block_stats { + /* total number of entries written */ + int entries; + /* total number of key restarts */ + int restarts; + /* total number of blocks */ + int blocks; + /* total number of index blocks */ + int index_blocks; + /* depth of the index */ + int max_index_level; + + /* offset of the first block for this type */ + uint64_t offset; + /* offset of the top level index block for this type, or 0 if not + * present */ + uint64_t index_offset; +}; + +/* stats holds overall statistics for a single reftable */ +struct reftable_stats { + /* total number of blocks written. */ + int blocks; + /* stats for ref data */ + struct reftable_block_stats ref_stats; + /* stats for the SHA1 to ref map. */ + struct reftable_block_stats obj_stats; + /* stats for index blocks */ + struct reftable_block_stats idx_stats; + /* stats for log blocks */ + struct reftable_block_stats log_stats; + + /* disambiguation length of shortened object IDs. */ + int object_id_len; +}; + +/* reftable_new_writer creates a new writer */ +struct reftable_writer * +reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + void *writer_arg, struct reftable_write_options *opts); + +/* Set the range of update indices for the records we will add. When writing a + table into a stack, the min should be at least + reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned. + + For transactional updates to a stack, typically min==max, and the + update_index can be obtained by inspeciting the stack. When converting an + existing ref database into a single reftable, this would be a range of + update-index timestamps. + */ +void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min, + uint64_t max); + +/* + Add a reftable_ref_record. The record should have names that come after + already added records. + + The update_index must be within the limits set by + reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned. It is an + REFTABLE_API_ERROR error to write a ref record after a log record. +*/ +int reftable_writer_add_ref(struct reftable_writer *w, + struct reftable_ref_record *ref); + +/* + Convenience function to add multiple reftable_ref_records; the function sorts + the records before adding them, reordering the records array passed in. +*/ +int reftable_writer_add_refs(struct reftable_writer *w, + struct reftable_ref_record *refs, int n); + +/* + adds reftable_log_records. Log records are keyed by (refname, decreasing + update_index). The key for the record added must come after the already added + log records. +*/ +int reftable_writer_add_log(struct reftable_writer *w, + struct reftable_log_record *log); + +/* + Convenience function to add multiple reftable_log_records; the function sorts + the records before adding them, reordering records array passed in. +*/ +int reftable_writer_add_logs(struct reftable_writer *w, + struct reftable_log_record *logs, int n); + +/* reftable_writer_close finalizes the reftable. The writer is retained so + * statistics can be inspected. */ +int reftable_writer_close(struct reftable_writer *w); + +/* writer_stats returns the statistics on the reftable being written. + + This struct becomes invalid when the writer is freed. + */ +const struct reftable_stats *writer_stats(struct reftable_writer *w); + +/* reftable_writer_free deallocates memory for the writer */ +void reftable_writer_free(struct reftable_writer *w); + +#endif diff --git a/reftable/reftable.c b/reftable/reftable.c new file mode 100644 index 0000000000..0e4607a7cd --- /dev/null +++ b/reftable/reftable.c @@ -0,0 +1,115 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "record.h" +#include "generic.h" +#include "reftable-iterator.h" +#include "reftable-generic.h" + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref) +{ + struct reftable_iterator it = { NULL }; + int err = reftable_table_seek_ref(tab, &it, name); + if (err) + goto done; + + err = reftable_iterator_next_ref(&it, ref); + if (err) + goto done; + + if (strcmp(ref->refname, name) || + reftable_ref_record_is_deletion(ref)) { + reftable_ref_record_release(ref); + err = 1; + goto done; + } + +done: + reftable_iterator_destroy(&it); + return err; +} + +uint64_t reftable_table_max_update_index(struct reftable_table *tab) +{ + return tab->ops->max_update_index(tab->table_arg); +} + +uint64_t reftable_table_min_update_index(struct reftable_table *tab) +{ + return tab->ops->min_update_index(tab->table_arg); +} + +uint32_t reftable_table_hash_id(struct reftable_table *tab) +{ + return tab->ops->hash_id(tab->table_arg); +} + +void reftable_iterator_destroy(struct reftable_iterator *it) +{ + if (!it->ops) { + return; + } + it->ops->close(it->iter_arg); + it->ops = NULL; + FREE_AND_NULL(it->iter_arg); +} + +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, ref); + return iterator_next(it, &rec); +} + +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, log); + return iterator_next(it, &rec); +} + +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) +{ + return it->ops->next(it->iter_arg, rec); +} + +static int empty_iterator_next(void *arg, struct reftable_record *rec) +{ + return 1; +} + +static void empty_iterator_close(void *arg) +{ +} + +static struct reftable_iterator_vtable empty_vtable = { + .next = &empty_iterator_next, + .close = &empty_iterator_close, +}; + +void iterator_set_empty(struct reftable_iterator *it) +{ + assert(!it->ops); + it->iter_arg = NULL; + it->ops = &empty_vtable; +} diff --git a/reftable/stack.c b/reftable/stack.c new file mode 100644 index 0000000000..df5021ebf0 --- /dev/null +++ b/reftable/stack.c @@ -0,0 +1,1396 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "stack.h" + +#include "system.h" +#include "merged.h" +#include "reader.h" +#include "refname.h" +#include "reftable-error.h" +#include "reftable-record.h" +#include "reftable-merged.h" +#include "writer.h" + +static int stack_try_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg); +static int stack_write_compact(struct reftable_stack *st, + struct reftable_writer *wr, int first, int last, + struct reftable_log_expiry_config *config); +static int stack_check_addition(struct reftable_stack *st, + const char *new_tab_name); +static void reftable_addition_close(struct reftable_addition *add); +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, + int reuse_open); + +static void stack_filename(struct strbuf *dest, struct reftable_stack *st, + const char *name) +{ + strbuf_reset(dest); + strbuf_addstr(dest, st->reftable_dir); + strbuf_addstr(dest, "/"); + strbuf_addstr(dest, name); +} + +static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz) +{ + int *fdp = (int *)arg; + return write(*fdp, data, sz); +} + +int reftable_new_stack(struct reftable_stack **dest, const char *dir, + struct reftable_write_options config) +{ + struct reftable_stack *p = + reftable_calloc(sizeof(struct reftable_stack)); + struct strbuf list_file_name = STRBUF_INIT; + int err = 0; + + if (config.hash_id == 0) { + config.hash_id = GIT_SHA1_FORMAT_ID; + } + + *dest = NULL; + + strbuf_reset(&list_file_name); + strbuf_addstr(&list_file_name, dir); + strbuf_addstr(&list_file_name, "/tables.list"); + + p->list_file = strbuf_detach(&list_file_name, NULL); + p->reftable_dir = xstrdup(dir); + p->config = config; + + err = reftable_stack_reload_maybe_reuse(p, 1); + if (err < 0) { + reftable_stack_destroy(p); + } else { + *dest = p; + } + return err; +} + +static int fd_read_lines(int fd, char ***namesp) +{ + off_t size = lseek(fd, 0, SEEK_END); + char *buf = NULL; + int err = 0; + if (size < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + err = lseek(fd, 0, SEEK_SET); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + buf = reftable_malloc(size + 1); + if (read(fd, buf, size) != size) { + err = REFTABLE_IO_ERROR; + goto done; + } + buf[size] = 0; + + parse_names(buf, size, namesp); + +done: + reftable_free(buf); + return err; +} + +int read_lines(const char *filename, char ***namesp) +{ + int fd = open(filename, O_RDONLY); + int err = 0; + if (fd < 0) { + if (errno == ENOENT) { + *namesp = reftable_calloc(sizeof(char *)); + return 0; + } + + return REFTABLE_IO_ERROR; + } + err = fd_read_lines(fd, namesp); + close(fd); + return err; +} + +struct reftable_merged_table * +reftable_stack_merged_table(struct reftable_stack *st) +{ + return st->merged; +} + +static int has_name(char **names, const char *name) +{ + while (*names) { + if (!strcmp(*names, name)) + return 1; + names++; + } + return 0; +} + +/* Close and free the stack */ +void reftable_stack_destroy(struct reftable_stack *st) +{ + char **names = NULL; + int err = 0; + if (st->merged) { + reftable_merged_table_free(st->merged); + st->merged = NULL; + } + + err = read_lines(st->list_file, &names); + if (err < 0) { + FREE_AND_NULL(names); + } + + if (st->readers) { + int i = 0; + struct strbuf filename = STRBUF_INIT; + for (i = 0; i < st->readers_len; i++) { + const char *name = reader_name(st->readers[i]); + strbuf_reset(&filename); + if (names && !has_name(names, name)) { + stack_filename(&filename, st, name); + } + reftable_reader_free(st->readers[i]); + + if (filename.len) { + /* On Windows, can only unlink after closing. */ + unlink(filename.buf); + } + } + strbuf_release(&filename); + st->readers_len = 0; + FREE_AND_NULL(st->readers); + } + FREE_AND_NULL(st->list_file); + FREE_AND_NULL(st->reftable_dir); + reftable_free(st); + free_names(names); +} + +static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, + int cur_len) +{ + struct reftable_reader **cur = + reftable_calloc(sizeof(struct reftable_reader *) * cur_len); + int i = 0; + for (i = 0; i < cur_len; i++) { + cur[i] = st->readers[i]; + } + return cur; +} + +static int reftable_stack_reload_once(struct reftable_stack *st, char **names, + int reuse_open) +{ + int cur_len = !st->merged ? 0 : st->merged->stack_len; + struct reftable_reader **cur = stack_copy_readers(st, cur_len); + int err = 0; + int names_len = names_length(names); + struct reftable_reader **new_readers = + reftable_calloc(sizeof(struct reftable_reader *) * names_len); + struct reftable_table *new_tables = + reftable_calloc(sizeof(struct reftable_table) * names_len); + int new_readers_len = 0; + struct reftable_merged_table *new_merged = NULL; + int i; + + while (*names) { + struct reftable_reader *rd = NULL; + char *name = *names++; + + /* this is linear; we assume compaction keeps the number of + tables under control so this is not quadratic. */ + int j = 0; + for (j = 0; reuse_open && j < cur_len; j++) { + if (cur[j] && 0 == strcmp(cur[j]->name, name)) { + rd = cur[j]; + cur[j] = NULL; + break; + } + } + + if (!rd) { + struct reftable_block_source src = { NULL }; + struct strbuf table_path = STRBUF_INIT; + stack_filename(&table_path, st, name); + + err = reftable_block_source_from_file(&src, + table_path.buf); + strbuf_release(&table_path); + + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, name); + if (err < 0) + goto done; + } + + new_readers[new_readers_len] = rd; + reftable_table_from_reader(&new_tables[new_readers_len], rd); + new_readers_len++; + } + + /* success! */ + err = reftable_new_merged_table(&new_merged, new_tables, + new_readers_len, st->config.hash_id); + if (err < 0) + goto done; + + new_tables = NULL; + st->readers_len = new_readers_len; + if (st->merged) { + merged_table_release(st->merged); + reftable_merged_table_free(st->merged); + } + if (st->readers) { + reftable_free(st->readers); + } + st->readers = new_readers; + new_readers = NULL; + new_readers_len = 0; + + new_merged->suppress_deletions = 1; + st->merged = new_merged; + for (i = 0; i < cur_len; i++) { + if (cur[i]) { + const char *name = reader_name(cur[i]); + struct strbuf filename = STRBUF_INIT; + stack_filename(&filename, st, name); + + reader_close(cur[i]); + reftable_reader_free(cur[i]); + + /* On Windows, can only unlink after closing. */ + unlink(filename.buf); + + strbuf_release(&filename); + } + } + +done: + for (i = 0; i < new_readers_len; i++) { + reader_close(new_readers[i]); + reftable_reader_free(new_readers[i]); + } + reftable_free(new_readers); + reftable_free(new_tables); + reftable_free(cur); + return err; +} + +/* return negative if a before b. */ +static int tv_cmp(struct timeval *a, struct timeval *b) +{ + time_t diff = a->tv_sec - b->tv_sec; + int udiff = a->tv_usec - b->tv_usec; + + if (diff != 0) + return diff; + + return udiff; +} + +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, + int reuse_open) +{ + struct timeval deadline = { 0 }; + int err = gettimeofday(&deadline, NULL); + int64_t delay = 0; + int tries = 0; + if (err < 0) + return err; + + deadline.tv_sec += 3; + while (1) { + char **names = NULL; + char **names_after = NULL; + struct timeval now = { 0 }; + int err = gettimeofday(&now, NULL); + int err2 = 0; + if (err < 0) { + return err; + } + + /* Only look at deadlines after the first few times. This + simplifies debugging in GDB */ + tries++; + if (tries > 3 && tv_cmp(&now, &deadline) >= 0) { + break; + } + + err = read_lines(st->list_file, &names); + if (err < 0) { + free_names(names); + return err; + } + err = reftable_stack_reload_once(st, names, reuse_open); + if (err == 0) { + free_names(names); + break; + } + if (err != REFTABLE_NOT_EXIST_ERROR) { + free_names(names); + return err; + } + + /* err == REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent + writer. Check if there was one by checking if the name list + changed. + */ + err2 = read_lines(st->list_file, &names_after); + if (err2 < 0) { + free_names(names); + return err2; + } + + if (names_equal(names_after, names)) { + free_names(names); + free_names(names_after); + return err; + } + free_names(names); + free_names(names_after); + + delay = delay + (delay * rand()) / RAND_MAX + 1; + sleep_millisec(delay); + } + + return 0; +} + +/* -1 = error + 0 = up to date + 1 = changed. */ +static int stack_uptodate(struct reftable_stack *st) +{ + char **names = NULL; + int err = read_lines(st->list_file, &names); + int i = 0; + if (err < 0) + return err; + + for (i = 0; i < st->readers_len; i++) { + if (!names[i]) { + err = 1; + goto done; + } + + if (strcmp(st->readers[i]->name, names[i])) { + err = 1; + goto done; + } + } + + if (names[st->merged->stack_len]) { + err = 1; + goto done; + } + +done: + free_names(names); + return err; +} + +int reftable_stack_reload(struct reftable_stack *st) +{ + int err = stack_uptodate(st); + if (err > 0) + return reftable_stack_reload_maybe_reuse(st, 1); + return err; +} + +int reftable_stack_add(struct reftable_stack *st, + int (*write)(struct reftable_writer *wr, void *arg), + void *arg) +{ + int err = stack_try_add(st, write, arg); + if (err < 0) { + if (err == REFTABLE_LOCK_ERROR) { + /* Ignore error return, we want to propagate + REFTABLE_LOCK_ERROR. + */ + reftable_stack_reload(st); + } + return err; + } + + if (!st->disable_auto_compact) + return reftable_stack_auto_compact(st); + + return 0; +} + +static void format_name(struct strbuf *dest, uint64_t min, uint64_t max) +{ + char buf[100]; + uint32_t rnd = (uint32_t)rand(); + snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64 "-%08x", + min, max, rnd); + strbuf_reset(dest); + strbuf_addstr(dest, buf); +} + +struct reftable_addition { + int lock_file_fd; + struct strbuf lock_file_name; + struct reftable_stack *stack; + + char **new_tables; + int new_tables_len; + uint64_t next_update_index; +}; + +#define REFTABLE_ADDITION_INIT \ + { \ + .lock_file_name = STRBUF_INIT \ + } + +static int reftable_stack_init_addition(struct reftable_addition *add, + struct reftable_stack *st) +{ + int err = 0; + add->stack = st; + + strbuf_reset(&add->lock_file_name); + strbuf_addstr(&add->lock_file_name, st->list_file); + strbuf_addstr(&add->lock_file_name, ".lock"); + + add->lock_file_fd = open(add->lock_file_name.buf, + O_EXCL | O_CREAT | O_WRONLY, 0644); + if (add->lock_file_fd < 0) { + if (errno == EEXIST) { + err = REFTABLE_LOCK_ERROR; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + err = stack_uptodate(st); + if (err < 0) + goto done; + + if (err > 1) { + err = REFTABLE_LOCK_ERROR; + goto done; + } + + add->next_update_index = reftable_stack_next_update_index(st); +done: + if (err) { + reftable_addition_close(add); + } + return err; +} + +static void reftable_addition_close(struct reftable_addition *add) +{ + int i = 0; + struct strbuf nm = STRBUF_INIT; + for (i = 0; i < add->new_tables_len; i++) { + stack_filename(&nm, add->stack, add->new_tables[i]); + unlink(nm.buf); + reftable_free(add->new_tables[i]); + add->new_tables[i] = NULL; + } + reftable_free(add->new_tables); + add->new_tables = NULL; + add->new_tables_len = 0; + + if (add->lock_file_fd > 0) { + close(add->lock_file_fd); + add->lock_file_fd = 0; + } + if (add->lock_file_name.len > 0) { + unlink(add->lock_file_name.buf); + strbuf_release(&add->lock_file_name); + } + + strbuf_release(&nm); +} + +void reftable_addition_destroy(struct reftable_addition *add) +{ + if (!add) { + return; + } + reftable_addition_close(add); + reftable_free(add); +} + +int reftable_addition_commit(struct reftable_addition *add) +{ + struct strbuf table_list = STRBUF_INIT; + int i = 0; + int err = 0; + if (add->new_tables_len == 0) + goto done; + + for (i = 0; i < add->stack->merged->stack_len; i++) { + strbuf_addstr(&table_list, add->stack->readers[i]->name); + strbuf_addstr(&table_list, "\n"); + } + for (i = 0; i < add->new_tables_len; i++) { + strbuf_addstr(&table_list, add->new_tables[i]); + strbuf_addstr(&table_list, "\n"); + } + + err = write(add->lock_file_fd, table_list.buf, table_list.len); + strbuf_release(&table_list); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = close(add->lock_file_fd); + add->lock_file_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = rename(add->lock_file_name.buf, add->stack->list_file); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + /* success, no more state to clean up. */ + strbuf_release(&add->lock_file_name); + for (i = 0; i < add->new_tables_len; i++) { + reftable_free(add->new_tables[i]); + } + reftable_free(add->new_tables); + add->new_tables = NULL; + add->new_tables_len = 0; + + err = reftable_stack_reload(add->stack); +done: + reftable_addition_close(add); + return err; +} + +int reftable_stack_new_addition(struct reftable_addition **dest, + struct reftable_stack *st) +{ + int err = 0; + struct reftable_addition empty = REFTABLE_ADDITION_INIT; + *dest = reftable_calloc(sizeof(**dest)); + **dest = empty; + err = reftable_stack_init_addition(*dest, st); + if (err) { + reftable_free(*dest); + *dest = NULL; + } + return err; +} + +static int stack_try_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg) +{ + struct reftable_addition add = REFTABLE_ADDITION_INIT; + int err = reftable_stack_init_addition(&add, st); + if (err < 0) + goto done; + if (err > 0) { + err = REFTABLE_LOCK_ERROR; + goto done; + } + + err = reftable_addition_add(&add, write_table, arg); + if (err < 0) + goto done; + + err = reftable_addition_commit(&add); +done: + reftable_addition_close(&add); + return err; +} + +int reftable_addition_add(struct reftable_addition *add, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg) +{ + struct strbuf temp_tab_file_name = STRBUF_INIT; + struct strbuf tab_file_name = STRBUF_INIT; + struct strbuf next_name = STRBUF_INIT; + struct reftable_writer *wr = NULL; + int err = 0; + int tab_fd = 0; + + strbuf_reset(&next_name); + format_name(&next_name, add->next_update_index, add->next_update_index); + + stack_filename(&temp_tab_file_name, add->stack, next_name.buf); + strbuf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + + tab_fd = mkstemp(temp_tab_file_name.buf); + if (tab_fd < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + wr = reftable_new_writer(reftable_fd_write, &tab_fd, + &add->stack->config); + err = write_table(wr, arg); + if (err < 0) + goto done; + + err = reftable_writer_close(wr); + if (err == REFTABLE_EMPTY_TABLE_ERROR) { + err = 0; + goto done; + } + if (err < 0) + goto done; + + err = close(tab_fd); + tab_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = stack_check_addition(add->stack, temp_tab_file_name.buf); + if (err < 0) + goto done; + + if (wr->min_update_index < add->next_update_index) { + err = REFTABLE_API_ERROR; + goto done; + } + + format_name(&next_name, wr->min_update_index, wr->max_update_index); + strbuf_addstr(&next_name, ".ref"); + + stack_filename(&tab_file_name, add->stack, next_name.buf); + + /* + On windows, this relies on rand() picking a unique destination name. + Maybe we should do retry loop as well? + */ + err = rename(temp_tab_file_name.buf, tab_file_name.buf); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + add->new_tables = reftable_realloc(add->new_tables, + sizeof(*add->new_tables) * + (add->new_tables_len + 1)); + add->new_tables[add->new_tables_len] = strbuf_detach(&next_name, NULL); + add->new_tables_len++; +done: + if (tab_fd > 0) { + close(tab_fd); + tab_fd = 0; + } + if (temp_tab_file_name.len > 0) { + unlink(temp_tab_file_name.buf); + } + + strbuf_release(&temp_tab_file_name); + strbuf_release(&tab_file_name); + strbuf_release(&next_name); + reftable_writer_free(wr); + return err; +} + +uint64_t reftable_stack_next_update_index(struct reftable_stack *st) +{ + int sz = st->merged->stack_len; + if (sz > 0) + return reftable_reader_max_update_index(st->readers[sz - 1]) + + 1; + return 1; +} + +static int stack_compact_locked(struct reftable_stack *st, int first, int last, + struct strbuf *temp_tab, + struct reftable_log_expiry_config *config) +{ + struct strbuf next_name = STRBUF_INIT; + int tab_fd = -1; + struct reftable_writer *wr = NULL; + int err = 0; + + format_name(&next_name, + reftable_reader_min_update_index(st->readers[first]), + reftable_reader_max_update_index(st->readers[last])); + + stack_filename(temp_tab, st, next_name.buf); + strbuf_addstr(temp_tab, ".temp.XXXXXX"); + + tab_fd = mkstemp(temp_tab->buf); + wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config); + + err = stack_write_compact(st, wr, first, last, config); + if (err < 0) + goto done; + err = reftable_writer_close(wr); + if (err < 0) + goto done; + + err = close(tab_fd); + tab_fd = 0; + +done: + reftable_writer_free(wr); + if (tab_fd > 0) { + close(tab_fd); + tab_fd = 0; + } + if (err != 0 && temp_tab->len > 0) { + unlink(temp_tab->buf); + strbuf_release(temp_tab); + } + strbuf_release(&next_name); + return err; +} + +static int stack_write_compact(struct reftable_stack *st, + struct reftable_writer *wr, int first, int last, + struct reftable_log_expiry_config *config) +{ + int subtabs_len = last - first + 1; + struct reftable_table *subtabs = reftable_calloc( + sizeof(struct reftable_table) * (last - first + 1)); + struct reftable_merged_table *mt = NULL; + int err = 0; + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + struct reftable_log_record log = { NULL }; + + uint64_t entries = 0; + + int i = 0, j = 0; + for (i = first, j = 0; i <= last; i++) { + struct reftable_reader *t = st->readers[i]; + reftable_table_from_reader(&subtabs[j++], t); + st->stats.bytes += t->size; + } + reftable_writer_set_limits(wr, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + + err = reftable_new_merged_table(&mt, subtabs, subtabs_len, + st->config.hash_id); + if (err < 0) { + reftable_free(subtabs); + goto done; + } + + err = reftable_merged_table_seek_ref(mt, &it, ""); + if (err < 0) + goto done; + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + err = 0; + break; + } + if (err < 0) { + break; + } + + if (first == 0 && reftable_ref_record_is_deletion(&ref)) { + continue; + } + + err = reftable_writer_add_ref(wr, &ref); + if (err < 0) { + break; + } + entries++; + } + reftable_iterator_destroy(&it); + + err = reftable_merged_table_seek_log(mt, &it, ""); + if (err < 0) + goto done; + + while (1) { + err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + err = 0; + break; + } + if (err < 0) { + break; + } + if (first == 0 && reftable_log_record_is_deletion(&log)) { + continue; + } + + if (config && config->min_update_index > 0 && + log.update_index < config->min_update_index) { + continue; + } + + if (config && config->time > 0 && + log.value.update.time < config->time) { + continue; + } + + err = reftable_writer_add_log(wr, &log); + if (err < 0) { + break; + } + entries++; + } + +done: + reftable_iterator_destroy(&it); + if (mt) { + merged_table_release(mt); + reftable_merged_table_free(mt); + } + reftable_ref_record_release(&ref); + reftable_log_record_release(&log); + st->stats.entries_written += entries; + return err; +} + +/* < 0: error. 0 == OK, > 0 attempt failed; could retry. */ +static int stack_compact_range(struct reftable_stack *st, int first, int last, + struct reftable_log_expiry_config *expiry) +{ + struct strbuf temp_tab_file_name = STRBUF_INIT; + struct strbuf new_table_name = STRBUF_INIT; + struct strbuf lock_file_name = STRBUF_INIT; + struct strbuf ref_list_contents = STRBUF_INIT; + struct strbuf new_table_path = STRBUF_INIT; + int err = 0; + int have_lock = 0; + int lock_file_fd = 0; + int compact_count = last - first + 1; + char **listp = NULL; + char **delete_on_success = + reftable_calloc(sizeof(char *) * (compact_count + 1)); + char **subtable_locks = + reftable_calloc(sizeof(char *) * (compact_count + 1)); + int i = 0; + int j = 0; + int is_empty_table = 0; + + if (first > last || (!expiry && first == last)) { + err = 0; + goto done; + } + + st->stats.attempts++; + + strbuf_reset(&lock_file_name); + strbuf_addstr(&lock_file_name, st->list_file); + strbuf_addstr(&lock_file_name, ".lock"); + + lock_file_fd = + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644); + if (lock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + /* Don't want to write to the lock for now. */ + close(lock_file_fd); + lock_file_fd = 0; + + have_lock = 1; + err = stack_uptodate(st); + if (err != 0) + goto done; + + for (i = first, j = 0; i <= last; i++) { + struct strbuf subtab_file_name = STRBUF_INIT; + struct strbuf subtab_lock = STRBUF_INIT; + int sublock_file_fd = -1; + + stack_filename(&subtab_file_name, st, + reader_name(st->readers[i])); + + strbuf_reset(&subtab_lock); + strbuf_addbuf(&subtab_lock, &subtab_file_name); + strbuf_addstr(&subtab_lock, ".lock"); + + sublock_file_fd = open(subtab_lock.buf, + O_EXCL | O_CREAT | O_WRONLY, 0644); + if (sublock_file_fd > 0) { + close(sublock_file_fd); + } else if (sublock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + } + + subtable_locks[j] = subtab_lock.buf; + delete_on_success[j] = subtab_file_name.buf; + j++; + + if (err != 0) + goto done; + } + + err = unlink(lock_file_name.buf); + if (err < 0) + goto done; + have_lock = 0; + + err = stack_compact_locked(st, first, last, &temp_tab_file_name, + expiry); + /* Compaction + tombstones can create an empty table out of non-empty + * tables. */ + is_empty_table = (err == REFTABLE_EMPTY_TABLE_ERROR); + if (is_empty_table) { + err = 0; + } + if (err < 0) + goto done; + + lock_file_fd = + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644); + if (lock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + have_lock = 1; + + format_name(&new_table_name, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + strbuf_addstr(&new_table_name, ".ref"); + + stack_filename(&new_table_path, st, new_table_name.buf); + + if (!is_empty_table) { + /* retry? */ + err = rename(temp_tab_file_name.buf, new_table_path.buf); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + } + + for (i = 0; i < first; i++) { + strbuf_addstr(&ref_list_contents, st->readers[i]->name); + strbuf_addstr(&ref_list_contents, "\n"); + } + if (!is_empty_table) { + strbuf_addbuf(&ref_list_contents, &new_table_name); + strbuf_addstr(&ref_list_contents, "\n"); + } + for (i = last + 1; i < st->merged->stack_len; i++) { + strbuf_addstr(&ref_list_contents, st->readers[i]->name); + strbuf_addstr(&ref_list_contents, "\n"); + } + + err = write(lock_file_fd, ref_list_contents.buf, ref_list_contents.len); + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + err = close(lock_file_fd); + lock_file_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + + err = rename(lock_file_name.buf, st->list_file); + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + have_lock = 0; + + /* Reload the stack before deleting. On windows, we can only delete the + files after we closed them. + */ + err = reftable_stack_reload_maybe_reuse(st, first < last); + + listp = delete_on_success; + while (*listp) { + if (strcmp(*listp, new_table_path.buf)) { + unlink(*listp); + } + listp++; + } + +done: + free_names(delete_on_success); + + listp = subtable_locks; + while (*listp) { + unlink(*listp); + listp++; + } + free_names(subtable_locks); + if (lock_file_fd > 0) { + close(lock_file_fd); + lock_file_fd = 0; + } + if (have_lock) { + unlink(lock_file_name.buf); + } + strbuf_release(&new_table_name); + strbuf_release(&new_table_path); + strbuf_release(&ref_list_contents); + strbuf_release(&temp_tab_file_name); + strbuf_release(&lock_file_name); + return err; +} + +int reftable_stack_compact_all(struct reftable_stack *st, + struct reftable_log_expiry_config *config) +{ + return stack_compact_range(st, 0, st->merged->stack_len - 1, config); +} + +static int stack_compact_range_stats(struct reftable_stack *st, int first, + int last, + struct reftable_log_expiry_config *config) +{ + int err = stack_compact_range(st, first, last, config); + if (err > 0) { + st->stats.failures++; + } + return err; +} + +static int segment_size(struct segment *s) +{ + return s->end - s->start; +} + +int fastlog2(uint64_t sz) +{ + int l = 0; + if (sz == 0) + return 0; + for (; sz; sz /= 2) { + l++; + } + return l - 1; +} + +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n) +{ + struct segment *segs = reftable_calloc(sizeof(struct segment) * n); + int next = 0; + struct segment cur = { 0 }; + int i = 0; + + if (n == 0) { + *seglen = 0; + return segs; + } + for (i = 0; i < n; i++) { + int log = fastlog2(sizes[i]); + if (cur.log != log && cur.bytes > 0) { + struct segment fresh = { + .start = i, + }; + + segs[next++] = cur; + cur = fresh; + } + + cur.log = log; + cur.end = i + 1; + cur.bytes += sizes[i]; + } + segs[next++] = cur; + *seglen = next; + return segs; +} + +struct segment suggest_compaction_segment(uint64_t *sizes, int n) +{ + int seglen = 0; + struct segment *segs = sizes_to_segments(&seglen, sizes, n); + struct segment min_seg = { + .log = 64, + }; + int i = 0; + for (i = 0; i < seglen; i++) { + if (segment_size(&segs[i]) == 1) { + continue; + } + + if (segs[i].log < min_seg.log) { + min_seg = segs[i]; + } + } + + while (min_seg.start > 0) { + int prev = min_seg.start - 1; + if (fastlog2(min_seg.bytes) < fastlog2(sizes[prev])) { + break; + } + + min_seg.start = prev; + min_seg.bytes += sizes[prev]; + } + + reftable_free(segs); + return min_seg; +} + +static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) +{ + uint64_t *sizes = + reftable_calloc(sizeof(uint64_t) * st->merged->stack_len); + int version = (st->config.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; + int overhead = header_size(version) - 1; + int i = 0; + for (i = 0; i < st->merged->stack_len; i++) { + sizes[i] = st->readers[i]->size - overhead; + } + return sizes; +} + +int reftable_stack_auto_compact(struct reftable_stack *st) +{ + uint64_t *sizes = stack_table_sizes_for_compaction(st); + struct segment seg = + suggest_compaction_segment(sizes, st->merged->stack_len); + reftable_free(sizes); + if (segment_size(&seg) > 0) + return stack_compact_range_stats(st, seg.start, seg.end - 1, + NULL); + + return 0; +} + +struct reftable_compaction_stats * +reftable_stack_compaction_stats(struct reftable_stack *st) +{ + return &st->stats; +} + +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, + struct reftable_ref_record *ref) +{ + struct reftable_table tab = { NULL }; + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st)); + return reftable_table_read_ref(&tab, refname, ref); +} + +int reftable_stack_read_log(struct reftable_stack *st, const char *refname, + struct reftable_log_record *log) +{ + struct reftable_iterator it = { NULL }; + struct reftable_merged_table *mt = reftable_stack_merged_table(st); + int err = reftable_merged_table_seek_log(mt, &it, refname); + if (err) + goto done; + + err = reftable_iterator_next_log(&it, log); + if (err) + goto done; + + if (strcmp(log->refname, refname) || + reftable_log_record_is_deletion(log)) { + err = 1; + goto done; + } + +done: + if (err) { + reftable_log_record_release(log); + } + reftable_iterator_destroy(&it); + return err; +} + +static int stack_check_addition(struct reftable_stack *st, + const char *new_tab_name) +{ + int err = 0; + struct reftable_block_source src = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_table tab = { NULL }; + struct reftable_ref_record *refs = NULL; + struct reftable_iterator it = { NULL }; + int cap = 0; + int len = 0; + int i = 0; + + if (st->config.skip_name_check) + return 0; + + err = reftable_block_source_from_file(&src, new_tab_name); + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, new_tab_name); + if (err < 0) + goto done; + + err = reftable_reader_seek_ref(rd, &it, ""); + if (err > 0) { + err = 0; + goto done; + } + if (err < 0) + goto done; + + while (1) { + struct reftable_ref_record ref = { NULL }; + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) + goto done; + + if (len >= cap) { + cap = 2 * cap + 1; + refs = reftable_realloc(refs, cap * sizeof(refs[0])); + } + + refs[len++] = ref; + } + + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st)); + + err = validate_ref_record_addition(tab, refs, len); + +done: + for (i = 0; i < len; i++) { + reftable_ref_record_release(&refs[i]); + } + + free(refs); + reftable_iterator_destroy(&it); + reftable_reader_free(rd); + return err; +} + +static int is_table_name(const char *s) +{ + const char *dot = strrchr(s, '.'); + return dot && !strcmp(dot, ".ref"); +} + +static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, + const char *name) +{ + int err = 0; + uint64_t update_idx = 0; + struct reftable_block_source src = { NULL }; + struct reftable_reader *rd = NULL; + struct strbuf table_path = STRBUF_INIT; + stack_filename(&table_path, st, name); + + err = reftable_block_source_from_file(&src, table_path.buf); + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, name); + if (err < 0) + goto done; + + update_idx = reftable_reader_max_update_index(rd); + reftable_reader_free(rd); + + if (update_idx <= max) { + unlink(table_path.buf); + } +done: + strbuf_release(&table_path); +} + +static int reftable_stack_clean_locked(struct reftable_stack *st) +{ + uint64_t max = reftable_merged_table_max_update_index( + reftable_stack_merged_table(st)); + DIR *dir = opendir(st->reftable_dir); + struct dirent *d = NULL; + if (!dir) { + return REFTABLE_IO_ERROR; + } + + while ((d = readdir(dir))) { + int i = 0; + int found = 0; + if (!is_table_name(d->d_name)) + continue; + + for (i = 0; !found && i < st->readers_len; i++) { + found = !strcmp(reader_name(st->readers[i]), d->d_name); + } + if (found) + continue; + + remove_maybe_stale_table(st, max, d->d_name); + } + + closedir(dir); + return 0; +} + +int reftable_stack_clean(struct reftable_stack *st) +{ + struct reftable_addition *add = NULL; + int err = reftable_stack_new_addition(&add, st); + if (err < 0) { + goto done; + } + + err = reftable_stack_reload(st); + if (err < 0) { + goto done; + } + + err = reftable_stack_clean_locked(st); + +done: + reftable_addition_destroy(add); + return err; +} + +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id) +{ + struct reftable_stack *stack = NULL; + struct reftable_write_options cfg = { .hash_id = hash_id }; + struct reftable_merged_table *merged = NULL; + struct reftable_table table = { NULL }; + + int err = reftable_new_stack(&stack, stackdir, cfg); + if (err < 0) + goto done; + + merged = reftable_stack_merged_table(stack); + reftable_table_from_merged_table(&table, merged); + err = reftable_table_print(&table); +done: + if (stack) + reftable_stack_destroy(stack); + return err; +} diff --git a/reftable/stack.h b/reftable/stack.h new file mode 100644 index 0000000000..f57005846e --- /dev/null +++ b/reftable/stack.h @@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef STACK_H +#define STACK_H + +#include "system.h" +#include "reftable-writer.h" +#include "reftable-stack.h" + +struct reftable_stack { + char *list_file; + char *reftable_dir; + int disable_auto_compact; + + struct reftable_write_options config; + + struct reftable_reader **readers; + size_t readers_len; + struct reftable_merged_table *merged; + struct reftable_compaction_stats stats; +}; + +int read_lines(const char *filename, char ***lines); + +struct segment { + int start, end; + int log; + uint64_t bytes; +}; + +int fastlog2(uint64_t sz); +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n); +struct segment suggest_compaction_segment(uint64_t *sizes, int n); + +#endif diff --git a/reftable/stack_test.c b/reftable/stack_test.c new file mode 100644 index 0000000000..eb0b7228b0 --- /dev/null +++ b/reftable/stack_test.c @@ -0,0 +1,953 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "stack.h" + +#include "system.h" + +#include "reftable-reader.h" +#include "merged.h" +#include "basics.h" +#include "constants.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +#include <sys/types.h> +#include <dirent.h> + +static void clear_dir(const char *dirname) +{ + struct strbuf path = STRBUF_INIT; + strbuf_addstr(&path, dirname); + remove_dir_recursively(&path, 0); + strbuf_release(&path); +} + +static int count_dir_entries(const char *dirname) +{ + DIR *dir = opendir(dirname); + int len = 0; + struct dirent *d; + if (dir == NULL) + return 0; + + while ((d = readdir(dir))) { + if (!strcmp(d->d_name, "..") || !strcmp(d->d_name, ".")) + continue; + len++; + } + closedir(dir); + return len; +} + +/* + * Work linenumber into the tempdir, so we can see which tests forget to + * cleanup. + */ +static char *get_tmp_template(int linenumber) +{ + const char *tmp = getenv("TMPDIR"); + static char template[1024]; + snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX", + tmp ? tmp : "/tmp", linenumber); + return template; +} + +static char *get_tmp_dir(int linenumber) +{ + char *dir = get_tmp_template(linenumber); + EXPECT(mkdtemp(dir)); + return dir; +} + +static void test_read_file(void) +{ + char *fn = get_tmp_template(__LINE__); + int fd = mkstemp(fn); + char out[1024] = "line1\n\nline2\nline3"; + int n, err; + char **names = NULL; + char *want[] = { "line1", "line2", "line3" }; + int i = 0; + + EXPECT(fd > 0); + n = write(fd, out, strlen(out)); + EXPECT(n == strlen(out)); + err = close(fd); + EXPECT(err >= 0); + + err = read_lines(fn, &names); + EXPECT_ERR(err); + + for (i = 0; names[i]; i++) { + EXPECT(0 == strcmp(want[i], names[i])); + } + free_names(names); + remove(fn); +} + +static void test_parse_names(void) +{ + char buf[] = "line\n"; + char **names = NULL; + parse_names(buf, strlen(buf), &names); + + EXPECT(NULL != names[0]); + EXPECT(0 == strcmp(names[0], "line")); + EXPECT(NULL == names[1]); + free_names(names); +} + +static void test_names_equal(void) +{ + char *a[] = { "a", "b", "c", NULL }; + char *b[] = { "a", "b", "d", NULL }; + char *c[] = { "a", "b", NULL }; + + EXPECT(names_equal(a, a)); + EXPECT(!names_equal(a, b)); + EXPECT(!names_equal(a, c)); +} + +static int write_test_ref(struct reftable_writer *wr, void *arg) +{ + struct reftable_ref_record *ref = arg; + reftable_writer_set_limits(wr, ref->update_index, ref->update_index); + return reftable_writer_add_ref(wr, ref); +} + +struct write_log_arg { + struct reftable_log_record *log; + uint64_t update_index; +}; + +static int write_test_log(struct reftable_writer *wr, void *arg) +{ + struct write_log_arg *wla = arg; + + reftable_writer_set_limits(wr, wla->update_index, wla->update_index); + return reftable_writer_add_log(wr, wla->log); +} + +static void test_reftable_stack_add_one(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp("master", dest.value.symref)); + + printf("testing print functionality:\n"); + err = reftable_stack_print_directory(dir, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + + err = reftable_stack_print_directory(dir, GIT_SHA256_FORMAT_ID); + EXPECT(err == REFTABLE_FORMAT_ERROR); + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_uptodate(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL; + struct reftable_stack *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err; + struct reftable_ref_record ref1 = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record ref2 = { + .refname = "branch2", + .update_index = 2, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + + /* simulate multi-process access to the same stack + by creating two stacks for the same directory. + */ + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st1, &write_test_ref, &ref1); + EXPECT_ERR(err); + + err = reftable_stack_add(st2, &write_test_ref, &ref2); + EXPECT(err == REFTABLE_LOCK_ERROR); + + err = reftable_stack_reload(st2); + EXPECT_ERR(err); + + err = reftable_stack_add(st2, &write_test_ref, &ref2); + EXPECT_ERR(err); + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + clear_dir(dir); +} + +static void test_reftable_stack_transaction_api(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_addition *add = NULL; + + struct reftable_ref_record ref = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + reftable_addition_destroy(add); + + err = reftable_stack_new_addition(&add, st); + EXPECT_ERR(err); + + err = reftable_addition_add(add, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_addition_commit(add); + EXPECT_ERR(err); + + reftable_addition_destroy(add); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + EXPECT_ERR(err); + EXPECT(REFTABLE_REF_SYMREF == dest.value_type); + EXPECT(0 == strcmp("master", dest.value.symref)); + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_validate_refname(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + char *dir = get_tmp_dir(__LINE__); + + int i; + struct reftable_ref_record ref = { + .refname = "a/b", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + char *additions[] = { "a", "a/b/c" }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + for (i = 0; i < ARRAY_SIZE(additions); i++) { + struct reftable_ref_record ref = { + .refname = additions[i], + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT(err == REFTABLE_NAME_CONFLICT); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static int write_error(struct reftable_writer *wr, void *arg) +{ + return *((int *)arg); +} + +static void test_reftable_stack_update_index_check(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref1 = { + .refname = "name1", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record ref2 = { + .refname = "name2", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref1); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref2); + EXPECT(err == REFTABLE_API_ERROR); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_lock_failure(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err, i; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) { + err = reftable_stack_add(st, &write_error, &i); + EXPECT(err == i); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_add(void) +{ + int i = 0; + int err = 0; + struct reftable_write_options cfg = { + .exact_log_message = 1, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_ref_record refs[2] = { { NULL } }; + struct reftable_log_record logs[2] = { { NULL } }; + int N = ARRAY_SIZE(refs); + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + st->disable_auto_compact = 1; + + for (i = 0; i < N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02d", i); + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + refs[i].value_type = REFTABLE_REF_VAL1; + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(refs[i].value.val1, i); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = N + i + 1; + logs[i].value_type = REFTABLE_LOG_UPDATE; + + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ); + logs[i].value.update.email = xstrdup("identity@invalid"); + set_test_hash(logs[i].value.update.new_hash, i); + } + + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, &write_test_ref, &refs[i]); + EXPECT_ERR(err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + struct reftable_ref_record dest = { NULL }; + + int err = reftable_stack_read_ref(st, refs[i].refname, &dest); + EXPECT_ERR(err); + EXPECT(reftable_ref_record_equal(&dest, refs + i, + GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + } + + for (i = 0; i < N; i++) { + struct reftable_log_record dest = { NULL }; + int err = reftable_stack_read_log(st, refs[i].refname, &dest); + EXPECT_ERR(err); + EXPECT(reftable_log_record_equal(&dest, logs + i, + GIT_SHA1_RAWSZ)); + reftable_log_record_release(&dest); + } + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void test_reftable_stack_log_normalize(void) +{ + int err = 0; + struct reftable_write_options cfg = { + 0, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + uint8_t h1[GIT_SHA1_RAWSZ] = { 0x01 }, h2[GIT_SHA1_RAWSZ] = { 0x02 }; + + struct reftable_log_record input = { .refname = "branch", + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value = { .update = { + .new_hash = h1, + .old_hash = h2, + } } }; + struct reftable_log_record dest = { + .update_index = 0, + }; + struct write_log_arg arg = { + .log = &input, + .update_index = 1, + }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + input.value.update.message = "one\ntwo"; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT(err == REFTABLE_API_ERROR); + + input.value.update.message = "one"; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, input.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp(dest.value.update.message, "one\n")); + + input.value.update.message = "two\n"; + arg.update_index = 2; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + err = reftable_stack_read_log(st, input.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp(dest.value.update.message, "two\n")); + + /* cleanup */ + reftable_stack_destroy(st); + reftable_log_record_release(&dest); + clear_dir(dir); +} + +static void test_reftable_stack_tombstone(void) +{ + int i = 0; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record refs[2] = { { NULL } }; + struct reftable_log_record logs[2] = { { NULL } }; + int N = ARRAY_SIZE(refs); + struct reftable_ref_record dest = { NULL }; + struct reftable_log_record log_dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + /* even entries add the refs, odd entries delete them. */ + for (i = 0; i < N; i++) { + const char *buf = "branch"; + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + if (i % 2 == 0) { + refs[i].value_type = REFTABLE_REF_VAL1; + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(refs[i].value.val1, i); + } + + logs[i].refname = xstrdup(buf); + /* update_index is part of the key. */ + logs[i].update_index = 42; + if (i % 2 == 0) { + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.new_hash = + reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(logs[i].value.update.new_hash, i); + logs[i].value.update.email = + xstrdup("identity@invalid"); + } + } + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, &write_test_ref, &refs[i]); + EXPECT_ERR(err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_read_ref(st, "branch", &dest); + EXPECT(err == 1); + reftable_ref_record_release(&dest); + + err = reftable_stack_read_log(st, "branch", &log_dest); + EXPECT(err == 1); + reftable_log_record_release(&log_dest); + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st, "branch", &dest); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, "branch", &log_dest); + EXPECT(err == 1); + reftable_ref_record_release(&dest); + reftable_log_record_release(&log_dest); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void test_reftable_stack_hash_id(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + + struct reftable_ref_record ref = { + .refname = "master", + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "target", + .update_index = 1, + }; + struct reftable_write_options cfg32 = { .hash_id = GIT_SHA256_FORMAT_ID }; + struct reftable_stack *st32 = NULL; + struct reftable_write_options cfg_default = { 0 }; + struct reftable_stack *st_default = NULL; + struct reftable_ref_record dest = { NULL }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + /* can't read it with the wrong hash ID. */ + err = reftable_new_stack(&st32, dir, cfg32); + EXPECT(err == REFTABLE_FORMAT_ERROR); + + /* check that we can read it back with default config too. */ + err = reftable_new_stack(&st_default, dir, cfg_default); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st_default, "master", &dest); + EXPECT_ERR(err); + + EXPECT(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + reftable_stack_destroy(st_default); + clear_dir(dir); +} + +static void test_log2(void) +{ + EXPECT(1 == fastlog2(3)); + EXPECT(2 == fastlog2(4)); + EXPECT(2 == fastlog2(5)); +} + +static void test_sizes_to_segments(void) +{ + uint64_t sizes[] = { 2, 3, 4, 5, 7, 9 }; + /* .................0 1 2 3 4 5 */ + + int seglen = 0; + struct segment *segs = + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes)); + EXPECT(segs[2].log == 3); + EXPECT(segs[2].start == 5); + EXPECT(segs[2].end == 6); + + EXPECT(segs[1].log == 2); + EXPECT(segs[1].start == 2); + EXPECT(segs[1].end == 5); + reftable_free(segs); +} + +static void test_sizes_to_segments_empty(void) +{ + int seglen = 0; + struct segment *segs = sizes_to_segments(&seglen, NULL, 0); + EXPECT(seglen == 0); + reftable_free(segs); +} + +static void test_sizes_to_segments_all_equal(void) +{ + uint64_t sizes[] = { 5, 5 }; + + int seglen = 0; + struct segment *segs = + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes)); + EXPECT(seglen == 1); + EXPECT(segs[0].start == 0); + EXPECT(segs[0].end == 2); + reftable_free(segs); +} + +static void test_suggest_compaction_segment(void) +{ + uint64_t sizes[] = { 128, 64, 17, 16, 9, 9, 9, 16, 16 }; + /* .................0 1 2 3 4 5 6 */ + struct segment min = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + EXPECT(min.start == 2); + EXPECT(min.end == 7); +} + +static void test_suggest_compaction_segment_nothing(void) +{ + uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; + struct segment result = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + EXPECT(result.start == result.end); +} + +static void test_reflog_expire(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + struct reftable_log_record logs[20] = { { NULL } }; + int N = ARRAY_SIZE(logs) - 1; + int i = 0; + int err; + struct reftable_log_expiry_config expiry = { + .time = 10, + }; + struct reftable_log_record log = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + for (i = 1; i <= N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02d", i); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = i; + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.time = i; + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ); + logs[i].value.update.email = xstrdup("identity@invalid"); + set_test_hash(logs[i].value.update.new_hash, i); + } + + for (i = 1; i <= N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st, &expiry); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, logs[9].refname, &log); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, logs[11].refname, &log); + EXPECT_ERR(err); + + expiry.min_update_index = 15; + err = reftable_stack_compact_all(st, &expiry); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, logs[14].refname, &log); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, logs[16].refname, &log); + EXPECT_ERR(err); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i <= N; i++) { + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); + reftable_log_record_release(&log); +} + +static int write_nothing(struct reftable_writer *wr, void *arg) +{ + reftable_writer_set_limits(wr, 1, 1); + return 0; +} + +static void test_empty_add(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_stack *st2 = NULL; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_nothing, NULL); + EXPECT_ERR(err); + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + clear_dir(dir); + reftable_stack_destroy(st); + reftable_stack_destroy(st2); +} + +static void test_reftable_stack_auto_compaction(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 100; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + st->disable_auto_compact = 1; /* call manually below for coverage. */ + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_stack_auto_compact(st); + EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i)); + } + + EXPECT(reftable_stack_compaction_stats(st)->entries_written < + (uint64_t)(N * fastlog2(N))); + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_compaction_concurrent(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 3; + + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st1), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st1, &write_test_ref, &ref); + EXPECT_ERR(err); + } + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st1, NULL); + EXPECT_ERR(err); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + + EXPECT(count_dir_entries(dir) == 2); + clear_dir(dir); +} + +static void unclean_stack_close(struct reftable_stack *st) +{ + /* break abstraction boundary to simulate unclean shutdown. */ + int i = 0; + for (; i < st->readers_len; i++) { + reftable_reader_free(st->readers[i]); + } + st->readers_len = 0; + FREE_AND_NULL(st->readers); +} + +static void test_reftable_stack_compaction_concurrent_clean(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 3; + + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st1), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st1, &write_test_ref, &ref); + EXPECT_ERR(err); + } + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st1, NULL); + EXPECT_ERR(err); + + unclean_stack_close(st1); + unclean_stack_close(st2); + + err = reftable_new_stack(&st3, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_clean(st3); + EXPECT_ERR(err); + EXPECT(count_dir_entries(dir) == 2); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + reftable_stack_destroy(st3); + + clear_dir(dir); +} + +int stack_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_empty_add); + RUN_TEST(test_log2); + RUN_TEST(test_names_equal); + RUN_TEST(test_parse_names); + RUN_TEST(test_read_file); + RUN_TEST(test_reflog_expire); + RUN_TEST(test_reftable_stack_add); + RUN_TEST(test_reftable_stack_add_one); + RUN_TEST(test_reftable_stack_auto_compaction); + RUN_TEST(test_reftable_stack_compaction_concurrent); + RUN_TEST(test_reftable_stack_compaction_concurrent_clean); + RUN_TEST(test_reftable_stack_hash_id); + RUN_TEST(test_reftable_stack_lock_failure); + RUN_TEST(test_reftable_stack_log_normalize); + RUN_TEST(test_reftable_stack_tombstone); + RUN_TEST(test_reftable_stack_transaction_api); + RUN_TEST(test_reftable_stack_update_index_check); + RUN_TEST(test_reftable_stack_uptodate); + RUN_TEST(test_reftable_stack_validate_refname); + RUN_TEST(test_sizes_to_segments); + RUN_TEST(test_sizes_to_segments_all_equal); + RUN_TEST(test_sizes_to_segments_empty); + RUN_TEST(test_suggest_compaction_segment); + RUN_TEST(test_suggest_compaction_segment_nothing); + return 0; +} diff --git a/reftable/system.h b/reftable/system.h new file mode 100644 index 0000000000..4907306c0c --- /dev/null +++ b/reftable/system.h @@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +#ifdef NO_UNCOMPRESS2 +/* + * This is uncompress2, which is only available in zlib >= 1.2.9 + * (released as of early 2017) + */ +int uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong *sourceLen); +#endif + +int hash_size(uint32_t id); + +#endif diff --git a/reftable/test_framework.c b/reftable/test_framework.c new file mode 100644 index 0000000000..84ac972cad --- /dev/null +++ b/reftable/test_framework.c @@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash(uint8_t *p, int i) +{ + memset(p, (uint8_t)i, hash_size(GIT_SHA1_FORMAT_ID)); +} + +ssize_t strbuf_add_void(void *b, const void *data, size_t sz) +{ + strbuf_add(b, data, sz); + return sz; +} diff --git a/reftable/test_framework.h b/reftable/test_framework.h new file mode 100644 index 0000000000..774cb275bf --- /dev/null +++ b/reftable/test_framework.h @@ -0,0 +1,53 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TEST_FRAMEWORK_H +#define TEST_FRAMEWORK_H + +#include "system.h" +#include "reftable-error.h" + +#define EXPECT_ERR(c) \ + if (c != 0) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s: %d: error == %d (%s), want 0\n", \ + __FILE__, __LINE__, c, reftable_error_str(c)); \ + abort(); \ + } + +#define EXPECT_STREQ(a, b) \ + if (strcmp(a, b)) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s:%d: %s (%s) != %s (%s)\n", __FILE__, \ + __LINE__, #a, a, #b, b); \ + abort(); \ + } + +#define EXPECT(c) \ + if (!(c)) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s: %d: failed assertion %s\n", __FILE__, \ + __LINE__, #c); \ + abort(); \ + } + +#define RUN_TEST(f) \ + fprintf(stderr, "running %s\n", #f); \ + fflush(stderr); \ + f(); + +void set_test_hash(uint8_t *p, int i); + +/* Like strbuf_add, but suitable for passing to reftable_new_writer + */ +ssize_t strbuf_add_void(void *b, const void *data, size_t sz); + +#endif diff --git a/reftable/tree.c b/reftable/tree.c new file mode 100644 index 0000000000..82db7995dd --- /dev/null +++ b/reftable/tree.c @@ -0,0 +1,63 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "tree.h" + +#include "basics.h" +#include "system.h" + +struct tree_node *tree_search(void *key, struct tree_node **rootp, + int (*compare)(const void *, const void *), + int insert) +{ + int res; + if (*rootp == NULL) { + if (!insert) { + return NULL; + } else { + struct tree_node *n = + reftable_calloc(sizeof(struct tree_node)); + n->key = key; + *rootp = n; + return *rootp; + } + } + + res = compare(key, (*rootp)->key); + if (res < 0) + return tree_search(key, &(*rootp)->left, compare, insert); + else if (res > 0) + return tree_search(key, &(*rootp)->right, compare, insert); + return *rootp; +} + +void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), + void *arg) +{ + if (t->left) { + infix_walk(t->left, action, arg); + } + action(arg, t->key); + if (t->right) { + infix_walk(t->right, action, arg); + } +} + +void tree_free(struct tree_node *t) +{ + if (t == NULL) { + return; + } + if (t->left) { + tree_free(t->left); + } + if (t->right) { + tree_free(t->right); + } + reftable_free(t); +} diff --git a/reftable/tree.h b/reftable/tree.h new file mode 100644 index 0000000000..fbdd002e23 --- /dev/null +++ b/reftable/tree.h @@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { + void *key; + struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert + * is set, insert the key if it's not found. Else, return NULL. + */ +struct tree_node *tree_search(void *key, struct tree_node **rootp, + int (*compare)(const void *, const void *), + int insert); + +/* performs an infix walk of the tree. */ +void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), + void *arg); + +/* + * deallocates the tree nodes recursively. Keys should be deallocated separately + * by walking over the tree. */ +void tree_free(struct tree_node *t); + +#endif diff --git a/reftable/tree_test.c b/reftable/tree_test.c new file mode 100644 index 0000000000..cbff125588 --- /dev/null +++ b/reftable/tree_test.c @@ -0,0 +1,61 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "tree.h" + +#include "basics.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static int test_compare(const void *a, const void *b) +{ + return (char *)a - (char *)b; +} + +struct curry { + void *last; +}; + +static void check_increasing(void *arg, void *key) +{ + struct curry *c = arg; + if (c->last) { + EXPECT(test_compare(c->last, key) < 0); + } + c->last = key; +} + +static void test_tree(void) +{ + struct tree_node *root = NULL; + + void *values[11] = { NULL }; + struct tree_node *nodes[11] = { NULL }; + int i = 1; + struct curry c = { NULL }; + do { + nodes[i] = tree_search(values + i, &root, &test_compare, 1); + i = (i * 7) % 11; + } while (i != 1); + + for (i = 1; i < ARRAY_SIZE(nodes); i++) { + EXPECT(values + i == nodes[i]->key); + EXPECT(nodes[i] == + tree_search(values + i, &root, &test_compare, 0)); + } + + infix_walk(root, check_increasing, &c); + tree_free(root); +} + +int tree_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_tree); + return 0; +} diff --git a/reftable/writer.c b/reftable/writer.c new file mode 100644 index 0000000000..3ca721e9f6 --- /dev/null +++ b/reftable/writer.c @@ -0,0 +1,690 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "writer.h" + +#include "system.h" + +#include "block.h" +#include "constants.h" +#include "record.h" +#include "tree.h" +#include "reftable-error.h" + +/* finishes a block, and writes it to storage */ +static int writer_flush_block(struct reftable_writer *w); + +/* deallocates memory related to the index */ +static void writer_clear_index(struct reftable_writer *w); + +/* finishes writing a 'r' (refs) or 'g' (reflogs) section */ +static int writer_finish_public_section(struct reftable_writer *w); + +static struct reftable_block_stats * +writer_reftable_block_stats(struct reftable_writer *w, uint8_t typ) +{ + switch (typ) { + case 'r': + return &w->stats.ref_stats; + case 'o': + return &w->stats.obj_stats; + case 'i': + return &w->stats.idx_stats; + case 'g': + return &w->stats.log_stats; + } + abort(); + return NULL; +} + +/* write data, queuing the padding for the next write. Returns negative for + * error. */ +static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len, + int padding) +{ + int n = 0; + if (w->pending_padding > 0) { + uint8_t *zeroed = reftable_calloc(w->pending_padding); + int n = w->write(w->write_arg, zeroed, w->pending_padding); + if (n < 0) + return n; + + w->pending_padding = 0; + reftable_free(zeroed); + } + + w->pending_padding = padding; + n = w->write(w->write_arg, data, len); + if (n < 0) + return n; + n += padding; + return 0; +} + +static void options_set_defaults(struct reftable_write_options *opts) +{ + if (opts->restart_interval == 0) { + opts->restart_interval = 16; + } + + if (opts->hash_id == 0) { + opts->hash_id = GIT_SHA1_FORMAT_ID; + } + if (opts->block_size == 0) { + opts->block_size = DEFAULT_BLOCK_SIZE; + } +} + +static int writer_version(struct reftable_writer *w) +{ + return (w->opts.hash_id == 0 || w->opts.hash_id == GIT_SHA1_FORMAT_ID) ? + 1 : + 2; +} + +static int writer_write_header(struct reftable_writer *w, uint8_t *dest) +{ + memcpy(dest, "REFT", 4); + + dest[4] = writer_version(w); + + put_be24(dest + 5, w->opts.block_size); + put_be64(dest + 8, w->min_update_index); + put_be64(dest + 16, w->max_update_index); + if (writer_version(w) == 2) { + put_be32(dest + 24, w->opts.hash_id); + } + return header_size(writer_version(w)); +} + +static void writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ) +{ + int block_start = 0; + if (w->next == 0) { + block_start = header_size(writer_version(w)); + } + + strbuf_release(&w->last_key); + block_writer_init(&w->block_writer_data, typ, w->block, + w->opts.block_size, block_start, + hash_size(w->opts.hash_id)); + w->block_writer = &w->block_writer_data; + w->block_writer->restart_interval = w->opts.restart_interval; +} + +static struct strbuf reftable_empty_strbuf = STRBUF_INIT; + +struct reftable_writer * +reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + void *writer_arg, struct reftable_write_options *opts) +{ + struct reftable_writer *wp = + reftable_calloc(sizeof(struct reftable_writer)); + strbuf_init(&wp->block_writer_data.last_key, 0); + options_set_defaults(opts); + if (opts->block_size >= (1 << 24)) { + /* TODO - error return? */ + abort(); + } + wp->last_key = reftable_empty_strbuf; + wp->block = reftable_calloc(opts->block_size); + wp->write = writer_func; + wp->write_arg = writer_arg; + wp->opts = *opts; + writer_reinit_block_writer(wp, BLOCK_TYPE_REF); + + return wp; +} + +void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min, + uint64_t max) +{ + w->min_update_index = min; + w->max_update_index = max; +} + +void reftable_writer_free(struct reftable_writer *w) +{ + reftable_free(w->block); + reftable_free(w); +} + +struct obj_index_tree_node { + struct strbuf hash; + uint64_t *offsets; + size_t offset_len; + size_t offset_cap; +}; + +#define OBJ_INDEX_TREE_NODE_INIT \ + { \ + .hash = STRBUF_INIT \ + } + +static int obj_index_tree_node_compare(const void *a, const void *b) +{ + return strbuf_cmp(&((const struct obj_index_tree_node *)a)->hash, + &((const struct obj_index_tree_node *)b)->hash); +} + +static void writer_index_hash(struct reftable_writer *w, struct strbuf *hash) +{ + uint64_t off = w->next; + + struct obj_index_tree_node want = { .hash = *hash }; + + struct tree_node *node = tree_search(&want, &w->obj_index_tree, + &obj_index_tree_node_compare, 0); + struct obj_index_tree_node *key = NULL; + if (node == NULL) { + struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT; + key = reftable_malloc(sizeof(struct obj_index_tree_node)); + *key = empty; + + strbuf_reset(&key->hash); + strbuf_addbuf(&key->hash, hash); + tree_search((void *)key, &w->obj_index_tree, + &obj_index_tree_node_compare, 1); + } else { + key = node->key; + } + + if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off) { + return; + } + + if (key->offset_len == key->offset_cap) { + key->offset_cap = 2 * key->offset_cap + 1; + key->offsets = reftable_realloc( + key->offsets, sizeof(uint64_t) * key->offset_cap); + } + + key->offsets[key->offset_len++] = off; +} + +static int writer_add_record(struct reftable_writer *w, + struct reftable_record *rec) +{ + struct strbuf key = STRBUF_INIT; + int err = -1; + reftable_record_key(rec, &key); + if (strbuf_cmp(&w->last_key, &key) >= 0) { + err = REFTABLE_API_ERROR; + goto done; + } + + strbuf_reset(&w->last_key); + strbuf_addbuf(&w->last_key, &key); + if (w->block_writer == NULL) { + writer_reinit_block_writer(w, reftable_record_type(rec)); + } + + assert(block_writer_type(w->block_writer) == reftable_record_type(rec)); + + if (block_writer_add(w->block_writer, rec) == 0) { + err = 0; + goto done; + } + + err = writer_flush_block(w); + if (err < 0) { + goto done; + } + + writer_reinit_block_writer(w, reftable_record_type(rec)); + err = block_writer_add(w->block_writer, rec); + if (err < 0) { + goto done; + } + + err = 0; +done: + strbuf_release(&key); + return err; +} + +int reftable_writer_add_ref(struct reftable_writer *w, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + struct reftable_ref_record copy = *ref; + int err = 0; + + if (ref->refname == NULL) + return REFTABLE_API_ERROR; + if (ref->update_index < w->min_update_index || + ref->update_index > w->max_update_index) + return REFTABLE_API_ERROR; + + reftable_record_from_ref(&rec, ©); + copy.update_index -= w->min_update_index; + + err = writer_add_record(w, &rec); + if (err < 0) + return err; + + if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) { + struct strbuf h = STRBUF_INIT; + strbuf_add(&h, (char *)reftable_ref_record_val1(ref), + hash_size(w->opts.hash_id)); + writer_index_hash(w, &h); + strbuf_release(&h); + } + + if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) { + struct strbuf h = STRBUF_INIT; + strbuf_add(&h, reftable_ref_record_val2(ref), + hash_size(w->opts.hash_id)); + writer_index_hash(w, &h); + strbuf_release(&h); + } + return 0; +} + +int reftable_writer_add_refs(struct reftable_writer *w, + struct reftable_ref_record *refs, int n) +{ + int err = 0; + int i = 0; + QSORT(refs, n, reftable_ref_record_compare_name); + for (i = 0; err == 0 && i < n; i++) { + err = reftable_writer_add_ref(w, &refs[i]); + } + return err; +} + +static int reftable_writer_add_log_verbatim(struct reftable_writer *w, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + if (w->block_writer && + block_writer_type(w->block_writer) == BLOCK_TYPE_REF) { + int err = writer_finish_public_section(w); + if (err < 0) + return err; + } + + w->next -= w->pending_padding; + w->pending_padding = 0; + + reftable_record_from_log(&rec, log); + return writer_add_record(w, &rec); +} + +int reftable_writer_add_log(struct reftable_writer *w, + struct reftable_log_record *log) +{ + char *input_log_message = NULL; + struct strbuf cleaned_message = STRBUF_INIT; + int err = 0; + + if (log->value_type == REFTABLE_LOG_DELETION) + return reftable_writer_add_log_verbatim(w, log); + + if (log->refname == NULL) + return REFTABLE_API_ERROR; + + input_log_message = log->value.update.message; + if (!w->opts.exact_log_message && log->value.update.message) { + strbuf_addstr(&cleaned_message, log->value.update.message); + while (cleaned_message.len && + cleaned_message.buf[cleaned_message.len - 1] == '\n') + strbuf_setlen(&cleaned_message, + cleaned_message.len - 1); + if (strchr(cleaned_message.buf, '\n')) { + /* multiple lines not allowed. */ + err = REFTABLE_API_ERROR; + goto done; + } + strbuf_addstr(&cleaned_message, "\n"); + log->value.update.message = cleaned_message.buf; + } + + err = reftable_writer_add_log_verbatim(w, log); + log->value.update.message = input_log_message; +done: + strbuf_release(&cleaned_message); + return err; +} + +int reftable_writer_add_logs(struct reftable_writer *w, + struct reftable_log_record *logs, int n) +{ + int err = 0; + int i = 0; + QSORT(logs, n, reftable_log_record_compare_key); + + for (i = 0; err == 0 && i < n; i++) { + err = reftable_writer_add_log(w, &logs[i]); + } + return err; +} + +static int writer_finish_section(struct reftable_writer *w) +{ + uint8_t typ = block_writer_type(w->block_writer); + uint64_t index_start = 0; + int max_level = 0; + int threshold = w->opts.unpadded ? 1 : 3; + int before_blocks = w->stats.idx_stats.blocks; + int err = writer_flush_block(w); + int i = 0; + struct reftable_block_stats *bstats = NULL; + if (err < 0) + return err; + + while (w->index_len > threshold) { + struct reftable_index_record *idx = NULL; + int idx_len = 0; + + max_level++; + index_start = w->next; + writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + + idx = w->index; + idx_len = w->index_len; + + w->index = NULL; + w->index_len = 0; + w->index_cap = 0; + for (i = 0; i < idx_len; i++) { + struct reftable_record rec = { NULL }; + reftable_record_from_index(&rec, idx + i); + if (block_writer_add(w->block_writer, &rec) == 0) { + continue; + } + + err = writer_flush_block(w); + if (err < 0) + return err; + + writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + + err = block_writer_add(w->block_writer, &rec); + if (err != 0) { + /* write into fresh block should always succeed + */ + abort(); + } + } + for (i = 0; i < idx_len; i++) { + strbuf_release(&idx[i].last_key); + } + reftable_free(idx); + } + + writer_clear_index(w); + + err = writer_flush_block(w); + if (err < 0) + return err; + + bstats = writer_reftable_block_stats(w, typ); + bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks; + bstats->index_offset = index_start; + bstats->max_index_level = max_level; + + /* Reinit lastKey, as the next section can start with any key. */ + w->last_key.len = 0; + + return 0; +} + +struct common_prefix_arg { + struct strbuf *last; + int max; +}; + +static void update_common(void *void_arg, void *key) +{ + struct common_prefix_arg *arg = void_arg; + struct obj_index_tree_node *entry = key; + if (arg->last) { + int n = common_prefix_size(&entry->hash, arg->last); + if (n > arg->max) { + arg->max = n; + } + } + arg->last = &entry->hash; +} + +struct write_record_arg { + struct reftable_writer *w; + int err; +}; + +static void write_object_record(void *void_arg, void *key) +{ + struct write_record_arg *arg = void_arg; + struct obj_index_tree_node *entry = key; + struct reftable_obj_record obj_rec = { + .hash_prefix = (uint8_t *)entry->hash.buf, + .hash_prefix_len = arg->w->stats.object_id_len, + .offsets = entry->offsets, + .offset_len = entry->offset_len, + }; + struct reftable_record rec = { NULL }; + if (arg->err < 0) + goto done; + + reftable_record_from_obj(&rec, &obj_rec); + arg->err = block_writer_add(arg->w->block_writer, &rec); + if (arg->err == 0) + goto done; + + arg->err = writer_flush_block(arg->w); + if (arg->err < 0) + goto done; + + writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); + arg->err = block_writer_add(arg->w->block_writer, &rec); + if (arg->err == 0) + goto done; + obj_rec.offset_len = 0; + arg->err = block_writer_add(arg->w->block_writer, &rec); + + /* Should be able to write into a fresh block. */ + assert(arg->err == 0); + +done:; +} + +static void object_record_free(void *void_arg, void *key) +{ + struct obj_index_tree_node *entry = key; + + FREE_AND_NULL(entry->offsets); + strbuf_release(&entry->hash); + reftable_free(entry); +} + +static int writer_dump_object_index(struct reftable_writer *w) +{ + struct write_record_arg closure = { .w = w }; + struct common_prefix_arg common = { NULL }; + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &update_common, &common); + } + w->stats.object_id_len = common.max + 1; + + writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); + + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &write_object_record, &closure); + } + + if (closure.err < 0) + return closure.err; + return writer_finish_section(w); +} + +static int writer_finish_public_section(struct reftable_writer *w) +{ + uint8_t typ = 0; + int err = 0; + + if (w->block_writer == NULL) + return 0; + + typ = block_writer_type(w->block_writer); + err = writer_finish_section(w); + if (err < 0) + return err; + if (typ == BLOCK_TYPE_REF && !w->opts.skip_index_objects && + w->stats.ref_stats.index_blocks > 0) { + err = writer_dump_object_index(w); + if (err < 0) + return err; + } + + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &object_record_free, NULL); + tree_free(w->obj_index_tree); + w->obj_index_tree = NULL; + } + + w->block_writer = NULL; + return 0; +} + +int reftable_writer_close(struct reftable_writer *w) +{ + uint8_t footer[72]; + uint8_t *p = footer; + int err = writer_finish_public_section(w); + int empty_table = w->next == 0; + if (err != 0) + goto done; + w->pending_padding = 0; + if (empty_table) { + /* Empty tables need a header anyway. */ + uint8_t header[28]; + int n = writer_write_header(w, header); + err = padded_write(w, header, n, 0); + if (err < 0) + goto done; + } + + p += writer_write_header(w, footer); + put_be64(p, w->stats.ref_stats.index_offset); + p += 8; + put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len); + p += 8; + put_be64(p, w->stats.obj_stats.index_offset); + p += 8; + + put_be64(p, w->stats.log_stats.offset); + p += 8; + put_be64(p, w->stats.log_stats.index_offset); + p += 8; + + put_be32(p, crc32(0, footer, p - footer)); + p += 4; + + err = padded_write(w, footer, footer_size(writer_version(w)), 0); + if (err < 0) + goto done; + + if (empty_table) { + err = REFTABLE_EMPTY_TABLE_ERROR; + goto done; + } + +done: + /* free up memory. */ + block_writer_release(&w->block_writer_data); + writer_clear_index(w); + strbuf_release(&w->last_key); + return err; +} + +static void writer_clear_index(struct reftable_writer *w) +{ + int i = 0; + for (i = 0; i < w->index_len; i++) { + strbuf_release(&w->index[i].last_key); + } + + FREE_AND_NULL(w->index); + w->index_len = 0; + w->index_cap = 0; +} + +static const int debug = 0; + +static int writer_flush_nonempty_block(struct reftable_writer *w) +{ + uint8_t typ = block_writer_type(w->block_writer); + struct reftable_block_stats *bstats = + writer_reftable_block_stats(w, typ); + uint64_t block_typ_off = (bstats->blocks == 0) ? w->next : 0; + int raw_bytes = block_writer_finish(w->block_writer); + int padding = 0; + int err = 0; + struct reftable_index_record ir = { .last_key = STRBUF_INIT }; + if (raw_bytes < 0) + return raw_bytes; + + if (!w->opts.unpadded && typ != BLOCK_TYPE_LOG) { + padding = w->opts.block_size - raw_bytes; + } + + if (block_typ_off > 0) { + bstats->offset = block_typ_off; + } + + bstats->entries += w->block_writer->entries; + bstats->restarts += w->block_writer->restart_len; + bstats->blocks++; + w->stats.blocks++; + + if (debug) { + fprintf(stderr, "block %c off %" PRIu64 " sz %d (%d)\n", typ, + w->next, raw_bytes, + get_be24(w->block + w->block_writer->header_off + 1)); + } + + if (w->next == 0) { + writer_write_header(w, w->block); + } + + err = padded_write(w, w->block, raw_bytes, padding); + if (err < 0) + return err; + + if (w->index_cap == w->index_len) { + w->index_cap = 2 * w->index_cap + 1; + w->index = reftable_realloc( + w->index, + sizeof(struct reftable_index_record) * w->index_cap); + } + + ir.offset = w->next; + strbuf_reset(&ir.last_key); + strbuf_addbuf(&ir.last_key, &w->block_writer->last_key); + w->index[w->index_len] = ir; + + w->index_len++; + w->next += padding + raw_bytes; + w->block_writer = NULL; + return 0; +} + +static int writer_flush_block(struct reftable_writer *w) +{ + if (w->block_writer == NULL) + return 0; + if (w->block_writer->entries == 0) + return 0; + return writer_flush_nonempty_block(w); +} + +const struct reftable_stats *writer_stats(struct reftable_writer *w) +{ + return &w->stats; +} diff --git a/reftable/writer.h b/reftable/writer.h new file mode 100644 index 0000000000..09b88673d9 --- /dev/null +++ b/reftable/writer.h @@ -0,0 +1,50 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef WRITER_H +#define WRITER_H + +#include "basics.h" +#include "block.h" +#include "tree.h" +#include "reftable-writer.h" + +struct reftable_writer { + ssize_t (*write)(void *, const void *, size_t); + void *write_arg; + int pending_padding; + struct strbuf last_key; + + /* offset of next block to write. */ + uint64_t next; + uint64_t min_update_index, max_update_index; + struct reftable_write_options opts; + + /* memory buffer for writing */ + uint8_t *block; + + /* writer for the current section. NULL or points to + * block_writer_data */ + struct block_writer *block_writer; + + struct block_writer block_writer_data; + + /* pending index records for the current section */ + struct reftable_index_record *index; + size_t index_len; + size_t index_cap; + + /* + * tree for use with tsearch; used to populate the 'o' inverse OID + * map */ + struct tree_node *obj_index_tree; + + struct reftable_stats stats; +}; + +#endif diff --git a/remote-curl.c b/remote-curl.c index d69156312b..0dabef2dd7 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1061,7 +1061,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads, client.in = -1; client.out = -1; client.git_cmd = 1; - client.argv = client_argv; + strvec_pushv(&client.args, client_argv); if (start_command(&client)) exit(1); write_or_die(client.in, preamble->buf, preamble->len); @@ -21,33 +21,6 @@ struct counted_string { size_t len; const char *s; }; -struct rewrite { - const char *base; - size_t baselen; - struct counted_string *instead_of; - int instead_of_nr; - int instead_of_alloc; -}; -struct rewrites { - struct rewrite **rewrite; - int rewrite_alloc; - int rewrite_nr; -}; - -static struct remote **remotes; -static int remotes_alloc; -static int remotes_nr; -static struct hashmap remotes_hash; - -static struct branch **branches; -static int branches_alloc; -static int branches_nr; - -static struct branch *current_branch; -static const char *pushremote_name; - -static struct rewrites rewrites; -static struct rewrites rewrites_push; static int valid_remote(const struct remote *remote) { @@ -92,17 +65,19 @@ static void add_pushurl(struct remote *remote, const char *pushurl) remote->pushurl[remote->pushurl_nr++] = pushurl; } -static void add_pushurl_alias(struct remote *remote, const char *url) +static void add_pushurl_alias(struct remote_state *remote_state, + struct remote *remote, const char *url) { - const char *pushurl = alias_url(url, &rewrites_push); + const char *pushurl = alias_url(url, &remote_state->rewrites_push); if (pushurl != url) add_pushurl(remote, pushurl); } -static void add_url_alias(struct remote *remote, const char *url) +static void add_url_alias(struct remote_state *remote_state, + struct remote *remote, const char *url) { - add_url(remote, alias_url(url, &rewrites)); - add_pushurl_alias(remote, url); + add_url(remote, alias_url(url, &remote_state->rewrites)); + add_pushurl_alias(remote_state, remote, url); } struct remotes_hash_key { @@ -127,13 +102,8 @@ static int remotes_hash_cmp(const void *unused_cmp_data, return strcmp(a->name, b->name); } -static inline void init_remotes_hash(void) -{ - if (!remotes_hash.cmpfn) - hashmap_init(&remotes_hash, remotes_hash_cmp, NULL, 0); -} - -static struct remote *make_remote(const char *name, int len) +static struct remote *make_remote(struct remote_state *remote_state, + const char *name, int len) { struct remote *ret; struct remotes_hash_key lookup; @@ -142,12 +112,11 @@ static struct remote *make_remote(const char *name, int len) if (!len) len = strlen(name); - init_remotes_hash(); lookup.str = name; lookup.len = len; hashmap_entry_init(&lookup_entry, memhash(name, len)); - e = hashmap_get(&remotes_hash, &lookup_entry, &lookup); + e = hashmap_get(&remote_state->remotes_hash, &lookup_entry, &lookup); if (e) return container_of(e, struct remote, ent); @@ -158,15 +127,38 @@ static struct remote *make_remote(const char *name, int len) refspec_init(&ret->push, REFSPEC_PUSH); refspec_init(&ret->fetch, REFSPEC_FETCH); - ALLOC_GROW(remotes, remotes_nr + 1, remotes_alloc); - remotes[remotes_nr++] = ret; + ALLOC_GROW(remote_state->remotes, remote_state->remotes_nr + 1, + remote_state->remotes_alloc); + remote_state->remotes[remote_state->remotes_nr++] = ret; hashmap_entry_init(&ret->ent, lookup_entry.hash); - if (hashmap_put_entry(&remotes_hash, ret, ent)) + if (hashmap_put_entry(&remote_state->remotes_hash, ret, ent)) BUG("hashmap_put overwrote entry after hashmap_get returned NULL"); return ret; } +static void remote_clear(struct remote *remote) +{ + int i; + + free((char *)remote->name); + free((char *)remote->foreign_vcs); + + for (i = 0; i < remote->url_nr; i++) { + free((char *)remote->url[i]); + } + FREE_AND_NULL(remote->pushurl); + + for (i = 0; i < remote->pushurl_nr; i++) { + free((char *)remote->pushurl[i]); + } + FREE_AND_NULL(remote->pushurl); + free((char *)remote->receivepack); + free((char *)remote->uploadpack); + FREE_AND_NULL(remote->http_proxy); + FREE_AND_NULL(remote->http_proxy_authmethod); +} + static void add_merge(struct branch *branch, const char *name) { ALLOC_GROW(branch->merge_name, branch->merge_nr + 1, @@ -174,23 +166,74 @@ static void add_merge(struct branch *branch, const char *name) branch->merge_name[branch->merge_nr++] = name; } -static struct branch *make_branch(const char *name, size_t len) +struct branches_hash_key { + const char *str; + int len; +}; + +static int branches_hash_cmp(const void *unused_cmp_data, + const struct hashmap_entry *eptr, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + const struct branch *a, *b; + const struct branches_hash_key *key = keydata; + + a = container_of(eptr, const struct branch, ent); + b = container_of(entry_or_key, const struct branch, ent); + + if (key) + return strncmp(a->name, key->str, key->len) || + a->name[key->len]; + else + return strcmp(a->name, b->name); +} + +static struct branch *find_branch(struct remote_state *remote_state, + const char *name, size_t len) +{ + struct branches_hash_key lookup; + struct hashmap_entry lookup_entry, *e; + + if (!len) + len = strlen(name); + + lookup.str = name; + lookup.len = len; + hashmap_entry_init(&lookup_entry, memhash(name, len)); + + e = hashmap_get(&remote_state->branches_hash, &lookup_entry, &lookup); + if (e) + return container_of(e, struct branch, ent); + + return NULL; +} + +static void die_on_missing_branch(struct repository *repo, + struct branch *branch) +{ + /* branch == NULL is always valid because it represents detached HEAD. */ + if (branch && + branch != find_branch(repo->remote_state, branch->name, 0)) + die("branch %s was not found in the repository", branch->name); +} + +static struct branch *make_branch(struct remote_state *remote_state, + const char *name, size_t len) { struct branch *ret; - int i; - for (i = 0; i < branches_nr; i++) { - if (!strncmp(name, branches[i]->name, len) && - !branches[i]->name[len]) - return branches[i]; - } + ret = find_branch(remote_state, name, len); + if (ret) + return ret; - ALLOC_GROW(branches, branches_nr + 1, branches_alloc); CALLOC_ARRAY(ret, 1); - branches[branches_nr++] = ret; ret->name = xstrndup(name, len); ret->refname = xstrfmt("refs/heads/%s", ret->name); + hashmap_entry_init(&ret->ent, memhash(name, len)); + if (hashmap_put_entry(&remote_state->branches_hash, ret, ent)) + BUG("hashmap_put overwrote entry after hashmap_get returned NULL"); return ret; } @@ -229,7 +272,8 @@ static const char *skip_spaces(const char *s) return s; } -static void read_remotes_file(struct remote *remote) +static void read_remotes_file(struct remote_state *remote_state, + struct remote *remote) { struct strbuf buf = STRBUF_INIT; FILE *f = fopen_or_warn(git_path("remotes/%s", remote->name), "r"); @@ -244,7 +288,8 @@ static void read_remotes_file(struct remote *remote) strbuf_rtrim(&buf); if (skip_prefix(buf.buf, "URL:", &v)) - add_url_alias(remote, xstrdup(skip_spaces(v))); + add_url_alias(remote_state, remote, + xstrdup(skip_spaces(v))); else if (skip_prefix(buf.buf, "Push:", &v)) refspec_append(&remote->push, skip_spaces(v)); else if (skip_prefix(buf.buf, "Pull:", &v)) @@ -254,7 +299,8 @@ static void read_remotes_file(struct remote *remote) fclose(f); } -static void read_branches_file(struct remote *remote) +static void read_branches_file(struct remote_state *remote_state, + struct remote *remote) { char *frag; struct strbuf buf = STRBUF_INIT; @@ -286,7 +332,7 @@ static void read_branches_file(struct remote *remote) else frag = (char *)git_default_branch_name(0); - add_url_alias(remote, strbuf_detach(&buf, NULL)); + add_url_alias(remote_state, remote, strbuf_detach(&buf, NULL)); refspec_appendf(&remote->fetch, "refs/heads/%s:refs/heads/%s", frag, remote->name); @@ -305,10 +351,12 @@ static int handle_config(const char *key, const char *value, void *cb) const char *subkey; struct remote *remote; struct branch *branch; + struct remote_state *remote_state = cb; + if (parse_config_key(key, "branch", &name, &namelen, &subkey) >= 0) { if (!name) return 0; - branch = make_branch(name, namelen); + branch = make_branch(remote_state, name, namelen); if (!strcmp(subkey, "remote")) { return git_config_string(&branch->remote_name, key, value); } else if (!strcmp(subkey, "pushremote")) { @@ -327,12 +375,14 @@ static int handle_config(const char *key, const char *value, void *cb) if (!strcmp(subkey, "insteadof")) { if (!value) return config_error_nonbool(key); - rewrite = make_rewrite(&rewrites, name, namelen); + rewrite = make_rewrite(&remote_state->rewrites, name, + namelen); add_instead_of(rewrite, xstrdup(value)); } else if (!strcmp(subkey, "pushinsteadof")) { if (!value) return config_error_nonbool(key); - rewrite = make_rewrite(&rewrites_push, name, namelen); + rewrite = make_rewrite(&remote_state->rewrites_push, + name, namelen); add_instead_of(rewrite, xstrdup(value)); } } @@ -342,7 +392,8 @@ static int handle_config(const char *key, const char *value, void *cb) /* Handle remote.* variables */ if (!name && !strcmp(subkey, "pushdefault")) - return git_config_string(&pushremote_name, key, value); + return git_config_string(&remote_state->pushremote_name, key, + value); if (!name) return 0; @@ -352,7 +403,7 @@ static int handle_config(const char *key, const char *value, void *cb) name); return 0; } - remote = make_remote(name, namelen); + remote = make_remote(remote_state, name, namelen); remote->origin = REMOTE_CONFIG; if (current_config_scope() == CONFIG_SCOPE_LOCAL || current_config_scope() == CONFIG_SCOPE_WORKTREE) @@ -422,44 +473,52 @@ static int handle_config(const char *key, const char *value, void *cb) return 0; } -static void alias_all_urls(void) +static void alias_all_urls(struct remote_state *remote_state) { int i, j; - for (i = 0; i < remotes_nr; i++) { + for (i = 0; i < remote_state->remotes_nr; i++) { int add_pushurl_aliases; - if (!remotes[i]) + if (!remote_state->remotes[i]) continue; - for (j = 0; j < remotes[i]->pushurl_nr; j++) { - remotes[i]->pushurl[j] = alias_url(remotes[i]->pushurl[j], &rewrites); + for (j = 0; j < remote_state->remotes[i]->pushurl_nr; j++) { + remote_state->remotes[i]->pushurl[j] = + alias_url(remote_state->remotes[i]->pushurl[j], + &remote_state->rewrites); } - add_pushurl_aliases = remotes[i]->pushurl_nr == 0; - for (j = 0; j < remotes[i]->url_nr; j++) { + add_pushurl_aliases = remote_state->remotes[i]->pushurl_nr == 0; + for (j = 0; j < remote_state->remotes[i]->url_nr; j++) { if (add_pushurl_aliases) - add_pushurl_alias(remotes[i], remotes[i]->url[j]); - remotes[i]->url[j] = alias_url(remotes[i]->url[j], &rewrites); + add_pushurl_alias( + remote_state, remote_state->remotes[i], + remote_state->remotes[i]->url[j]); + remote_state->remotes[i]->url[j] = + alias_url(remote_state->remotes[i]->url[j], + &remote_state->rewrites); } } } -static void read_config(void) +static void read_config(struct repository *repo) { - static int loaded; int flag; - if (loaded) + if (repo->remote_state->initialized) return; - loaded = 1; + repo->remote_state->initialized = 1; - current_branch = NULL; + repo->remote_state->current_branch = NULL; if (startup_info->have_repository) { - const char *head_ref = resolve_ref_unsafe("HEAD", 0, NULL, &flag); + int ignore_errno; + const char *head_ref = refs_resolve_ref_unsafe( + get_main_ref_store(repo), "HEAD", 0, NULL, &flag, &ignore_errno); if (head_ref && (flag & REF_ISSYMREF) && skip_prefix(head_ref, "refs/heads/", &head_ref)) { - current_branch = make_branch(head_ref, strlen(head_ref)); + repo->remote_state->current_branch = make_branch( + repo->remote_state, head_ref, strlen(head_ref)); } } - git_config(handle_config, NULL); - alias_all_urls(); + repo_config(repo, handle_config, repo->remote_state); + alias_all_urls(repo->remote_state); } static int valid_remote_nick(const char *name) @@ -474,7 +533,9 @@ static int valid_remote_nick(const char *name) return 1; } -const char *remote_for_branch(struct branch *branch, int *explicit) +static const char *remotes_remote_for_branch(struct remote_state *remote_state, + struct branch *branch, + int *explicit) { if (branch && branch->remote_name) { if (explicit) @@ -486,32 +547,61 @@ const char *remote_for_branch(struct branch *branch, int *explicit) return "origin"; } -const char *pushremote_for_branch(struct branch *branch, int *explicit) +const char *remote_for_branch(struct branch *branch, int *explicit) +{ + read_config(the_repository); + die_on_missing_branch(the_repository, branch); + + return remotes_remote_for_branch(the_repository->remote_state, branch, + explicit); +} + +static const char * +remotes_pushremote_for_branch(struct remote_state *remote_state, + struct branch *branch, int *explicit) { if (branch && branch->pushremote_name) { if (explicit) *explicit = 1; return branch->pushremote_name; } - if (pushremote_name) { + if (remote_state->pushremote_name) { if (explicit) *explicit = 1; - return pushremote_name; + return remote_state->pushremote_name; } - return remote_for_branch(branch, explicit); + return remotes_remote_for_branch(remote_state, branch, explicit); +} + +const char *pushremote_for_branch(struct branch *branch, int *explicit) +{ + read_config(the_repository); + die_on_missing_branch(the_repository, branch); + + return remotes_pushremote_for_branch(the_repository->remote_state, + branch, explicit); } +static struct remote *remotes_remote_get(struct remote_state *remote_state, + const char *name); + const char *remote_ref_for_branch(struct branch *branch, int for_push) { + read_config(the_repository); + die_on_missing_branch(the_repository, branch); + if (branch) { if (!for_push) { if (branch->merge_nr) { return branch->merge_name[0]; } } else { - const char *dst, *remote_name = - pushremote_for_branch(branch, NULL); - struct remote *remote = remote_get(remote_name); + const char *dst, + *remote_name = remotes_pushremote_for_branch( + the_repository->remote_state, branch, + NULL); + struct remote *remote = remotes_remote_get( + the_repository->remote_state, remote_name); if (remote && remote->push.nr && (dst = apply_refspecs(&remote->push, @@ -523,41 +613,58 @@ const char *remote_ref_for_branch(struct branch *branch, int for_push) return NULL; } -static struct remote *remote_get_1(const char *name, - const char *(*get_default)(struct branch *, int *)) +static struct remote * +remotes_remote_get_1(struct remote_state *remote_state, const char *name, + const char *(*get_default)(struct remote_state *, + struct branch *, int *)) { struct remote *ret; int name_given = 0; - read_config(); - if (name) name_given = 1; else - name = get_default(current_branch, &name_given); + name = get_default(remote_state, remote_state->current_branch, + &name_given); - ret = make_remote(name, 0); + ret = make_remote(remote_state, name, 0); if (valid_remote_nick(name) && have_git_dir()) { if (!valid_remote(ret)) - read_remotes_file(ret); + read_remotes_file(remote_state, ret); if (!valid_remote(ret)) - read_branches_file(ret); + read_branches_file(remote_state, ret); } if (name_given && !valid_remote(ret)) - add_url_alias(ret, name); + add_url_alias(remote_state, ret, name); if (!valid_remote(ret)) return NULL; return ret; } +static inline struct remote * +remotes_remote_get(struct remote_state *remote_state, const char *name) +{ + return remotes_remote_get_1(remote_state, name, + remotes_remote_for_branch); +} + struct remote *remote_get(const char *name) { - return remote_get_1(name, remote_for_branch); + read_config(the_repository); + return remotes_remote_get(the_repository->remote_state, name); +} + +static inline struct remote * +remotes_pushremote_get(struct remote_state *remote_state, const char *name) +{ + return remotes_remote_get_1(remote_state, name, + remotes_pushremote_for_branch); } struct remote *pushremote_get(const char *name) { - return remote_get_1(name, pushremote_for_branch); + read_config(the_repository); + return remotes_pushremote_get(the_repository->remote_state, name); } int remote_is_configured(struct remote *remote, int in_repo) @@ -572,12 +679,14 @@ int remote_is_configured(struct remote *remote, int in_repo) int for_each_remote(each_remote_fn fn, void *priv) { int i, result = 0; - read_config(); - for (i = 0; i < remotes_nr && !result; i++) { - struct remote *r = remotes[i]; - if (!r) + read_config(the_repository); + for (i = 0; i < the_repository->remote_state->remotes_nr && !result; + i++) { + struct remote *remote = + the_repository->remote_state->remotes[i]; + if (!remote) continue; - result = fn(r, priv); + result = fn(remote, priv); } return result; } @@ -1642,7 +1751,7 @@ void set_ref_status_for_push(struct ref *remote_refs, int send_mirror, } } -static void set_merge(struct branch *ret) +static void set_merge(struct remote_state *remote_state, struct branch *ret) { struct remote *remote; char *ref; @@ -1662,7 +1771,7 @@ static void set_merge(struct branch *ret) return; } - remote = remote_get(ret->remote_name); + remote = remotes_remote_get(remote_state, ret->remote_name); CALLOC_ARRAY(ret->merge, ret->merge_nr); for (i = 0; i < ret->merge_nr; i++) { @@ -1683,12 +1792,13 @@ struct branch *branch_get(const char *name) { struct branch *ret; - read_config(); + read_config(the_repository); if (!name || !*name || !strcmp(name, "HEAD")) - ret = current_branch; + ret = the_repository->remote_state->current_branch; else - ret = make_branch(name, strlen(name)); - set_merge(ret); + ret = make_branch(the_repository->remote_state, name, + strlen(name)); + set_merge(the_repository->remote_state, ret); return ret; } @@ -1759,11 +1869,14 @@ static const char *tracking_for_push_dest(struct remote *remote, return ret; } -static const char *branch_get_push_1(struct branch *branch, struct strbuf *err) +static const char *branch_get_push_1(struct remote_state *remote_state, + struct branch *branch, struct strbuf *err) { struct remote *remote; - remote = remote_get(pushremote_for_branch(branch, NULL)); + remote = remotes_remote_get( + remote_state, + remotes_pushremote_for_branch(remote_state, branch, NULL)); if (!remote) return error_buf(err, _("branch '%s' has no remote for pushing"), @@ -1821,11 +1934,15 @@ static const char *branch_get_push_1(struct branch *branch, struct strbuf *err) const char *branch_get_push(struct branch *branch, struct strbuf *err) { + read_config(the_repository); + die_on_missing_branch(the_repository, branch); + if (!branch) return error_buf(err, _("HEAD does not point to a branch")); if (!branch->push_tracking_ref) - branch->push_tracking_ref = branch_get_push_1(branch, err); + branch->push_tracking_ref = branch_get_push_1( + the_repository->remote_state, branch, err); return branch->push_tracking_ref; } @@ -2585,3 +2702,29 @@ void apply_push_cas(struct push_cas_option *cas, check_if_includes_upstream(ref); } } + +struct remote_state *remote_state_new(void) +{ + struct remote_state *r = xmalloc(sizeof(*r)); + + memset(r, 0, sizeof(*r)); + + hashmap_init(&r->remotes_hash, remotes_hash_cmp, NULL, 0); + hashmap_init(&r->branches_hash, branches_hash_cmp, NULL, 0); + return r; +} + +void remote_state_clear(struct remote_state *remote_state) +{ + int i; + + for (i = 0; i < remote_state->remotes_nr; i++) { + remote_clear(remote_state->remotes[i]); + } + FREE_AND_NULL(remote_state->remotes); + remote_state->remotes_alloc = 0; + remote_state->remotes_nr = 0; + + hashmap_clear_and_free(&remote_state->remotes_hash, struct remote, ent); + hashmap_clear_and_free(&remote_state->branches_hash, struct remote, ent); +} @@ -23,6 +23,40 @@ enum { REMOTE_BRANCHES }; +struct rewrite { + const char *base; + size_t baselen; + struct counted_string *instead_of; + int instead_of_nr; + int instead_of_alloc; +}; + +struct rewrites { + struct rewrite **rewrite; + int rewrite_alloc; + int rewrite_nr; +}; + +struct remote_state { + struct remote **remotes; + int remotes_alloc; + int remotes_nr; + struct hashmap remotes_hash; + + struct hashmap branches_hash; + + struct branch *current_branch; + const char *pushremote_name; + + struct rewrites rewrites; + struct rewrites rewrites_push; + + int initialized; +}; + +void remote_state_clear(struct remote_state *remote_state); +struct remote_state *remote_state_new(void); + struct remote { struct hashmap_entry ent; @@ -256,6 +290,7 @@ int remote_find_tracking(struct remote *remote, struct refspec_item *refspec); * branch_get(name) for "refs/heads/{name}", or with branch_get(NULL) for HEAD. */ struct branch { + struct hashmap_entry ent; /* The short name of the branch. */ const char *name; diff --git a/repository.c b/repository.c index c5b90ba93e..c7ea706c20 100644 --- a/repository.c +++ b/repository.c @@ -9,6 +9,7 @@ #include "config.h" #include "object.h" #include "lockfile.h" +#include "remote.h" #include "submodule-config.h" #include "sparse-index.h" #include "promisor-remote.h" @@ -24,6 +25,7 @@ void initialize_the_repository(void) the_repo.index = &the_index; the_repo.objects = raw_object_store_new(); + the_repo.remote_state = remote_state_new(); the_repo.parsed_objects = parsed_object_pool_new(); repo_set_hash_algo(&the_repo, GIT_HASH_SHA1); @@ -164,6 +166,7 @@ int repo_init(struct repository *repo, repo->objects = raw_object_store_new(); repo->parsed_objects = parsed_object_pool_new(); + repo->remote_state = remote_state_new(); if (repo_init_gitdir(repo, gitdir)) goto error; @@ -270,6 +273,11 @@ void repo_clear(struct repository *repo) promisor_remote_clear(repo->promisor_remote_config); FREE_AND_NULL(repo->promisor_remote_config); } + + if (repo->remote_state) { + remote_state_clear(repo->remote_state); + FREE_AND_NULL(repo->remote_state); + } } int repo_read_index(struct repository *repo) diff --git a/repository.h b/repository.h index a057653981..98f9583470 100644 --- a/repository.h +++ b/repository.h @@ -11,6 +11,7 @@ struct pathspec; struct raw_object_store; struct submodule_cache; struct promisor_remote_config; +struct remote_state; enum untracked_cache_setting { UNTRACKED_CACHE_KEEP, @@ -127,6 +128,9 @@ struct repository { */ struct index_state *index; + /* Repository's remotes and associated structures. */ + struct remote_state *remote_state; + /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; diff --git a/run-command.c b/run-command.c index f40df01c77..3ea2c2fc10 100644 --- a/run-command.c +++ b/run-command.c @@ -380,7 +380,7 @@ static void child_err_spew(struct child_process *cmd, struct child_err *cerr) switch (cerr->err) { case CHILD_ERR_CHDIR: error_errno("exec '%s': cd to '%s' failed", - cmd->argv[0], cmd->dir); + cmd->args.v[0], cmd->dir); break; case CHILD_ERR_DUP2: error_errno("dup2() in child failed"); @@ -392,12 +392,12 @@ static void child_err_spew(struct child_process *cmd, struct child_err *cerr) error_errno("sigprocmask failed restoring signals"); break; case CHILD_ERR_ENOENT: - error_errno("cannot run %s", cmd->argv[0]); + error_errno("cannot run %s", cmd->args.v[0]); break; case CHILD_ERR_SILENT: break; case CHILD_ERR_ERRNO: - error_errno("cannot exec '%s'", cmd->argv[0]); + error_errno("cannot exec '%s'", cmd->args.v[0]); break; } set_error_routine(old_errfn); @@ -405,7 +405,7 @@ static void child_err_spew(struct child_process *cmd, struct child_err *cerr) static int prepare_cmd(struct strvec *out, const struct child_process *cmd) { - if (!cmd->argv[0]) + if (!cmd->args.v[0]) BUG("command is empty"); /* @@ -415,11 +415,11 @@ static int prepare_cmd(struct strvec *out, const struct child_process *cmd) strvec_push(out, SHELL_PATH); if (cmd->git_cmd) { - prepare_git_cmd(out, cmd->argv); + prepare_git_cmd(out, cmd->args.v); } else if (cmd->use_shell) { - prepare_shell_cmd(out, cmd->argv); + prepare_shell_cmd(out, cmd->args.v); } else { - strvec_pushv(out, cmd->argv); + strvec_pushv(out, cmd->args.v); } /* @@ -552,20 +552,17 @@ static int wait_or_whine(pid_t pid, const char *argv0, int in_signal) while ((waiting = waitpid(pid, &status, 0)) < 0 && errno == EINTR) ; /* nothing */ - if (in_signal) { - if (WIFEXITED(status)) - code = WEXITSTATUS(status); - return code; - } if (waiting < 0) { failed_errno = errno; - error_errno("waitpid for %s failed", argv0); + if (!in_signal) + error_errno("waitpid for %s failed", argv0); } else if (waiting != pid) { - error("waitpid is confused (%s)", argv0); + if (!in_signal) + error("waitpid is confused (%s)", argv0); } else if (WIFSIGNALED(status)) { code = WTERMSIG(status); - if (code != SIGINT && code != SIGQUIT && code != SIGPIPE) + if (!in_signal && code != SIGINT && code != SIGQUIT && code != SIGPIPE) error("%s died of signal %d", argv0, code); /* * This return value is chosen so that code & 0xff @@ -576,10 +573,12 @@ static int wait_or_whine(pid_t pid, const char *argv0, int in_signal) } else if (WIFEXITED(status)) { code = WEXITSTATUS(status); } else { - error("waitpid is confused (%s)", argv0); + if (!in_signal) + error("waitpid is confused (%s)", argv0); } - clear_child_for_cleanup(pid); + if (!in_signal) + clear_child_for_cleanup(pid); errno = failed_errno; return code; @@ -655,15 +654,10 @@ static void trace_run_command(const struct child_process *cp) sq_quote_buf_pretty(&buf, cp->dir); strbuf_addch(&buf, ';'); } - /* - * The caller is responsible for initializing cp->env from - * cp->env_array if needed. We only check one place. - */ - if (cp->env) - trace_add_env(&buf, cp->env); + trace_add_env(&buf, cp->env_array.v); if (cp->git_cmd) strbuf_addstr(&buf, " git"); - sq_quote_argv_pretty(&buf, cp->argv); + sq_quote_argv_pretty(&buf, cp->args.v); trace_printf("%s", buf.buf); strbuf_release(&buf); @@ -676,11 +670,6 @@ int start_command(struct child_process *cmd) int failed_errno; char *str; - if (!cmd->argv) - cmd->argv = cmd->args.v; - if (!cmd->env) - cmd->env = cmd->env_array.v; - /* * In case of errors we must keep the promise to close FDs * that have been passed in via ->in and ->out. @@ -729,7 +718,7 @@ int start_command(struct child_process *cmd) str = "standard error"; fail_pipe: error("cannot create %s pipe for %s: %s", - str, cmd->argv[0], strerror(failed_errno)); + str, cmd->args.v[0], strerror(failed_errno)); child_process_clear(cmd); errno = failed_errno; return -1; @@ -758,7 +747,7 @@ fail_pipe: failed_errno = errno; cmd->pid = -1; if (!cmd->silent_exec_failure) - error_errno("cannot run %s", cmd->argv[0]); + error_errno("cannot run %s", cmd->args.v[0]); goto end_of_spawn; } @@ -770,7 +759,7 @@ fail_pipe: set_cloexec(null_fd); } - childenv = prep_childenv(cmd->env); + childenv = prep_childenv(cmd->env_array.v); atfork_prepare(&as); /* @@ -868,7 +857,7 @@ fail_pipe: } atfork_parent(&as); if (cmd->pid < 0) - error_errno("cannot fork() for %s", cmd->argv[0]); + error_errno("cannot fork() for %s", cmd->args.v[0]); else if (cmd->clean_on_exit) mark_child_for_cleanup(cmd->pid, cmd); @@ -885,7 +874,7 @@ fail_pipe: * At this point we know that fork() succeeded, but exec() * failed. Errors have been reported to our stderr. */ - wait_or_whine(cmd->pid, cmd->argv[0], 0); + wait_or_whine(cmd->pid, cmd->args.v[0], 0); child_err_spew(cmd, &cerr); failed_errno = errno; cmd->pid = -1; @@ -902,7 +891,7 @@ end_of_spawn: #else { int fhin = 0, fhout = 1, fherr = 2; - const char **sargv = cmd->argv; + const char **sargv = cmd->args.v; struct strvec nargv = STRVEC_INIT; if (cmd->no_stdin) @@ -929,20 +918,20 @@ end_of_spawn: fhout = dup(cmd->out); if (cmd->git_cmd) - cmd->argv = prepare_git_cmd(&nargv, cmd->argv); + cmd->args.v = prepare_git_cmd(&nargv, sargv); else if (cmd->use_shell) - cmd->argv = prepare_shell_cmd(&nargv, cmd->argv); + cmd->args.v = prepare_shell_cmd(&nargv, sargv); - cmd->pid = mingw_spawnvpe(cmd->argv[0], cmd->argv, (char**) cmd->env, + cmd->pid = mingw_spawnvpe(cmd->args.v[0], cmd->args.v, (char**) cmd->env_array.v, cmd->dir, fhin, fhout, fherr); failed_errno = errno; if (cmd->pid < 0 && (!cmd->silent_exec_failure || errno != ENOENT)) - error_errno("cannot spawn %s", cmd->argv[0]); + error_errno("cannot spawn %s", cmd->args.v[0]); if (cmd->clean_on_exit && cmd->pid >= 0) mark_child_for_cleanup(cmd->pid, cmd); strvec_clear(&nargv); - cmd->argv = sargv; + cmd->args.v = sargv; if (fhin != 0) close(fhin); if (fhout != 1) @@ -992,7 +981,7 @@ end_of_spawn: int finish_command(struct child_process *cmd) { - int ret = wait_or_whine(cmd->pid, cmd->argv[0], 0); + int ret = wait_or_whine(cmd->pid, cmd->args.v[0], 0); trace2_child_exit(cmd, ret); child_process_clear(cmd); invalidate_lstat_cache(); @@ -1001,7 +990,7 @@ int finish_command(struct child_process *cmd) int finish_command_in_signal(struct child_process *cmd) { - int ret = wait_or_whine(cmd->pid, cmd->argv[0], 1); + int ret = wait_or_whine(cmd->pid, cmd->args.v[0], 1); trace2_child_exit(cmd, ret); return ret; } @@ -1039,7 +1028,7 @@ int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir, const char *const *env, const char *tr2_class) { struct child_process cmd = CHILD_PROCESS_INIT; - cmd.argv = argv; + strvec_pushv(&cmd.args, argv); cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0; cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0; cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0; @@ -1049,7 +1038,8 @@ int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir, cmd.wait_after_clean = opt & RUN_WAIT_AFTER_CLEAN ? 1 : 0; cmd.close_object_store = opt & RUN_CLOSE_OBJECT_STORE ? 1 : 0; cmd.dir = dir; - cmd.env = env; + if (env) + strvec_pushv(&cmd.env_array, (const char **)env); cmd.trace2_child_class = tr2_class; return run_command(&cmd); } @@ -1335,7 +1325,8 @@ int run_hook_ve(const char *const *env, const char *name, va_list args) strvec_push(&hook.args, p); while ((p = va_arg(args, const char *))) strvec_push(&hook.args, p); - hook.env = env; + if (env) + strvec_pushv(&hook.env_array, (const char **)env); hook.no_stdin = 1; hook.stdout_to_stderr = 1; hook.trace2_hook_name = name; diff --git a/run-command.h b/run-command.h index 4987826258..2be5f5d642 100644 --- a/run-command.h +++ b/run-command.h @@ -44,22 +44,35 @@ struct child_process { /** - * The .argv member is set up as an array of string pointers (NULL - * terminated), of which .argv[0] is the program name to run (usually - * without a path). If the command to run is a git command, set argv[0] to - * the command name without the 'git-' prefix and set .git_cmd = 1. + * The .args is a `struct strvec', use that API to manipulate + * it, e.g. strvec_pushv() to add an existing "const char **" + * vector. * - * Note that the ownership of the memory pointed to by .argv stays with the - * caller, but it should survive until `finish_command` completes. If the - * .argv member is NULL, `start_command` will point it at the .args - * `strvec` (so you may use one or the other, but you must use exactly - * one). The memory in .args will be cleaned up automatically during - * `finish_command` (or during `start_command` when it is unsuccessful). + * If the command to run is a git command, set the first + * element in the strvec to the command name without the + * 'git-' prefix and set .git_cmd = 1. * + * The memory in .args will be cleaned up automatically during + * `finish_command` (or during `start_command` when it is unsuccessful). */ - const char **argv; - struct strvec args; + + /** + * Like .args the .env_array is a `struct strvec'. + * + * To modify the environment of the sub-process, specify an array of + * environment settings. Each string in the array manipulates the + * environment. + * + * - If the string is of the form "VAR=value", i.e. it contains '=' + * the variable is added to the child process's environment. + * + * - If the string does not contain '=', it names an environment + * variable that will be removed from the child process's environment. + * + * The memory in .env_array will be cleaned up automatically during + * `finish_command` (or during `start_command` when it is unsuccessful). + */ struct strvec env_array; pid_t pid; @@ -96,23 +109,6 @@ struct child_process { */ const char *dir; - /** - * To modify the environment of the sub-process, specify an array of - * string pointers (NULL terminated) in .env: - * - * - If the string is of the form "VAR=value", i.e. it contains '=' - * the variable is added to the child process's environment. - * - * - If the string does not contain '=', it names an environment - * variable that will be removed from the child process's environment. - * - * If the .env member is NULL, `start_command` will point it at the - * .env_array `strvec` (so you may use one or the other, but not both). - * The memory in .env_array will be cleaned up automatically during - * `finish_command` (or during `start_command` when it is unsuccessful). - */ - const char *const *env; - unsigned no_stdin:1; unsigned no_stdout:1; unsigned no_stderr:1; diff --git a/sequencer.c b/sequencer.c index ea96837cde..b69ef4dd4e 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1164,18 +1164,14 @@ static int run_rewrite_hook(const struct object_id *oldoid, const struct object_id *newoid) { struct child_process proc = CHILD_PROCESS_INIT; - const char *argv[3]; int code; struct strbuf sb = STRBUF_INIT; + const char *hook_path = find_hook("post-rewrite"); - argv[0] = find_hook("post-rewrite"); - if (!argv[0]) + if (!hook_path) return 0; - argv[1] = "amend"; - argv[2] = NULL; - - proc.argv = argv; + strvec_pushl(&proc.args, hook_path, "amend", NULL); proc.in = -1; proc.stdout_to_stderr = 1; proc.trace2_hook_name = "post-rewrite"; @@ -1284,6 +1280,8 @@ void print_commit_summary(struct repository *r, struct pretty_print_context pctx = {0}; struct strbuf author_ident = STRBUF_INIT; struct strbuf committer_ident = STRBUF_INIT; + struct ref_store *refs; + int resolve_errno; commit = lookup_commit(r, oid); if (!commit) @@ -1333,9 +1331,13 @@ void print_commit_summary(struct repository *r, rev.diffopt.break_opt = 0; diff_setup_done(&rev.diffopt); - head = resolve_ref_unsafe("HEAD", 0, NULL, NULL); - if (!head) + refs = get_main_ref_store(the_repository); + head = refs_resolve_ref_unsafe(refs, "HEAD", 0, NULL, NULL, + &resolve_errno); + if (!head) { + errno = resolve_errno; die_errno(_("unable to resolve HEAD after creating commit")); + } if (!strcmp(head, "HEAD")) head = _("detached HEAD"); else @@ -5496,7 +5498,7 @@ static void todo_list_add_exec_commands(struct todo_list *todo_list, } /* insert or append final <commands> */ - if (insert || nr == todo_list->nr) { + if (insert) { ALLOC_GROW(items, nr + commands->nr, alloc); COPY_ARRAY(items + nr, base_items, commands->nr); nr += commands->nr; diff --git a/sparse-index.c b/sparse-index.c index 7b7ff79e04..a1d505d50e 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -122,17 +122,17 @@ static int index_has_unmerged_entries(struct index_state *istate) return 0; } -int convert_to_sparse(struct index_state *istate, int flags) +static int is_sparse_index_allowed(struct index_state *istate, int flags) { - int test_env; - if (istate->sparse_index || !istate->cache_nr || - !core_apply_sparse_checkout || !core_sparse_checkout_cone) + if (!core_apply_sparse_checkout || !core_sparse_checkout_cone) return 0; if (!istate->repo) istate->repo = the_repository; if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) { + int test_env; + /* * The sparse index is not (yet) integrated with a split index. */ @@ -168,25 +168,41 @@ int convert_to_sparse(struct index_state *istate, int flags) if (!istate->sparse_checkout_patterns->use_cone_patterns) return 0; + return 1; +} + +int convert_to_sparse(struct index_state *istate, int flags) +{ /* - * NEEDSWORK: If we have unmerged entries, then stay full. - * Unmerged entries prevent the cache-tree extension from working. + * If the index is already sparse, empty, or otherwise + * cannot be converted to sparse, do not convert. */ - if (index_has_unmerged_entries(istate)) + if (istate->sparse_index || !istate->cache_nr || + !is_sparse_index_allowed(istate, flags)) return 0; - /* Clear and recompute the cache-tree */ - cache_tree_free(&istate->cache_tree); /* - * Silently return if there is a problem with the cache tree update, - * which might just be due to a conflict state in some entry. - * - * This might create new tree objects, so be sure to use - * WRITE_TREE_MISSING_OK. + * NEEDSWORK: If we have unmerged entries, then stay full. + * Unmerged entries prevent the cache-tree extension from working. */ - if (cache_tree_update(istate, WRITE_TREE_MISSING_OK)) + if (index_has_unmerged_entries(istate)) return 0; + if (!cache_tree_fully_valid(istate->cache_tree)) { + /* Clear and recompute the cache-tree */ + cache_tree_free(&istate->cache_tree); + + /* + * Silently return if there is a problem with the cache tree update, + * which might just be due to a conflict state in some entry. + * + * This might create new tree objects, so be sure to use + * WRITE_TREE_MISSING_OK. + */ + if (cache_tree_update(istate, WRITE_TREE_MISSING_OK)) + return 0; + } + remove_fsmonitor(istate); trace2_region_enter("index", "convert_to_sparse", istate->repo); @@ -313,6 +329,18 @@ void ensure_full_index(struct index_state *istate) trace2_region_leave("index", "ensure_full_index", istate->repo); } +void ensure_correct_sparsity(struct index_state *istate) +{ + /* + * If the index can be sparse, make it sparse. Otherwise, + * ensure the index is full. + */ + if (is_sparse_index_allowed(istate, 0)) + convert_to_sparse(istate, 0); + else + ensure_full_index(istate); +} + /* * This static global helps avoid infinite recursion between * expand_to_path() and index_file_exists(). diff --git a/sparse-index.h b/sparse-index.h index 9f3d7bc7fa..656bd835b2 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -4,6 +4,7 @@ struct index_state; #define SPARSE_INDEX_MEMORY_ONLY (1 << 0) int convert_to_sparse(struct index_state *istate, int flags); +void ensure_correct_sparsity(struct index_state *istate); /* * Some places in the codebase expect to search for a specific path. @@ -1006,7 +1006,12 @@ void strbuf_addftime(struct strbuf *sb, const char *fmt, const struct tm *tm, /* * There is no portable way to pass timezone information to - * strftime, so we handle %z and %Z here. + * strftime, so we handle %z and %Z here. Likewise '%s', because + * going back to an epoch time requires knowing the zone. + * + * Note that tz_offset is in the "[-+]HHMM" decimal form; this is what + * we want for %z, but the computation for %s has to convert to number + * of seconds. */ for (;;) { const char *percent = strchrnul(fmt, '%'); @@ -1019,6 +1024,13 @@ void strbuf_addftime(struct strbuf *sb, const char *fmt, const struct tm *tm, strbuf_addstr(&munged_fmt, "%%"); fmt++; break; + case 's': + strbuf_addf(&munged_fmt, "%"PRItime, + (timestamp_t)tm_to_time_t(tm) - + 3600 * (tz_offset / 100) - + 60 * (tz_offset % 100)); + fmt++; + break; case 'z': strbuf_addf(&munged_fmt, "%+05d", tz_offset); fmt++; diff --git a/sub-process.c b/sub-process.c index dfa790d3ff..cae56ae6b8 100644 --- a/sub-process.c +++ b/sub-process.c @@ -187,7 +187,7 @@ static int handshake_capabilities(struct child_process *process, *supported_capabilities |= capabilities[i].flag; } else { die("subprocess '%s' requested unsupported capability '%s'", - process->argv[0], p); + process->args.v[0], p); } } @@ -466,6 +466,12 @@ explicitly providing repositories when accessing submodule objects is complete or needs to be abandoned for whatever reason (in which case the migrated codepaths still retain their performance benefits). +GIT_TEST_REQUIRE_PREREQ=<list> allows specifying a space speparated list of +prereqs that are required to succeed. If a prereq in this list is triggered by +a test and then fails then the whole test run will abort. This can help to make +sure the expected tests are executed and not silently skipped when their +dependency breaks or is simply not present in a new environment. + Naming Tests ------------ diff --git a/t/aggregate-results.sh b/t/aggregate-results.sh index 7913e206ed..7f2b83bdc8 100755 --- a/t/aggregate-results.sh +++ b/t/aggregate-results.sh @@ -6,6 +6,7 @@ success=0 failed=0 broken=0 total=0 +missing_prereq= while read file do @@ -30,10 +31,26 @@ do broken=$(($broken + $value)) ;; total) total=$(($total + $value)) ;; + missing_prereq) + missing_prereq="$missing_prereq,$value" ;; esac done <"$file" done +if test -n "$missing_prereq" +then + unique_missing_prereq=$( + echo $missing_prereq | + tr -s "," "\n" | + grep -v '^$' | + sort -u | + paste -s -d ' ') + if test -n "$unique_missing_prereq" + then + printf "\nmissing prereq: $unique_missing_prereq\n\n" + fi +fi + if test -n "$failed_tests" then printf "\nfailed test(s):$failed_tests\n\n" diff --git a/t/helper/test-genzeros.c b/t/helper/test-genzeros.c index 9532f5bac9..8ca988d621 100644 --- a/t/helper/test-genzeros.c +++ b/t/helper/test-genzeros.c @@ -3,18 +3,31 @@ int cmd__genzeros(int argc, const char **argv) { - long count; + /* static, so that it is NUL-initialized */ + static const char zeros[256 * 1024]; + intmax_t count; + ssize_t n; if (argc > 2) { fprintf(stderr, "usage: %s [<count>]\n", argv[0]); return 1; } - count = argc > 1 ? strtol(argv[1], NULL, 0) : -1L; + count = argc > 1 ? strtoimax(argv[1], NULL, 0) : -1; - while (count < 0 || count--) { - if (putchar(0) == EOF) + /* Writing out individual NUL bytes is slow... */ + while (count < 0) + if (write(1, zeros, ARRAY_SIZE(zeros)) < 0) return -1; + + while (count > 0) { + n = write(1, zeros, count < ARRAY_SIZE(zeros) ? + count : ARRAY_SIZE(zeros)); + + if (n < 0) + return -1; + + count -= n; } return 0; diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index b52c174acc..0d9f08931a 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -39,8 +39,6 @@ int cmd__read_cache(int argc, const char **argv) int table = 0, expand = 0; initialize_the_repository(); - prepare_repo_settings(r); - r->settings.command_requires_full_index = 0; for (++argv, --argc; *argv && starts_with(*argv, "--"); ++argv, --argc) { if (skip_prefix(*argv, "--print-and-refresh=", &name)) @@ -56,6 +54,9 @@ int cmd__read_cache(int argc, const char **argv) setup_git_directory(); git_config(git_default_config, NULL); + prepare_repo_settings(r); + r->settings.command_requires_full_index = 0; + for (i = 0; i < cnt; i++) { repo_read_index(r); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 9d6fa7a377..27072ba94d 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -55,9 +55,10 @@ static int read_midx_file(const char *object_dir, int show_objects) printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); } - return 0; } + close_midx(m); + return 0; } diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index b314b81a45..971b755579 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -123,9 +123,10 @@ static int cmd_resolve_ref(struct ref_store *refs, const char **argv) int resolve_flags = arg_flags(*argv++, "resolve-flags"); int flags; const char *ref; + int ignore_errno; ref = refs_resolve_ref_unsafe(refs, refname, resolve_flags, - &oid, &flags); + &oid, &flags, &ignore_errno); printf("%s %s 0x%x\n", oid_to_hex(&oid), ref ? ref : "(null)", flags); return ref ? 0 : 1; } @@ -151,9 +152,9 @@ static int each_reflog(struct object_id *old_oid, struct object_id *new_oid, const char *committer, timestamp_t timestamp, int tz, const char *msg, void *cb_data) { - printf("%s %s %s %"PRItime" %d %s\n", - oid_to_hex(old_oid), oid_to_hex(new_oid), - committer, timestamp, tz, msg); + printf("%s %s %s %" PRItime " %+05d%s%s", oid_to_hex(old_oid), + oid_to_hex(new_oid), committer, timestamp, tz, + *msg == '\n' ? "" : "\t", msg); return 0; } @@ -181,11 +182,10 @@ static int cmd_reflog_exists(struct ref_store *refs, const char **argv) static int cmd_create_reflog(struct ref_store *refs, const char **argv) { const char *refname = notnull(*argv++, "refname"); - int force_create = arg_flags(*argv++, "force-create"); struct strbuf err = STRBUF_INIT; int ret; - ret = refs_create_reflog(refs, refname, force_create, &err); + ret = refs_create_reflog(refs, refname, &err); if (err.len) puts(err.buf); return ret; diff --git a/t/helper/test-reftable.c b/t/helper/test-reftable.c new file mode 100644 index 0000000000..26b03d7b78 --- /dev/null +++ b/t/helper/test-reftable.c @@ -0,0 +1,21 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable(int argc, const char **argv) +{ + basics_test_main(argc, argv); + block_test_main(argc, argv); + merged_test_main(argc, argv); + pq_test_main(argc, argv); + record_test_main(argc, argv); + refname_test_main(argc, argv); + readwrite_test_main(argc, argv); + stack_test_main(argc, argv); + tree_test_main(argc, argv); + return 0; +} + +int cmd__dump_reftable(int argc, const char **argv) +{ + return reftable_dump_main(argc, (char *const *)argv); +} diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c index 3c4fb86223..913775a14b 100644 --- a/t/helper/test-run-command.c +++ b/t/helper/test-run-command.c @@ -31,7 +31,7 @@ static int parallel_next(struct child_process *cp, if (number_callbacks >= 4) return 0; - strvec_pushv(&cp->args, d->argv); + strvec_pushv(&cp->args, d->args.v); strbuf_addstr(err, "preloaded output of a child\n"); number_callbacks++; return 1; @@ -274,7 +274,7 @@ static int quote_stress_test(int argc, const char **argv) if (i < skip) continue; - cp.argv = args.v; + strvec_pushv(&cp.args, args.v); strbuf_reset(&out); if (pipe_command(&cp, NULL, 0, &out, 0, NULL, 0) < 0) return error("Failed to spawn child process"); @@ -396,7 +396,7 @@ int cmd__run_command(int argc, const char **argv) } if (argc < 3) return 1; - proc.argv = (const char **)argv + 2; + strvec_pushv(&proc.args, (const char **)argv + 2); if (!strcmp(argv[1], "start-command-ENOENT")) { if (start_command(&proc) < 0 && errno == ENOENT) @@ -408,7 +408,8 @@ int cmd__run_command(int argc, const char **argv) exit(run_command(&proc)); jobs = atoi(argv[2]); - proc.argv = (const char **)argv + 3; + strvec_clear(&proc.args); + strvec_pushv(&proc.args, (const char **)argv + 3); if (!strcmp(argv[1], "run-command-parallel")) exit(run_processes_parallel(jobs, parallel_next, diff --git a/t/helper/test-subprocess.c b/t/helper/test-subprocess.c index 92b69de635..ff22f2fa2c 100644 --- a/t/helper/test-subprocess.c +++ b/t/helper/test-subprocess.c @@ -15,6 +15,6 @@ int cmd__subprocess(int argc, const char **argv) argv++; } cp.git_cmd = 1; - cp.argv = (const char **)argv + 1; + strvec_pushv(&cp.args, (const char **)argv + 1); return run_command(&cp); } diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 3ce5585e53..338a57b104 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -53,13 +53,15 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 9f0f522850..48cee1f4a2 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -19,6 +19,7 @@ int cmd__dump_cache_tree(int argc, const char **argv); int cmd__dump_fsmonitor(int argc, const char **argv); int cmd__dump_split_index(int argc, const char **argv); int cmd__dump_untracked_cache(int argc, const char **argv); +int cmd__dump_reftable(int argc, const char **argv); int cmd__example_decorate(int argc, const char **argv); int cmd__fast_rebase(int argc, const char **argv); int cmd__genrandom(int argc, const char **argv); @@ -49,6 +50,7 @@ int cmd__read_cache(int argc, const char **argv); int cmd__read_graph(int argc, const char **argv); int cmd__read_midx(int argc, const char **argv); int cmd__ref_store(int argc, const char **argv); +int cmd__reftable(int argc, const char **argv); int cmd__regex(int argc, const char **argv); int cmd__repository(int argc, const char **argv); int cmd__revision_walking(int argc, const char **argv); diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index 597626276f..bfd332120c 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -110,5 +110,8 @@ test_perf_on_all git add -A test_perf_on_all git add . test_perf_on_all git commit -a -m A test_perf_on_all git checkout -f - +test_perf_on_all git reset +test_perf_on_all git reset --hard +test_perf_on_all git reset -- does-not-exist test_done diff --git a/t/t0006-date.sh b/t/t0006-date.sh index 6b757d7169..794186961e 100755 --- a/t/t0006-date.sh +++ b/t/t0006-date.sh @@ -63,6 +63,10 @@ check_show 'format-local:%%z' "$TIME" '%z' check_show 'format:%Y-%m-%d %H:%M:%S' "$TIME" '2016-06-15 16:13:20' check_show 'format-local:%Y-%m-%d %H:%M:%S' "$TIME" '2016-06-15 09:13:20' '' EST5 +check_show 'format:%s' '123456789 +1234' 123456789 +check_show 'format:%s' '123456789 -1234' 123456789 +check_show 'format-local:%s' '123456789 -1234' 123456789 + # arbitrary time absurdly far in the future FUTURE="5758122296 -0400" check_show iso "$FUTURE" "2152-06-19 18:24:56 -0400" TIME_IS_64BIT,TIME_T_IS_64BIT diff --git a/t/t0007-git-var.sh b/t/t0007-git-var.sh index 53af92d571..e56f4b9ac5 100755 --- a/t/t0007-git-var.sh +++ b/t/t0007-git-var.sh @@ -27,6 +27,26 @@ test_expect_success !FAIL_PREREQS,!AUTOIDENT 'requested identities are strict' ' ) ' +test_expect_success 'get GIT_DEFAULT_BRANCH without configuration' ' + ( + sane_unset GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && + git init defbranch && + git -C defbranch symbolic-ref --short HEAD >expect && + git var GIT_DEFAULT_BRANCH >actual && + test_cmp expect actual + ) +' + +test_expect_success 'get GIT_DEFAULT_BRANCH with configuration' ' + test_config init.defaultbranch foo && + ( + sane_unset GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME && + echo foo >expect && + git var GIT_DEFAULT_BRANCH >actual && + test_cmp expect actual + ) +' + # For git var -l, we check only a representative variable; # testing the whole output would make our test too brittle with # respect to unrelated changes in the test suite's environment. diff --git a/t/t0020-crlf.sh b/t/t0020-crlf.sh index f25ae8b5e1..4125ab8b88 100755 --- a/t/t0020-crlf.sh +++ b/t/t0020-crlf.sh @@ -5,6 +5,7 @@ test_description='CRLF conversion' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh has_cr() { diff --git a/t/t0032-reftable-unittest.sh b/t/t0032-reftable-unittest.sh new file mode 100755 index 0000000000..0ed14971a5 --- /dev/null +++ b/t/t0032-reftable-unittest.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done diff --git a/t/t0071-sort.sh b/t/t0071-sort.sh index a8ab174879..6f9a501c72 100755 --- a/t/t0071-sort.sh +++ b/t/t0071-sort.sh @@ -2,6 +2,7 @@ test_description='verify sort functions' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'llist_mergesort()' ' diff --git a/t/t0110-urlmatch-normalization.sh b/t/t0110-urlmatch-normalization.sh index f99529d838..4dc9fecf72 100755 --- a/t/t0110-urlmatch-normalization.sh +++ b/t/t0110-urlmatch-normalization.sh @@ -47,7 +47,7 @@ test_expect_success 'url authority' ' test-tool urlmatch-normalization "scheme://@host" && test-tool urlmatch-normalization "scheme://%00@host" && ! test-tool urlmatch-normalization "scheme://%%@host" && - ! test-tool urlmatch-normalization "scheme://host_" && + test-tool urlmatch-normalization "scheme://host_" && test-tool urlmatch-normalization "scheme://user:pass@host/" && test-tool urlmatch-normalization "scheme://@host/" && test-tool urlmatch-normalization "scheme://host/" && diff --git a/t/t0200-gettext-basic.sh b/t/t0200-gettext-basic.sh index 8853d8afb9..522fb2ae69 100755 --- a/t/t0200-gettext-basic.sh +++ b/t/t0200-gettext-basic.sh @@ -5,6 +5,7 @@ test_description='Gettext support for Git' +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh test_expect_success "sanity: \$GIT_INTERNAL_GETTEXT_SH_SCHEME is set (to $GIT_INTERNAL_GETTEXT_SH_SCHEME)" ' diff --git a/t/t0201-gettext-fallbacks.sh b/t/t0201-gettext-fallbacks.sh index 6c74df0dc6..8724ce1052 100755 --- a/t/t0201-gettext-fallbacks.sh +++ b/t/t0201-gettext-fallbacks.sh @@ -8,6 +8,7 @@ test_description='Gettext Shell fallbacks' GIT_INTERNAL_GETTEXT_TEST_FALLBACKS=YesPlease export GIT_INTERNAL_GETTEXT_TEST_FALLBACKS +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh test_expect_success "sanity: \$GIT_INTERNAL_GETTEXT_SH_SCHEME is set (to $GIT_INTERNAL_GETTEXT_SH_SCHEME)" ' diff --git a/t/t0202-gettext-perl.sh b/t/t0202-gettext-perl.sh index a29d166e00..df2ea34932 100755 --- a/t/t0202-gettext-perl.sh +++ b/t/t0202-gettext-perl.sh @@ -5,6 +5,7 @@ test_description='Perl gettext interface (Git::I18N)' +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh if ! test_have_prereq PERL; then diff --git a/t/t0204-gettext-reencode-sanity.sh b/t/t0204-gettext-reencode-sanity.sh index 8437e51eb5..4f2e0dcb02 100755 --- a/t/t0204-gettext-reencode-sanity.sh +++ b/t/t0204-gettext-reencode-sanity.sh @@ -5,6 +5,7 @@ test_description="Gettext reencoding of our *.po/*.mo files works" +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh # The constants used in a tricky observation for undefined behaviour diff --git a/t/t1002-read-tree-m-u-2way.sh b/t/t1002-read-tree-m-u-2way.sh index 9c05f5e1f5..ca5c5510c7 100755 --- a/t/t1002-read-tree-m-u-2way.sh +++ b/t/t1002-read-tree-m-u-2way.sh @@ -8,6 +8,8 @@ test_description='Two way merge with read-tree -m -u $H $M This is identical to t1001, but uses -u to update the work tree as well. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1005-read-tree-reset.sh b/t/t1005-read-tree-reset.sh index 83b09e1310..12e30d77d0 100755 --- a/t/t1005-read-tree-reset.sh +++ b/t/t1005-read-tree-reset.sh @@ -2,6 +2,7 @@ test_description='read-tree -u --reset' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1008-read-tree-overlay.sh b/t/t1008-read-tree-overlay.sh index 4512fb0b6e..ad5936e54d 100755 --- a/t/t1008-read-tree-overlay.sh +++ b/t/t1008-read-tree-overlay.sh @@ -5,6 +5,7 @@ test_description='test multi-tree read-tree without merging' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1022-read-tree-partial-clone.sh b/t/t1022-read-tree-partial-clone.sh index a763e27c7d..a9953b6a71 100755 --- a/t/t1022-read-tree-partial-clone.sh +++ b/t/t1022-read-tree-partial-clone.sh @@ -4,9 +4,6 @@ test_description='git read-tree in partial clones' TEST_NO_CREATE_REPO=1 -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh test_expect_success 'read-tree in partial clone prefetches in one batch' ' diff --git a/t/t1051-large-conversion.sh b/t/t1051-large-conversion.sh index 8b7640b3ba..042b0e4429 100755 --- a/t/t1051-large-conversion.sh +++ b/t/t1051-large-conversion.sh @@ -83,4 +83,30 @@ test_expect_success 'ident converts on output' ' test_cmp small.clean large.clean ' +# This smudge filter prepends 5GB of zeros to the file it checks out. This +# ensures that smudging doesn't mangle large files on 64-bit Windows. +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'files over 4GB convert on output' ' + test_commit test small "a small file" && + small_size=$(test_file_size small) && + test_config filter.makelarge.smudge \ + "test-tool genzeros $((5*1024*1024*1024)) && cat" && + echo "small filter=makelarge" >.gitattributes && + rm small && + git checkout -- small && + size=$(test_file_size small) && + test "$size" -eq $((5 * 1024 * 1024 * 1024 + $small_size)) +' + +# This clean filter writes down the size of input it receives. By checking against +# the actual size, we ensure that cleaning doesn't mangle large files on 64-bit Windows. +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'files over 4GB convert on input' ' + test-tool genzeros $((5*1024*1024*1024)) >big && + test_config filter.checklarge.clean "wc -c >big.size" && + echo "big filter=checklarge" >.gitattributes && + git add big && + test $(test_file_size big) -eq $(cat big.size) +' + test_done diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 16fbd2c6db..f8270943aa 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -19,6 +19,8 @@ test_expect_success 'setup' ' mkdir folder1 folder2 deep x && mkdir deep/deeper1 deep/deeper2 deep/before deep/later && mkdir deep/deeper1/deepest && + mkdir deep/deeper1/deepest2 && + mkdir deep/deeper1/deepest3 && echo "after deeper1" >deep/e && echo "after deepest" >deep/deeper1/e && cp a folder1 && @@ -30,7 +32,9 @@ test_expect_success 'setup' ' cp a deep/deeper2 && cp a deep/later && cp a deep/deeper1/deepest && - cp -r deep/deeper1/deepest deep/deeper2 && + cp a deep/deeper1/deepest2 && + cp a deep/deeper1/deepest3 && + cp -r deep/deeper1/ deep/deeper2 && mkdir deep/deeper1/0 && mkdir deep/deeper1/0/0 && touch deep/deeper1/0/1 && @@ -126,6 +130,8 @@ test_expect_success 'setup' ' git checkout -b deepest base && echo "updated deepest" >deep/deeper1/deepest/a && + echo "updated deepest2" >deep/deeper1/deepest2/a && + echo "updated deepest3" >deep/deeper1/deepest3/a && git commit -a -m "update deepest" && git checkout -f base && @@ -301,6 +307,14 @@ test_expect_success 'add, commit, checkout' ' test_all_match git checkout - ' +test_expect_success 'deep changes during checkout' ' + init_repos && + + test_sparse_match git sparse-checkout set deep/deeper1/deepest && + test_all_match git checkout deepest && + test_all_match git checkout base +' + test_expect_success 'add outside sparse cone' ' init_repos && @@ -489,26 +503,118 @@ test_expect_failure 'blame with pathspec outside sparse definition' ' test_all_match git blame deep/deeper2/deepest/a ' -# NEEDSWORK: a sparse-checkout behaves differently from a full checkout -# in this scenario, but it shouldn't. -test_expect_failure 'checkout and reset (mixed)' ' +test_expect_success 'checkout and reset (mixed)' ' init_repos && test_all_match git checkout -b reset-test update-deep && test_all_match git reset deepest && - test_all_match git reset update-folder1 && - test_all_match git reset update-folder2 + + # Because skip-worktree is preserved, resetting to update-folder1 + # will show worktree changes for folder1/a in full-checkout, but not + # in sparse-checkout or sparse-index. + git -C full-checkout reset update-folder1 >full-checkout-out && + test_sparse_match git reset update-folder1 && + grep "M folder1/a" full-checkout-out && + ! grep "M folder1/a" sparse-checkout-out && + run_on_sparse test_path_is_missing folder1 ' -# NEEDSWORK: a sparse-checkout behaves differently from a full checkout -# in this scenario, but it shouldn't. -test_expect_success 'checkout and reset (mixed) [sparse]' ' +test_expect_success 'checkout and reset (merge)' ' init_repos && - test_sparse_match git checkout -b reset-test update-deep && - test_sparse_match git reset deepest && - test_sparse_match git reset update-folder1 && - test_sparse_match git reset update-folder2 + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents a && + test_all_match git reset --merge deepest && + test_all_match git status --porcelain=v2 && + + test_all_match git reset --hard update-deep && + run_on_all ../edit-contents deep/a && + test_all_match test_must_fail git reset --merge deepest +' + +test_expect_success 'checkout and reset (keep)' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents a && + test_all_match git reset --keep deepest && + test_all_match git status --porcelain=v2 && + + test_all_match git reset --hard update-deep && + run_on_all ../edit-contents deep/a && + test_all_match test_must_fail git reset --keep deepest +' + +test_expect_success 'reset with pathspecs inside sparse definition' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents deep/a && + + test_all_match git reset base -- deep/a && + test_all_match git status --porcelain=v2 && + + test_all_match git reset base -- nonexistent-file && + test_all_match git status --porcelain=v2 && + + test_all_match git reset deepest -- deep && + test_all_match git status --porcelain=v2 +' + +# Although the working tree differs between full and sparse checkouts after +# reset, the state of the index is the same. +test_expect_success 'reset with pathspecs outside sparse definition' ' + init_repos && + test_all_match git checkout -b reset-test base && + + test_sparse_match git reset update-folder1 -- folder1 && + git -C full-checkout reset update-folder1 -- folder1 && + test_sparse_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder1 && + + test_sparse_match git reset update-folder2 -- folder2/a && + git -C full-checkout reset update-folder2 -- folder2/a && + test_sparse_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder2/a +' + +test_expect_success 'reset with wildcard pathspec' ' + init_repos && + + test_all_match git reset update-deep -- deep\* && + test_all_match git ls-files -s -- deep && + + test_all_match git reset deepest -- deep\*\*\* && + test_all_match git ls-files -s -- deep && + + # The following `git reset`s result in updating the index on files with + # `skip-worktree` enabled. To avoid failing due to discrepencies in reported + # "modified" files, `test_sparse_match` reset is performed separately from + # "full-checkout" reset, then the index contents of all repos are verified. + + test_sparse_match git reset update-folder1 -- \*/a && + git -C full-checkout reset update-folder1 -- \*/a && + test_all_match git ls-files -s -- deep/a folder1/a && + + test_sparse_match git reset update-folder2 -- folder\* && + git -C full-checkout reset update-folder2 -- folder\* && + test_all_match git ls-files -s -- folder10 folder1 folder2 && + + test_sparse_match git reset base -- folder1/\* && + git -C full-checkout reset base -- folder1/\* && + test_all_match git ls-files -s -- folder1 ' test_expect_success 'merge, cherry-pick, and rebase' ' @@ -685,15 +791,50 @@ test_expect_success 'submodule handling' ' grep "160000 commit $(git -C initial-repo rev-parse HEAD) modules/sub" cache ' +# When working with a sparse index, some commands will need to expand the +# index to operate properly. If those commands also write the index back +# to disk, they need to convert the index to sparse before writing. +# This test verifies that both of these events are logged in trace2 logs. test_expect_success 'sparse-index is expanded and converted back' ' init_repos && - GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ - git -C sparse-index -c core.fsmonitor="" reset --hard && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -C sparse-index reset -- folder1/a && test_region index convert_to_sparse trace2.txt && test_region index ensure_full_index trace2.txt ' +test_expect_success 'index.sparse disabled inline uses full index' ' + init_repos && + + # When index.sparse is disabled inline with `git status`, the + # index is expanded at the beginning of the execution then never + # converted back to sparse. It is then written to disk as a full index. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index -c index.sparse=false status && + ! test_region index convert_to_sparse trace2.txt && + test_region index ensure_full_index trace2.txt && + + # Since index.sparse is set to true at a repo level, the index + # is converted from full to sparse when read, then never expanded + # over the course of `git status`. It is written to disk as a sparse + # index. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index status && + test_region index convert_to_sparse trace2.txt && + ! test_region index ensure_full_index trace2.txt && + + # Now that the index has been written to disk as sparse, it is not + # converted to sparse (or expanded to full) when read by `git status`. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index status && + ! test_region index convert_to_sparse trace2.txt && + ! test_region index ensure_full_index trace2.txt +' + ensure_not_expanded () { rm -f trace2.txt && echo >>sparse-index/untracked.txt && @@ -702,10 +843,10 @@ ensure_not_expanded () { then shift && test_must_fail env \ - GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ git -C sparse-index "$@" || return 1 else - GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ git -C sparse-index "$@" || return 1 fi && test_region ! index ensure_full_index trace2.txt @@ -726,9 +867,9 @@ test_expect_success 'sparse-index is not expanded' ' ensure_not_expanded checkout - && ensure_not_expanded switch rename-out-to-out && ensure_not_expanded switch - && - git -C sparse-index reset --hard && + ensure_not_expanded reset --hard && ensure_not_expanded checkout rename-out-to-out -- deep/deeper1 && - git -C sparse-index reset --hard && + ensure_not_expanded reset --hard && ensure_not_expanded restore -s rename-out-to-out -- deep/deeper1 && echo >>sparse-index/README.md && @@ -738,6 +879,34 @@ test_expect_success 'sparse-index is not expanded' ' echo >>sparse-index/untracked.txt && ensure_not_expanded add . && + for ref in update-deep update-folder1 update-folder2 update-deep + do + echo >>sparse-index/README.md && + ensure_not_expanded reset --hard $ref || return 1 + done && + + ensure_not_expanded reset --mixed base && + ensure_not_expanded reset --hard update-deep && + ensure_not_expanded reset --keep base && + ensure_not_expanded reset --merge update-deep && + ensure_not_expanded reset --hard && + + ensure_not_expanded reset base -- deep/a && + ensure_not_expanded reset base -- nonexistent-file && + ensure_not_expanded reset deepest -- deep && + + # Although folder1 is outside the sparse definition, it exists as a + # directory entry in the index, so the pathspec will not force the + # index to be expanded. + ensure_not_expanded reset deepest -- folder1 && + ensure_not_expanded reset deepest -- folder1/ && + + # Wildcard identifies only in-cone files, no index expansion + ensure_not_expanded reset deepest -- deep/\* && + + # Wildcard identifies only full sparse directories, no index expansion + ensure_not_expanded reset deepest -- folder\* && + ensure_not_expanded checkout -f update-deep && test_config -C sparse-index pull.twohead ort && ( diff --git a/t/t1300-config.sh b/t/t1300-config.sh index 9ff46f3b04..f8031afaaf 100755 --- a/t/t1300-config.sh +++ b/t/t1300-config.sh @@ -8,6 +8,7 @@ test_description='Test git config in different settings' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'clear default config' ' diff --git a/t/t1303-wacky-config.sh b/t/t1303-wacky-config.sh index 0000e664e7..0506f3d6bb 100755 --- a/t/t1303-wacky-config.sh +++ b/t/t1303-wacky-config.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='Test wacky input to git config' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Leaving off the newline is intentional! diff --git a/t/t1307-config-blob.sh b/t/t1307-config-blob.sh index 930dce06f0..0a7099d6f5 100755 --- a/t/t1307-config-blob.sh +++ b/t/t1307-config-blob.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='support for reading config from a blob' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create config blob' ' diff --git a/t/t1308-config-set.sh b/t/t1308-config-set.sh index 88b119a0a3..b38e158d3b 100755 --- a/t/t1308-config-set.sh +++ b/t/t1308-config-set.sh @@ -2,6 +2,7 @@ test_description='Test git config-set API in different settings' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # 'check_config get_* section.key value' verifies that the entry for diff --git a/t/t1309-early-config.sh b/t/t1309-early-config.sh index b4a9158307..537435b90a 100755 --- a/t/t1309-early-config.sh +++ b/t/t1309-early-config.sh @@ -2,6 +2,7 @@ test_description='Test read_early_config()' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'read early config' ' diff --git a/t/t1310-config-default.sh b/t/t1310-config-default.sh index 6049d91708..09b10c144b 100755 --- a/t/t1310-config-default.sh +++ b/t/t1310-config-default.sh @@ -2,6 +2,7 @@ test_description='Test git config in different settings (with --default)' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'uses --default when entry missing' ' diff --git a/t/t1400-update-ref.sh b/t/t1400-update-ref.sh index 0d4f73acaa..640659c9d7 100755 --- a/t/t1400-update-ref.sh +++ b/t/t1400-update-ref.sh @@ -4,9 +4,6 @@ # test_description='Test git update-ref and basic ref logging' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh Z=$ZERO_OID @@ -321,8 +318,9 @@ $A $B $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117150260 +0000 Switch $B $A $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117150860 +0000 EOF test_expect_success "verifying $m's log (logged by touch)" ' - test_when_finished "rm -rf .git/$m .git/logs expect" && - test_cmp expect .git/logs/$m + test_when_finished "git update-ref -d $m && rm -rf .git/logs actual expect" && + test-tool ref-store main for-each-reflog-ent $m >actual && + test_cmp actual expect ' test_expect_success "create $m (logged by config)" ' @@ -350,8 +348,9 @@ $A $B $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117150380 +0000 Switch $B $A $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117150980 +0000 EOF test_expect_success "verifying $m's log (logged by config)" ' - test_when_finished "rm -f .git/$m .git/logs/$m expect" && - test_cmp expect .git/logs/$m + test_when_finished "git update-ref -d $m && rm -rf .git/logs actual expect" && + test-tool ref-store main for-each-reflog-ent $m >actual && + test_cmp actual expect ' test_expect_success 'set up for querying the reflog' ' @@ -467,7 +466,8 @@ $h_OTHER $h_FIXED $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117151040 +0000 co $h_FIXED $h_MERGED $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> 1117151100 +0000 commit (merge): Merged initial commit and a later commit. EOF test_expect_success 'git commit logged updates' ' - test_cmp expect .git/logs/$m + test-tool ref-store main for-each-reflog-ent $m >actual && + test_cmp expect actual ' unset h_TEST h_OTHER h_FIXED h_MERGED diff --git a/t/t1403-show-ref.sh b/t/t1403-show-ref.sh index 17d3cc1405..4d261e80c6 100755 --- a/t/t1403-show-ref.sh +++ b/t/t1403-show-ref.sh @@ -4,6 +4,7 @@ test_description='show-ref' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1404-update-ref-errors.sh b/t/t1404-update-ref-errors.sh index b729c1f480..13c2b43bba 100755 --- a/t/t1404-update-ref-errors.sh +++ b/t/t1404-update-ref-errors.sh @@ -261,69 +261,69 @@ test_expect_success REFFILES 'empty directory should not fool 1-arg delete' ' git update-ref --stdin ' -test_expect_success 'D/F conflict prevents add long + delete short' ' +test_expect_success REFFILES 'D/F conflict prevents add long + delete short' ' df_test refs/df-al-ds --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents add short + delete long' ' +test_expect_success REFFILES 'D/F conflict prevents add short + delete long' ' df_test refs/df-as-dl --add-del foo foo/bar ' -test_expect_success 'D/F conflict prevents delete long + add short' ' +test_expect_success REFFILES 'D/F conflict prevents delete long + add short' ' df_test refs/df-dl-as --del-add foo/bar foo ' -test_expect_success 'D/F conflict prevents delete short + add long' ' +test_expect_success REFFILES 'D/F conflict prevents delete short + add long' ' df_test refs/df-ds-al --del-add foo foo/bar ' -test_expect_success 'D/F conflict prevents add long + delete short packed' ' +test_expect_success REFFILES 'D/F conflict prevents add long + delete short packed' ' df_test refs/df-al-dsp --pack --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents add short + delete long packed' ' +test_expect_success REFFILES 'D/F conflict prevents add short + delete long packed' ' df_test refs/df-as-dlp --pack --add-del foo foo/bar ' -test_expect_success 'D/F conflict prevents delete long packed + add short' ' +test_expect_success REFFILES 'D/F conflict prevents delete long packed + add short' ' df_test refs/df-dlp-as --pack --del-add foo/bar foo ' -test_expect_success 'D/F conflict prevents delete short packed + add long' ' +test_expect_success REFFILES 'D/F conflict prevents delete short packed + add long' ' df_test refs/df-dsp-al --pack --del-add foo foo/bar ' # Try some combinations involving symbolic refs... -test_expect_success 'D/F conflict prevents indirect add long + delete short' ' +test_expect_success REFFILES 'D/F conflict prevents indirect add long + delete short' ' df_test refs/df-ial-ds --sym-add --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents indirect add long + indirect delete short' ' +test_expect_success REFFILES 'D/F conflict prevents indirect add long + indirect delete short' ' df_test refs/df-ial-ids --sym-add --sym-del --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents indirect add short + indirect delete long' ' +test_expect_success REFFILES 'D/F conflict prevents indirect add short + indirect delete long' ' df_test refs/df-ias-idl --sym-add --sym-del --add-del foo foo/bar ' -test_expect_success 'D/F conflict prevents indirect delete long + indirect add short' ' +test_expect_success REFFILES 'D/F conflict prevents indirect delete long + indirect add short' ' df_test refs/df-idl-ias --sym-add --sym-del --del-add foo/bar foo ' -test_expect_success 'D/F conflict prevents indirect add long + delete short packed' ' +test_expect_success REFFILES 'D/F conflict prevents indirect add long + delete short packed' ' df_test refs/df-ial-dsp --sym-add --pack --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents indirect add long + indirect delete short packed' ' +test_expect_success REFFILES 'D/F conflict prevents indirect add long + indirect delete short packed' ' df_test refs/df-ial-idsp --sym-add --sym-del --pack --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents add long + indirect delete short packed' ' +test_expect_success REFFILES 'D/F conflict prevents add long + indirect delete short packed' ' df_test refs/df-al-idsp --sym-del --pack --add-del foo/bar foo ' -test_expect_success 'D/F conflict prevents indirect delete long packed + indirect add short' ' +test_expect_success REFFILES 'D/F conflict prevents indirect delete long packed + indirect add short' ' df_test refs/df-idlp-ias --sym-add --sym-del --pack --del-add foo/bar foo ' diff --git a/t/t1405-main-ref-store.sh b/t/t1405-main-ref-store.sh index 49718b7ea7..3cb5e23d6d 100755 --- a/t/t1405-main-ref-store.sh +++ b/t/t1405-main-ref-store.sh @@ -89,13 +89,13 @@ test_expect_success 'for_each_reflog()' ' test_expect_success 'for_each_reflog_ent()' ' $RUN for-each-reflog-ent HEAD >actual && head -n1 actual | grep one && - tail -n2 actual | head -n1 | grep recreate-main + tail -n1 actual | grep recreate-main ' test_expect_success 'for_each_reflog_ent_reverse()' ' $RUN for-each-reflog-ent-reverse HEAD >actual && head -n1 actual | grep recreate-main && - tail -n2 actual | head -n1 | grep one + tail -n1 actual | grep one ' test_expect_success 'reflog_exists(HEAD)' ' @@ -108,7 +108,7 @@ test_expect_success 'delete_reflog(HEAD)' ' ' test_expect_success 'create-reflog(HEAD)' ' - $RUN create-reflog HEAD 1 && + $RUN create-reflog HEAD && git reflog exists HEAD ' diff --git a/t/t1406-submodule-ref-store.sh b/t/t1406-submodule-ref-store.sh index 0a87058971..e6a7f7334b 100755 --- a/t/t1406-submodule-ref-store.sh +++ b/t/t1406-submodule-ref-store.sh @@ -5,6 +5,7 @@ test_description='test submodule ref store api' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh RUN="test-tool ref-store submodule:sub" @@ -74,13 +75,13 @@ test_expect_success 'for_each_reflog()' ' test_expect_success 'for_each_reflog_ent()' ' $RUN for-each-reflog-ent HEAD >actual && head -n1 actual | grep first && - tail -n2 actual | head -n1 | grep main.to.new + tail -n1 actual | grep main.to.new ' test_expect_success 'for_each_reflog_ent_reverse()' ' $RUN for-each-reflog-ent-reverse HEAD >actual && head -n1 actual | grep main.to.new && - tail -n2 actual | head -n1 | grep first + tail -n1 actual | grep first ' test_expect_success 'reflog_exists(HEAD)' ' @@ -92,7 +93,7 @@ test_expect_success 'delete_reflog() not allowed' ' ' test_expect_success 'create-reflog() not allowed' ' - test_must_fail $RUN create-reflog HEAD 1 + test_must_fail $RUN create-reflog HEAD ' test_done diff --git a/t/t1417-reflog-updateref.sh b/t/t1417-reflog-updateref.sh new file mode 100755 index 0000000000..14f13b57c6 --- /dev/null +++ b/t/t1417-reflog-updateref.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +test_description='git reflog --updateref' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_expect_success 'setup' ' + git init -b main repo && + ( + cd repo && + + test_commit A && + test_commit B && + test_commit C && + + cp .git/logs/HEAD HEAD.old && + git reset --hard HEAD~ && + cp HEAD.old .git/logs/HEAD + ) +' + +test_reflog_updateref () { + exp=$1 + shift + args="$@" + + test_expect_success REFFILES "get '$exp' with '$args'" ' + test_when_finished "rm -rf copy" && + cp -R repo copy && + + ( + cd copy && + + $args && + git rev-parse $exp >expect && + git rev-parse HEAD >actual && + + test_cmp expect actual + ) + ' +} + +test_reflog_updateref B git reflog delete --updateref HEAD@{0} +test_reflog_updateref B git reflog delete --updateref HEAD@{1} +test_reflog_updateref C git reflog delete --updateref main@{0} +test_reflog_updateref B git reflog delete --updateref main@{1} +test_reflog_updateref B git reflog delete --updateref --rewrite HEAD@{0} +test_reflog_updateref B git reflog delete --updateref --rewrite HEAD@{1} +test_reflog_updateref C git reflog delete --updateref --rewrite main@{0} +test_reflog_updateref B git reflog delete --updateref --rewrite main@{1} +test_reflog_updateref B test_must_fail git reflog expire HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire main@{0} +test_reflog_updateref B test_must_fail git reflog expire main@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref main@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref main@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite main@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite main@{1} + +test_done diff --git a/t/t1420-lost-found.sh b/t/t1420-lost-found.sh index dc9e402c55..dbe15a0be1 100755 --- a/t/t1420-lost-found.sh +++ b/t/t1420-lost-found.sh @@ -4,6 +4,8 @@ # test_description='Test fsck --lost-found' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1503-rev-parse-verify.sh b/t/t1503-rev-parse-verify.sh index 40958615eb..94fe413ee3 100755 --- a/t/t1503-rev-parse-verify.sh +++ b/t/t1503-rev-parse-verify.sh @@ -9,6 +9,7 @@ exec </dev/null GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh add_line_into_file() diff --git a/t/t1505-rev-parse-last.sh b/t/t1505-rev-parse-last.sh index 2803ca9489..4a5758f08a 100755 --- a/t/t1505-rev-parse-last.sh +++ b/t/t1505-rev-parse-last.sh @@ -5,6 +5,7 @@ test_description='test @{-N} syntax' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh diff --git a/t/t1506-rev-parse-diagnosis.sh b/t/t1506-rev-parse-diagnosis.sh index 65a154a8a2..18688cae17 100755 --- a/t/t1506-rev-parse-diagnosis.sh +++ b/t/t1506-rev-parse-diagnosis.sh @@ -7,6 +7,7 @@ exec </dev/null GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_did_you_mean () diff --git a/t/t1513-rev-parse-prefix.sh b/t/t1513-rev-parse-prefix.sh index 5f437be8c9..ba43387bf1 100755 --- a/t/t1513-rev-parse-prefix.sh +++ b/t/t1513-rev-parse-prefix.sh @@ -5,6 +5,7 @@ test_description='Tests for rev-parse --prefix' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t1515-rev-parse-outside-repo.sh b/t/t1515-rev-parse-outside-repo.sh index 3ec2971ee5..cdb26a30d7 100755 --- a/t/t1515-rev-parse-outside-repo.sh +++ b/t/t1515-rev-parse-outside-repo.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='check that certain rev-parse options work outside repo' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'set up non-repo directory' ' diff --git a/t/t1600-index.sh b/t/t1600-index.sh index 46329c488b..010989f90e 100755 --- a/t/t1600-index.sh +++ b/t/t1600-index.sh @@ -2,6 +2,7 @@ test_description='index file specific tests' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh sane_unset GIT_TEST_SPLIT_INDEX diff --git a/t/t2000-conflict-when-checking-files-out.sh b/t/t2000-conflict-when-checking-files-out.sh index f18616ad2b..79fc97f1d7 100755 --- a/t/t2000-conflict-when-checking-files-out.sh +++ b/t/t2000-conflict-when-checking-files-out.sh @@ -21,6 +21,7 @@ test_description='git conflicts when checking files out test.' # path1 is occupied by a non-directory. With "-f" flag, it should remove # the conflicting paths and succeed. +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh show_files() { diff --git a/t/t2007-checkout-symlink.sh b/t/t2007-checkout-symlink.sh index 6f0b90ce12..bd9e9e7530 100755 --- a/t/t2007-checkout-symlink.sh +++ b/t/t2007-checkout-symlink.sh @@ -7,6 +7,7 @@ test_description='git checkout to switch between branches with symlink<->dir' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2008-checkout-subdir.sh b/t/t2008-checkout-subdir.sh index eadb9434ae..8a518a44ea 100755 --- a/t/t2008-checkout-subdir.sh +++ b/t/t2008-checkout-subdir.sh @@ -4,6 +4,7 @@ test_description='git checkout from subdirectories' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2009-checkout-statinfo.sh b/t/t2009-checkout-statinfo.sh index b0540636ae..71195dd28f 100755 --- a/t/t2009-checkout-statinfo.sh +++ b/t/t2009-checkout-statinfo.sh @@ -5,6 +5,7 @@ test_description='checkout should leave clean stat info' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2010-checkout-ambiguous.sh b/t/t2010-checkout-ambiguous.sh index 6e8757387d..9d4b37526a 100755 --- a/t/t2010-checkout-ambiguous.sh +++ b/t/t2010-checkout-ambiguous.sh @@ -5,6 +5,7 @@ test_description='checkout and pathspecs/refspecs ambiguities' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2011-checkout-invalid-head.sh b/t/t2011-checkout-invalid-head.sh index e52022e152..d9997e7b6b 100755 --- a/t/t2011-checkout-invalid-head.sh +++ b/t/t2011-checkout-invalid-head.sh @@ -5,6 +5,7 @@ test_description='checkout switching away from an invalid branch' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2014-checkout-switch.sh b/t/t2014-checkout-switch.sh index ccfb147113..c138bdde4f 100755 --- a/t/t2014-checkout-switch.sh +++ b/t/t2014-checkout-switch.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='Peter MacMillan' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2017-checkout-orphan.sh b/t/t2017-checkout-orphan.sh index 88d6992a5e..f3371e2646 100755 --- a/t/t2017-checkout-orphan.sh +++ b/t/t2017-checkout-orphan.sh @@ -10,6 +10,7 @@ Main Tests for --orphan functionality.' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh TEST_FILE=foo diff --git a/t/t2019-checkout-ambiguous-ref.sh b/t/t2019-checkout-ambiguous-ref.sh index b99d5192a9..2c8c926b4d 100755 --- a/t/t2019-checkout-ambiguous-ref.sh +++ b/t/t2019-checkout-ambiguous-ref.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='checkout handling of ambiguous (branch/tag) refs' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup ambiguous refs' ' diff --git a/t/t2021-checkout-overwrite.sh b/t/t2021-checkout-overwrite.sh index 660132ff8d..713c3fa603 100755 --- a/t/t2021-checkout-overwrite.sh +++ b/t/t2021-checkout-overwrite.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='checkout must not overwrite an untracked objects' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2022-checkout-paths.sh b/t/t2022-checkout-paths.sh index c49ba7f9bd..f1b709d58b 100755 --- a/t/t2022-checkout-paths.sh +++ b/t/t2022-checkout-paths.sh @@ -4,6 +4,7 @@ test_description='checkout $tree -- $paths' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2026-checkout-pathspec-file.sh b/t/t2026-checkout-pathspec-file.sh index 43d31d7948..9db11f86dd 100755 --- a/t/t2026-checkout-pathspec-file.sh +++ b/t/t2026-checkout-pathspec-file.sh @@ -2,6 +2,7 @@ test_description='checkout --pathspec-from-file' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_tick diff --git a/t/t2100-update-cache-badpath.sh b/t/t2100-update-cache-badpath.sh index 2df3fdde8b..7915e7b821 100755 --- a/t/t2100-update-cache-badpath.sh +++ b/t/t2100-update-cache-badpath.sh @@ -22,6 +22,7 @@ and tries to git update-index --add the following: All of the attempts should fail. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh mkdir path2 path3 diff --git a/t/t2101-update-index-reupdate.sh b/t/t2101-update-index-reupdate.sh index 6c32d42c8c..e3c7acdbf9 100755 --- a/t/t2101-update-index-reupdate.sh +++ b/t/t2101-update-index-reupdate.sh @@ -6,6 +6,7 @@ test_description='git update-index --again test. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'update-index --add' ' diff --git a/t/t2102-update-index-symlinks.sh b/t/t2102-update-index-symlinks.sh index 22f2c730ae..d7a3485582 100755 --- a/t/t2102-update-index-symlinks.sh +++ b/t/t2102-update-index-symlinks.sh @@ -8,6 +8,7 @@ test_description='git update-index on filesystem w/o symlinks test. This tests that git update-index keeps the symbolic link property even if a plain file is in the working tree if core.symlinks is false.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success \ diff --git a/t/t2103-update-index-ignore-missing.sh b/t/t2103-update-index-ignore-missing.sh index 0114f05228..b304714fdb 100755 --- a/t/t2103-update-index-ignore-missing.sh +++ b/t/t2103-update-index-ignore-missing.sh @@ -2,6 +2,7 @@ test_description='update-index with options' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success basics ' diff --git a/t/t2104-update-index-skip-worktree.sh b/t/t2104-update-index-skip-worktree.sh index 30666fc70d..b8686aabd3 100755 --- a/t/t2104-update-index-skip-worktree.sh +++ b/t/t2104-update-index-skip-worktree.sh @@ -5,6 +5,7 @@ test_description='skip-worktree bit test' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh sane_unset GIT_TEST_SPLIT_INDEX diff --git a/t/t2105-update-index-gitfile.sh b/t/t2105-update-index-gitfile.sh index a7f3d47aec..963ebe77eb 100755 --- a/t/t2105-update-index-gitfile.sh +++ b/t/t2105-update-index-gitfile.sh @@ -6,6 +6,7 @@ test_description='git update-index for gitlink to .git file. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'submodule with absolute .git file' ' diff --git a/t/t2106-update-index-assume-unchanged.sh b/t/t2106-update-index-assume-unchanged.sh index 2d450daf5c..d943ddf47e 100755 --- a/t/t2106-update-index-assume-unchanged.sh +++ b/t/t2106-update-index-assume-unchanged.sh @@ -3,6 +3,7 @@ test_description='git update-index --assume-unchanged test. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2200-add-update.sh b/t/t2200-add-update.sh index 94c4cb0672..a8297c2943 100755 --- a/t/t2200-add-update.sh +++ b/t/t2200-add-update.sh @@ -14,6 +14,7 @@ only the updates to dir/sub. Also tested are "git add -u" without limiting, and "git add -u" without contents changes, and other conditions' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2201-add-update-typechange.sh b/t/t2201-add-update-typechange.sh index a4eec0a346..c6876b120f 100755 --- a/t/t2201-add-update-typechange.sh +++ b/t/t2201-add-update-typechange.sh @@ -2,6 +2,7 @@ test_description='more git add -u' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2202-add-addremove.sh b/t/t2202-add-addremove.sh index 9ee659098c..24c60bfd79 100755 --- a/t/t2202-add-addremove.sh +++ b/t/t2202-add-addremove.sh @@ -2,6 +2,7 @@ test_description='git add --all' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2204-add-ignored.sh b/t/t2204-add-ignored.sh index 2e07365bbb..89424abccd 100755 --- a/t/t2204-add-ignored.sh +++ b/t/t2204-add-ignored.sh @@ -2,6 +2,7 @@ test_description='giving ignored paths to git add' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2401-worktree-prune.sh b/t/t2401-worktree-prune.sh index a615d3b483..3d28c7f06b 100755 --- a/t/t2401-worktree-prune.sh +++ b/t/t2401-worktree-prune.sh @@ -19,7 +19,7 @@ test_expect_success 'worktree prune on normal repo' ' test_expect_success 'prune files inside $GIT_DIR/worktrees' ' mkdir .git/worktrees && : >.git/worktrees/abc && - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && cat >expect <<EOF && Removing worktrees/abc: not a valid directory EOF @@ -34,7 +34,7 @@ test_expect_success 'prune directories without gitdir' ' cat >expect <<EOF && Removing worktrees/def: gitdir file does not exist EOF - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && test_cmp expect actual && ! test -d .git/worktrees/def && ! test -d .git/worktrees @@ -45,7 +45,7 @@ test_expect_success SANITY 'prune directories with unreadable gitdir' ' : >.git/worktrees/def/def && : >.git/worktrees/def/gitdir && chmod u-r .git/worktrees/def/gitdir && - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && test_i18ngrep "Removing worktrees/def: unable to read gitdir file" actual && ! test -d .git/worktrees/def && ! test -d .git/worktrees @@ -55,7 +55,7 @@ test_expect_success 'prune directories with invalid gitdir' ' mkdir -p .git/worktrees/def/abc && : >.git/worktrees/def/def && : >.git/worktrees/def/gitdir && - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && test_i18ngrep "Removing worktrees/def: invalid gitdir file" actual && ! test -d .git/worktrees/def && ! test -d .git/worktrees @@ -65,7 +65,7 @@ test_expect_success 'prune directories with gitdir pointing to nowhere' ' mkdir -p .git/worktrees/def/abc && : >.git/worktrees/def/def && echo "$(pwd)"/nowhere >.git/worktrees/def/gitdir && - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && test_i18ngrep "Removing worktrees/def: gitdir file points to non-existent location" actual && ! test -d .git/worktrees/def && ! test -d .git/worktrees @@ -101,7 +101,7 @@ test_expect_success 'prune duplicate (linked/linked)' ' git worktree add --detach w2 && sed "s/w2/w1/" .git/worktrees/w2/gitdir >.git/worktrees/w2/gitdir.new && mv .git/worktrees/w2/gitdir.new .git/worktrees/w2/gitdir && - git worktree prune --verbose >actual && + git worktree prune --verbose 2>actual && test_i18ngrep "duplicate entry" actual && test -d .git/worktrees/w1 && ! test -d .git/worktrees/w2 @@ -114,7 +114,7 @@ test_expect_success 'prune duplicate (main/linked)' ' git -C repo worktree add --detach ../wt && rm -fr wt && mv repo wt && - git -C wt worktree prune --verbose >actual && + git -C wt worktree prune --verbose 2>actual && test_i18ngrep "duplicate entry" actual && ! test -d .git/worktrees/wt ' diff --git a/t/t2402-worktree-list.sh b/t/t2402-worktree-list.sh index 4012bd67b0..c8a5a0aac6 100755 --- a/t/t2402-worktree-list.sh +++ b/t/t2402-worktree-list.sh @@ -134,7 +134,7 @@ test_expect_success '"list" all worktrees with prunable consistent with "prune"' git worktree list >out && grep "/prunable *[0-9a-f].* prunable$" out && ! grep "/unprunable *[0-9a-f].* unprunable$" out && - git worktree prune --verbose >out && + git worktree prune --verbose 2>out && test_i18ngrep "^Removing worktrees/prunable" out && test_i18ngrep ! "^Removing worktrees/unprunable" out ' diff --git a/t/t2404-worktree-config.sh b/t/t2404-worktree-config.sh index 9536d10919..842937bfb9 100755 --- a/t/t2404-worktree-config.sh +++ b/t/t2404-worktree-config.sh @@ -2,6 +2,7 @@ test_description="config file in multi worktree" +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2406-worktree-repair.sh b/t/t2406-worktree-repair.sh index f73741886b..5c44453e1c 100755 --- a/t/t2406-worktree-repair.sh +++ b/t/t2406-worktree-repair.sh @@ -45,9 +45,8 @@ test_corrupt_gitfile () { git worktree add --detach corrupt && git -C corrupt rev-parse --absolute-git-dir >expect && eval "$butcher" && - git -C "$repairdir" worktree repair >out 2>err && - test_i18ngrep "$problem" out && - test_must_be_empty err && + git -C "$repairdir" worktree repair 2>err && + test_i18ngrep "$problem" err && git -C corrupt rev-parse --absolute-git-dir >actual && test_cmp expect actual } @@ -130,10 +129,9 @@ test_expect_success 'repair broken gitdir' ' sed s,orig/\.git$,moved/.git, .git/worktrees/orig/gitdir >expect && rm .git/worktrees/orig/gitdir && mv orig moved && - git worktree repair moved >out 2>err && + git worktree repair moved 2>err && test_cmp expect .git/worktrees/orig/gitdir && - test_i18ngrep "gitdir unreadable" out && - test_must_be_empty err + test_i18ngrep "gitdir unreadable" err ' test_expect_success 'repair incorrect gitdir' ' @@ -141,10 +139,9 @@ test_expect_success 'repair incorrect gitdir' ' git worktree add --detach orig && sed s,orig/\.git$,moved/.git, .git/worktrees/orig/gitdir >expect && mv orig moved && - git worktree repair moved >out 2>err && + git worktree repair moved 2>err && test_cmp expect .git/worktrees/orig/gitdir && - test_i18ngrep "gitdir incorrect" out && - test_must_be_empty err + test_i18ngrep "gitdir incorrect" err ' test_expect_success 'repair gitdir (implicit) from linked worktree' ' @@ -152,10 +149,9 @@ test_expect_success 'repair gitdir (implicit) from linked worktree' ' git worktree add --detach orig && sed s,orig/\.git$,moved/.git, .git/worktrees/orig/gitdir >expect && mv orig moved && - git -C moved worktree repair >out 2>err && + git -C moved worktree repair 2>err && test_cmp expect .git/worktrees/orig/gitdir && - test_i18ngrep "gitdir incorrect" out && - test_must_be_empty err + test_i18ngrep "gitdir incorrect" err ' test_expect_success 'unable to repair gitdir (implicit) from main worktree' ' @@ -163,9 +159,8 @@ test_expect_success 'unable to repair gitdir (implicit) from main worktree' ' git worktree add --detach orig && cat .git/worktrees/orig/gitdir >expect && mv orig moved && - git worktree repair >out 2>err && + git worktree repair 2>err && test_cmp expect .git/worktrees/orig/gitdir && - test_must_be_empty out && test_must_be_empty err ' @@ -178,12 +173,11 @@ test_expect_success 'repair multiple gitdir files' ' sed s,orig2/\.git$,moved2/.git, .git/worktrees/orig2/gitdir >expect2 && mv orig1 moved1 && mv orig2 moved2 && - git worktree repair moved1 moved2 >out 2>err && + git worktree repair moved1 moved2 2>err && test_cmp expect1 .git/worktrees/orig1/gitdir && test_cmp expect2 .git/worktrees/orig2/gitdir && - test_i18ngrep "gitdir incorrect:.*orig1/gitdir$" out && - test_i18ngrep "gitdir incorrect:.*orig2/gitdir$" out && - test_must_be_empty err + test_i18ngrep "gitdir incorrect:.*orig1/gitdir$" err && + test_i18ngrep "gitdir incorrect:.*orig2/gitdir$" err ' test_expect_success 'repair moved main and linked worktrees' ' diff --git a/t/t3040-subprojects-basic.sh b/t/t3040-subprojects-basic.sh index 6abdcbbc94..bd65dfcffc 100755 --- a/t/t3040-subprojects-basic.sh +++ b/t/t3040-subprojects-basic.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='Basic subproject functionality' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup: create superproject' ' diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index e575ffb4ff..8c5c1ccf33 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -731,6 +731,28 @@ test_expect_success SYMLINKS 'git branch -m u v should fail when the reflog for test_must_fail git branch -m u v ' +test_expect_success SYMLINKS 'git branch -m with symlinked .git/refs' ' + test_when_finished "rm -rf subdir" && + git init --bare subdir && + + rm -rfv subdir/refs subdir/objects subdir/packed-refs && + ln -s ../.git/refs subdir/refs && + ln -s ../.git/objects subdir/objects && + ln -s ../.git/packed-refs subdir/packed-refs && + + git -C subdir rev-parse --absolute-git-dir >subdir.dir && + git rev-parse --absolute-git-dir >our.dir && + ! test_cmp subdir.dir our.dir && + + git -C subdir log && + git -C subdir branch rename-src && + git rev-parse rename-src >expect && + git -C subdir branch -m rename-src rename-dest && + git rev-parse rename-dest >actual && + test_cmp expect actual && + git branch -D rename-dest +' + test_expect_success 'test tracking setup via --track' ' git config remote.local.url . && git config remote.local.fetch refs/heads/*:refs/remotes/local/* && @@ -1418,7 +1440,17 @@ test_expect_success 'invalid sort parameter in configuration' ' ( cd sort && git config branch.sort "v:notvalid" && - test_must_fail git branch + + # this works in the "listing" mode, so bad sort key + # is a dying offence. + test_must_fail git branch && + + # these do not need to use sorting, and should all + # succeed + git branch newone main && + git branch -c newone newerone && + git branch -m newone newestone && + git branch -d newerone newestone ) ' diff --git a/t/t3201-branch-contains.sh b/t/t3201-branch-contains.sh index 349a810cee..800fc33165 100755 --- a/t/t3201-branch-contains.sh +++ b/t/t3201-branch-contains.sh @@ -2,9 +2,6 @@ test_description='branch --contains <commit>, --no-contains <commit> --merged, and --no-merged' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh test_expect_success setup ' diff --git a/t/t3202-show-branch.sh b/t/t3202-show-branch.sh index ad9902a06b..d4d64401e4 100755 --- a/t/t3202-show-branch.sh +++ b/t/t3202-show-branch.sh @@ -4,6 +4,9 @@ test_description='test show-branch' . ./test-lib.sh +# arbitrary reference time: 2009-08-30 19:20:00 +GIT_TEST_DATE_NOW=1251660000; export GIT_TEST_DATE_NOW + test_expect_success 'setup' ' test_commit initial && for i in $(test_seq 1 10) @@ -146,4 +149,16 @@ test_expect_success 'show branch --merge-base with N arguments' ' test_cmp expect actual ' +test_expect_success 'show branch --reflog=2' ' + sed "s/^> //" >expect <<-\EOF && + > ! [refs/heads/branch10@{0}] (4 years, 5 months ago) commit: branch10 + > ! [refs/heads/branch10@{1}] (4 years, 5 months ago) commit: branch10 + > -- + > + [refs/heads/branch10@{0}] branch10 + > ++ [refs/heads/branch10@{1}] initial + EOF + git show-branch --reflog=2 >actual && + test_cmp actual expect +' + test_done diff --git a/t/t3203-branch-output.sh b/t/t3203-branch-output.sh index 6e94c6db7b..d34d77f893 100755 --- a/t/t3203-branch-output.sh +++ b/t/t3203-branch-output.sh @@ -1,9 +1,6 @@ #!/bin/sh test_description='git branch display tests' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh . "$TEST_DIRECTORY"/lib-terminal.sh diff --git a/t/t3205-branch-color.sh b/t/t3205-branch-color.sh index 6a521c1a3e..0b61da92b3 100755 --- a/t/t3205-branch-color.sh +++ b/t/t3205-branch-color.sh @@ -1,9 +1,6 @@ #!/bin/sh test_description='basic branch output coloring' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh diff --git a/t/t3302-notes-index-expensive.sh b/t/t3302-notes-index-expensive.sh index ef8b63952e..bc9d8ee1e6 100755 --- a/t/t3302-notes-index-expensive.sh +++ b/t/t3302-notes-index-expensive.sh @@ -8,6 +8,7 @@ test_description='Test commit notes index (expensive!)' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh create_repo () { diff --git a/t/t3303-notes-subtrees.sh b/t/t3303-notes-subtrees.sh index d47ce00f69..f5fd80d4d3 100755 --- a/t/t3303-notes-subtrees.sh +++ b/t/t3303-notes-subtrees.sh @@ -5,6 +5,7 @@ test_description='Test commit notes organized in subtrees' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh number_of_commits=100 diff --git a/t/t3305-notes-fanout.sh b/t/t3305-notes-fanout.sh index 94c1b02251..960d0587e1 100755 --- a/t/t3305-notes-fanout.sh +++ b/t/t3305-notes-fanout.sh @@ -2,6 +2,7 @@ test_description='Test that adding/removing many notes triggers automatic fanout restructuring' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh path_has_fanout() { diff --git a/t/t3320-notes-merge-worktrees.sh b/t/t3320-notes-merge-worktrees.sh index 6b2d507f3e..bff0aea550 100755 --- a/t/t3320-notes-merge-worktrees.sh +++ b/t/t3320-notes-merge-worktrees.sh @@ -8,6 +8,7 @@ test_description='Test merging of notes trees in multiple worktrees' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup commit' ' diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index 12eb226957..0baa68e9b4 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -25,8 +25,6 @@ Initial setup: where A, B, D and G all touch file1, and one, two, three, four all touch file "conflict". ' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh diff --git a/t/t3422-rebase-incompatible-options.sh b/t/t3422-rebase-incompatible-options.sh index eb0a3d9d48..6dabb05a2a 100755 --- a/t/t3422-rebase-incompatible-options.sh +++ b/t/t3422-rebase-incompatible-options.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test if rebase detects and aborts on incompatible options' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3429-rebase-edit-todo.sh b/t/t3429-rebase-edit-todo.sh index 7024d49ae7..abd66f3602 100755 --- a/t/t3429-rebase-edit-todo.sh +++ b/t/t3429-rebase-edit-todo.sh @@ -13,10 +13,15 @@ test_expect_success 'setup' ' test_expect_success 'rebase exec modifies rebase-todo' ' todo=.git/rebase-merge/git-rebase-todo && - git rebase HEAD -x "echo exec touch F >>$todo" && + git rebase HEAD~1 -x "echo exec touch F >>$todo" && test -e F ' +test_expect_success 'rebase exec with an empty list does not exec anything' ' + git rebase HEAD -x "true" 2>output && + ! grep "Executing: true" output +' + test_expect_success 'loose object cache vs re-reading todo list' ' GIT_REBASE_TODO=.git/rebase-merge/git-rebase-todo && export GIT_REBASE_TODO && diff --git a/t/t3702-add-edit.sh b/t/t3702-add-edit.sh index 6c676645d8..a1801a8cbd 100755 --- a/t/t3702-add-edit.sh +++ b/t/t3702-add-edit.sh @@ -4,6 +4,8 @@ # test_description='add -e basic tests' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh diff --git a/t/t3703-add-magic-pathspec.sh b/t/t3703-add-magic-pathspec.sh index 3ef525a559..d84071038e 100755 --- a/t/t3703-add-magic-pathspec.sh +++ b/t/t3703-add-magic-pathspec.sh @@ -2,6 +2,7 @@ test_description='magic pathspec tests using git-add' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3704-add-pathspec-file.sh b/t/t3704-add-pathspec-file.sh index 9e35c1fbca..5d5164d1fc 100755 --- a/t/t3704-add-pathspec-file.sh +++ b/t/t3704-add-pathspec-file.sh @@ -2,6 +2,7 @@ test_description='add --pathspec-from-file' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_tick diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index f0a82be9de..2c66cfbc3b 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -288,6 +288,17 @@ test_expect_success 'stash --no-keep-index' ' test bar,bar2 = $(cat file),$(cat file2) ' +test_expect_success 'stash --staged' ' + echo bar3 >file && + echo bar4 >file2 && + git add file2 && + git stash --staged && + test bar3,bar2 = $(cat file),$(cat file2) && + git reset --hard && + git stash pop && + test bar,bar4 = $(cat file),$(cat file2) +' + test_expect_success 'dont assume push with non-option args' ' test_must_fail git stash -q drop 2>err && test_i18ngrep -e "subcommand wasn'\''t specified; '\''push'\'' can'\''t be assumed due to unexpected token '\''drop'\''" err diff --git a/t/t3908-stash-in-worktree.sh b/t/t3908-stash-in-worktree.sh index 2b2b366ef9..347a89b030 100755 --- a/t/t3908-stash-in-worktree.sh +++ b/t/t3908-stash-in-worktree.sh @@ -5,6 +5,7 @@ test_description='Test git stash in a worktree' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4000-diff-format.sh b/t/t4000-diff-format.sh index cce334981e..bfcaae390f 100755 --- a/t/t4000-diff-format.sh +++ b/t/t4000-diff-format.sh @@ -6,6 +6,8 @@ test_description='Test built-in diff output engine. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh diff --git a/t/t4003-diff-rename-1.sh b/t/t4003-diff-rename-1.sh index f4485a87c6..181e9683a7 100755 --- a/t/t4003-diff-rename-1.sh +++ b/t/t4003-diff-rename-1.sh @@ -6,6 +6,8 @@ test_description='More rename detection ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash diff --git a/t/t4004-diff-rename-symlink.sh b/t/t4004-diff-rename-symlink.sh index 3d495e37bb..8def4d4aee 100755 --- a/t/t4004-diff-rename-symlink.sh +++ b/t/t4004-diff-rename-symlink.sh @@ -9,6 +9,8 @@ The rename detection logic should be able to detect pure rename or copy of symbolic links, but should not produce rename/copy followed by an edit for them. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh diff --git a/t/t4005-diff-rename-2.sh b/t/t4005-diff-rename-2.sh index 6f1b323f97..5c756dc243 100755 --- a/t/t4005-diff-rename-2.sh +++ b/t/t4005-diff-rename-2.sh @@ -5,6 +5,8 @@ test_description='Same rename detection as t4003 but testing diff-raw.' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash diff --git a/t/t4006-diff-mode.sh b/t/t4006-diff-mode.sh index 6cdee2a216..dbd4c0da21 100755 --- a/t/t4006-diff-mode.sh +++ b/t/t4006-diff-mode.sh @@ -6,6 +6,8 @@ test_description='Test mode change diffs. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh sed_script='s/\(:100644 100755\) \('"$OID_REGEX"'\) \2 /\1 X X /' diff --git a/t/t4007-rename-3.sh b/t/t4007-rename-3.sh index c634653b5b..b86165cbac 100755 --- a/t/t4007-rename-3.sh +++ b/t/t4007-rename-3.sh @@ -6,6 +6,8 @@ test_description='Rename interaction with pathspec. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash diff --git a/t/t4009-diff-rename-4.sh b/t/t4009-diff-rename-4.sh index 59b7f44f05..3480781dab 100755 --- a/t/t4009-diff-rename-4.sh +++ b/t/t4009-diff-rename-4.sh @@ -6,6 +6,8 @@ test_description='Same rename detection as t4003 but testing diff-raw -z. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash diff --git a/t/t4010-diff-pathspec.sh b/t/t4010-diff-pathspec.sh index 1bbced79ec..9d9650eba7 100755 --- a/t/t4010-diff-pathspec.sh +++ b/t/t4010-diff-pathspec.sh @@ -9,6 +9,8 @@ Prepare: file0 path1/file1 ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash diff --git a/t/t4011-diff-symlink.sh b/t/t4011-diff-symlink.sh index 5a25c259fe..d7a5f7ae78 100755 --- a/t/t4011-diff-symlink.sh +++ b/t/t4011-diff-symlink.sh @@ -6,6 +6,8 @@ test_description='Test diff of symlinks. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh diff --git a/t/t4012-diff-binary.sh b/t/t4012-diff-binary.sh index 33ff588ebc..00eeafb2ac 100755 --- a/t/t4012-diff-binary.sh +++ b/t/t4012-diff-binary.sh @@ -6,6 +6,7 @@ test_description='Binary diff and apply ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >expect.binary-numstat <<\EOF diff --git a/t/t4020-diff-external.sh b/t/t4020-diff-external.sh index e009826fcb..54bb8ef27e 100755 --- a/t/t4020-diff-external.sh +++ b/t/t4020-diff-external.sh @@ -2,6 +2,7 @@ test_description='external diff interface test' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4024-diff-optimize-common.sh b/t/t4024-diff-optimize-common.sh index 6b44ce1493..540f93715e 100755 --- a/t/t4024-diff-optimize-common.sh +++ b/t/t4024-diff-optimize-common.sh @@ -2,6 +2,7 @@ test_description='common tail optimization' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh z=zzzzzzzz ;# 8 diff --git a/t/t4027-diff-submodule.sh b/t/t4027-diff-submodule.sh index 94ef77e1df..6cef0da982 100755 --- a/t/t4027-diff-submodule.sh +++ b/t/t4027-diff-submodule.sh @@ -2,6 +2,7 @@ test_description='difference in submodules' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh diff --git a/t/t4029-diff-trailing-space.sh b/t/t4029-diff-trailing-space.sh index 32b6e9a4e7..5f8ffef74b 100755 --- a/t/t4029-diff-trailing-space.sh +++ b/t/t4029-diff-trailing-space.sh @@ -4,6 +4,7 @@ # test_description='diff honors config option, diff.suppressBlankEmpty' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat <<\EOF >expected || diff --git a/t/t4032-diff-inter-hunk-context.sh b/t/t4032-diff-inter-hunk-context.sh index bada0cbd32..7db92d0d9f 100755 --- a/t/t4032-diff-inter-hunk-context.sh +++ b/t/t4032-diff-inter-hunk-context.sh @@ -2,6 +2,7 @@ test_description='diff hunk fusing' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh f() { diff --git a/t/t4033-diff-patience.sh b/t/t4033-diff-patience.sh index 113304dc59..f7be7f5ef0 100755 --- a/t/t4033-diff-patience.sh +++ b/t/t4033-diff-patience.sh @@ -2,6 +2,7 @@ test_description='patience diff algorithm' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff-alternative.sh diff --git a/t/t4034-diff-words.sh b/t/t4034-diff-words.sh index 561c582d16..d5abcf4b4c 100755 --- a/t/t4034-diff-words.sh +++ b/t/t4034-diff-words.sh @@ -2,6 +2,7 @@ test_description='word diff colors' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff.sh diff --git a/t/t4035-diff-quiet.sh b/t/t4035-diff-quiet.sh index 0352bf81a9..76f8034c60 100755 --- a/t/t4035-diff-quiet.sh +++ b/t/t4035-diff-quiet.sh @@ -2,6 +2,7 @@ test_description='Return value of diffs' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4037-diff-r-t-dirs.sh b/t/t4037-diff-r-t-dirs.sh index f5ce3b29a2..b5f96fe23b 100755 --- a/t/t4037-diff-r-t-dirs.sh +++ b/t/t4037-diff-r-t-dirs.sh @@ -2,6 +2,7 @@ test_description='diff -r -t shows directory additions and deletions' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4040-whitespace-status.sh b/t/t4040-whitespace-status.sh index 3c728a3ebf..e70e020ae9 100755 --- a/t/t4040-whitespace-status.sh +++ b/t/t4040-whitespace-status.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='diff --exit-code with whitespace' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4046-diff-unmerged.sh b/t/t4046-diff-unmerged.sh index ff7cfd884a..fd3f3a7260 100755 --- a/t/t4046-diff-unmerged.sh +++ b/t/t4046-diff-unmerged.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='diff with unmerged index entries' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4049-diff-stat-count.sh b/t/t4049-diff-stat-count.sh index 53061b104e..f5b35e7860 100755 --- a/t/t4049-diff-stat-count.sh +++ b/t/t4049-diff-stat-count.sh @@ -2,6 +2,8 @@ # Copyright (c) 2011, Google Inc. test_description='diff --stat-count' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4050-diff-histogram.sh b/t/t4050-diff-histogram.sh index fd3e86a74f..c61b30f96d 100755 --- a/t/t4050-diff-histogram.sh +++ b/t/t4050-diff-histogram.sh @@ -2,6 +2,7 @@ test_description='histogram diff algorithm' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff-alternative.sh diff --git a/t/t4054-diff-bogus-tree.sh b/t/t4054-diff-bogus-tree.sh index 8c95f152b2..294fb55313 100755 --- a/t/t4054-diff-bogus-tree.sh +++ b/t/t4054-diff-bogus-tree.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test diff with a bogus tree containing the null sha1' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create bogus tree' ' diff --git a/t/t4062-diff-pickaxe.sh b/t/t4062-diff-pickaxe.sh index 1130c8019b..9aaa068ed9 100755 --- a/t/t4062-diff-pickaxe.sh +++ b/t/t4062-diff-pickaxe.sh @@ -5,6 +5,7 @@ test_description='Pickaxe options' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4063-diff-blobs.sh b/t/t4063-diff-blobs.sh index bc69e26c52..7e6c9d6384 100755 --- a/t/t4063-diff-blobs.sh +++ b/t/t4063-diff-blobs.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test direct comparison of blobs via git-diff' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh run_diff () { diff --git a/t/t4100-apply-stat.sh b/t/t4100-apply-stat.sh index 9b433de836..d503547732 100755 --- a/t/t4100-apply-stat.sh +++ b/t/t4100-apply-stat.sh @@ -6,6 +6,8 @@ test_description='git apply --stat --summary test, with --recount ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh UNC='s/^\(@@ -[1-9][0-9]*\),[0-9]* \(+[1-9][0-9]*\),[0-9]* @@/\1,999 \2,999 @@/' diff --git a/t/t4101-apply-nonl.sh b/t/t4101-apply-nonl.sh index e3443d004d..b1169193ef 100755 --- a/t/t4101-apply-nonl.sh +++ b/t/t4101-apply-nonl.sh @@ -6,6 +6,8 @@ test_description='git apply should handle files with incomplete lines. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # setup diff --git a/t/t4102-apply-rename.sh b/t/t4102-apply-rename.sh index fae305979a..d1e06fc1ac 100755 --- a/t/t4102-apply-rename.sh +++ b/t/t4102-apply-rename.sh @@ -6,6 +6,8 @@ test_description='git apply handling copy/rename patch. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # setup diff --git a/t/t4105-apply-fuzz.sh b/t/t4105-apply-fuzz.sh index 3266e39400..76e2c01b0f 100755 --- a/t/t4105-apply-fuzz.sh +++ b/t/t4105-apply-fuzz.sh @@ -2,6 +2,8 @@ test_description='apply with fuzz and offset' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh dotest () { diff --git a/t/t4106-apply-stdin.sh b/t/t4106-apply-stdin.sh index 72467a1e8e..a57a318699 100755 --- a/t/t4106-apply-stdin.sh +++ b/t/t4106-apply-stdin.sh @@ -2,6 +2,8 @@ test_description='git apply --numstat - <patch' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4109-apply-multifrag.sh b/t/t4109-apply-multifrag.sh index ac58083fe2..4dc6d8e7d3 100755 --- a/t/t4109-apply-multifrag.sh +++ b/t/t4109-apply-multifrag.sh @@ -6,6 +6,8 @@ test_description='git apply test patches with multiple fragments.' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cp "$TEST_DIRECTORY/t4109/patch1.patch" . diff --git a/t/t4110-apply-scan.sh b/t/t4110-apply-scan.sh index 09f58112e0..266302a182 100755 --- a/t/t4110-apply-scan.sh +++ b/t/t4110-apply-scan.sh @@ -7,6 +7,8 @@ test_description='git apply test for patches which require scanning forwards and backwards. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'git apply scan' ' diff --git a/t/t4112-apply-renames.sh b/t/t4112-apply-renames.sh index f9ad183758..d53aa4222e 100755 --- a/t/t4112-apply-renames.sh +++ b/t/t4112-apply-renames.sh @@ -7,6 +7,8 @@ test_description='git apply should not get confused with rename/copy. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # setup diff --git a/t/t4115-apply-symlink.sh b/t/t4115-apply-symlink.sh index 872fcda6cb..d0f3edef54 100755 --- a/t/t4115-apply-symlink.sh +++ b/t/t4115-apply-symlink.sh @@ -7,6 +7,7 @@ test_description='git apply symlinks and partial files ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4116-apply-reverse.sh b/t/t4116-apply-reverse.sh index b99e65c086..f3b635475a 100755 --- a/t/t4116-apply-reverse.sh +++ b/t/t4116-apply-reverse.sh @@ -7,6 +7,8 @@ test_description='git apply in reverse ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4118-apply-empty-context.sh b/t/t4118-apply-empty-context.sh index 65f2e4c3ef..5fc6d3e4e7 100755 --- a/t/t4118-apply-empty-context.sh +++ b/t/t4118-apply-empty-context.sh @@ -7,6 +7,8 @@ test_description='git apply with new style GNU diff with empty context ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4119-apply-config.sh b/t/t4119-apply-config.sh index a9a0583811..208c961d37 100755 --- a/t/t4119-apply-config.sh +++ b/t/t4119-apply-config.sh @@ -7,6 +7,8 @@ test_description='git apply --whitespace=strip and configuration file. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4121-apply-diffs.sh b/t/t4121-apply-diffs.sh index b45454aaf4..a80cec9d11 100755 --- a/t/t4121-apply-diffs.sh +++ b/t/t4121-apply-diffs.sh @@ -4,6 +4,7 @@ test_description='git apply for contextually independent diffs' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh echo '1 diff --git a/t/t4123-apply-shrink.sh b/t/t4123-apply-shrink.sh index 984157f03b..ef57cd3aeb 100755 --- a/t/t4123-apply-shrink.sh +++ b/t/t4123-apply-shrink.sh @@ -2,6 +2,8 @@ test_description='apply a patch that is larger than the preimage' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >F <<\EOF diff --git a/t/t4126-apply-empty.sh b/t/t4126-apply-empty.sh index ceb6a79fe0..a361e79a81 100755 --- a/t/t4126-apply-empty.sh +++ b/t/t4126-apply-empty.sh @@ -2,6 +2,8 @@ test_description='apply empty' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4127-apply-same-fn.sh b/t/t4127-apply-same-fn.sh index 305b7e649e..c27e9aec57 100755 --- a/t/t4127-apply-same-fn.sh +++ b/t/t4127-apply-same-fn.sh @@ -2,6 +2,8 @@ test_description='apply same filename' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh modify () { diff --git a/t/t4128-apply-root.sh b/t/t4128-apply-root.sh index 6cc741a634..cb3181e8b7 100755 --- a/t/t4128-apply-root.sh +++ b/t/t4128-apply-root.sh @@ -2,6 +2,8 @@ test_description='apply same filename' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4129-apply-samemode.sh b/t/t4129-apply-samemode.sh index 576632f868..a1c7686519 100755 --- a/t/t4129-apply-samemode.sh +++ b/t/t4129-apply-samemode.sh @@ -2,6 +2,8 @@ test_description='applying patch with mode bits' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4130-apply-criss-cross-rename.sh b/t/t4130-apply-criss-cross-rename.sh index f8a313bcb9..f3ea632742 100755 --- a/t/t4130-apply-criss-cross-rename.sh +++ b/t/t4130-apply-criss-cross-rename.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='git apply handling criss-cross rename patch.' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh create_file() { diff --git a/t/t4132-apply-removal.sh b/t/t4132-apply-removal.sh index fec1d6fa51..c1e3049c04 100755 --- a/t/t4132-apply-removal.sh +++ b/t/t4132-apply-removal.sh @@ -4,6 +4,8 @@ test_description='git-apply notices removal patches generated by GNU diff' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4133-apply-filenames.sh b/t/t4133-apply-filenames.sh index c5ed3b17c4..35f1060bc8 100755 --- a/t/t4133-apply-filenames.sh +++ b/t/t4133-apply-filenames.sh @@ -5,6 +5,8 @@ test_description='git apply filename consistency check' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4134-apply-submodule.sh b/t/t4134-apply-submodule.sh index d1c16ba33c..aceb4c42b0 100755 --- a/t/t4134-apply-submodule.sh +++ b/t/t4134-apply-submodule.sh @@ -5,6 +5,8 @@ test_description='git apply submodule tests' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4136-apply-check.sh b/t/t4136-apply-check.sh index 4c3f264a63..dfec1c5f0f 100755 --- a/t/t4136-apply-check.sh +++ b/t/t4136-apply-check.sh @@ -2,6 +2,8 @@ test_description='git apply should exit non-zero with unrecognized input.' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4139-apply-escape.sh b/t/t4139-apply-escape.sh index 45b5660a47..e5c7439df1 100755 --- a/t/t4139-apply-escape.sh +++ b/t/t4139-apply-escape.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='paths written by git-apply cannot escape the working tree' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # tests will try to write to ../foo, and we do not diff --git a/t/t4204-patch-id.sh b/t/t4204-patch-id.sh index f120857c20..e78d8097f3 100755 --- a/t/t4204-patch-id.sh +++ b/t/t4204-patch-id.sh @@ -5,6 +5,7 @@ test_description='git patch-id' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4205-log-pretty-formats.sh b/t/t4205-log-pretty-formats.sh index 5865daa8f8..35eef4c865 100755 --- a/t/t4205-log-pretty-formats.sh +++ b/t/t4205-log-pretty-formats.sh @@ -1002,4 +1002,20 @@ test_expect_success '%(describe:exclude=...) vs git describe --exclude ...' ' test_cmp expect actual ' +test_expect_success '%(describe:tags) vs git describe --tags' ' + test_when_finished "git tag -d tagname" && + git tag tagname && + git describe --tags >expect && + git log -1 --format="%(describe:tags)" >actual && + test_cmp expect actual +' + +test_expect_success '%(describe:abbrev=...) vs git describe --abbrev=...' ' + test_when_finished "git tag -d tagname" && + git tag -a -m tagged tagname && + git describe --abbrev=15 >expect && + git log -1 --format="%(describe:abbrev=15)" >actual && + test_cmp expect actual +' + test_done diff --git a/t/t4216-log-bloom.sh b/t/t4216-log-bloom.sh index 50f206db55..8018c12a6a 100755 --- a/t/t4216-log-bloom.sh +++ b/t/t4216-log-bloom.sh @@ -175,13 +175,11 @@ test_expect_success 'persist filter settings' ' test_when_finished rm -rf .git/objects/info/commit-graph* && rm -rf .git/objects/info/commit-graph* && GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ - GIT_TRACE2_EVENT_NESTING=5 \ GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=9 \ GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY=15 \ git commit-graph write --reachable --changed-paths && grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2.txt && GIT_TRACE2_EVENT="$(pwd)/trace2-auto.txt" \ - GIT_TRACE2_EVENT_NESTING=5 \ git commit-graph write --reachable --changed-paths && grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2-auto.txt ' diff --git a/t/t5002-archive-attr-pattern.sh b/t/t5002-archive-attr-pattern.sh index bda6d7d7e9..a66b5ba27e 100755 --- a/t/t5002-archive-attr-pattern.sh +++ b/t/t5002-archive-attr-pattern.sh @@ -2,6 +2,7 @@ test_description='git archive attribute pattern tests' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_exists() { diff --git a/t/t5200-update-server-info.sh b/t/t5200-update-server-info.sh index 21a58eecb9..ed9dfd624c 100755 --- a/t/t5200-update-server-info.sh +++ b/t/t5200-update-server-info.sh @@ -2,6 +2,7 @@ test_description='Test git update-server-info' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' 'test_commit file' diff --git a/t/t5307-pack-missing-commit.sh b/t/t5307-pack-missing-commit.sh index f4338abb78..d3482ab279 100755 --- a/t/t5307-pack-missing-commit.sh +++ b/t/t5307-pack-missing-commit.sh @@ -2,6 +2,7 @@ test_description='pack should notice missing commit objects' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5310-pack-bitmaps.sh b/t/t5310-pack-bitmaps.sh index dcf03d324a..cbd33d5bde 100755 --- a/t/t5310-pack-bitmaps.sh +++ b/t/t5310-pack-bitmaps.sh @@ -1,8 +1,6 @@ #!/bin/sh test_description='exercise basic bitmap functionality' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bitmap.sh @@ -35,7 +33,7 @@ test_expect_success 'setup writing bitmaps during repack' ' ' test_expect_success 'full repack creates bitmaps' ' - GIT_TRACE2_EVENT_NESTING=4 GIT_TRACE2_EVENT="$(pwd)/trace" \ + GIT_TRACE2_EVENT="$(pwd)/trace" \ git repack -ad && ls .git/objects/pack/ | grep bitmap >output && test_line_count = 1 output && diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 3f69e43178..a612e44547 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -482,8 +482,10 @@ test_expect_success 'corrupt MIDX is not reused' ' ' test_expect_success 'verify incorrect checksum' ' - pos=$(($(wc -c <$objdir/pack/multi-pack-index) - 1)) && - corrupt_midx_and_verify $pos "\377" $objdir "incorrect checksum" + pos=$(($(wc -c <$objdir/pack/multi-pack-index) - 10)) && + corrupt_midx_and_verify $pos \ + "\377\377\377\377\377\377\377\377\377\377" \ + $objdir "incorrect checksum" ' test_expect_success 'repack progress off for redirected stderr' ' diff --git a/t/t5410-receive-pack-alternates.sh b/t/t5410-receive-pack-alternates.sh index 0b28e4e452..7a45d4c311 100755 --- a/t/t5410-receive-pack-alternates.sh +++ b/t/t5410-receive-pack-alternates.sh @@ -5,6 +5,7 @@ test_description='git receive-pack with alternate ref filtering' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index e6e3c8f552..7ce1a28ef6 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -2,9 +2,6 @@ test_description='git remote porcelain-ish' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh setup_repository () { diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index a0faf0dd94..8c33755c26 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -5,9 +5,6 @@ test_description='Per branch config variables affects "git fetch". ' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bundle.sh diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 8212ca56dc..7831a38dde 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -541,6 +541,15 @@ do done +test_expect_success "push to remote with no explicit refspec and config remote.*.push = src:dest" ' + mk_test testrepo heads/main && + git checkout $the_first_commit && + test_config remote.there.url testrepo && + test_config remote.there.push refs/heads/main:refs/heads/main && + git push there && + check_push_result testrepo $the_commit heads/main +' + test_expect_success 'push with remote.pushdefault' ' mk_test up_repo heads/main && mk_test down_repo heads/main && diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh index 2dc75b80db..840c89cc8b 100755 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@ -3,9 +3,6 @@ test_description='Recursive "git fetch" for submodules' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB diff --git a/t/t5555-http-smart-common.sh b/t/t5555-http-smart-common.sh index 49faf5e283..b1cfe8b7db 100755 --- a/t/t5555-http-smart-common.sh +++ b/t/t5555-http-smart-common.sh @@ -2,6 +2,7 @@ test_description='test functionality common to smart fetch & push' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t5602-clone-remote-exec.sh b/t/t5602-clone-remote-exec.sh index cbcceab9d5..56329aa160 100755 --- a/t/t5602-clone-remote-exec.sh +++ b/t/t5602-clone-remote-exec.sh @@ -2,6 +2,7 @@ test_description=clone +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5603-clone-dirname.sh b/t/t5603-clone-dirname.sh index 13b5e5eb9b..8ca1f09423 100755 --- a/t/t5603-clone-dirname.sh +++ b/t/t5603-clone-dirname.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='check output directory names used by git-clone' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # we use a fake ssh wrapper that ignores the arguments diff --git a/t/t5609-clone-branch.sh b/t/t5609-clone-branch.sh index f86a674a03..252e1f7c20 100755 --- a/t/t5609-clone-branch.sh +++ b/t/t5609-clone-branch.sh @@ -4,6 +4,7 @@ test_description='clone --branch option' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh check_HEAD() { diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index aa1827d841..1896f671cb 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -5,6 +5,7 @@ test_description='test protocol v2 server commands' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'test capability advertisement' ' diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index d527cf6c49..78f85b0714 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -1107,6 +1107,57 @@ test_expect_success 'packfile-uri with transfer.fsckobjects fails when .gitmodul test_i18ngrep "disallowed submodule name" err ' +test_expect_success 'packfile-uri path redacted in trace' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child log && + + git init "$P" && + git -C "$P" config "uploadpack.allowsidebandall" "true" && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + + git -C "$P" hash-object my-blob >objh && + git -C "$P" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" <objh >packh && + git -C "$P" config --add \ + "uploadpack.blobpackfileuri" \ + "$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" && + + GIT_TRACE_PACKET="$(pwd)/log" \ + git -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + grep -F "clone< \\1$(cat packh) $HTTPD_URL/<redacted>" log +' + +test_expect_success 'packfile-uri path not redacted in trace when GIT_TRACE_REDACT=0' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child log && + + git init "$P" && + git -C "$P" config "uploadpack.allowsidebandall" "true" && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + + git -C "$P" hash-object my-blob >objh && + git -C "$P" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" <objh >packh && + git -C "$P" config --add \ + "uploadpack.blobpackfileuri" \ + "$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" && + + GIT_TRACE_PACKET="$(pwd)/log" \ + GIT_TRACE_REDACT=0 \ + git -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + grep -F "clone< \\1$(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" log +' + test_expect_success 'http:// --negotiate-only' ' SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && URI="$HTTPD_URL/smart/server" && diff --git a/t/t5703-upload-pack-ref-in-want.sh b/t/t5703-upload-pack-ref-in-want.sh index 220098523a..9d6cd7d986 100755 --- a/t/t5703-upload-pack-ref-in-want.sh +++ b/t/t5703-upload-pack-ref-in-want.sh @@ -2,9 +2,6 @@ test_description='upload-pack ref-in-want' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh get_actual_refs () { diff --git a/t/t5704-protocol-violations.sh b/t/t5704-protocol-violations.sh index bc393d7c31..ae1a00afb0 100755 --- a/t/t5704-protocol-violations.sh +++ b/t/t5704-protocol-violations.sh @@ -4,6 +4,8 @@ test_description='Test responses to violations of the network protocol. In most of these cases it will generally be acceptable for one side to break off communications if the other side says something unexpected. We are mostly making sure that we do not segfault or otherwise behave badly.' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'extra delim packet in v2 ls-refs args' ' diff --git a/t/t5705-session-id-in-capabilities.sh b/t/t5705-session-id-in-capabilities.sh index eb8c79aafd..ed38c76c29 100755 --- a/t/t5705-session-id-in-capabilities.sh +++ b/t/t5705-session-id-in-capabilities.sh @@ -32,7 +32,6 @@ do test_when_finished "git -C local push --delete origin new-branch" && cp -r "$LOCAL_PRISTINE" local && git -C local pull --no-rebase origin && - GIT_TRACE2_EVENT_NESTING=5 \ GIT_TRACE2_EVENT="$(pwd)/tr2-client-events" \ git -c protocol.version=$PROTO -C local push \ --receive-pack "GIT_TRACE2_EVENT=\"$(pwd)/tr2-server-events\" git-receive-pack" \ @@ -65,7 +64,6 @@ do test_when_finished "git -C local push --delete origin new-branch" && cp -r "$LOCAL_PRISTINE" local && git -C local pull --no-rebase origin && - GIT_TRACE2_EVENT_NESTING=5 \ GIT_TRACE2_EVENT="$(pwd)/tr2-client-events" \ git -c protocol.version=$PROTO -C local push \ --receive-pack "GIT_TRACE2_EVENT=\"$(pwd)/tr2-server-events\" git-receive-pack" \ diff --git a/t/t6005-rev-list-count.sh b/t/t6005-rev-list-count.sh index 0b64822bf6..d763de0041 100755 --- a/t/t6005-rev-list-count.sh +++ b/t/t6005-rev-list-count.sh @@ -2,6 +2,7 @@ test_description='git rev-list --max-count and --skip test' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t6102-rev-list-unexpected-objects.sh b/t/t6102-rev-list-unexpected-objects.sh index 52cde097dd..6f0902b863 100755 --- a/t/t6102-rev-list-unexpected-objects.sh +++ b/t/t6102-rev-list-unexpected-objects.sh @@ -2,6 +2,7 @@ test_description='git rev-list should handle unexpected object types' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup well-formed objects' ' diff --git a/t/t6136-pathspec-in-bare.sh b/t/t6136-pathspec-in-bare.sh index b117251366..ae8b5379e2 100755 --- a/t/t6136-pathspec-in-bare.sh +++ b/t/t6136-pathspec-in-bare.sh @@ -2,6 +2,7 @@ test_description='diagnosing out-of-scope pathspec' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup a bare and non-bare repository' ' diff --git a/t/t6300-for-each-ref.sh b/t/t6300-for-each-ref.sh index 80679d5e12..6aa9fb17ea 100755 --- a/t/t6300-for-each-ref.sh +++ b/t/t6300-for-each-ref.sh @@ -5,9 +5,6 @@ test_description='for-each-ref test' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh . "$TEST_DIRECTORY"/lib-gpg.sh . "$TEST_DIRECTORY"/lib-terminal.sh @@ -419,6 +416,11 @@ test_expect_success 'Verify descending sort' ' test_cmp expected actual ' +test_expect_success 'Give help even with invalid sort atoms' ' + test_expect_code 129 git for-each-ref --sort=bogus -h >actual 2>&1 && + grep "^usage: git for-each-ref" actual +' + cat >expected <<\EOF refs/tags/testtag refs/tags/testtag-2 @@ -1019,6 +1021,27 @@ test_expect_success 'equivalent sorts fall back on refname' ' test_cmp expected actual ' +test_expect_success '--no-sort cancels the previous sort keys' ' + cat >expected <<-\EOF && + 100000 <user1@example.com> refs/tags/multi-ref1-100000-user1 + 100000 <user2@example.com> refs/tags/multi-ref1-100000-user2 + 100000 <user1@example.com> refs/tags/multi-ref2-100000-user1 + 100000 <user2@example.com> refs/tags/multi-ref2-100000-user2 + 200000 <user1@example.com> refs/tags/multi-ref1-200000-user1 + 200000 <user2@example.com> refs/tags/multi-ref1-200000-user2 + 200000 <user1@example.com> refs/tags/multi-ref2-200000-user1 + 200000 <user2@example.com> refs/tags/multi-ref2-200000-user2 + EOF + git for-each-ref \ + --format="%(taggerdate:unix) %(taggeremail) %(refname)" \ + --sort=-refname \ + --sort=taggeremail \ + --no-sort \ + --sort=taggerdate \ + "refs/tags/multi-*" >actual && + test_cmp expected actual +' + test_expect_success 'do not dereference NULL upon %(HEAD) on unborn branch' ' test_when_finished "git checkout main" && git for-each-ref --format="%(HEAD) %(refname:short)" refs/heads/ >actual && diff --git a/t/t6302-for-each-ref-filter.sh b/t/t6302-for-each-ref-filter.sh index 1537aa2179..1ce5f490e9 100755 --- a/t/t6302-for-each-ref-filter.sh +++ b/t/t6302-for-each-ref-filter.sh @@ -2,9 +2,6 @@ test_description='test for-each-refs usage of ref-filter APIs' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh . "$TEST_DIRECTORY"/lib-gpg.sh diff --git a/t/t6407-merge-binary.sh b/t/t6407-merge-binary.sh index d4273f2575..e34676c204 100755 --- a/t/t6407-merge-binary.sh +++ b/t/t6407-merge-binary.sh @@ -5,6 +5,7 @@ test_description='ask merge-recursive to merge binary files' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t6414-merge-rename-nocruft.sh b/t/t6414-merge-rename-nocruft.sh index d7e3c1fa6e..69fc1c9e69 100755 --- a/t/t6414-merge-rename-nocruft.sh +++ b/t/t6414-merge-rename-nocruft.sh @@ -4,6 +4,7 @@ test_description='Merge-recursive merging renames' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t6427-diff3-conflict-markers.sh b/t/t6427-diff3-conflict-markers.sh index 25c4b720e7..a9ee4cb207 100755 --- a/t/t6427-diff3-conflict-markers.sh +++ b/t/t6427-diff3-conflict-markers.sh @@ -211,4 +211,94 @@ test_expect_success 'rebase --apply describes fake ancestor base' ' ) ' +test_setup_zdiff3 () { + test_create_repo zdiff3 && + ( + cd zdiff3 && + + test_write_lines 1 2 3 4 5 6 7 8 9 >basic && + test_write_lines 1 2 3 AA 4 5 BB 6 7 8 >middle-common && + test_write_lines 1 2 3 4 5 6 7 8 9 >interesting && + test_write_lines 1 2 3 4 5 6 7 8 9 >evil && + + git add basic middle-common interesting evil && + git commit -m base && + + git branch left && + git branch right && + + git checkout left && + test_write_lines 1 2 3 4 A B C D E 7 8 9 >basic && + test_write_lines 1 2 3 CC 4 5 DD 6 7 8 >middle-common && + test_write_lines 1 2 3 4 A B C D E F G H I J 7 8 9 >interesting && + test_write_lines 1 2 3 4 X A B C 7 8 9 >evil && + git add -u && + git commit -m letters && + + git checkout right && + test_write_lines 1 2 3 4 A X C Y E 7 8 9 >basic && + test_write_lines 1 2 3 EE 4 5 FF 6 7 8 >middle-common && + test_write_lines 1 2 3 4 A B C 5 6 G H I J 7 8 9 >interesting && + test_write_lines 1 2 3 4 Y A B C B C 7 8 9 >evil && + git add -u && + git commit -m permuted + ) +} + +test_expect_success 'check zdiff3 markers' ' + test_setup_zdiff3 && + ( + cd zdiff3 && + + git checkout left^0 && + + base=$(git rev-parse --short HEAD^1) && + test_must_fail git -c merge.conflictstyle=zdiff3 merge -s recursive right^0 && + + test_write_lines 1 2 3 4 A \ + "<<<<<<< HEAD" B C D \ + "||||||| $base" 5 6 \ + ======= X C Y \ + ">>>>>>> right^0" \ + E 7 8 9 \ + >expect && + test_cmp expect basic && + + test_write_lines 1 2 3 \ + "<<<<<<< HEAD" CC \ + "||||||| $base" AA \ + ======= EE \ + ">>>>>>> right^0" \ + 4 5 \ + "<<<<<<< HEAD" DD \ + "||||||| $base" BB \ + ======= FF \ + ">>>>>>> right^0" \ + 6 7 8 \ + >expect && + test_cmp expect middle-common && + + test_write_lines 1 2 3 4 A B C \ + "<<<<<<< HEAD" D E F \ + "||||||| $base" 5 6 \ + ======= 5 6 \ + ">>>>>>> right^0" \ + G H I J 7 8 9 \ + >expect && + test_cmp expect interesting && + + # Not passing this one yet; the common "B C" lines is still + # being left in the conflict blocks on the left and right + # sides. + test_write_lines 1 2 3 4 \ + "<<<<<<< HEAD" X A \ + "||||||| $base" 5 6 \ + ======= Y A B C \ + ">>>>>>> right^0" \ + B C 7 8 9 \ + >expect && + test_cmp expect evil + ) +' + test_done diff --git a/t/t7006-pager.sh b/t/t7006-pager.sh index 0e7cf75435..e56ca5b0fa 100755 --- a/t/t7006-pager.sh +++ b/t/t7006-pager.sh @@ -661,6 +661,13 @@ test_expect_success 'setup trace2' ' export GIT_TRACE2_BRIEF ' +test_expect_success 'setup large log output' ' + perl -e " + print \"this is a long commit message\" x 50000 + " >commit-msg && + git commit --allow-empty -F commit-msg +' + test_expect_success TTY 'git returns SIGPIPE on early pager exit' ' test_when_finished "rm pager-used trace.normal" && test_config core.pager ">pager-used; head -n 1; exit 0" && @@ -670,7 +677,7 @@ test_expect_success TTY 'git returns SIGPIPE on early pager exit' ' if test_have_prereq !MINGW then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && + { test_terminal git log >/dev/null; OUT=$?; } && test_match_signal 13 "$OUT" else test_terminal git log @@ -691,7 +698,7 @@ test_expect_success TTY 'git returns SIGPIPE on early pager non-zero exit' ' if test_have_prereq !MINGW then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && + { test_terminal git log >/dev/null; OUT=$?; } && test_match_signal 13 "$OUT" else test_terminal git log @@ -710,13 +717,7 @@ test_expect_success TTY 'git discards pager non-zero exit without SIGPIPE' ' export GIT_TRACE2 && test_when_finished "unset GIT_TRACE2" && - if test_have_prereq !MINGW - then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && - test "$OUT" -eq 0 - else - test_terminal git log - fi && + test_terminal git log && grep child_exit trace.normal >child-exits && test_line_count = 1 child-exits && @@ -724,41 +725,14 @@ test_expect_success TTY 'git discards pager non-zero exit without SIGPIPE' ' test_path_is_file pager-used ' -test_expect_success TTY 'git discards nonexisting pager without SIGPIPE' ' - test_when_finished "rm pager-used trace.normal" && - test_config core.pager "wc >pager-used; does-not-exist" && - GIT_TRACE2="$(pwd)/trace.normal" && - export GIT_TRACE2 && - test_when_finished "unset GIT_TRACE2" && - - if test_have_prereq !MINGW - then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && - test "$OUT" -eq 0 - else - test_terminal git log - fi && - - grep child_exit trace.normal >child-exits && - test_line_count = 1 child-exits && - grep " code:127 " child-exits && - test_path_is_file pager-used -' - -test_expect_success TTY 'git attempts to page to nonexisting pager command, gets SIGPIPE' ' +test_expect_success TTY 'git skips paging nonexisting command' ' test_when_finished "rm trace.normal" && test_config core.pager "does-not-exist" && GIT_TRACE2="$(pwd)/trace.normal" && export GIT_TRACE2 && test_when_finished "unset GIT_TRACE2" && - if test_have_prereq !MINGW - then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && - test_match_signal 13 "$OUT" - else - test_terminal git log - fi && + test_terminal git log && grep child_exit trace.normal >child-exits && test_line_count = 1 child-exits && @@ -767,14 +741,14 @@ test_expect_success TTY 'git attempts to page to nonexisting pager command, gets test_expect_success TTY 'git returns SIGPIPE on propagated signals from pager' ' test_when_finished "rm pager-used trace.normal" && - test_config core.pager ">pager-used; test-tool sigchain" && + test_config core.pager ">pager-used; exec test-tool sigchain" && GIT_TRACE2="$(pwd)/trace.normal" && export GIT_TRACE2 && test_when_finished "unset GIT_TRACE2" && if test_have_prereq !MINGW then - OUT=$( ((test_terminal git log; echo $? 1>&3) | :) 3>&1 ) && + { test_terminal git log >/dev/null; OUT=$?; } && test_match_signal 13 "$OUT" else test_terminal git log @@ -786,4 +760,9 @@ test_expect_success TTY 'git returns SIGPIPE on propagated signals from pager' ' test_path_is_file pager-used ' +test_expect_success TTY 'non-existent pager doesnt cause crash' ' + test_config pager.show invalid-pager && + test_terminal git show +' + test_done diff --git a/t/t7064-wtstatus-pv2.sh b/t/t7064-wtstatus-pv2.sh index eeb0534163..20a0d2afc2 100755 --- a/t/t7064-wtstatus-pv2.sh +++ b/t/t7064-wtstatus-pv2.sh @@ -4,10 +4,6 @@ test_description='git status --porcelain=v2 This test exercises porcelain V2 output for git status.' - -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh @@ -113,6 +109,21 @@ test_expect_success 'after first commit, create unstaged changes' ' test_cmp expect actual ' +test_expect_success 'after first commit, stash existing changes' ' + cat >expect <<-EOF && + # branch.oid $H0 + # branch.head initial-branch + # stash 2 + EOF + + test_when_finished "git stash pop && git stash pop" && + + git stash -- file_x && + git stash && + git status --porcelain=v2 --branch --show-stash --untracked-files=no >actual && + test_cmp expect actual +' + test_expect_success 'after first commit but omit untracked files and branch' ' cat >expect <<-EOF && 1 .M N... 100644 100644 100644 $OID_X $OID_X file_x diff --git a/t/t7101-reset-empty-subdirs.sh b/t/t7101-reset-empty-subdirs.sh index 5530651eea..638bb04e21 100755 --- a/t/t7101-reset-empty-subdirs.sh +++ b/t/t7101-reset-empty-subdirs.sh @@ -4,6 +4,8 @@ # test_description='git reset should cull empty subdirs' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-diff-data.sh diff --git a/t/t7102-reset.sh b/t/t7102-reset.sh index 601b2bf97f..d05426062e 100755 --- a/t/t7102-reset.sh +++ b/t/t7102-reset.sh @@ -472,6 +472,23 @@ test_expect_success '--mixed refreshes the index' ' test_cmp expect output ' +test_expect_success '--mixed preserves skip-worktree' ' + echo 123 >>file2 && + git add file2 && + git update-index --skip-worktree file2 && + git reset --mixed HEAD >output && + test_must_be_empty output && + + cat >expect <<-\EOF && + Unstaged changes after reset: + M file2 + EOF + git update-index --no-skip-worktree file2 && + git add file2 && + git reset --mixed HEAD >output && + test_cmp expect output +' + test_expect_success 'resetting specific path that is unmerged' ' git rm --cached file2 && F1=$(git rev-parse HEAD:file1) && diff --git a/t/t7103-reset-bare.sh b/t/t7103-reset-bare.sh index afe36a533c..0de83e3619 100755 --- a/t/t7103-reset-bare.sh +++ b/t/t7103-reset-bare.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='git reset in a bare repository' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup non-bare' ' diff --git a/t/t7113-post-index-change-hook.sh b/t/t7113-post-index-change-hook.sh index 688fa995c9..a21781d68a 100755 --- a/t/t7113-post-index-change-hook.sh +++ b/t/t7113-post-index-change-hook.sh @@ -5,6 +5,7 @@ test_description='post index change hook' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh index cb1b8e35db..e7cec2e457 100755 --- a/t/t7400-submodule-basic.sh +++ b/t/t7400-submodule-basic.sh @@ -1182,18 +1182,17 @@ test_expect_success 'submodule deinit is silent when used on an uninitialized su rmdir init example2 ' -test_expect_success 'submodule deinit fails when submodule has a .git directory even when forced' ' +test_expect_success 'submodule deinit absorbs .git directory if .git is a directory' ' git submodule update --init && ( cd init && rm .git && - cp -R ../.git/modules/example .git && + mv ../.git/modules/example .git && GIT_WORK_TREE=. git config --unset core.worktree ) && - test_must_fail git submodule deinit init && - test_must_fail git submodule deinit -f init && - test -d init/.git && - test -n "$(git config --get-regexp "submodule\.example\.")" + git submodule deinit init && + test_path_is_missing init/.git && + test -z "$(git config --get-regexp "submodule\.example\.")" ' test_expect_success 'submodule with UTF-8 name' ' diff --git a/t/t7509-commit-authorship.sh b/t/t7509-commit-authorship.sh index d568593382..21c668f75e 100755 --- a/t/t7509-commit-authorship.sh +++ b/t/t7509-commit-authorship.sh @@ -5,6 +5,7 @@ test_description='commit tests of various authorhip options. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh author_header () { diff --git a/t/t7511-status-index.sh b/t/t7511-status-index.sh index b5fdc048a5..4ffa45a7bf 100755 --- a/t/t7511-status-index.sh +++ b/t/t7511-status-index.sh @@ -2,6 +2,7 @@ test_description='git status with certain file name lengths' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh files="0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z" diff --git a/t/t7515-status-symlinks.sh b/t/t7515-status-symlinks.sh index 9f989be01b..e3d6bb67bf 100755 --- a/t/t7515-status-symlinks.sh +++ b/t/t7515-status-symlinks.sh @@ -2,6 +2,7 @@ test_description='git status and symlinks' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index f488d930df..a5e2233cb1 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -390,7 +390,7 @@ test_expect_success 'status succeeds after staging/unstaging' ' # during a call to 'git status'. Otherwise, we verify that we _do_ call it. check_sparse_index_behavior () { git -C full status --porcelain=v2 >expect && - GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ git -C sparse status --porcelain=v2 >actual && test_region $1 index ensure_full_index trace2.txt && test_region fsm_hook query trace2.txt && diff --git a/t/t7525-status-rename.sh b/t/t7525-status-rename.sh index a62736dce0..22bf5c7e5d 100755 --- a/t/t7525-status-rename.sh +++ b/t/t7525-status-rename.sh @@ -2,6 +2,7 @@ test_description='git status rename detection options' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t7526-commit-pathspec-file.sh b/t/t7526-commit-pathspec-file.sh index 5fbe47ebcd..dca62fc48e 100755 --- a/t/t7526-commit-pathspec-file.sh +++ b/t/t7526-commit-pathspec-file.sh @@ -2,6 +2,7 @@ test_description='commit --pathspec-from-file' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_tick diff --git a/t/t7815-grep-binary.sh b/t/t7815-grep-binary.sh index 90ebb64f46..ac871287c0 100755 --- a/t/t7815-grep-binary.sh +++ b/t/t7815-grep-binary.sh @@ -2,6 +2,7 @@ test_description='git grep in binary files' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' " diff --git a/t/t9102-git-svn-deep-rmdir.sh b/t/t9102-git-svn-deep-rmdir.sh index 66cd51102c..7b2049caa0 100755 --- a/t/t9102-git-svn-deep-rmdir.sh +++ b/t/t9102-git-svn-deep-rmdir.sh @@ -1,5 +1,7 @@ #!/bin/sh test_description='git svn rmdir' + +TEST_PASSES_SANITIZE_LEAK=true . ./lib-git-svn.sh test_expect_success 'initialize repo' ' diff --git a/t/t9123-git-svn-rebuild-with-rewriteroot.sh b/t/t9123-git-svn-rebuild-with-rewriteroot.sh index ead404589e..3320b1f39c 100755 --- a/t/t9123-git-svn-rebuild-with-rewriteroot.sh +++ b/t/t9123-git-svn-rebuild-with-rewriteroot.sh @@ -5,6 +5,7 @@ test_description='git svn respects rewriteRoot during rebuild' +TEST_PASSES_SANITIZE_LEAK=true . ./lib-git-svn.sh mkdir import diff --git a/t/t9128-git-svn-cmd-branch.sh b/t/t9128-git-svn-cmd-branch.sh index 4e95f791db..9871f5abc9 100755 --- a/t/t9128-git-svn-cmd-branch.sh +++ b/t/t9128-git-svn-cmd-branch.sh @@ -4,6 +4,8 @@ # test_description='git svn partial-rebuild tests' + +TEST_PASSES_SANITIZE_LEAK=true . ./lib-git-svn.sh test_expect_success 'initialize svnrepo' ' diff --git a/t/t9151-svn-mergeinfo.sh b/t/t9151-svn-mergeinfo.sh index 1fbe84feb1..c93a5beab2 100755 --- a/t/t9151-svn-mergeinfo.sh +++ b/t/t9151-svn-mergeinfo.sh @@ -5,9 +5,6 @@ test_description='git-svn svn mergeinfo properties' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./lib-git-svn.sh test_expect_success 'load svn dump' " diff --git a/t/t9167-git-svn-cmd-branch-subproject.sh b/t/t9167-git-svn-cmd-branch-subproject.sh index ba35fc06fc..d9fd111c10 100755 --- a/t/t9167-git-svn-cmd-branch-subproject.sh +++ b/t/t9167-git-svn-cmd-branch-subproject.sh @@ -4,6 +4,8 @@ # test_description='git svn branch for subproject clones' + +TEST_PASSES_SANITIZE_LEAK=true . ./lib-git-svn.sh test_expect_success 'initialize svnrepo' ' diff --git a/t/t9302-fast-import-unpack-limit.sh b/t/t9302-fast-import-unpack-limit.sh index f519e4f1bf..d8b1f9442e 100755 --- a/t/t9302-fast-import-unpack-limit.sh +++ b/t/t9302-fast-import-unpack-limit.sh @@ -1,5 +1,7 @@ #!/bin/sh test_description='test git fast-import unpack limit' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create loose objects on import' ' diff --git a/t/t9303-fast-import-compression.sh b/t/t9303-fast-import-compression.sh index 57d916524e..4f5bf40587 100755 --- a/t/t9303-fast-import-compression.sh +++ b/t/t9303-fast-import-compression.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='compression setting of fast-import utility' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh import_large () { diff --git a/t/t9603-cvsimport-patchsets.sh b/t/t9603-cvsimport-patchsets.sh index 0e9daa5768..19f38f78f2 100755 --- a/t/t9603-cvsimport-patchsets.sh +++ b/t/t9603-cvsimport-patchsets.sh @@ -12,9 +12,6 @@ # bug. test_description='git cvsimport testing for correct patchset estimation' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./lib-cvs.sh setup_cvs_test_repository t9603 diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index 5decc3b269..0f28c4ad94 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -5,9 +5,6 @@ test_description='test bash completion' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./lib-bash.sh complete () @@ -2148,6 +2145,9 @@ test_expect_success PERL 'send-email' ' --cover-from-description=Z --cover-letter Z EOF + test_completion "git send-email --val" <<-\EOF && + --validate Z + EOF test_completion "git send-email ma" "main " ' diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index eef2262a36..389153e591 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -680,6 +680,17 @@ test_have_prereq () { # Keep a list of missing prerequisites; restore # the negative marker if necessary. prerequisite=${negative_prereq:+!}$prerequisite + + # Abort if this prereq was marked as required + if test -n "$GIT_TEST_REQUIRE_PREREQ" + then + case " $GIT_TEST_REQUIRE_PREREQ " in + *" $prerequisite "*) + BAIL_OUT "required prereq $prerequisite failed" + ;; + esac + fi + if test -z "$missing_prereq" then missing_prereq=$prerequisite diff --git a/t/test-lib.sh b/t/test-lib.sh index 2679a7596a..d1750bf615 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -476,6 +476,13 @@ export GIT_TEST_MERGE_ALGORITHM GIT_TRACE_BARE=1 export GIT_TRACE_BARE +# Some tests scan the GIT_TRACE2_EVENT feed for events, but the +# default depth is 2, which frequently causes issues when the +# events are wrapped in new regions. Set it to a sufficiently +# large depth to avoid custom changes in the test suite. +GIT_TRACE2_EVENT_NESTING=100 +export GIT_TRACE2_EVENT_NESTING + # Use specific version of the index file format if test -n "${GIT_TEST_INDEX_VERSION:+isset}" then @@ -589,6 +596,15 @@ USER_TERM="$TERM" TERM=dumb export TERM USER_TERM +# What is written by tests to stdout and stderr is sent to different places +# depending on the test mode (e.g. /dev/null in non-verbose mode, piped to tee +# with --tee option, etc.). We save the original stdin to FD #6 and stdout and +# stderr to #5 and #7, so that the test framework can use them (e.g. for +# printing errors within the test framework) independently of the test mode. +exec 5>&1 +exec 6<&0 +exec 7>&2 + _error_exit () { finalize_junit_xml GIT_EXIT_OK=t @@ -612,7 +628,7 @@ BAIL_OUT () { local bail_out="Bail out! " local message="$1" - say_color error $bail_out "$message" + say_color >&5 error $bail_out "$message" _error_exit } @@ -637,9 +653,6 @@ then exit 0 fi -exec 5>&1 -exec 6<&0 -exec 7>&2 if test "$verbose_log" = "t" then exec 3>>"$GIT_TEST_TEE_OUTPUT_FILE" 4>&3 @@ -669,6 +682,8 @@ test_fixed=0 test_broken=0 test_success=0 +test_missing_prereq= + test_external_has_tap=0 die () { @@ -1069,6 +1084,14 @@ test_skip () { of_prereq=" of $test_prereq" fi skipped_reason="missing $missing_prereq${of_prereq}" + + # Keep a list of all the missing prereq for result aggregation + if test -z "$missing_prereq" + then + test_missing_prereq=$missing_prereq + else + test_missing_prereq="$test_missing_prereq,$missing_prereq" + fi fi case "$to_skip" in @@ -1175,6 +1198,7 @@ test_done () { fixed $test_fixed broken $test_broken failed $test_failure + missing_prereq $test_missing_prereq EOF fi @@ -1734,6 +1758,10 @@ build_option () { sed -ne "s/^$1: //p" } +test_lazy_prereq SIZE_T_IS_64BIT ' + test 8 -eq "$(build_option sizeof-size_t)" +' + test_lazy_prereq LONG_IS_64BIT ' test 8 -le "$(build_option sizeof-long)" ' diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index bda283e7f4..8a21dd2972 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -1,4 +1,5 @@ #include "cache.h" +#include "sigchain.h" #include "trace2/tr2_dst.h" #include "trace2/tr2_sid.h" #include "trace2/tr2_sysenv.h" @@ -360,6 +361,7 @@ int tr2_dst_trace_want(struct tr2_dst *dst) void tr2_dst_write_line(struct tr2_dst *dst, struct strbuf *buf_line) { int fd = tr2_dst_get_trace_fd(dst); + ssize_t bytes; strbuf_complete_line(buf_line); /* ensure final NL on buffer */ @@ -378,12 +380,15 @@ void tr2_dst_write_line(struct tr2_dst *dst, struct strbuf *buf_line) * * If we get an IO error, just close the trace dst. */ - if (write(fd, buf_line->buf, buf_line->len) >= 0) + sigchain_push(SIGPIPE, SIG_IGN); + bytes = write(fd, buf_line->buf, buf_line->len); + sigchain_pop(SIGPIPE); + if (bytes >= 0) return; + tr2_dst_trace_disable(dst); if (tr2_dst_want_warning()) warning("unable to write trace to '%s': %s", tr2_sysenv_display_name(dst->sysenv_var), strerror(errno)); - tr2_dst_trace_disable(dst); } diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 3a0014417c..bd17ecdc32 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -354,7 +354,7 @@ static void fn_child_start_fl(const char *file, int line, jw_object_inline_begin_array(&jw, "argv"); if (cmd->git_cmd) jw_array_string(&jw, "git"); - jw_array_argv(&jw, cmd->argv); + jw_array_argv(&jw, cmd->args.v); jw_end(&jw); jw_end(&jw); diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c index 58d9e430f0..6e429a3fb9 100644 --- a/trace2/tr2_tgt_normal.c +++ b/trace2/tr2_tgt_normal.c @@ -232,7 +232,7 @@ static void fn_child_start_fl(const char *file, int line, strbuf_addch(&buf_payload, ' '); if (cmd->git_cmd) strbuf_addstr(&buf_payload, "git "); - sq_append_quote_argv_pretty(&buf_payload, cmd->argv); + sq_append_quote_argv_pretty(&buf_payload, cmd->args.v); normal_io_write_fl(file, line, &buf_payload); strbuf_release(&buf_payload); diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c index e4acca13d6..2ff9cf7083 100644 --- a/trace2/tr2_tgt_perf.c +++ b/trace2/tr2_tgt_perf.c @@ -335,10 +335,10 @@ static void fn_child_start_fl(const char *file, int line, strbuf_addstr(&buf_payload, " argv:["); if (cmd->git_cmd) { strbuf_addstr(&buf_payload, "git"); - if (cmd->argv[0]) + if (cmd->args.nr) strbuf_addch(&buf_payload, ' '); } - sq_append_quote_argv_pretty(&buf_payload, cmd->argv); + sq_append_quote_argv_pretty(&buf_payload, cmd->args.v); strbuf_addch(&buf_payload, ']'); perf_io_write_fl(file, line, event_name, NULL, &us_elapsed_absolute, @@ -236,7 +236,7 @@ static char *apply_command(struct conf_info *conf, const char *arg) strbuf_replace(&cmd, TRAILER_ARG_STRING, arg); strvec_push(&cp.args, cmd.buf); } - cp.env = local_repo_env; + strvec_pushv(&cp.env_array, (const char **)local_repo_env); cp.no_stdin = 1; cp.use_shell = 1; diff --git a/transport.c b/transport.c index e4f1decae2..92ab9a3fa6 100644 --- a/transport.c +++ b/transport.c @@ -1204,16 +1204,15 @@ static int run_pre_push_hook(struct transport *transport, struct ref *r; struct child_process proc = CHILD_PROCESS_INIT; struct strbuf buf; - const char *argv[4]; + const char *hook_path = find_hook("pre-push"); - if (!(argv[0] = find_hook("pre-push"))) + if (!hook_path) return 0; - argv[1] = transport->remote->name; - argv[2] = transport->url; - argv[3] = NULL; + strvec_push(&proc.args, hook_path); + strvec_push(&proc.args, transport->remote->name); + strvec_push(&proc.args, transport->url); - proc.argv = argv; proc.in = -1; proc.trace2_hook_name = "pre-push"; diff --git a/unpack-trees.c b/unpack-trees.c index 89ca95ce90..98e2f2e0e6 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -645,17 +645,24 @@ static void mark_ce_used_same_name(struct cache_entry *ce, } } -static struct cache_entry *next_cache_entry(struct unpack_trees_options *o) +static struct cache_entry *next_cache_entry(struct unpack_trees_options *o, int *hint) { const struct index_state *index = o->src_index; int pos = o->cache_bottom; + if (*hint > pos) + pos = *hint; + while (pos < index->cache_nr) { struct cache_entry *ce = index->cache[pos]; - if (!(ce->ce_flags & CE_UNPACKED)) + if (!(ce->ce_flags & CE_UNPACKED)) { + *hint = pos + 1; return ce; + } pos++; } + + *hint = pos; return NULL; } @@ -1231,7 +1238,9 @@ static int find_cache_pos(struct traverse_info *info, /* * Given a sparse directory entry 'ce', compare ce->name to - * info->name + '/' + p->path + '/' if info->name is non-empty. + * info->traverse_path + p->path + '/' if info->traverse_path + * is non-empty. + * * Compare ce->name to p->path + '/' otherwise. Note that * ce->name must end in a trailing '/' because it is a sparse * directory entry. @@ -1243,11 +1252,11 @@ static int sparse_dir_matches_path(const struct cache_entry *ce, assert(S_ISSPARSEDIR(ce->ce_mode)); assert(ce->name[ce->ce_namelen - 1] == '/'); - if (info->namelen) - return ce->ce_namelen == info->namelen + p->pathlen + 2 && - ce->name[info->namelen] == '/' && - !strncmp(ce->name, info->name, info->namelen) && - !strncmp(ce->name + info->namelen + 1, p->path, p->pathlen); + if (info->pathlen) + return ce->ce_namelen == info->pathlen + p->pathlen + 1 && + ce->name[info->pathlen - 1] == '/' && + !strncmp(ce->name, info->traverse_path, info->pathlen) && + !strncmp(ce->name + info->pathlen, p->path, p->pathlen); return ce->ce_namelen == p->pathlen + 1 && !strncmp(ce->name, p->path, p->pathlen); } @@ -1365,12 +1374,13 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str /* Are we supposed to look at the index too? */ if (o->merge) { + int hint = -1; while (1) { int cmp; struct cache_entry *ce; if (o->diff_index_cached) - ce = next_cache_entry(o); + ce = next_cache_entry(o, &hint); else ce = find_cache_entry(info, p); @@ -1690,7 +1700,7 @@ static int verify_absent(const struct cache_entry *, int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o) { struct repository *repo = the_repository; - int i, ret; + int i, hint, ret; static struct cache_entry *dfc; struct pattern_list pl; int free_pattern_list = 0; @@ -1779,13 +1789,15 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options info.pathspec = o->pathspec; if (o->prefix) { + hint = -1; + /* * Unpack existing index entries that sort before the * prefix the tree is spliced into. Note that o->merge * is always true in this case. */ while (1) { - struct cache_entry *ce = next_cache_entry(o); + struct cache_entry *ce = next_cache_entry(o, &hint); if (!ce) break; if (ce_in_traverse_path(ce, &info)) @@ -1806,8 +1818,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options /* Any left-over entries in the index? */ if (o->merge) { + hint = -1; while (1) { - struct cache_entry *ce = next_cache_entry(o); + struct cache_entry *ce = next_cache_entry(o, &hint); if (!ce) break; if (unpack_index_entry(ce, o) < 0) diff --git a/upload-pack.c b/upload-pack.c index c78d55bc67..9b5db32623 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -596,14 +596,11 @@ static int do_reachable_revlist(struct child_process *cmd, struct object_array *reachable, enum allow_uor allow_uor) { - static const char *argv[] = { - "rev-list", "--stdin", NULL, - }; struct object *o; FILE *cmd_in = NULL; int i; - cmd->argv = argv; + strvec_pushl(&cmd->args, "rev-list", "--stdin", NULL); cmd->git_cmd = 1; cmd->no_stderr = 1; cmd->in = -1; diff --git a/urlmatch.c b/urlmatch.c index 33a2ccd306..03ad3f30a9 100644 --- a/urlmatch.c +++ b/urlmatch.c @@ -5,7 +5,7 @@ #define URL_DIGIT "0123456789" #define URL_ALPHADIGIT URL_ALPHA URL_DIGIT #define URL_SCHEME_CHARS URL_ALPHADIGIT "+.-" -#define URL_HOST_CHARS URL_ALPHADIGIT ".-[:]" /* IPv6 literals need [:] */ +#define URL_HOST_CHARS URL_ALPHADIGIT ".-_[:]" /* IPv6 literals need [:] */ #define URL_UNSAFE_CHARS " <>\"%{}|\\^`" /* plus 0x00-0x1F,0x7F-0xFF */ #define URL_GEN_RESERVED ":/?#[]@" #define URL_SUB_RESERVED "!$&'()*+,;=" diff --git a/worktree.c b/worktree.c index 092a4f92ad..2c155b1015 100644 --- a/worktree.c +++ b/worktree.c @@ -28,11 +28,13 @@ static void add_head_info(struct worktree *wt) { int flags; const char *target; + int ignore_errno; target = refs_resolve_ref_unsafe(get_worktree_ref_store(wt), "HEAD", 0, - &wt->head_oid, &flags); + &wt->head_oid, &flags, + &ignore_errno); if (!target) return; @@ -418,6 +420,7 @@ const struct worktree *find_shared_symref(const char *symref, const char *symref_target; struct ref_store *refs; int flags; + int ignore_errno; if (wt->is_bare) continue; @@ -435,7 +438,8 @@ const struct worktree *find_shared_symref(const char *symref, refs = get_worktree_ref_store(wt); symref_target = refs_resolve_ref_unsafe(refs, symref, 0, - NULL, &flags); + NULL, &flags, + &ignore_errno); if ((flags & REF_ISSYMREF) && symref_target && !strcmp(symref_target, target)) { existing = wt; @@ -563,16 +567,17 @@ int other_head_refs(each_ref_fn fn, void *cb_data) struct worktree *wt = *p; struct object_id oid; int flag; + int ignore_errno; if (wt->is_current) continue; strbuf_reset(&refname); strbuf_worktree_ref(wt, &refname, "HEAD"); - if (!refs_read_ref_full(get_main_ref_store(the_repository), - refname.buf, - RESOLVE_REF_READING, - &oid, &flag)) + if (refs_resolve_ref_unsafe(get_main_ref_store(the_repository), + refname.buf, + RESOLVE_REF_READING, + &oid, &flag, &ignore_errno)) ret = fn(refname.buf, &oid, flag, cb_data); if (ret) break; diff --git a/wt-status.c b/wt-status.c index e4f29b2b4c..5d215f4e4f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -948,11 +948,17 @@ static int stash_count_refs(struct object_id *ooid, struct object_id *noid, return 0; } +static int count_stash_entries(void) +{ + int n = 0; + for_each_reflog_ent("refs/stash", stash_count_refs, &n); + return n; +} + static void wt_longstatus_print_stash_summary(struct wt_status *s) { - int stash_count = 0; + int stash_count = count_stash_entries(); - for_each_reflog_ent("refs/stash", stash_count_refs, &stash_count); if (stash_count > 0) status_printf_ln(s, GIT_COLOR_NORMAL, Q_("Your stash currently has %d entry", @@ -2177,6 +2183,18 @@ static void wt_porcelain_v2_print_tracking(struct wt_status *s) } /* + * Print the stash count in a porcelain-friendly format + */ +static void wt_porcelain_v2_print_stash(struct wt_status *s) +{ + int stash_count = count_stash_entries(); + char eol = s->null_termination ? '\0' : '\n'; + + if (stash_count > 0) + fprintf(s->fp, "# stash %d%c", stash_count, eol); +} + +/* * Convert various submodule status values into a * fixed-length string of characters in the buffer provided. */ @@ -2437,6 +2455,9 @@ static void wt_porcelain_v2_print(struct wt_status *s) if (s->show_branch) wt_porcelain_v2_print_tracking(s); + if (s->show_stash) + wt_porcelain_v2_print_stash(s); + for (i = 0; i < s->change.nr; i++) { it = &(s->change.items[i]); d = it->util; diff --git a/xdiff-interface.c b/xdiff-interface.c index 75b32aef51..2e3a5a2943 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -313,6 +313,8 @@ int git_xmerge_config(const char *var, const char *value, void *cb) die("'%s' is not a boolean", var); if (!strcmp(value, "diff3")) git_xmerge_style = XDL_MERGE_DIFF3; + else if (!strcmp(value, "zdiff3")) + git_xmerge_style = XDL_MERGE_ZEALOUS_DIFF3; else if (!strcmp(value, "merge")) git_xmerge_style = 0; /* diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index b29deca5de..72e25a9ffa 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -66,6 +66,7 @@ extern "C" { /* merge output styles */ #define XDL_MERGE_DIFF3 1 +#define XDL_MERGE_ZEALOUS_DIFF3 2 typedef struct s_mmfile { char *ptr; diff --git a/xdiff/xmerge.c b/xdiff/xmerge.c index 1659edb453..fff0b594f9 100644 --- a/xdiff/xmerge.c +++ b/xdiff/xmerge.c @@ -230,7 +230,7 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1, size += xdl_recs_copy(xe1, m->i1, m->chg1, needs_cr, 1, dest ? dest + size : NULL); - if (style == XDL_MERGE_DIFF3) { + if (style == XDL_MERGE_DIFF3 || style == XDL_MERGE_ZEALOUS_DIFF3) { /* Shared preimage */ if (!dest) { size += marker_size + 1 + needs_cr + marker3_size; @@ -322,6 +322,40 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1, return size; } +static int recmatch(xrecord_t *rec1, xrecord_t *rec2, unsigned long flags) +{ + return xdl_recmatch(rec1->ptr, rec1->size, + rec2->ptr, rec2->size, flags); +} + +/* + * Remove any common lines from the beginning and end of the conflicted region. + */ +static void xdl_refine_zdiff3_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m, + xpparam_t const *xpp) +{ + xrecord_t **rec1 = xe1->xdf2.recs, **rec2 = xe2->xdf2.recs; + for (; m; m = m->next) { + /* let's handle just the conflicts */ + if (m->mode) + continue; + + while(m->chg1 && m->chg2 && + recmatch(rec1[m->i1], rec2[m->i2], xpp->flags)) { + m->chg1--; + m->chg2--; + m->i1++; + m->i2++; + } + while (m->chg1 && m->chg2 && + recmatch(rec1[m->i1 + m->chg1 - 1], + rec2[m->i2 + m->chg2 - 1], xpp->flags)) { + m->chg1--; + m->chg2--; + } + } +} + /* * Sometimes, changes are not quite identical, but differ in only a few * lines. Try hard to show only these few lines as conflicting. @@ -482,7 +516,22 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, int style = xmp->style; int favor = xmp->favor; - if (style == XDL_MERGE_DIFF3) { + /* + * XDL_MERGE_DIFF3 does not attempt to refine conflicts by looking + * at common areas of sides 1 & 2, because the base (side 0) does + * not match and is being shown. Similarly, simplification of + * non-conflicts is also skipped due to the skipping of conflict + * refinement. + * + * XDL_MERGE_ZEALOUS_DIFF3, on the other hand, will attempt to + * refine conflicts looking for common areas of sides 1 & 2. + * However, since the base is being shown and does not match, + * it will only look for common areas at the beginning or end + * of the conflict block. Since XDL_MERGE_ZEALOUS_DIFF3's + * conflict refinement is much more limited in this fashion, the + * conflict simplification will be skipped. + */ + if (style == XDL_MERGE_DIFF3 || style == XDL_MERGE_ZEALOUS_DIFF3) { /* * "diff3 -m" output does not make sense for anything * more aggressive than XDL_MERGE_EAGER. @@ -603,10 +652,12 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, if (!changes) changes = c; /* refine conflicts */ - if (XDL_MERGE_ZEALOUS <= level && - (xdl_refine_conflicts(xe1, xe2, changes, xpp) < 0 || - xdl_simplify_non_conflicts(xe1, changes, - XDL_MERGE_ZEALOUS < level) < 0)) { + if (style == XDL_MERGE_ZEALOUS_DIFF3) { + xdl_refine_zdiff3_conflicts(xe1, xe2, changes, xpp); + } else if (XDL_MERGE_ZEALOUS <= level && + (xdl_refine_conflicts(xe1, xe2, changes, xpp) < 0 || + xdl_simplify_non_conflicts(xe1, changes, + XDL_MERGE_ZEALOUS < level) < 0)) { xdl_cleanup_merge(changes); return -1; } |