diff options
| -rw-r--r-- | Documentation/git-multi-pack-index.txt | 19 | ||||
| -rw-r--r-- | Documentation/git-repack.txt | 18 | ||||
| -rw-r--r-- | builtin/multi-pack-index.c | 36 | ||||
| -rw-r--r-- | builtin/repack.c | 282 | ||||
| -rw-r--r-- | midx.c | 110 | ||||
| -rw-r--r-- | midx.h | 15 | ||||
| -rw-r--r-- | pack-bitmap.c | 2 | ||||
| -rw-r--r-- | pack-bitmap.h | 1 | ||||
| -rw-r--r-- | t/helper/test-read-midx.c | 29 | ||||
| -rw-r--r-- | t/lib-midx.sh | 8 | ||||
| -rwxr-xr-x | t/t5319-multi-pack-index.sh | 15 | ||||
| -rwxr-xr-x | t/t5326-multi-pack-bitmaps.sh | 82 | ||||
| -rwxr-xr-x | t/t7700-repack.sh | 138 | ||||
| -rwxr-xr-x | t/t7703-repack-geometric.sh | 24 |
14 files changed, 723 insertions, 56 deletions
diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index 3b0b55cd75..b008ce2850 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -45,6 +45,25 @@ write:: --[no-]bitmap:: Control whether or not a multi-pack bitmap is written. + + --stdin-packs:: + Write a multi-pack index containing only the set of + line-delimited pack index basenames provided over stdin. + + --refs-snapshot=<path>:: + With `--bitmap`, optionally specify a file which + contains a "refs snapshot" taken prior to repacking. ++ +A reference snapshot is composed of line-delimited OIDs corresponding to +the reference tips, usually taken by `git repack` prior to generating a +new pack. A line may optionally start with a `+` character to indicate +that the reference which corresponds to that OID is "preferred" (see +linkgit:git-config[1]'s `pack.preferBitmapTips`.) ++ +The file given at `<path>` is expected to be readable, and can contain +duplicates. (If a given OID is given more than once, it is marked as +preferred if at least one instance of it begins with the special `+` +marker). -- verify:: diff --git a/Documentation/git-repack.txt b/Documentation/git-repack.txt index 24c00c9384..7183fb498f 100644 --- a/Documentation/git-repack.txt +++ b/Documentation/git-repack.txt @@ -9,7 +9,7 @@ git-repack - Pack unpacked objects in a repository SYNOPSIS -------- [verse] -'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [--window=<n>] [--depth=<n>] [--threads=<n>] [--keep-pack=<pack-name>] +'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [-m] [--window=<n>] [--depth=<n>] [--threads=<n>] [--keep-pack=<pack-name>] [--write-midx] DESCRIPTION ----------- @@ -128,10 +128,11 @@ depth is 4095. -b:: --write-bitmap-index:: Write a reachability bitmap index as part of the repack. This - only makes sense when used with `-a` or `-A`, as the bitmaps + only makes sense when used with `-a`, `-A` or `-m`, as the bitmaps must be able to refer to all reachable objects. This option - overrides the setting of `repack.writeBitmaps`. This option - has no effect if multiple packfiles are created. + overrides the setting of `repack.writeBitmaps`. This option + has no effect if multiple packfiles are created, unless writing a + MIDX (in which case a multi-pack bitmap is created). --pack-kept-objects:: Include objects in `.keep` files when repacking. Note that we @@ -189,6 +190,15 @@ this "roll-up", without respect to their reachability. This is subject to change in the future. This option (implying a drastically different repack mode) is not guaranteed to work with all other combinations of option to `git repack`. ++ +When writing a multi-pack bitmap, `git repack` selects the largest resulting +pack as the preferred pack for object selection by the MIDX (see +linkgit:git-multi-pack-index[1]). + +-m:: +--write-midx:: + Write a multi-pack index (see linkgit:git-multi-pack-index[1]) + containing the non-redundant packs. CONFIGURATION ------------- diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 6426bbdace..075d15d706 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -7,7 +7,8 @@ #include "object-store.h" #define BUILTIN_MIDX_WRITE_USAGE \ - N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]") + N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]" \ + "[--refs-snapshot=<path>]") #define BUILTIN_MIDX_VERIFY_USAGE \ N_("git multi-pack-index [<options>] verify") @@ -45,8 +46,10 @@ static char const * const builtin_multi_pack_index_usage[] = { static struct opts_multi_pack_index { const char *object_dir; const char *preferred_pack; + const char *refs_snapshot; unsigned long batch_size; unsigned flags; + int stdin_packs; } opts; static struct option common_opts[] = { @@ -77,6 +80,16 @@ static int git_multi_pack_index_write_config(const char *var, const char *value, return 0; } +static void read_packs_from_stdin(struct string_list *to) +{ + struct strbuf buf = STRBUF_INIT; + while (strbuf_getline(&buf, stdin) != EOF) + string_list_append(to, buf.buf); + string_list_sort(to); + + strbuf_release(&buf); +} + static int cmd_multi_pack_index_write(int argc, const char **argv) { struct option *options; @@ -88,6 +101,10 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX), OPT_BIT(0, "progress", &opts.flags, N_("force progress reporting"), MIDX_PROGRESS), + OPT_BOOL(0, "stdin-packs", &opts.stdin_packs, + N_("write multi-pack index containing only given indexes")), + OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot, + N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; @@ -110,8 +127,23 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) FREE_AND_NULL(options); + if (opts.stdin_packs) { + struct string_list packs = STRING_LIST_INIT_DUP; + int ret; + + read_packs_from_stdin(&packs); + + ret = write_midx_file_only(opts.object_dir, &packs, + opts.preferred_pack, + opts.refs_snapshot, opts.flags); + + string_list_clear(&packs, 0); + + return ret; + + } return write_midx_file(opts.object_dir, opts.preferred_pack, - opts.flags); + opts.refs_snapshot, opts.flags); } static int cmd_multi_pack_index_verify(int argc, const char **argv) diff --git a/builtin/repack.c b/builtin/repack.c index cb9f4bfed3..0b2d1e5d82 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -15,6 +15,8 @@ #include "promisor-remote.h" #include "shallow.h" #include "pack.h" +#include "pack-bitmap.h" +#include "refs.h" static int delta_base_offset = 1; static int pack_kept_objects = -1; @@ -94,12 +96,14 @@ static void remove_pack_on_signal(int signo) } /* - * Adds all packs hex strings to the fname list, which do not - * have a corresponding .keep file. These packs are not to - * be kept if we are going to pack everything into one file. + * Adds all packs hex strings to either fname_nonkept_list or + * fname_kept_list based on whether each pack has a corresponding + * .keep file or not. Packs without a .keep file are not to be kept + * if we are going to pack everything into one file. */ -static void get_non_kept_pack_filenames(struct string_list *fname_list, - const struct string_list *extra_keep) +static void collect_pack_filenames(struct string_list *fname_nonkept_list, + struct string_list *fname_kept_list, + const struct string_list *extra_keep) { DIR *dir; struct dirent *e; @@ -112,21 +116,20 @@ static void get_non_kept_pack_filenames(struct string_list *fname_list, size_t len; int i; + if (!strip_suffix(e->d_name, ".pack", &len)) + continue; + for (i = 0; i < extra_keep->nr; i++) if (!fspathcmp(e->d_name, extra_keep->items[i].string)) break; - if (extra_keep->nr > 0 && i < extra_keep->nr) - continue; - - if (!strip_suffix(e->d_name, ".pack", &len)) - continue; fname = xmemdupz(e->d_name, len); - if (!file_exists(mkpath("%s/%s.keep", packdir, fname))) - string_list_append_nodup(fname_list, fname); + if ((extra_keep->nr > 0 && i < extra_keep->nr) || + (file_exists(mkpath("%s/%s.keep", packdir, fname)))) + string_list_append_nodup(fname_kept_list, fname); else - free(fname); + string_list_append_nodup(fname_nonkept_list, fname); } closedir(dir); } @@ -422,6 +425,25 @@ static void split_pack_geometry(struct pack_geometry *geometry, int factor) geometry->split = split; } +static struct packed_git *get_largest_active_pack(struct pack_geometry *geometry) +{ + if (!geometry) { + /* + * No geometry means either an all-into-one repack (in which + * case there is only one pack left and it is the largest) or an + * incremental one. + * + * If repacking incrementally, then we could check the size of + * all packs to determine which should be preferred, but leave + * this for later. + */ + return NULL; + } + if (geometry->split == geometry->pack_nr) + return NULL; + return geometry->pack[geometry->pack_nr - 1]; +} + static void clear_pack_geometry(struct pack_geometry *geometry) { if (!geometry) @@ -433,17 +455,162 @@ static void clear_pack_geometry(struct pack_geometry *geometry) geometry->split = 0; } +struct midx_snapshot_ref_data { + struct tempfile *f; + struct oidset seen; + int preferred; +}; + +static int midx_snapshot_ref_one(const char *refname, + const struct object_id *oid, + int flag, void *_data) +{ + struct midx_snapshot_ref_data *data = _data; + struct object_id peeled; + + if (!peel_iterated_oid(oid, &peeled)) + oid = &peeled; + + if (oidset_insert(&data->seen, oid)) + return 0; /* already seen */ + + if (oid_object_info(the_repository, oid, NULL) != OBJ_COMMIT) + return 0; + + fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", + oid_to_hex(oid)); + + return 0; +} + +static void midx_snapshot_refs(struct tempfile *f) +{ + struct midx_snapshot_ref_data data; + const struct string_list *preferred = bitmap_preferred_tips(the_repository); + + data.f = f; + data.preferred = 0; + oidset_init(&data.seen, 0); + + if (!fdopen_tempfile(f, "w")) + die(_("could not open tempfile %s for writing"), + get_tempfile_path(f)); + + if (preferred) { + struct string_list_item *item; + + data.preferred = 1; + for_each_string_list_item(item, preferred) + for_each_ref_in(item->string, midx_snapshot_ref_one, &data); + data.preferred = 0; + } + + for_each_ref(midx_snapshot_ref_one, &data); + + if (close_tempfile_gently(f)) { + int save_errno = errno; + delete_tempfile(&f); + errno = save_errno; + die_errno(_("could not close refs snapshot tempfile")); + } + + oidset_clear(&data.seen); +} + +static void midx_included_packs(struct string_list *include, + struct string_list *existing_nonkept_packs, + struct string_list *existing_kept_packs, + struct string_list *names, + struct pack_geometry *geometry) +{ + struct string_list_item *item; + + for_each_string_list_item(item, existing_kept_packs) + string_list_insert(include, xstrfmt("%s.idx", item->string)); + for_each_string_list_item(item, names) + string_list_insert(include, xstrfmt("pack-%s.idx", item->string)); + if (geometry) { + struct strbuf buf = STRBUF_INIT; + uint32_t i; + for (i = geometry->split; i < geometry->pack_nr; i++) { + struct packed_git *p = geometry->pack[i]; + + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + strbuf_addstr(&buf, ".idx"); + + string_list_insert(include, strbuf_detach(&buf, NULL)); + } + } else { + for_each_string_list_item(item, existing_nonkept_packs) { + if (item->util) + continue; + string_list_insert(include, xstrfmt("%s.idx", item->string)); + } + } +} + +static int write_midx_included_packs(struct string_list *include, + struct pack_geometry *geometry, + const char *refs_snapshot, + int show_progress, int write_bitmaps) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list_item *item; + struct packed_git *largest = get_largest_active_pack(geometry); + FILE *in; + int ret; + + if (!include->nr) + return 0; + + cmd.in = -1; + cmd.git_cmd = 1; + + strvec_push(&cmd.args, "multi-pack-index"); + strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL); + + if (show_progress) + strvec_push(&cmd.args, "--progress"); + else + strvec_push(&cmd.args, "--no-progress"); + + if (write_bitmaps) + strvec_push(&cmd.args, "--bitmap"); + + if (largest) + strvec_pushf(&cmd.args, "--preferred-pack=%s", + pack_basename(largest)); + + if (refs_snapshot) + strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot); + + ret = start_command(&cmd); + if (ret) + return ret; + + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, include) + fprintf(in, "%s\n", item->string); + fclose(in); + + return finish_command(&cmd); +} + int cmd_repack(int argc, const char **argv, const char *prefix) { struct child_process cmd = CHILD_PROCESS_INIT; struct string_list_item *item; struct string_list names = STRING_LIST_INIT_DUP; struct string_list rollback = STRING_LIST_INIT_NODUP; - struct string_list existing_packs = STRING_LIST_INIT_DUP; + struct string_list existing_nonkept_packs = STRING_LIST_INIT_DUP; + struct string_list existing_kept_packs = STRING_LIST_INIT_DUP; struct pack_geometry *geometry = NULL; struct strbuf line = STRBUF_INIT; + struct tempfile *refs_snapshot = NULL; int i, ext, ret; FILE *out; + int show_progress = isatty(2); /* variables to be filled by option parsing */ int pack_everything = 0; @@ -454,6 +621,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) int no_update_server_info = 0; struct pack_objects_args po_args = {NULL}; int geometric_factor = 0; + int write_midx = 0; struct option builtin_repack_options[] = { OPT_BIT('a', NULL, &pack_everything, @@ -496,6 +664,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) N_("do not repack this pack")), OPT_INTEGER('g', "geometric", &geometric_factor, N_("find a geometric progression with factor <N>")), + OPT_BOOL('m', "write-midx", &write_midx, + N_("write a multi-pack index of the resulting packs")), OPT_END() }; @@ -512,8 +682,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) die(_("--keep-unreachable and -A are incompatible")); if (write_bitmaps < 0) { - if (!(pack_everything & ALL_INTO_ONE) || - !is_bare_repository()) + if (!write_midx && + (!(pack_everything & ALL_INTO_ONE) || !is_bare_repository())) write_bitmaps = 0; } else if (write_bitmaps && git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0) && @@ -523,9 +693,21 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (pack_kept_objects < 0) pack_kept_objects = write_bitmaps > 0; - if (write_bitmaps && !(pack_everything & ALL_INTO_ONE)) + if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx) die(_(incremental_bitmap_conflict_error)); + if (write_midx && write_bitmaps) { + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/%s_XXXXXX", get_object_directory(), + "bitmap-ref-tips"); + + refs_snapshot = xmks_tempfile(path.buf); + midx_snapshot_refs(refs_snapshot); + + strbuf_release(&path); + } + if (geometric_factor) { if (pack_everything) die(_("--geometric is incompatible with -A, -a")); @@ -565,19 +747,22 @@ int cmd_repack(int argc, const char **argv, const char *prefix) } if (has_promisor_remote()) strvec_push(&cmd.args, "--exclude-promisor-objects"); - if (write_bitmaps > 0) - strvec_push(&cmd.args, "--write-bitmap-index"); - else if (write_bitmaps < 0) - strvec_push(&cmd.args, "--write-bitmap-index-quiet"); + if (!write_midx) { + if (write_bitmaps > 0) + strvec_push(&cmd.args, "--write-bitmap-index"); + else if (write_bitmaps < 0) + strvec_push(&cmd.args, "--write-bitmap-index-quiet"); + } if (use_delta_islands) strvec_push(&cmd.args, "--delta-islands"); - if (pack_everything & ALL_INTO_ONE) { - get_non_kept_pack_filenames(&existing_packs, &keep_pack_list); + collect_pack_filenames(&existing_nonkept_packs, &existing_kept_packs, + &keep_pack_list); + if (pack_everything & ALL_INTO_ONE) { repack_promisor_objects(&po_args, &names); - if (existing_packs.nr && delete_redundant) { + if (existing_nonkept_packs.nr && delete_redundant) { for_each_string_list_item(item, &names) { strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack", packtmp_name, item->string); @@ -677,20 +862,48 @@ int cmd_repack(int argc, const char **argv, const char *prefix) } /* End of pack replacement. */ - reprepare_packed_git(the_repository); - - if (delete_redundant) { + if (delete_redundant && pack_everything & ALL_INTO_ONE) { const int hexsz = the_hash_algo->hexsz; - int opts = 0; string_list_sort(&names); - for_each_string_list_item(item, &existing_packs) { + for_each_string_list_item(item, &existing_nonkept_packs) { char *sha1; size_t len = strlen(item->string); if (len < hexsz) continue; sha1 = item->string + len - hexsz; - if (!string_list_has_string(&names, sha1)) - remove_redundant_pack(packdir, item->string); + /* + * Mark this pack for deletion, which ensures that this + * pack won't be included in a MIDX (if `--write-midx` + * was given) and that we will actually delete this pack + * (if `-d` was given). + */ + item->util = (void*)(intptr_t)!string_list_has_string(&names, sha1); + } + } + + if (write_midx) { + struct string_list include = STRING_LIST_INIT_NODUP; + midx_included_packs(&include, &existing_nonkept_packs, + &existing_kept_packs, &names, geometry); + + ret = write_midx_included_packs(&include, geometry, + refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, + show_progress, write_bitmaps > 0); + + string_list_clear(&include, 0); + + if (ret) + return ret; + } + + reprepare_packed_git(the_repository); + + if (delete_redundant) { + int opts = 0; + for_each_string_list_item(item, &existing_nonkept_packs) { + if (!item->util) + continue; + remove_redundant_pack(packdir, item->string); } if (geometry) { @@ -711,7 +924,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) } strbuf_release(&buf); } - if (!po_args.quiet && isatty(2)) + if (!po_args.quiet && show_progress) opts |= PRUNE_PACKED_VERBOSE; prune_packed_objects(opts); @@ -730,12 +943,13 @@ int cmd_repack(int argc, const char **argv, const char *prefix) unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP, 0)) flags |= MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX; - write_midx_file(get_object_directory(), NULL, flags); + write_midx_file(get_object_directory(), NULL, NULL, flags); } string_list_clear(&names, 0); string_list_clear(&rollback, 0); - string_list_clear(&existing_packs, 0); + string_list_clear(&existing_nonkept_packs, 0); + string_list_clear(&existing_kept_packs, 0); clear_pack_geometry(geometry); strbuf_release(&line); @@ -460,6 +460,8 @@ struct write_midx_context { uint32_t num_large_offsets; int preferred_pack_idx; + + struct string_list *to_include; }; static void add_pack_to_midx(const char *full_path, size_t full_path_len, @@ -469,8 +471,26 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, if (ends_with(file_name, ".idx")) { display_progress(ctx->progress, ++ctx->pack_paths_checked); + /* + * Note that at most one of ctx->m and ctx->to_include are set, + * so we are testing midx_contains_pack() and + * string_list_has_string() independently (guarded by the + * appropriate NULL checks). + * + * We could support passing to_include while reusing an existing + * MIDX, but don't currently since the reuse process drags + * forward all packs from an existing MIDX (without checking + * whether or not they appear in the to_include list). + * + * If we added support for that, these next two conditional + * should be performed independently (likely checking + * to_include before the existing MIDX). + */ if (ctx->m && midx_contains_pack(ctx->m, file_name)) return; + else if (ctx->to_include && + !string_list_has_string(ctx->to_include, file_name)) + return; ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc); @@ -948,7 +968,43 @@ static void bitmap_show_commit(struct commit *commit, void *_data) data->commits[data->commits_nr++] = commit; } +static int read_refs_snapshot(const char *refs_snapshot, + struct rev_info *revs) +{ + struct strbuf buf = STRBUF_INIT; + struct object_id oid; + FILE *f = xfopen(refs_snapshot, "r"); + + while (strbuf_getline(&buf, f) != EOF) { + struct object *object; + int preferred = 0; + char *hex = buf.buf; + const char *end = NULL; + + if (buf.len && *buf.buf == '+') { + preferred = 1; + hex = &buf.buf[1]; + } + + if (parse_oid_hex(hex, &oid, &end) < 0) + die(_("could not parse line: %s"), buf.buf); + if (*end) + die(_("malformed line: %s"), buf.buf); + + object = parse_object_or_die(&oid, NULL); + if (preferred) + object->flags |= NEEDS_BITMAP; + + add_pending_object(revs, object, ""); + } + + fclose(f); + strbuf_release(&buf); + return 0; +} + static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p, + const char *refs_snapshot, struct write_midx_context *ctx) { struct rev_info revs; @@ -957,8 +1013,12 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr cb.ctx = ctx; repo_init_revisions(the_repository, &revs, NULL); - setup_revisions(0, NULL, &revs, NULL); - for_each_ref(add_ref_to_pending, &revs); + if (refs_snapshot) { + read_refs_snapshot(refs_snapshot, &revs); + } else { + setup_revisions(0, NULL, &revs, NULL); + for_each_ref(add_ref_to_pending, &revs); + } /* * Skipping promisor objects here is intentional, since it only excludes @@ -987,6 +1047,7 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash, struct write_midx_context *ctx, + const char *refs_snapshot, unsigned flags) { struct packing_data pdata; @@ -1002,7 +1063,7 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash, prepare_midx_packing_data(&pdata, ctx); - commits = find_commits_for_midx_bitmap(&commits_nr, ctx); + commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, ctx); /* * Build the MIDX-order index based on pdata.objects (which is already @@ -1047,8 +1108,10 @@ cleanup: } static int write_midx_internal(const char *object_dir, + struct string_list *packs_to_include, struct string_list *packs_to_drop, const char *preferred_pack_name, + const char *refs_snapshot, unsigned flags) { char *midx_name; @@ -1071,10 +1134,17 @@ static int write_midx_internal(const char *object_dir, die_errno(_("unable to create leading directories of %s"), midx_name); - for (cur = get_multi_pack_index(the_repository); cur; cur = cur->next) { - if (!strcmp(object_dir, cur->object_dir)) { - ctx.m = cur; - break; + if (!packs_to_include) { + /* + * Only reference an existing MIDX when not filtering which + * packs to include, since all packs and objects are copied + * blindly from an existing MIDX if one is present. + */ + for (cur = get_multi_pack_index(the_repository); cur; cur = cur->next) { + if (!strcmp(object_dir, cur->object_dir)) { + ctx.m = cur; + break; + } } } @@ -1125,10 +1195,13 @@ static int write_midx_internal(const char *object_dir, else ctx.progress = NULL; + ctx.to_include = packs_to_include; + for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx); stop_progress(&ctx.progress); - if (ctx.m && ctx.nr == ctx.m->num_packs && !packs_to_drop) { + if ((ctx.m && ctx.nr == ctx.m->num_packs) && + !(packs_to_include || packs_to_drop)) { struct bitmap_index *bitmap_git; int bitmap_exists; int want_bitmap = flags & MIDX_WRITE_BITMAP; @@ -1332,7 +1405,8 @@ static int write_midx_internal(const char *object_dir, if (flags & MIDX_WRITE_REV_INDEX) write_midx_reverse_index(midx_name, midx_hash, &ctx); if (flags & MIDX_WRITE_BITMAP) { - if (write_midx_bitmap(midx_name, midx_hash, &ctx, flags) < 0) { + if (write_midx_bitmap(midx_name, midx_hash, &ctx, + refs_snapshot, flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; goto cleanup; @@ -1367,9 +1441,21 @@ cleanup: int write_midx_file(const char *object_dir, const char *preferred_pack_name, + const char *refs_snapshot, unsigned flags) { - return write_midx_internal(object_dir, NULL, preferred_pack_name, flags); + return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name, + refs_snapshot, flags); +} + +int write_midx_file_only(const char *object_dir, + struct string_list *packs_to_include, + const char *preferred_pack_name, + const char *refs_snapshot, + unsigned flags) +{ + return write_midx_internal(object_dir, packs_to_include, NULL, + preferred_pack_name, refs_snapshot, flags); } struct clear_midx_data { @@ -1649,7 +1735,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla free(count); if (packs_to_drop.nr) { - result = write_midx_internal(object_dir, &packs_to_drop, NULL, flags); + result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags); m = NULL; } @@ -1840,7 +1926,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, goto cleanup; } - result = write_midx_internal(object_dir, NULL, NULL, flags); + result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags); m = NULL; cleanup: @@ -2,6 +2,7 @@ #define MIDX_H #include "repository.h" +#include "string-list.h" struct object_id; struct pack_entry; @@ -62,7 +63,19 @@ int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pa int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local); -int write_midx_file(const char *object_dir, const char *preferred_pack_name, unsigned flags); +/* + * Variant of write_midx_file which writes a MIDX containing only the packs + * specified in packs_to_include. + */ +int write_midx_file(const char *object_dir, + const char *preferred_pack_name, + const char *refs_snapshot, + unsigned flags); +int write_midx_file_only(const char *object_dir, + struct string_list *packs_to_include, + const char *preferred_pack_name, + const char *refs_snapshot, + unsigned flags); void clear_midx_file(struct repository *r); int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags); int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); diff --git a/pack-bitmap.c b/pack-bitmap.c index 33a3732992..f47a0a7db4 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -1418,7 +1418,7 @@ static int try_partial_reuse(struct packed_git *pack, return 0; } -static uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git) +uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git) { struct multi_pack_index *m = bitmap_git->midx; if (!m) diff --git a/pack-bitmap.h b/pack-bitmap.h index ed46d27077..19a63fa1ab 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -56,6 +56,7 @@ int test_bitmap_hashes(struct repository *r); struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs, struct list_objects_filter_options *filter, int filter_provided_objects); +uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git); int reuse_partial_packfile_from_bitmap(struct bitmap_index *, struct packed_git **packfile, uint32_t *entries, diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index cb0d27049a..9d6fa7a377 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -3,6 +3,7 @@ #include "midx.h" #include "repository.h" #include "object-store.h" +#include "pack-bitmap.h" static int read_midx_file(const char *object_dir, int show_objects) { @@ -72,14 +73,40 @@ static int read_midx_checksum(const char *object_dir) return 0; } +static int read_midx_preferred_pack(const char *object_dir) +{ + struct multi_pack_index *midx = NULL; + struct bitmap_index *bitmap = NULL; + + setup_git_directory(); + + midx = load_multi_pack_index(object_dir, 1); + if (!midx) + return 1; + + bitmap = prepare_bitmap_git(the_repository); + if (!bitmap) + return 1; + if (!bitmap_is_midx(bitmap)) { + free_bitmap_index(bitmap); + return 1; + } + + printf("%s\n", midx->pack_names[midx_preferred_pack(bitmap)]); + free_bitmap_index(bitmap); + return 0; +} + int cmd__read_midx(int argc, const char **argv) { if (!(argc == 2 || argc == 3)) - usage("read-midx [--show-objects|--checksum] <object-dir>"); + usage("read-midx [--show-objects|--checksum|--preferred-pack] <object-dir>"); if (!strcmp(argv[1], "--show-objects")) return read_midx_file(argv[2], 1); else if (!strcmp(argv[1], "--checksum")) return read_midx_checksum(argv[2]); + else if (!strcmp(argv[1], "--preferred-pack")) + return read_midx_preferred_pack(argv[2]); return read_midx_file(argv[1], 0); } diff --git a/t/lib-midx.sh b/t/lib-midx.sh new file mode 100644 index 0000000000..1261994744 --- /dev/null +++ b/t/lib-midx.sh @@ -0,0 +1,8 @@ +# test_midx_consistent <objdir> +test_midx_consistent () { + ls $1/pack/pack-*.idx | xargs -n 1 basename | sort >expect && + test-tool read-midx $1 | grep ^pack-.*\.idx$ | sort >actual && + + test_cmp expect actual && + git multi-pack-index --object-dir=$1 verify +} diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index bd17f308b3..a3c72b68f7 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -168,6 +168,21 @@ test_expect_success 'write midx with two packs' ' compare_results_with_midx "two packs" +test_expect_success 'write midx with --stdin-packs' ' + rm -fr $objdir/pack/multi-pack-index && + + idx="$(find $objdir/pack -name "test-2-*.idx")" && + basename "$idx" >in && + + git multi-pack-index write --stdin-packs <in && + + test-tool read-midx $objdir | grep "\.idx$" >packs && + + test_cmp packs in +' + +compare_results_with_midx "mixed mode (one pack + extra)" + test_expect_success 'write progress off for redirected stderr' ' git multi-pack-index --object-dir=$objdir write 2>err && test_line_count = 0 err diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index ec4aa89f63..e187f90f29 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -283,6 +283,88 @@ test_expect_success 'pack.preferBitmapTips' ' ) ' +test_expect_success 'writing a bitmap with --refs-snapshot' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit one && + test_commit two && + + git rev-parse one >snapshot && + + git repack -ad && + + # First, write a MIDX which see both refs/tags/one and + # refs/tags/two (causing both of those commits to receive + # bitmaps). + git multi-pack-index write --bitmap && + + test_path_is_file $midx && + test_path_is_file $midx-$(midx_checksum $objdir).bitmap && + + test-tool bitmap list-commits | sort >bitmaps && + grep "$(git rev-parse one)" bitmaps && + grep "$(git rev-parse two)" bitmaps && + + rm -fr $midx-$(midx_checksum $objdir).bitmap && + rm -fr $midx-$(midx_checksum $objdir).rev && + rm -fr $midx && + + # Then again, but with a refs snapshot which only sees + # refs/tags/one. + git multi-pack-index write --bitmap --refs-snapshot=snapshot && + + test_path_is_file $midx && + test_path_is_file $midx-$(midx_checksum $objdir).bitmap && + + test-tool bitmap list-commits | sort >bitmaps && + grep "$(git rev-parse one)" bitmaps && + ! grep "$(git rev-parse two)" bitmaps + ) +' + +test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' ' + git init repo && + test_when_finished "rm -fr repo" && + ( + cd repo && + + test_commit_bulk --message="%s" 103 && + + git log --format="%H" >commits.raw && + sort <commits.raw >commits && + + git log --format="create refs/tags/%s %H" HEAD >refs && + git update-ref --stdin <refs && + + git multi-pack-index write --bitmap && + test_path_is_file $midx && + test_path_is_file $midx-$(midx_checksum $objdir).bitmap && + + test-tool bitmap list-commits | sort >bitmaps && + comm -13 bitmaps commits >before && + test_line_count = 1 before && + + ( + grep -vf before commits.raw && + # mark missing commits as preferred + sed "s/^/+/" before + ) >snapshot && + + rm -fr $midx-$(midx_checksum $objdir).bitmap && + rm -fr $midx-$(midx_checksum $objdir).rev && + rm -fr $midx && + + git multi-pack-index write --bitmap --refs-snapshot=snapshot && + test-tool bitmap list-commits | sort >bitmaps && + comm -13 bitmaps commits >after && + + ! test_cmp before after + ) +' + test_expect_success 'hash-cache values are propagated from pack bitmaps' ' rm -fr repo && git init repo && diff --git a/t/t7700-repack.sh b/t/t7700-repack.sh index 98eda3bfeb..0260ad6f0e 100755 --- a/t/t7700-repack.sh +++ b/t/t7700-repack.sh @@ -3,6 +3,8 @@ test_description='git repack works correctly' . ./test-lib.sh +. "${TEST_DIRECTORY}/lib-bitmap.sh" +. "${TEST_DIRECTORY}/lib-midx.sh" commit_and_pack () { test_commit "$@" 1>&2 && @@ -234,4 +236,140 @@ test_expect_success 'auto-bitmaps do not complain if unavailable' ' test_must_be_empty actual ' +objdir=.git/objects +midx=$objdir/pack/multi-pack-index + +test_expect_success 'setup for --write-midx tests' ' + git init midx && + ( + cd midx && + git config core.multiPackIndex true && + + test_commit base + ) +' + +test_expect_success '--write-midx unchanged' ' + ( + cd midx && + GIT_TEST_MULTI_PACK_INDEX=0 git repack && + test_path_is_missing $midx && + test_path_is_missing $midx-*.bitmap && + + GIT_TEST_MULTI_PACK_INDEX=0 git repack --write-midx && + + test_path_is_file $midx && + test_path_is_missing $midx-*.bitmap && + test_midx_consistent $objdir + ) +' + +test_expect_success '--write-midx with a new pack' ' + ( + cd midx && + test_commit loose && + + GIT_TEST_MULTI_PACK_INDEX=0 git repack --write-midx && + + test_path_is_file $midx && + test_path_is_missing $midx-*.bitmap && + test_midx_consistent $objdir + ) +' + +test_expect_success '--write-midx with -b' ' + ( + cd midx && + GIT_TEST_MULTI_PACK_INDEX=0 git repack -mb && + + test_path_is_file $midx && + test_path_is_file $midx-*.bitmap && + test_midx_consistent $objdir + ) +' + +test_expect_success '--write-midx with -d' ' + ( + cd midx && + test_commit repack && + + GIT_TEST_MULTI_PACK_INDEX=0 git repack -Ad --write-midx && + + test_path_is_file $midx && + test_path_is_missing $midx-*.bitmap && + test_midx_consistent $objdir + ) +' + +test_expect_success 'cleans up MIDX when appropriate' ' + ( + cd midx && + + test_commit repack-2 && + GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb --write-midx && + + checksum=$(midx_checksum $objdir) && + test_path_is_file $midx && + test_path_is_file $midx-$checksum.bitmap && + test_path_is_file $midx-$checksum.rev && + + test_commit repack-3 && + GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb --write-midx && + + test_path_is_file $midx && + test_path_is_missing $midx-$checksum.bitmap && + test_path_is_missing $midx-$checksum.rev && + test_path_is_file $midx-$(midx_checksum $objdir).bitmap && + test_path_is_file $midx-$(midx_checksum $objdir).rev && + + test_commit repack-4 && + GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb && + + find $objdir/pack -type f -name "multi-pack-index*" >files && + test_must_be_empty files + ) +' + +test_expect_success '--write-midx with preferred bitmap tips' ' + git init midx-preferred-tips && + test_when_finished "rm -fr midx-preferred-tips" && + ( + cd midx-preferred-tips && + + test_commit_bulk --message="%s" 103 && + + git log --format="%H" >commits.raw && + sort <commits.raw >commits && + + git log --format="create refs/tags/%s/%s %H" HEAD >refs && + git update-ref --stdin <refs && + + git repack --write-midx --write-bitmap-index && + test_path_is_file $midx && + test_path_is_file $midx-$(midx_checksum $objdir).bitmap && + + test-tool bitmap list-commits | sort >bitmaps && + comm -13 bitmaps commits >before && + test_line_count = 1 before && + + rm -fr $midx-$(midx_checksum $objdir).bitmap && + rm -fr $midx-$(midx_checksum $objdir).rev && + rm -fr $midx && + + # instead of constructing the snapshot ourselves (c.f., the test + # "write a bitmap with --refs-snapshot (preferred tips)" in + # t5326), mark the missing commit as preferred by adding it to + # the pack.preferBitmapTips configuration. + git for-each-ref --format="%(refname:rstrip=1)" \ + --points-at="$(cat before)" >missing && + git config pack.preferBitmapTips "$(cat missing)" && + git repack --write-midx --write-bitmap-index && + + test-tool bitmap list-commits | sort >bitmaps && + comm -13 bitmaps commits >after && + + ! test_cmp before after + ) +' + test_done diff --git a/t/t7703-repack-geometric.sh b/t/t7703-repack-geometric.sh index 5ccaa440e0..bdbbcbf1ec 100755 --- a/t/t7703-repack-geometric.sh +++ b/t/t7703-repack-geometric.sh @@ -15,7 +15,7 @@ test_expect_success '--geometric with no packs' ' ( cd geometric && - git repack --geometric 2 >out && + git repack --write-midx --geometric 2 >out && test_i18ngrep "Nothing new to pack" out ) ' @@ -180,4 +180,26 @@ test_expect_success '--geometric ignores kept packs' ' ) ' +test_expect_success '--geometric chooses largest MIDX preferred pack' ' + git init geometric && + test_when_finished "rm -fr geometric" && + ( + cd geometric && + + # These packs already form a geometric progression. + test_commit_bulk --start=1 1 && # 3 objects + test_commit_bulk --start=2 2 && # 6 objects + ls $objdir/pack/pack-*.idx >before && + test_commit_bulk --start=4 4 && # 12 objects + ls $objdir/pack/pack-*.idx >after && + + git repack --geometric 2 -dbm && + + comm -3 before after | xargs -n 1 basename >expect && + test-tool read-midx --preferred-pack $objdir >actual && + + test_cmp expect actual + ) +' + test_done |