summaryrefslogtreecommitdiff
path: root/builtin/repack.c
diff options
context:
space:
mode:
Diffstat (limited to 'builtin/repack.c')
-rw-r--r--builtin/repack.c297
1 files changed, 258 insertions, 39 deletions
diff --git a/builtin/repack.c b/builtin/repack.c
index 5f9bc74adc..0b2d1e5d82 100644
--- a/builtin/repack.c
+++ b/builtin/repack.c
@@ -15,6 +15,8 @@
#include "promisor-remote.h"
#include "shallow.h"
#include "pack.h"
+#include "pack-bitmap.h"
+#include "refs.h"
static int delta_base_offset = 1;
static int pack_kept_objects = -1;
@@ -94,12 +96,14 @@ static void remove_pack_on_signal(int signo)
}
/*
- * Adds all packs hex strings to the fname list, which do not
- * have a corresponding .keep file. These packs are not to
- * be kept if we are going to pack everything into one file.
+ * Adds all packs hex strings to either fname_nonkept_list or
+ * fname_kept_list based on whether each pack has a corresponding
+ * .keep file or not. Packs without a .keep file are not to be kept
+ * if we are going to pack everything into one file.
*/
-static void get_non_kept_pack_filenames(struct string_list *fname_list,
- const struct string_list *extra_keep)
+static void collect_pack_filenames(struct string_list *fname_nonkept_list,
+ struct string_list *fname_kept_list,
+ const struct string_list *extra_keep)
{
DIR *dir;
struct dirent *e;
@@ -112,21 +116,20 @@ static void get_non_kept_pack_filenames(struct string_list *fname_list,
size_t len;
int i;
+ if (!strip_suffix(e->d_name, ".pack", &len))
+ continue;
+
for (i = 0; i < extra_keep->nr; i++)
if (!fspathcmp(e->d_name, extra_keep->items[i].string))
break;
- if (extra_keep->nr > 0 && i < extra_keep->nr)
- continue;
-
- if (!strip_suffix(e->d_name, ".pack", &len))
- continue;
fname = xmemdupz(e->d_name, len);
- if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
- string_list_append_nodup(fname_list, fname);
+ if ((extra_keep->nr > 0 && i < extra_keep->nr) ||
+ (file_exists(mkpath("%s/%s.keep", packdir, fname))))
+ string_list_append_nodup(fname_kept_list, fname);
else
- free(fname);
+ string_list_append_nodup(fname_nonkept_list, fname);
}
closedir(dir);
}
@@ -208,10 +211,10 @@ static struct {
unsigned optional:1;
} exts[] = {
{".pack"},
- {".idx"},
{".rev", 1},
{".bitmap", 1},
{".promisor", 1},
+ {".idx"},
};
static unsigned populate_pack_exts(char *name)
@@ -422,6 +425,25 @@ static void split_pack_geometry(struct pack_geometry *geometry, int factor)
geometry->split = split;
}
+static struct packed_git *get_largest_active_pack(struct pack_geometry *geometry)
+{
+ if (!geometry) {
+ /*
+ * No geometry means either an all-into-one repack (in which
+ * case there is only one pack left and it is the largest) or an
+ * incremental one.
+ *
+ * If repacking incrementally, then we could check the size of
+ * all packs to determine which should be preferred, but leave
+ * this for later.
+ */
+ return NULL;
+ }
+ if (geometry->split == geometry->pack_nr)
+ return NULL;
+ return geometry->pack[geometry->pack_nr - 1];
+}
+
static void clear_pack_geometry(struct pack_geometry *geometry)
{
if (!geometry)
@@ -433,17 +455,162 @@ static void clear_pack_geometry(struct pack_geometry *geometry)
geometry->split = 0;
}
+struct midx_snapshot_ref_data {
+ struct tempfile *f;
+ struct oidset seen;
+ int preferred;
+};
+
+static int midx_snapshot_ref_one(const char *refname,
+ const struct object_id *oid,
+ int flag, void *_data)
+{
+ struct midx_snapshot_ref_data *data = _data;
+ struct object_id peeled;
+
+ if (!peel_iterated_oid(oid, &peeled))
+ oid = &peeled;
+
+ if (oidset_insert(&data->seen, oid))
+ return 0; /* already seen */
+
+ if (oid_object_info(the_repository, oid, NULL) != OBJ_COMMIT)
+ return 0;
+
+ fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "",
+ oid_to_hex(oid));
+
+ return 0;
+}
+
+static void midx_snapshot_refs(struct tempfile *f)
+{
+ struct midx_snapshot_ref_data data;
+ const struct string_list *preferred = bitmap_preferred_tips(the_repository);
+
+ data.f = f;
+ data.preferred = 0;
+ oidset_init(&data.seen, 0);
+
+ if (!fdopen_tempfile(f, "w"))
+ die(_("could not open tempfile %s for writing"),
+ get_tempfile_path(f));
+
+ if (preferred) {
+ struct string_list_item *item;
+
+ data.preferred = 1;
+ for_each_string_list_item(item, preferred)
+ for_each_ref_in(item->string, midx_snapshot_ref_one, &data);
+ data.preferred = 0;
+ }
+
+ for_each_ref(midx_snapshot_ref_one, &data);
+
+ if (close_tempfile_gently(f)) {
+ int save_errno = errno;
+ delete_tempfile(&f);
+ errno = save_errno;
+ die_errno(_("could not close refs snapshot tempfile"));
+ }
+
+ oidset_clear(&data.seen);
+}
+
+static void midx_included_packs(struct string_list *include,
+ struct string_list *existing_nonkept_packs,
+ struct string_list *existing_kept_packs,
+ struct string_list *names,
+ struct pack_geometry *geometry)
+{
+ struct string_list_item *item;
+
+ for_each_string_list_item(item, existing_kept_packs)
+ string_list_insert(include, xstrfmt("%s.idx", item->string));
+ for_each_string_list_item(item, names)
+ string_list_insert(include, xstrfmt("pack-%s.idx", item->string));
+ if (geometry) {
+ struct strbuf buf = STRBUF_INIT;
+ uint32_t i;
+ for (i = geometry->split; i < geometry->pack_nr; i++) {
+ struct packed_git *p = geometry->pack[i];
+
+ strbuf_addstr(&buf, pack_basename(p));
+ strbuf_strip_suffix(&buf, ".pack");
+ strbuf_addstr(&buf, ".idx");
+
+ string_list_insert(include, strbuf_detach(&buf, NULL));
+ }
+ } else {
+ for_each_string_list_item(item, existing_nonkept_packs) {
+ if (item->util)
+ continue;
+ string_list_insert(include, xstrfmt("%s.idx", item->string));
+ }
+ }
+}
+
+static int write_midx_included_packs(struct string_list *include,
+ struct pack_geometry *geometry,
+ const char *refs_snapshot,
+ int show_progress, int write_bitmaps)
+{
+ struct child_process cmd = CHILD_PROCESS_INIT;
+ struct string_list_item *item;
+ struct packed_git *largest = get_largest_active_pack(geometry);
+ FILE *in;
+ int ret;
+
+ if (!include->nr)
+ return 0;
+
+ cmd.in = -1;
+ cmd.git_cmd = 1;
+
+ strvec_push(&cmd.args, "multi-pack-index");
+ strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL);
+
+ if (show_progress)
+ strvec_push(&cmd.args, "--progress");
+ else
+ strvec_push(&cmd.args, "--no-progress");
+
+ if (write_bitmaps)
+ strvec_push(&cmd.args, "--bitmap");
+
+ if (largest)
+ strvec_pushf(&cmd.args, "--preferred-pack=%s",
+ pack_basename(largest));
+
+ if (refs_snapshot)
+ strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot);
+
+ ret = start_command(&cmd);
+ if (ret)
+ return ret;
+
+ in = xfdopen(cmd.in, "w");
+ for_each_string_list_item(item, include)
+ fprintf(in, "%s\n", item->string);
+ fclose(in);
+
+ return finish_command(&cmd);
+}
+
int cmd_repack(int argc, const char **argv, const char *prefix)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct string_list_item *item;
struct string_list names = STRING_LIST_INIT_DUP;
struct string_list rollback = STRING_LIST_INIT_NODUP;
- struct string_list existing_packs = STRING_LIST_INIT_DUP;
+ struct string_list existing_nonkept_packs = STRING_LIST_INIT_DUP;
+ struct string_list existing_kept_packs = STRING_LIST_INIT_DUP;
struct pack_geometry *geometry = NULL;
struct strbuf line = STRBUF_INIT;
+ struct tempfile *refs_snapshot = NULL;
int i, ext, ret;
FILE *out;
+ int show_progress = isatty(2);
/* variables to be filled by option parsing */
int pack_everything = 0;
@@ -454,6 +621,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
int no_update_server_info = 0;
struct pack_objects_args po_args = {NULL};
int geometric_factor = 0;
+ int write_midx = 0;
struct option builtin_repack_options[] = {
OPT_BIT('a', NULL, &pack_everything,
@@ -496,6 +664,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
N_("do not repack this pack")),
OPT_INTEGER('g', "geometric", &geometric_factor,
N_("find a geometric progression with factor <N>")),
+ OPT_BOOL('m', "write-midx", &write_midx,
+ N_("write a multi-pack index of the resulting packs")),
OPT_END()
};
@@ -512,16 +682,32 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
die(_("--keep-unreachable and -A are incompatible"));
if (write_bitmaps < 0) {
- if (!(pack_everything & ALL_INTO_ONE) ||
- !is_bare_repository())
+ if (!write_midx &&
+ (!(pack_everything & ALL_INTO_ONE) || !is_bare_repository()))
write_bitmaps = 0;
+ } else if (write_bitmaps &&
+ git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0) &&
+ git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP, 0)) {
+ write_bitmaps = 0;
}
if (pack_kept_objects < 0)
pack_kept_objects = write_bitmaps > 0;
- if (write_bitmaps && !(pack_everything & ALL_INTO_ONE))
+ if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx)
die(_(incremental_bitmap_conflict_error));
+ if (write_midx && write_bitmaps) {
+ struct strbuf path = STRBUF_INIT;
+
+ strbuf_addf(&path, "%s/%s_XXXXXX", get_object_directory(),
+ "bitmap-ref-tips");
+
+ refs_snapshot = xmks_tempfile(path.buf);
+ midx_snapshot_refs(refs_snapshot);
+
+ strbuf_release(&path);
+ }
+
if (geometric_factor) {
if (pack_everything)
die(_("--geometric is incompatible with -A, -a"));
@@ -561,19 +747,22 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
}
if (has_promisor_remote())
strvec_push(&cmd.args, "--exclude-promisor-objects");
- if (write_bitmaps > 0)
- strvec_push(&cmd.args, "--write-bitmap-index");
- else if (write_bitmaps < 0)
- strvec_push(&cmd.args, "--write-bitmap-index-quiet");
+ if (!write_midx) {
+ if (write_bitmaps > 0)
+ strvec_push(&cmd.args, "--write-bitmap-index");
+ else if (write_bitmaps < 0)
+ strvec_push(&cmd.args, "--write-bitmap-index-quiet");
+ }
if (use_delta_islands)
strvec_push(&cmd.args, "--delta-islands");
- if (pack_everything & ALL_INTO_ONE) {
- get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
+ collect_pack_filenames(&existing_nonkept_packs, &existing_kept_packs,
+ &keep_pack_list);
+ if (pack_everything & ALL_INTO_ONE) {
repack_promisor_objects(&po_args, &names);
- if (existing_packs.nr && delete_redundant) {
+ if (existing_nonkept_packs.nr && delete_redundant) {
for_each_string_list_item(item, &names) {
strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack",
packtmp_name, item->string);
@@ -582,15 +771,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
strvec_pushf(&cmd.args,
"--unpack-unreachable=%s",
unpack_unreachable);
- strvec_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
} else if (pack_everything & LOOSEN_UNREACHABLE) {
strvec_push(&cmd.args,
"--unpack-unreachable");
} else if (keep_unreachable) {
strvec_push(&cmd.args, "--keep-unreachable");
strvec_push(&cmd.args, "--pack-loose-unreachable");
- } else {
- strvec_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
}
}
} else if (geometry) {
@@ -676,20 +862,48 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
}
/* End of pack replacement. */
- reprepare_packed_git(the_repository);
-
- if (delete_redundant) {
+ if (delete_redundant && pack_everything & ALL_INTO_ONE) {
const int hexsz = the_hash_algo->hexsz;
- int opts = 0;
string_list_sort(&names);
- for_each_string_list_item(item, &existing_packs) {
+ for_each_string_list_item(item, &existing_nonkept_packs) {
char *sha1;
size_t len = strlen(item->string);
if (len < hexsz)
continue;
sha1 = item->string + len - hexsz;
- if (!string_list_has_string(&names, sha1))
- remove_redundant_pack(packdir, item->string);
+ /*
+ * Mark this pack for deletion, which ensures that this
+ * pack won't be included in a MIDX (if `--write-midx`
+ * was given) and that we will actually delete this pack
+ * (if `-d` was given).
+ */
+ item->util = (void*)(intptr_t)!string_list_has_string(&names, sha1);
+ }
+ }
+
+ if (write_midx) {
+ struct string_list include = STRING_LIST_INIT_NODUP;
+ midx_included_packs(&include, &existing_nonkept_packs,
+ &existing_kept_packs, &names, geometry);
+
+ ret = write_midx_included_packs(&include, geometry,
+ refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL,
+ show_progress, write_bitmaps > 0);
+
+ string_list_clear(&include, 0);
+
+ if (ret)
+ return ret;
+ }
+
+ reprepare_packed_git(the_repository);
+
+ if (delete_redundant) {
+ int opts = 0;
+ for_each_string_list_item(item, &existing_nonkept_packs) {
+ if (!item->util)
+ continue;
+ remove_redundant_pack(packdir, item->string);
}
if (geometry) {
@@ -710,7 +924,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
}
strbuf_release(&buf);
}
- if (!po_args.quiet && isatty(2))
+ if (!po_args.quiet && show_progress)
opts |= PRUNE_PACKED_VERBOSE;
prune_packed_objects(opts);
@@ -725,12 +939,17 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
update_server_info(0);
remove_temporary_files();
- if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
- write_midx_file(get_object_directory(), NULL, 0);
+ if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) {
+ unsigned flags = 0;
+ if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP, 0))
+ flags |= MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX;
+ write_midx_file(get_object_directory(), NULL, NULL, flags);
+ }
string_list_clear(&names, 0);
string_list_clear(&rollback, 0);
- string_list_clear(&existing_packs, 0);
+ string_list_clear(&existing_nonkept_packs, 0);
+ string_list_clear(&existing_kept_packs, 0);
clear_pack_geometry(geometry);
strbuf_release(&line);