summaryrefslogtreecommitdiff
path: root/builtin/pack-objects.c
diff options
context:
space:
mode:
Diffstat (limited to 'builtin/pack-objects.c')
-rw-r--r--builtin/pack-objects.c859
1 files changed, 607 insertions, 252 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 807f034365..5617c01b5a 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -24,14 +24,18 @@
#include "streaming.h"
#include "thread-utils.h"
#include "pack-bitmap.h"
+#include "delta-islands.h"
#include "reachable.h"
-#include "sha1-array.h"
-#include "argv-array.h"
+#include "oid-array.h"
+#include "strvec.h"
#include "list.h"
#include "packfile.h"
#include "object-store.h"
#include "dir.h"
#include "midx.h"
+#include "trace2.h"
+#include "shallow.h"
+#include "promisor-remote.h"
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
#define SIZE(obj) oe_size(&to_pack, obj)
@@ -41,6 +45,7 @@
#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
+#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
@@ -60,6 +65,8 @@ static struct packing_data to_pack;
static struct pack_idx_entry **written_list;
static uint32_t nr_result, nr_written, nr_seen;
+static struct bitmap_index *bitmap_git;
+static uint32_t write_layer;
static int non_empty;
static int reuse_delta = 1, reuse_object = 1;
@@ -80,20 +87,29 @@ static unsigned long pack_size_limit;
static int depth = 50;
static int delta_search_threads;
static int pack_to_stdout;
+static int sparse;
+static int thin;
static int num_preferred_base;
static struct progress *progress_state;
static struct packed_git *reuse_packfile;
static uint32_t reuse_packfile_objects;
-static off_t reuse_packfile_offset;
+static struct bitmap *reuse_packfile_bitmap;
static int use_bitmap_index_default = 1;
static int use_bitmap_index = -1;
-static int write_bitmap_index;
-static uint16_t write_bitmap_options;
+static int allow_pack_reuse = 1;
+static enum {
+ WRITE_BITMAP_FALSE = 0,
+ WRITE_BITMAP_QUIET,
+ WRITE_BITMAP_TRUE,
+} write_bitmap_index;
+static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
static int exclude_promisor_objects;
+static int use_delta_islands;
+
static unsigned long delta_cache_size = 0;
static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
static unsigned long cache_max_small_delta_size = 1000;
@@ -102,6 +118,8 @@ static unsigned long window_memory_limit = 0;
static struct list_objects_filter_options filter_options;
+static struct string_list uri_protocols = STRING_LIST_INIT_NODUP;
+
enum missing_action {
MA_ERROR = 0, /* fail if any missing objects are encountered */
MA_ALLOW_ANY, /* silently allow ALL missing objects */
@@ -110,6 +128,15 @@ enum missing_action {
static enum missing_action arg_missing_action;
static show_object_fn fn_show_object;
+struct configured_exclusion {
+ struct oidmap_entry e;
+ char *pack_hash_hex;
+ char *uri;
+};
+static struct oidmap configured_exclusions;
+
+static struct oidset excluded_by_config;
+
/*
* stats
*/
@@ -150,7 +177,7 @@ static void *get_delta(struct object_entry *entry)
delta_buf = diff_delta(base_buf, base_size,
buf, size, &delta_size, 0);
/*
- * We succesfully computed this delta once but dropped it for
+ * We successfully computed this delta once but dropped it for
* memory reasons. Something is very wrong if this time we
* recompute and create a different delta.
*/
@@ -290,7 +317,8 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
if (!usable_delta) {
if (oe_type(entry) == OBJ_BLOB &&
oe_size_greater_than(&to_pack, entry, big_file_threshold) &&
- (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL)
+ (st = open_istream(the_repository, &entry->idx.oid, &type,
+ &size, NULL)) != NULL)
buf = NULL;
else {
buf = read_object_file(&entry->idx.oid, &type, &size);
@@ -597,12 +625,12 @@ static int mark_tagged(const char *path, const struct object_id *oid, int flag,
void *cb_data)
{
struct object_id peeled;
- struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL);
+ struct object_entry *entry = packlist_find(&to_pack, oid);
if (entry)
entry->tagged = 1;
if (!peel_ref(path, &peeled)) {
- entry = packlist_find(&to_pack, peeled.hash, NULL);
+ entry = packlist_find(&to_pack, &peeled);
if (entry)
entry->tagged = 1;
}
@@ -613,7 +641,7 @@ static inline void add_to_write_order(struct object_entry **wo,
unsigned int *endp,
struct object_entry *e)
{
- if (e->filled)
+ if (e->filled || oe_layer(&to_pack, e) != write_layer)
return;
wo[(*endp)++] = e;
e->filled = 1;
@@ -673,48 +701,15 @@ static void add_family_to_write_order(struct object_entry **wo,
add_descendants_to_write_order(wo, endp, root);
}
-static struct object_entry **compute_write_order(void)
+static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end)
{
- unsigned int i, wo_end, last_untagged;
-
- struct object_entry **wo;
+ unsigned int i, last_untagged;
struct object_entry *objects = to_pack.objects;
for (i = 0; i < to_pack.nr_objects; i++) {
- objects[i].tagged = 0;
- objects[i].filled = 0;
- SET_DELTA_CHILD(&objects[i], NULL);
- SET_DELTA_SIBLING(&objects[i], NULL);
- }
-
- /*
- * Fully connect delta_child/delta_sibling network.
- * Make sure delta_sibling is sorted in the original
- * recency order.
- */
- for (i = to_pack.nr_objects; i > 0;) {
- struct object_entry *e = &objects[--i];
- if (!DELTA(e))
- continue;
- /* Mark me as the first child */
- e->delta_sibling_idx = DELTA(e)->delta_child_idx;
- SET_DELTA_CHILD(DELTA(e), e);
- }
-
- /*
- * Mark objects that are at the tip of tags.
- */
- for_each_tag_ref(mark_tagged, NULL);
-
- /*
- * Give the objects in the original recency order until
- * we see a tagged tip.
- */
- ALLOC_ARRAY(wo, to_pack.nr_objects);
- for (i = wo_end = 0; i < to_pack.nr_objects; i++) {
if (objects[i].tagged)
break;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
last_untagged = i;
@@ -723,7 +718,7 @@ static struct object_entry **compute_write_order(void)
*/
for (; i < to_pack.nr_objects; i++) {
if (objects[i].tagged)
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
@@ -733,7 +728,7 @@ static struct object_entry **compute_write_order(void)
if (oe_type(&objects[i]) != OBJ_COMMIT &&
oe_type(&objects[i]) != OBJ_TAG)
continue;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
@@ -742,16 +737,60 @@ static struct object_entry **compute_write_order(void)
for (i = last_untagged; i < to_pack.nr_objects; i++) {
if (oe_type(&objects[i]) != OBJ_TREE)
continue;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
* Finally all the rest in really tight order
*/
for (i = last_untagged; i < to_pack.nr_objects; i++) {
- if (!objects[i].filled)
- add_family_to_write_order(wo, &wo_end, &objects[i]);
+ if (!objects[i].filled && oe_layer(&to_pack, &objects[i]) == write_layer)
+ add_family_to_write_order(wo, wo_end, &objects[i]);
}
+}
+
+static struct object_entry **compute_write_order(void)
+{
+ uint32_t max_layers = 1;
+ unsigned int i, wo_end;
+
+ struct object_entry **wo;
+ struct object_entry *objects = to_pack.objects;
+
+ for (i = 0; i < to_pack.nr_objects; i++) {
+ objects[i].tagged = 0;
+ objects[i].filled = 0;
+ SET_DELTA_CHILD(&objects[i], NULL);
+ SET_DELTA_SIBLING(&objects[i], NULL);
+ }
+
+ /*
+ * Fully connect delta_child/delta_sibling network.
+ * Make sure delta_sibling is sorted in the original
+ * recency order.
+ */
+ for (i = to_pack.nr_objects; i > 0;) {
+ struct object_entry *e = &objects[--i];
+ if (!DELTA(e))
+ continue;
+ /* Mark me as the first child */
+ e->delta_sibling_idx = DELTA(e)->delta_child_idx;
+ SET_DELTA_CHILD(DELTA(e), e);
+ }
+
+ /*
+ * Mark objects that are at the tip of tags.
+ */
+ for_each_tag_ref(mark_tagged, NULL);
+
+ if (use_delta_islands)
+ max_layers = compute_pack_layers(&to_pack);
+
+ ALLOC_ARRAY(wo, to_pack.nr_objects);
+ wo_end = 0;
+
+ for (; write_layer < max_layers; ++write_layer)
+ compute_layer_order(wo, &wo_end);
if (wo_end != to_pack.nr_objects)
die(_("ordered %u objects, expected %"PRIu32),
@@ -760,57 +799,205 @@ static struct object_entry **compute_write_order(void)
return wo;
}
-static off_t write_reused_pack(struct hashfile *f)
+
+/*
+ * A reused set of objects. All objects in a chunk have the same
+ * relative position in the original packfile and the generated
+ * packfile.
+ */
+
+static struct reused_chunk {
+ /* The offset of the first object of this chunk in the original
+ * packfile. */
+ off_t original;
+ /* The offset of the first object of this chunk in the generated
+ * packfile minus "original". */
+ off_t difference;
+} *reused_chunks;
+static int reused_chunks_nr;
+static int reused_chunks_alloc;
+
+static void record_reused_object(off_t where, off_t offset)
+{
+ if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset)
+ return;
+
+ ALLOC_GROW(reused_chunks, reused_chunks_nr + 1,
+ reused_chunks_alloc);
+ reused_chunks[reused_chunks_nr].original = where;
+ reused_chunks[reused_chunks_nr].difference = offset;
+ reused_chunks_nr++;
+}
+
+/*
+ * Binary search to find the chunk that "where" is in. Note
+ * that we're not looking for an exact match, just the first
+ * chunk that contains it (which implicitly ends at the start
+ * of the next chunk.
+ */
+static off_t find_reused_offset(off_t where)
+{
+ int lo = 0, hi = reused_chunks_nr;
+ while (lo < hi) {
+ int mi = lo + ((hi - lo) / 2);
+ if (where == reused_chunks[mi].original)
+ return reused_chunks[mi].difference;
+ if (where < reused_chunks[mi].original)
+ hi = mi;
+ else
+ lo = mi + 1;
+ }
+
+ /*
+ * The first chunk starts at zero, so we can't have gone below
+ * there.
+ */
+ assert(lo);
+ return reused_chunks[lo-1].difference;
+}
+
+static void write_reused_pack_one(size_t pos, struct hashfile *out,
+ struct pack_window **w_curs)
{
- unsigned char buffer[8192];
- off_t to_write, total;
- int fd;
+ off_t offset, next, cur;
+ enum object_type type;
+ unsigned long size;
- if (!is_pack_valid(reuse_packfile))
- die(_("packfile is invalid: %s"), reuse_packfile->pack_name);
+ offset = reuse_packfile->revindex[pos].offset;
+ next = reuse_packfile->revindex[pos + 1].offset;
- fd = git_open(reuse_packfile->pack_name);
- if (fd < 0)
- die_errno(_("unable to open packfile for reuse: %s"),
- reuse_packfile->pack_name);
+ record_reused_object(offset, offset - hashfile_total(out));
- if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1)
- die_errno(_("unable to seek in reused packfile"));
+ cur = offset;
+ type = unpack_object_header(reuse_packfile, w_curs, &cur, &size);
+ assert(type >= 0);
- if (reuse_packfile_offset < 0)
- reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz;
+ if (type == OBJ_OFS_DELTA) {
+ off_t base_offset;
+ off_t fixup;
- total = to_write = reuse_packfile_offset - sizeof(struct pack_header);
+ unsigned char header[MAX_PACK_OBJECT_HEADER];
+ unsigned len;
- while (to_write) {
- int read_pack = xread(fd, buffer, sizeof(buffer));
+ base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset);
+ assert(base_offset != 0);
- if (read_pack <= 0)
- die_errno(_("unable to read from reused packfile"));
+ /* Convert to REF_DELTA if we must... */
+ if (!allow_ofs_delta) {
+ int base_pos = find_revindex_position(reuse_packfile, base_offset);
+ struct object_id base_oid;
- if (read_pack > to_write)
- read_pack = to_write;
+ nth_packed_object_id(&base_oid, reuse_packfile,
+ reuse_packfile->revindex[base_pos].nr);
- hashwrite(f, buffer, read_pack);
- to_write -= read_pack;
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_REF_DELTA, size);
+ hashwrite(out, header, len);
+ hashwrite(out, base_oid.hash, the_hash_algo->rawsz);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* Otherwise see if we need to rewrite the offset... */
+ fixup = find_reused_offset(offset) -
+ find_reused_offset(base_offset);
+ if (fixup) {
+ unsigned char ofs_header[10];
+ unsigned i, ofs_len;
+ off_t ofs = offset - base_offset - fixup;
+
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_OFS_DELTA, size);
+
+ i = sizeof(ofs_header) - 1;
+ ofs_header[i] = ofs & 127;
+ while (ofs >>= 7)
+ ofs_header[--i] = 128 | (--ofs & 127);
+
+ ofs_len = sizeof(ofs_header) - i;
+
+ hashwrite(out, header, len);
+ hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* ...otherwise we have no fixup, and can write it verbatim */
+ }
+
+ copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset);
+}
+
+static size_t write_reused_pack_verbatim(struct hashfile *out,
+ struct pack_window **w_curs)
+{
+ size_t pos = 0;
+
+ while (pos < reuse_packfile_bitmap->word_alloc &&
+ reuse_packfile_bitmap->words[pos] == (eword_t)~0)
+ pos++;
+
+ if (pos) {
+ off_t to_write;
+
+ written = (pos * BITS_IN_EWORD);
+ to_write = reuse_packfile->revindex[written].offset
+ - sizeof(struct pack_header);
+
+ /* We're recording one chunk, not one object. */
+ record_reused_object(sizeof(struct pack_header), 0);
+ hashflush(out);
+ copy_pack_data(out, reuse_packfile, w_curs,
+ sizeof(struct pack_header), to_write);
- /*
- * We don't know the actual number of objects written,
- * only how many bytes written, how many bytes total, and
- * how many objects total. So we can fake it by pretending all
- * objects we are writing are the same size. This gives us a
- * smooth progress meter, and at the end it matches the true
- * answer.
- */
- written = reuse_packfile_objects *
- (((double)(total - to_write)) / total);
display_progress(progress_state, written);
}
+ return pos;
+}
+
+static void write_reused_pack(struct hashfile *f)
+{
+ size_t i = 0;
+ uint32_t offset;
+ struct pack_window *w_curs = NULL;
+
+ if (allow_ofs_delta)
+ i = write_reused_pack_verbatim(f, &w_curs);
- close(fd);
- written = reuse_packfile_objects;
- display_progress(progress_state, written);
- return reuse_packfile_offset - sizeof(struct pack_header);
+ for (; i < reuse_packfile_bitmap->word_alloc; ++i) {
+ eword_t word = reuse_packfile_bitmap->words[i];
+ size_t pos = (i * BITS_IN_EWORD);
+
+ for (offset = 0; offset < BITS_IN_EWORD; ++offset) {
+ if ((word >> offset) == 0)
+ break;
+
+ offset += ewah_bit_ctz64(word >> offset);
+ write_reused_pack_one(pos + offset, f, &w_curs);
+ display_progress(progress_state, ++written);
+ }
+ }
+
+ unuse_pack(&w_curs);
+}
+
+static void write_excluded_by_configs(void)
+{
+ struct oidset_iter iter;
+ const struct object_id *oid;
+
+ oidset_iter_init(&excluded_by_config, &iter);
+ while ((oid = oidset_iter_next(&iter))) {
+ struct configured_exclusion *ex =
+ oidmap_get(&configured_exclusions, oid);
+
+ if (!ex)
+ BUG("configured exclusion wasn't configured");
+ write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex));
+ write_in_full(1, " ", 1);
+ write_in_full(1, ex->uri, strlen(ex->uri));
+ write_in_full(1, "\n", 1);
+ }
}
static const char no_split_warning[] = N_(
@@ -843,11 +1030,9 @@ static void write_pack_file(void)
offset = write_pack_header(f, nr_remaining);
if (reuse_packfile) {
- off_t packfile_size;
assert(pack_to_stdout);
-
- packfile_size = write_reused_pack(f);
- offset += packfile_size;
+ write_reused_pack(f);
+ offset = hashfile_total(f);
}
nr_written = 0;
@@ -872,7 +1057,8 @@ static void write_pack_file(void)
nr_written, oid.hash, offset);
close(fd);
if (write_bitmap_index) {
- warning(_(no_split_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_split_warning));
write_bitmap_index = 0;
}
}
@@ -944,6 +1130,8 @@ static void write_pack_file(void)
if (written != nr_result)
die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
written, nr_result);
+ trace2_data_intmax("pack-objects", the_repository,
+ "write_pack_file/wrote", nr_result);
}
static int no_try_delta(const char *path)
@@ -952,8 +1140,7 @@ static int no_try_delta(const char *path)
if (!check)
check = attr_check_initl("delta", NULL);
- if (git_check_attr(&the_index, path, check))
- return 0;
+ git_check_attr(the_repository->index, path, check);
if (ATTR_FALSE(check->items[0].value))
return 1;
return 0;
@@ -970,12 +1157,15 @@ static int no_try_delta(const char *path)
* few lines later when we want to add the new entry.
*/
static int have_duplicate_entry(const struct object_id *oid,
- int exclude,
- uint32_t *index_pos)
+ int exclude)
{
struct object_entry *entry;
- entry = packlist_find(&to_pack, oid->hash, index_pos);
+ if (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid))
+ return 1;
+
+ entry = packlist_find(&to_pack, oid);
if (!entry)
return 0;
@@ -1059,7 +1249,7 @@ static int want_object_in_pack(const struct object_id *oid,
for (m = get_multi_pack_index(the_repository); m; m = m->next) {
struct pack_entry e;
- if (fill_midx_entry(oid, &e, m)) {
+ if (fill_midx_entry(the_repository, oid, &e, m)) {
struct packed_git *p = e.p;
off_t offset;
@@ -1107,6 +1297,25 @@ static int want_object_in_pack(const struct object_id *oid,
}
}
+ if (uri_protocols.nr) {
+ struct configured_exclusion *ex =
+ oidmap_get(&configured_exclusions, oid);
+ int i;
+ const char *p;
+
+ if (ex) {
+ for (i = 0; i < uri_protocols.nr; i++) {
+ if (skip_prefix(ex->uri,
+ uri_protocols.items[i].string,
+ &p) &&
+ *p == ':') {
+ oidset_insert(&excluded_by_config, oid);
+ return 0;
+ }
+ }
+ }
+ }
+
return 1;
}
@@ -1115,13 +1324,12 @@ static void create_object_entry(const struct object_id *oid,
uint32_t hash,
int exclude,
int no_try_delta,
- uint32_t index_pos,
struct packed_git *found_pack,
off_t found_offset)
{
struct object_entry *entry;
- entry = packlist_alloc(&to_pack, oid->hash, index_pos);
+ entry = packlist_alloc(&to_pack, oid);
entry->hash = hash;
oe_set_type(entry, type);
if (exclude)
@@ -1145,17 +1353,17 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
{
struct packed_git *found_pack = NULL;
off_t found_offset = 0;
- uint32_t index_pos;
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, exclude, &index_pos))
+ if (have_duplicate_entry(oid, exclude))
return 0;
if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {
/* The pack is missing an object, so it will not have closure */
if (write_bitmap_index) {
- warning(_(no_closure_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_closure_warning));
write_bitmap_index = 0;
}
return 0;
@@ -1163,7 +1371,7 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
create_object_entry(oid, type, pack_name_hash(name),
exclude, name && no_try_delta(name),
- index_pos, found_pack, found_offset);
+ found_pack, found_offset);
return 1;
}
@@ -1172,17 +1380,15 @@ static int add_object_entry_from_bitmap(const struct object_id *oid,
int flags, uint32_t name_hash,
struct packed_git *pack, off_t offset)
{
- uint32_t index_pos;
-
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, 0, &index_pos))
+ if (have_duplicate_entry(oid, 0))
return 0;
if (!want_object_in_pack(oid, 0, &pack, &offset))
return 0;
- create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);
+ create_object_entry(oid, type, name_hash, 0, 0, pack, offset);
return 1;
}
@@ -1230,7 +1436,7 @@ static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)
*/
for (neigh = 0; neigh < 8; neigh++) {
ent = pbase_tree_cache[my_ix];
- if (ent && !oidcmp(&ent->oid, oid)) {
+ if (ent && oideq(&ent->oid, oid)) {
ent->ref++;
return ent;
}
@@ -1317,7 +1523,7 @@ static void add_pbase_object(struct tree_desc *tree,
if (cmp < 0)
return;
if (name[cmplen] != '/') {
- add_object_entry(entry.oid,
+ add_object_entry(&entry.oid,
object_type(entry.mode),
fullname, 1);
return;
@@ -1328,7 +1534,7 @@ static void add_pbase_object(struct tree_desc *tree,
const char *down = name+cmplen+1;
int downlen = name_cmp_len(down);
- tree = pbase_tree_get(entry.oid);
+ tree = pbase_tree_get(&entry.oid);
if (!tree)
return;
init_tree_desc(&sub, tree->tree_data, tree->tree_size);
@@ -1407,12 +1613,13 @@ static void add_preferred_base(struct object_id *oid)
if (window <= num_preferred_base++)
return;
- data = read_object_with_reference(oid, tree_type, &size, &tree_oid);
+ data = read_object_with_reference(the_repository, oid,
+ tree_type, &size, &tree_oid);
if (!data)
return;
for (it = pbase_tree; it; it = it->next) {
- if (!oidcmp(&it->pcache.oid, &tree_oid)) {
+ if (oideq(&it->pcache.oid, &tree_oid)) {
free(data);
return;
}
@@ -1452,14 +1659,82 @@ static void cleanup_preferred_base(void)
done_pbase_paths_num = done_pbase_paths_alloc = 0;
}
-static void check_object(struct object_entry *entry)
+/*
+ * Return 1 iff the object specified by "delta" can be sent
+ * literally as a delta against the base in "base_sha1". If
+ * so, then *base_out will point to the entry in our packing
+ * list, or NULL if we must use the external-base list.
+ *
+ * Depth value does not matter - find_deltas() will
+ * never consider reused delta as the base object to
+ * deltify other objects against, in order to avoid
+ * circular deltas.
+ */
+static int can_reuse_delta(const struct object_id *base_oid,
+ struct object_entry *delta,
+ struct object_entry **base_out)
+{
+ struct object_entry *base;
+
+ /*
+ * First see if we're already sending the base (or it's explicitly in
+ * our "excluded" list).
+ */
+ base = packlist_find(&to_pack, base_oid);
+ if (base) {
+ if (!in_same_island(&delta->idx.oid, &base->idx.oid))
+ return 0;
+ *base_out = base;
+ return 1;
+ }
+
+ /*
+ * Otherwise, reachability bitmaps may tell us if the receiver has it,
+ * even if it was buried too deep in history to make it into the
+ * packing list.
+ */
+ if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
+ if (use_delta_islands) {
+ if (!in_same_island(&delta->idx.oid, base_oid))
+ return 0;
+ }
+ *base_out = NULL;
+ return 1;
+ }
+
+ return 0;
+}
+
+static void prefetch_to_pack(uint32_t object_index_start) {
+ struct oid_array to_fetch = OID_ARRAY_INIT;
+ uint32_t i;
+
+ for (i = object_index_start; i < to_pack.nr_objects; i++) {
+ struct object_entry *entry = to_pack.objects + i;
+
+ if (!oid_object_info_extended(the_repository,
+ &entry->idx.oid,
+ NULL,
+ OBJECT_INFO_FOR_PREFETCH))
+ continue;
+ oid_array_append(&to_fetch, &entry->idx.oid);
+ }
+ promisor_remote_get_direct(the_repository,
+ to_fetch.oid, to_fetch.nr);
+ oid_array_clear(&to_fetch);
+}
+
+static void check_object(struct object_entry *entry, uint32_t object_index)
{
unsigned long canonical_size;
+ enum object_type type;
+ struct object_info oi = {.typep = &type, .sizep = &canonical_size};
if (IN_PACK(entry)) {
struct packed_git *p = IN_PACK(entry);
struct pack_window *w_curs = NULL;
- const unsigned char *base_ref = NULL;
+ int have_base = 0;
+ struct object_id base_ref;
struct object_entry *base_entry;
unsigned long used, used_0;
unsigned long avail;
@@ -1500,9 +1775,13 @@ static void check_object(struct object_entry *entry)
unuse_pack(&w_curs);
return;
case OBJ_REF_DELTA:
- if (reuse_delta && !entry->preferred_base)
- base_ref = use_pack(p, &w_curs,
- entry->in_pack_offset + used, NULL);
+ if (reuse_delta && !entry->preferred_base) {
+ oidread(&base_ref,
+ use_pack(p, &w_curs,
+ entry->in_pack_offset + used,
+ NULL));
+ have_base = 1;
+ }
entry->in_pack_header_size = used + the_hash_algo->rawsz;
break;
case OBJ_OFS_DELTA:
@@ -1532,29 +1811,27 @@ static void check_object(struct object_entry *entry)
revidx = find_pack_revindex(p, ofs);
if (!revidx)
goto give_up;
- base_ref = nth_packed_object_sha1(p, revidx->nr);
+ if (!nth_packed_object_id(&base_ref, p, revidx->nr))
+ have_base = 1;
}
entry->in_pack_header_size = used + used_0;
break;
}
- if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
- /*
- * If base_ref was set above that means we wish to
- * reuse delta data, and we even found that base
- * in the list of objects we want to pack. Goodie!
- *
- * Depth value does not matter - find_deltas() will
- * never consider reused delta as the base object to
- * deltify other objects against, in order to avoid
- * circular deltas.
- */
+ if (have_base &&
+ can_reuse_delta(&base_ref, entry, &base_entry)) {
oe_set_type(entry, entry->in_pack_type);
SET_SIZE(entry, in_pack_size); /* delta size */
- SET_DELTA(entry, base_entry);
SET_DELTA_SIZE(entry, in_pack_size);
- entry->delta_sibling_idx = base_entry->delta_child_idx;
- SET_DELTA_CHILD(base_entry, entry);
+
+ if (base_entry) {
+ SET_DELTA(entry, base_entry);
+ entry->delta_sibling_idx = base_entry->delta_child_idx;
+ SET_DELTA_CHILD(base_entry, entry);
+ } else {
+ SET_DELTA_EXT(entry, &base_ref);
+ }
+
unuse_pack(&w_curs);
return;
}
@@ -1578,15 +1855,25 @@ static void check_object(struct object_entry *entry)
/*
* No choice but to fall back to the recursive delta walk
- * with sha1_object_info() to find about the object type
+ * with oid_object_info() to find about the object type
* at this point...
*/
give_up:
unuse_pack(&w_curs);
}
- oe_set_type(entry,
- oid_object_info(the_repository, &entry->idx.oid, &canonical_size));
+ if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
+ OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) {
+ if (has_promisor_remote()) {
+ prefetch_to_pack(object_index);
+ if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
+ OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0)
+ type = -1;
+ } else {
+ type = -1;
+ }
+ }
+ oe_set_type(entry, type);
if (entry->type_valid) {
SET_SIZE(entry, canonical_size);
} else {
@@ -1654,7 +1941,7 @@ static void drop_reused_delta(struct object_entry *entry)
if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
/*
* We failed to get the info from this pack for some reason;
- * fall back to sha1_object_info, which may find another copy.
+ * fall back to oid_object_info, which may find another copy.
* And if that fails, the error will be recorded in oe_type(entry)
* and dealt with in prepare_pack().
*/
@@ -1806,7 +2093,7 @@ static void get_object_details(void)
for (i = 0; i < to_pack.nr_objects; i++) {
struct object_entry *entry = sorted_by_offset[i];
- check_object(entry);
+ check_object(entry, i);
if (entry->type_valid &&
oe_size_greater_than(&to_pack, entry, big_file_threshold))
entry->no_try_delta = 1;
@@ -1837,10 +2124,10 @@ static int type_size_sort(const void *_a, const void *_b)
{
const struct object_entry *a = *(struct object_entry **)_a;
const struct object_entry *b = *(struct object_entry **)_b;
- enum object_type a_type = oe_type(a);
- enum object_type b_type = oe_type(b);
- unsigned long a_size = SIZE(a);
- unsigned long b_size = SIZE(b);
+ const enum object_type a_type = oe_type(a);
+ const enum object_type b_type = oe_type(b);
+ const unsigned long a_size = SIZE(a);
+ const unsigned long b_size = SIZE(b);
if (a_type > b_type)
return -1;
@@ -1854,6 +2141,11 @@ static int type_size_sort(const void *_a, const void *_b)
return -1;
if (a->preferred_base < b->preferred_base)
return 1;
+ if (use_delta_islands) {
+ const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
+ if (island_cmp)
+ return island_cmp;
+ }
if (a_size > b_size)
return -1;
if (a_size < b_size)
@@ -1884,13 +2176,6 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
return 0;
}
-#ifndef NO_PTHREADS
-
-/* Protect access to object database */
-static pthread_mutex_t read_mutex;
-#define read_lock() pthread_mutex_lock(&read_mutex)
-#define read_unlock() pthread_mutex_unlock(&read_mutex)
-
/* Protect delta_cache_size */
static pthread_mutex_t cache_mutex;
#define cache_lock() pthread_mutex_lock(&cache_mutex)
@@ -1910,16 +2195,6 @@ static pthread_mutex_t progress_mutex;
* ahead in the list because they can be stolen and would need
* progress_mutex for protection.
*/
-#else
-
-#define read_lock() (void)0
-#define read_unlock() (void)0
-#define cache_lock() (void)0
-#define cache_unlock() (void)0
-#define progress_lock() (void)0
-#define progress_unlock() (void)0
-
-#endif
/*
* Return the size of the object without doing any delta
@@ -1936,11 +2211,11 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
unsigned long used, avail, size;
if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
- read_lock();
+ packing_data_lock(&to_pack);
if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)
die(_("unable to get size of %s"),
oid_to_hex(&e->idx.oid));
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
@@ -1948,7 +2223,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
if (!p)
BUG("when e->type is a delta, it must belong to a pack");
- read_lock();
+ packing_data_lock(&to_pack);
w_curs = NULL;
buf = use_pack(p, &w_curs, e->in_pack_offset, &avail);
used = unpack_object_header_buffer(buf, avail, &type, &size);
@@ -1957,7 +2232,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
oid_to_hex(&e->idx.oid));
unuse_pack(&w_curs);
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
@@ -2014,24 +2289,27 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
if (trg_size < src_size / 32)
return 0;
+ if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))
+ return 0;
+
/* Load data if not already done */
if (!trg->data) {
- read_lock();
+ packing_data_lock(&to_pack);
trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!trg->data)
die(_("object %s cannot be read"),
oid_to_hex(&trg_entry->idx.oid));
if (sz != trg_size)
- die(_("object %s inconsistent object length (%lu vs %lu)"),
- oid_to_hex(&trg_entry->idx.oid), sz,
- trg_size);
+ die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
+ oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,
+ (uintmax_t)trg_size);
*mem_usage += sz;
}
if (!src->data) {
- read_lock();
+ packing_data_lock(&to_pack);
src->data = read_object_file(&src_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!src->data) {
if (src_entry->preferred_base) {
static int warned = 0;
@@ -2050,9 +2328,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
oid_to_hex(&src_entry->idx.oid));
}
if (sz != src_size)
- die(_("object %s inconsistent object length (%lu vs %lu)"),
- oid_to_hex(&src_entry->idx.oid), sz,
- src_size);
+ die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
+ oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,
+ (uintmax_t)src_size);
*mem_usage += sz;
}
if (!src->index) {
@@ -2069,10 +2347,6 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
if (!delta_buf)
return 0;
- if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) {
- free(delta_buf);
- return 0;
- }
if (DELTA(trg_entry)) {
/* Prefer only shallower same-sized deltas. */
@@ -2115,7 +2389,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
struct object_entry *child = DELTA_CHILD(me);
unsigned int m = n;
while (child) {
- unsigned int c = check_delta_limit(child, n + 1);
+ const unsigned int c = check_delta_limit(child, n + 1);
if (m < c)
m = c;
child = DELTA_SIBLING(child);
@@ -2170,7 +2444,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
while (window_memory_limit &&
mem_usage > window_memory_limit &&
count > 1) {
- uint32_t tail = (idx + window - count) % window;
+ const uint32_t tail = (idx + window - count) % window;
mem_usage -= free_unpacked(array + tail);
count--;
}
@@ -2279,17 +2553,6 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
free(array);
}
-#ifndef NO_PTHREADS
-
-static void try_to_free_from_threads(size_t size)
-{
- read_lock();
- release_pack_memory(size);
- read_unlock();
-}
-
-static try_to_free_t old_try_to_free_routine;
-
/*
* The main object list is split into smaller lists, each is handed to
* one worker.
@@ -2327,18 +2590,14 @@ static pthread_cond_t progress_cond;
*/
static void init_threaded_search(void)
{
- init_recursive_mutex(&read_mutex);
pthread_mutex_init(&cache_mutex, NULL);
pthread_mutex_init(&progress_mutex, NULL);
pthread_cond_init(&progress_cond, NULL);
- old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads);
}
static void cleanup_threaded_search(void)
{
- set_try_to_free_routine(old_try_to_free_routine);
pthread_cond_destroy(&progress_cond);
- pthread_mutex_destroy(&read_mutex);
pthread_mutex_destroy(&cache_mutex);
pthread_mutex_destroy(&progress_mutex);
}
@@ -2509,9 +2768,12 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size,
free(p);
}
-#else
-#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
-#endif
+static int obj_is_packed(const struct object_id *oid)
+{
+ return packlist_find(&to_pack, oid) ||
+ (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid));
+}
static void add_tag_chain(const struct object_id *oid)
{
@@ -2524,7 +2786,7 @@ static void add_tag_chain(const struct object_id *oid)
* it was included via bitmaps, we would not have parsed it
* previously).
*/
- if (packlist_find(&to_pack, oid->hash, NULL))
+ if (obj_is_packed(oid))
return;
tag = lookup_tag(the_repository, oid);
@@ -2548,7 +2810,7 @@ static int add_ref_tag(const char *path, const struct object_id *oid, int flag,
if (starts_with(path, "refs/tags/") && /* is a tag? */
!peel_ref(path, &peeled) && /* peelable? */
- packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */
+ obj_is_packed(&peeled)) /* object packed? */
add_tag_chain(oid);
return 0;
}
@@ -2559,6 +2821,9 @@ static void prepare_pack(int window, int depth)
uint32_t i, nr_deltas;
unsigned n;
+ if (use_delta_islands)
+ resolve_tree_islands(the_repository, progress, &to_pack);
+
get_object_details();
/*
@@ -2613,6 +2878,7 @@ static void prepare_pack(int window, int depth)
if (nr_deltas && n > 1) {
unsigned nr_done = 0;
+
if (progress)
progress_state = start_progress(_("Compressing objects"),
nr_deltas);
@@ -2657,17 +2923,19 @@ static int git_pack_config(const char *k, const char *v, void *cb)
use_bitmap_index_default = git_config_bool(k, v);
return 0;
}
+ if (!strcmp(k, "pack.allowpackreuse")) {
+ allow_pack_reuse = git_config_bool(k, v);
+ return 0;
+ }
if (!strcmp(k, "pack.threads")) {
delta_search_threads = git_config_int(k, v);
if (delta_search_threads < 0)
die(_("invalid number of threads specified (%d)"),
delta_search_threads);
-#ifdef NO_PTHREADS
- if (delta_search_threads != 1) {
+ if (!HAVE_THREADS && delta_search_threads != 1) {
warning(_("no threads support, ignoring %s"), k);
delta_search_threads = 0;
}
-#endif
return 0;
}
if (!strcmp(k, "pack.indexversion")) {
@@ -2677,6 +2945,29 @@ static int git_pack_config(const char *k, const char *v, void *cb)
pack_idx_opts.version);
return 0;
}
+ if (!strcmp(k, "uploadpack.blobpackfileuri")) {
+ struct configured_exclusion *ex = xmalloc(sizeof(*ex));
+ const char *oid_end, *pack_end;
+ /*
+ * Stores the pack hash. This is not a true object ID, but is
+ * of the same form.
+ */
+ struct object_id pack_hash;
+
+ if (parse_oid_hex(v, &ex->e.oid, &oid_end) ||
+ *oid_end != ' ' ||
+ parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) ||
+ *pack_end != ' ')
+ die(_("value of uploadpack.blobpackfileuri must be "
+ "of the form '<object-hash> <pack-hash> <uri>' (got '%s')"), v);
+ if (oidmap_get(&configured_exclusions, &ex->e.oid))
+ die(_("object already configured in another "
+ "uploadpack.blobpackfileuri (got '%s')"), v);
+ ex->pack_hash_hex = xcalloc(1, pack_end - oid_end);
+ memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1);
+ ex->uri = xstrdup(pack_end + 1);
+ oidmap_put(&configured_exclusions, ex);
+ }
return git_default_config(k, v, cb);
}
@@ -2722,6 +3013,9 @@ static void show_commit(struct commit *commit, void *data)
if (write_bitmap_index)
index_commit_for_bitmap(commit);
+
+ if (use_delta_islands)
+ propagate_island_marks(commit);
}
static void show_object(struct object *obj, const char *name, void *data)
@@ -2729,6 +3023,21 @@ static void show_object(struct object *obj, const char *name, void *data)
add_preferred_base_object(name);
add_object_entry(&obj->oid, obj->type, name, 0);
obj->flags |= OBJECT_ADDED;
+
+ if (use_delta_islands) {
+ const char *p;
+ unsigned depth;
+ struct object_entry *ent;
+
+ /* the empty string is a root tree, which is depth 0 */
+ depth = *name ? 1 : 0;
+ for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
+ depth++;
+
+ ent = packlist_find(&to_pack, &obj->oid);
+ if (ent && depth > oe_tree_depth(&to_pack, ent))
+ oe_set_tree_depth(&to_pack, ent, depth);
+ }
}
static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
@@ -2739,7 +3048,7 @@ static void show_object__ma_allow_any(struct object *obj, const char *name, void
* Quietly ignore ALL missing objects. This avoids problems with
* staging them now and getting an odd error later.
*/
- if (!has_object_file(&obj->oid))
+ if (!has_object(the_repository, &obj->oid, 0))
return;
show_object(obj, name, data);
@@ -2753,7 +3062,7 @@ static void show_object__ma_allow_promisor(struct object *obj, const char *name,
* Quietly ignore EXPECTED missing objects. This avoids problems with
* staging them now and getting an odd error later.
*/
- if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid))
+ if (!has_object(the_repository, &obj->oid, 0) && is_promisor_object(&obj->oid))
return;
show_object(obj, name, data);
@@ -2829,7 +3138,7 @@ static int ofscmp(const void *a_, const void *b_)
return oidcmp(&a->object->oid, &b->object->oid);
}
-static void add_objects_in_unpacked_packs(struct rev_info *revs)
+static void add_objects_in_unpacked_packs(void)
{
struct packed_git *p;
struct in_pack in_pack;
@@ -2851,8 +3160,8 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
in_pack.alloc);
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- o = lookup_unknown_object(oid.hash);
+ nth_packed_object_id(&oid, p, i);
+ o = lookup_unknown_object(&oid);
if (!(o->flags & OBJECT_ADDED))
mark_in_pack_object(o, p, &in_pack);
o->flags |= OBJECT_ADDED;
@@ -2941,7 +3250,7 @@ static int loosened_object_can_be_discarded(const struct object_id *oid,
return 1;
}
-static void loosen_unused_packed_objects(struct rev_info *revs)
+static void loosen_unused_packed_objects(void)
{
struct packed_git *p;
uint32_t i;
@@ -2955,8 +3264,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
die(_("cannot open pack index"));
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- if (!packlist_find(&to_pack, oid.hash, NULL) &&
+ nth_packed_object_id(&oid, p, i);
+ if (!packlist_find(&to_pack, &oid) &&
!has_sha1_pack_kept_or_nonlocal(&oid) &&
!loosened_object_can_be_discarded(&oid, p->mtime))
if (force_object_loose(&oid, p->mtime))
@@ -2972,8 +3281,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
*/
static int pack_options_allow_reuse(void)
{
- return pack_to_stdout &&
- allow_ofs_delta &&
+ return allow_pack_reuse &&
+ pack_to_stdout &&
!ignore_packed_keep_on_disk &&
!ignore_packed_keep_in_core &&
(!local || !have_non_local_packs) &&
@@ -2982,8 +3291,7 @@ static int pack_options_allow_reuse(void)
static int get_object_list_from_bitmap(struct rev_info *revs)
{
- struct bitmap_index *bitmap_git;
- if (!(bitmap_git = prepare_bitmap_walk(revs)))
+ if (!(bitmap_git = prepare_bitmap_walk(revs, &filter_options)))
return -1;
if (pack_options_allow_reuse() &&
@@ -2991,14 +3299,14 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
bitmap_git,
&reuse_packfile,
&reuse_packfile_objects,
- &reuse_packfile_offset)) {
+ &reuse_packfile_bitmap)) {
assert(reuse_packfile_objects);
nr_result += reuse_packfile_objects;
display_progress(progress_state, nr_result);
}
- traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
- free_bitmap_index(bitmap_git);
+ traverse_bitmap_commit_list(bitmap_git, revs,
+ &add_object_entry_from_bitmap);
return 0;
}
@@ -3017,16 +3325,23 @@ static void record_recent_commit(struct commit *commit, void *data)
static void get_object_list(int ac, const char **av)
{
struct rev_info revs;
+ struct setup_revision_opt s_r_opt = {
+ .allow_exclude_promisor_objects = 1,
+ };
char line[1000];
int flags = 0;
+ int save_warning;
- init_revisions(&revs, NULL);
+ repo_init_revisions(the_repository, &revs, NULL);
save_commit_buffer = 0;
- setup_revisions(ac, av, &revs, NULL);
+ setup_revisions(ac, av, &revs, &s_r_opt);
/* make sure shallows are read */
is_repository_shallow(the_repository);
+ save_warning = warn_on_object_refname_ambiguity;
+ warn_on_object_refname_ambiguity = 0;
+
while (fgets(line, sizeof(line), stdin) != NULL) {
int len = strlen(line);
if (len && line[len - 1] == '\n')
@@ -3042,7 +3357,7 @@ static void get_object_list(int ac, const char **av)
if (starts_with(line, "--shallow ")) {
struct object_id oid;
if (get_oid_hex(line + 10, &oid))
- die("not an SHA-1 '%s'", line + 10);
+ die("not an object name '%s'", line + 10);
register_shallow(the_repository, &oid);
use_bitmap_index = 0;
continue;
@@ -3053,12 +3368,17 @@ static void get_object_list(int ac, const char **av)
die(_("bad revision '%s'"), line);
}
+ warn_on_object_refname_ambiguity = save_warning;
+
if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
return;
+ if (use_delta_islands)
+ load_delta_islands(the_repository, progress);
+
if (prepare_revision_walk(&revs))
die(_("revision walk setup failed"));
- mark_edges_uninteresting(&revs, show_edge);
+ mark_edges_uninteresting(&revs, show_edge, sparse);
if (!fn_show_object)
fn_show_object = show_object;
@@ -3078,11 +3398,11 @@ static void get_object_list(int ac, const char **av)
}
if (keep_unreachable)
- add_objects_in_unpacked_packs(&revs);
+ add_objects_in_unpacked_packs();
if (pack_loose_unreachable)
add_unreachable_loose_objects();
if (unpack_unreachable)
- loosen_unused_packed_objects(&revs);
+ loosen_unused_packed_objects();
oid_array_clear(&recent_objects);
}
@@ -3118,6 +3438,9 @@ static int option_parse_index_version(const struct option *opt,
{
char *c;
const char *val = arg;
+
+ BUG_ON_OPT_NEG(unset);
+
pack_idx_opts.version = strtoul(val, &c, 10);
if (pack_idx_opts.version > 2)
die(_("unsupported index version %s"), val);
@@ -3146,10 +3469,9 @@ static int option_parse_unpack_unreachable(const struct option *opt,
int cmd_pack_objects(int argc, const char **argv, const char *prefix)
{
int use_internal_rev_list = 0;
- int thin = 0;
int shallow = 0;
int all_progress_implied = 0;
- struct argv_array rp = ARGV_ARRAY_INIT;
+ struct strvec rp = STRVEC_INIT;
int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
int rev_list_index = 0;
struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
@@ -3163,9 +3485,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
OPT_BOOL(0, "all-progress-implied",
&all_progress_implied,
N_("similar to --all-progress when progress meter is shown")),
- { OPTION_CALLBACK, 0, "index-version", NULL, N_("<version>[,<offset>]"),
+ OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"),
N_("write the pack index file in the specified idx format version"),
- 0, option_parse_index_version },
+ PARSE_OPT_NONEG, option_parse_index_version),
OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,
N_("maximum size of each output pack file")),
OPT_BOOL(0, "local", &local,
@@ -3210,9 +3532,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
N_("keep unreachable objects")),
OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable,
N_("pack loose unreachable objects")),
- { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"),
+ OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"),
N_("unpack unreachable objects newer than <time>"),
- PARSE_OPT_OPTARG, option_parse_unpack_unreachable },
+ PARSE_OPT_OPTARG, option_parse_unpack_unreachable),
+ OPT_BOOL(0, "sparse", &sparse,
+ N_("use the sparse reachability algorithm")),
OPT_BOOL(0, "thin", &thin,
N_("create thin packs")),
OPT_BOOL(0, "shallow", &shallow,
@@ -3227,14 +3551,24 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
N_("do not hide commits by grafts"), 0),
OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,
N_("use a bitmap index if available to speed up counting objects")),
- OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
- N_("write a bitmap index together with the pack index")),
+ OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index,
+ N_("write a bitmap index together with the pack index"),
+ WRITE_BITMAP_TRUE),
+ OPT_SET_INT_F(0, "write-bitmap-index-quiet",
+ &write_bitmap_index,
+ N_("write a bitmap index if possible"),
+ WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
- { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
+ OPT_CALLBACK_F(0, "missing", NULL, N_("action"),
N_("handling for missing objects"), PARSE_OPT_NONEG,
- option_parse_missing_action },
+ option_parse_missing_action),
OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
N_("do not pack objects in promisor packfiles")),
+ OPT_BOOL(0, "delta-islands", &use_delta_islands,
+ N_("respect islands during delta compression")),
+ OPT_STRING_LIST(0, "uri-protocol", &uri_protocols,
+ N_("protocol"),
+ N_("exclude any configured uploadpack.blobpackfileuri with this protocol")),
OPT_END(),
};
@@ -3243,6 +3577,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
read_replace_refs = 0;
+ sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1);
+ prepare_repo_settings(the_repository);
+ if (sparse < 0)
+ sparse = the_repository->settings.pack_use_sparse;
+
reset_pack_idx_option(&pack_idx_opts);
git_config(git_pack_config, NULL);
@@ -3268,36 +3607,36 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
cache_max_small_delta_size = (1U << OE_Z_DELTA_BITS) - 1;
}
- argv_array_push(&rp, "pack-objects");
+ strvec_push(&rp, "pack-objects");
if (thin) {
use_internal_rev_list = 1;
- argv_array_push(&rp, shallow
+ strvec_push(&rp, shallow
? "--objects-edge-aggressive"
: "--objects-edge");
} else
- argv_array_push(&rp, "--objects");
+ strvec_push(&rp, "--objects");
if (rev_list_all) {
use_internal_rev_list = 1;
- argv_array_push(&rp, "--all");
+ strvec_push(&rp, "--all");
}
if (rev_list_reflog) {
use_internal_rev_list = 1;
- argv_array_push(&rp, "--reflog");
+ strvec_push(&rp, "--reflog");
}
if (rev_list_index) {
use_internal_rev_list = 1;
- argv_array_push(&rp, "--indexed-objects");
+ strvec_push(&rp, "--indexed-objects");
}
if (rev_list_unpacked) {
use_internal_rev_list = 1;
- argv_array_push(&rp, "--unpacked");
+ strvec_push(&rp, "--unpacked");
}
if (exclude_promisor_objects) {
use_internal_rev_list = 1;
fetch_if_missing = 0;
- argv_array_push(&rp, "--exclude-promisor-objects");
+ strvec_push(&rp, "--exclude-promisor-objects");
}
if (unpack_unreachable || keep_unreachable || pack_loose_unreachable)
use_internal_rev_list = 1;
@@ -3312,10 +3651,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (!delta_search_threads) /* --threads=0 means autodetect */
delta_search_threads = online_cpus();
-#ifdef NO_PTHREADS
- if (delta_search_threads != 1)
+ if (!HAVE_THREADS && delta_search_threads != 1)
warning(_("no threads support, ignoring --threads"));
-#endif
if (!pack_to_stdout && !pack_size_limit)
pack_size_limit = pack_size_limit_cfg;
if (pack_to_stdout && pack_size_limit)
@@ -3336,7 +3673,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (filter_options.choice) {
if (!pack_to_stdout)
die(_("cannot use --filter without --stdout"));
- use_bitmap_index = 0;
}
/*
@@ -3361,6 +3697,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (pack_to_stdout || !rev_list_all)
write_bitmap_index = 0;
+ if (use_delta_islands)
+ strvec_push(&rp, "--topo-order");
+
if (progress && all_progress_implied)
progress = 2;
@@ -3388,30 +3727,46 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
}
}
- prepare_packing_data(&to_pack);
+ trace2_region_enter("pack-objects", "enumerate-objects",
+ the_repository);
+ prepare_packing_data(the_repository, &to_pack);
if (progress)
progress_state = start_progress(_("Enumerating objects"), 0);
if (!use_internal_rev_list)
read_object_list_from_stdin();
else {
- get_object_list(rp.argc, rp.argv);
- argv_array_clear(&rp);
+ get_object_list(rp.nr, rp.v);
+ strvec_clear(&rp);
}
cleanup_preferred_base();
if (include_tag && nr_result)
for_each_ref(add_ref_tag, NULL);
stop_progress(&progress_state);
+ trace2_region_leave("pack-objects", "enumerate-objects",
+ the_repository);
if (non_empty && !nr_result)
return 0;
- if (nr_result)
+ if (nr_result) {
+ trace2_region_enter("pack-objects", "prepare-pack",
+ the_repository);
prepare_pack(window, depth);
+ trace2_region_leave("pack-objects", "prepare-pack",
+ the_repository);
+ }
+
+ trace2_region_enter("pack-objects", "write-pack-file", the_repository);
+ write_excluded_by_configs();
write_pack_file();
+ trace2_region_leave("pack-objects", "write-pack-file", the_repository);
+
if (progress)
fprintf_ln(stderr,
_("Total %"PRIu32" (delta %"PRIu32"),"
- " reused %"PRIu32" (delta %"PRIu32")"),
- written, written_delta, reused, reused_delta);
+ " reused %"PRIu32" (delta %"PRIu32"),"
+ " pack-reused %"PRIu32),
+ written, written_delta, reused, reused_delta,
+ reuse_packfile_objects);
return 0;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-28 06:26:13 +0000