summaryrefslogtreecommitdiff
path: root/builtin/pack-objects.c
diff options
context:
space:
mode:
Diffstat (limited to 'builtin/pack-objects.c')
-rw-r--r--builtin/pack-objects.c470
1 files changed, 317 insertions, 153 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 411aefd687..03b85f5166 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -26,13 +26,14 @@
#include "pack-bitmap.h"
#include "delta-islands.h"
#include "reachable.h"
-#include "sha1-array.h"
+#include "oid-array.h"
#include "argv-array.h"
#include "list.h"
#include "packfile.h"
#include "object-store.h"
#include "dir.h"
#include "midx.h"
+#include "trace2.h"
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
#define SIZE(obj) oe_size(&to_pack, obj)
@@ -84,18 +85,24 @@ static unsigned long pack_size_limit;
static int depth = 50;
static int delta_search_threads;
static int pack_to_stdout;
+static int sparse;
static int thin;
static int num_preferred_base;
static struct progress *progress_state;
static struct packed_git *reuse_packfile;
static uint32_t reuse_packfile_objects;
-static off_t reuse_packfile_offset;
+static struct bitmap *reuse_packfile_bitmap;
static int use_bitmap_index_default = 1;
static int use_bitmap_index = -1;
-static int write_bitmap_index;
-static uint16_t write_bitmap_options;
+static int allow_pack_reuse = 1;
+static enum {
+ WRITE_BITMAP_FALSE = 0,
+ WRITE_BITMAP_QUIET,
+ WRITE_BITMAP_TRUE,
+} write_bitmap_index;
+static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
static int exclude_promisor_objects;
@@ -157,7 +164,7 @@ static void *get_delta(struct object_entry *entry)
delta_buf = diff_delta(base_buf, base_size,
buf, size, &delta_size, 0);
/*
- * We succesfully computed this delta once but dropped it for
+ * We successfully computed this delta once but dropped it for
* memory reasons. Something is very wrong if this time we
* recompute and create a different delta.
*/
@@ -297,7 +304,8 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
if (!usable_delta) {
if (oe_type(entry) == OBJ_BLOB &&
oe_size_greater_than(&to_pack, entry, big_file_threshold) &&
- (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL)
+ (st = open_istream(the_repository, &entry->idx.oid, &type,
+ &size, NULL)) != NULL)
buf = NULL;
else {
buf = read_object_file(&entry->idx.oid, &type, &size);
@@ -604,12 +612,12 @@ static int mark_tagged(const char *path, const struct object_id *oid, int flag,
void *cb_data)
{
struct object_id peeled;
- struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL);
+ struct object_entry *entry = packlist_find(&to_pack, oid);
if (entry)
entry->tagged = 1;
if (!peel_ref(path, &peeled)) {
- entry = packlist_find(&to_pack, peeled.hash, NULL);
+ entry = packlist_find(&to_pack, &peeled);
if (entry)
entry->tagged = 1;
}
@@ -778,57 +786,186 @@ static struct object_entry **compute_write_order(void)
return wo;
}
-static off_t write_reused_pack(struct hashfile *f)
+
+/*
+ * A reused set of objects. All objects in a chunk have the same
+ * relative position in the original packfile and the generated
+ * packfile.
+ */
+
+static struct reused_chunk {
+ /* The offset of the first object of this chunk in the original
+ * packfile. */
+ off_t original;
+ /* The offset of the first object of this chunk in the generated
+ * packfile minus "original". */
+ off_t difference;
+} *reused_chunks;
+static int reused_chunks_nr;
+static int reused_chunks_alloc;
+
+static void record_reused_object(off_t where, off_t offset)
+{
+ if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset)
+ return;
+
+ ALLOC_GROW(reused_chunks, reused_chunks_nr + 1,
+ reused_chunks_alloc);
+ reused_chunks[reused_chunks_nr].original = where;
+ reused_chunks[reused_chunks_nr].difference = offset;
+ reused_chunks_nr++;
+}
+
+/*
+ * Binary search to find the chunk that "where" is in. Note
+ * that we're not looking for an exact match, just the first
+ * chunk that contains it (which implicitly ends at the start
+ * of the next chunk.
+ */
+static off_t find_reused_offset(off_t where)
+{
+ int lo = 0, hi = reused_chunks_nr;
+ while (lo < hi) {
+ int mi = lo + ((hi - lo) / 2);
+ if (where == reused_chunks[mi].original)
+ return reused_chunks[mi].difference;
+ if (where < reused_chunks[mi].original)
+ hi = mi;
+ else
+ lo = mi + 1;
+ }
+
+ /*
+ * The first chunk starts at zero, so we can't have gone below
+ * there.
+ */
+ assert(lo);
+ return reused_chunks[lo-1].difference;
+}
+
+static void write_reused_pack_one(size_t pos, struct hashfile *out,
+ struct pack_window **w_curs)
{
- unsigned char buffer[8192];
- off_t to_write, total;
- int fd;
+ off_t offset, next, cur;
+ enum object_type type;
+ unsigned long size;
- if (!is_pack_valid(reuse_packfile))
- die(_("packfile is invalid: %s"), reuse_packfile->pack_name);
+ offset = reuse_packfile->revindex[pos].offset;
+ next = reuse_packfile->revindex[pos + 1].offset;
- fd = git_open(reuse_packfile->pack_name);
- if (fd < 0)
- die_errno(_("unable to open packfile for reuse: %s"),
- reuse_packfile->pack_name);
+ record_reused_object(offset, offset - hashfile_total(out));
- if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1)
- die_errno(_("unable to seek in reused packfile"));
+ cur = offset;
+ type = unpack_object_header(reuse_packfile, w_curs, &cur, &size);
+ assert(type >= 0);
- if (reuse_packfile_offset < 0)
- reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz;
+ if (type == OBJ_OFS_DELTA) {
+ off_t base_offset;
+ off_t fixup;
- total = to_write = reuse_packfile_offset - sizeof(struct pack_header);
+ unsigned char header[MAX_PACK_OBJECT_HEADER];
+ unsigned len;
- while (to_write) {
- int read_pack = xread(fd, buffer, sizeof(buffer));
+ base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset);
+ assert(base_offset != 0);
- if (read_pack <= 0)
- die_errno(_("unable to read from reused packfile"));
+ /* Convert to REF_DELTA if we must... */
+ if (!allow_ofs_delta) {
+ int base_pos = find_revindex_position(reuse_packfile, base_offset);
+ struct object_id base_oid;
- if (read_pack > to_write)
- read_pack = to_write;
+ nth_packed_object_id(&base_oid, reuse_packfile,
+ reuse_packfile->revindex[base_pos].nr);
- hashwrite(f, buffer, read_pack);
- to_write -= read_pack;
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_REF_DELTA, size);
+ hashwrite(out, header, len);
+ hashwrite(out, base_oid.hash, the_hash_algo->rawsz);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* Otherwise see if we need to rewrite the offset... */
+ fixup = find_reused_offset(offset) -
+ find_reused_offset(base_offset);
+ if (fixup) {
+ unsigned char ofs_header[10];
+ unsigned i, ofs_len;
+ off_t ofs = offset - base_offset - fixup;
+
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_OFS_DELTA, size);
+
+ i = sizeof(ofs_header) - 1;
+ ofs_header[i] = ofs & 127;
+ while (ofs >>= 7)
+ ofs_header[--i] = 128 | (--ofs & 127);
+
+ ofs_len = sizeof(ofs_header) - i;
+
+ hashwrite(out, header, len);
+ hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* ...otherwise we have no fixup, and can write it verbatim */
+ }
+
+ copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset);
+}
+
+static size_t write_reused_pack_verbatim(struct hashfile *out,
+ struct pack_window **w_curs)
+{
+ size_t pos = 0;
+
+ while (pos < reuse_packfile_bitmap->word_alloc &&
+ reuse_packfile_bitmap->words[pos] == (eword_t)~0)
+ pos++;
+
+ if (pos) {
+ off_t to_write;
+
+ written = (pos * BITS_IN_EWORD);
+ to_write = reuse_packfile->revindex[written].offset
+ - sizeof(struct pack_header);
+
+ /* We're recording one chunk, not one object. */
+ record_reused_object(sizeof(struct pack_header), 0);
+ hashflush(out);
+ copy_pack_data(out, reuse_packfile, w_curs,
+ sizeof(struct pack_header), to_write);
- /*
- * We don't know the actual number of objects written,
- * only how many bytes written, how many bytes total, and
- * how many objects total. So we can fake it by pretending all
- * objects we are writing are the same size. This gives us a
- * smooth progress meter, and at the end it matches the true
- * answer.
- */
- written = reuse_packfile_objects *
- (((double)(total - to_write)) / total);
display_progress(progress_state, written);
}
+ return pos;
+}
+
+static void write_reused_pack(struct hashfile *f)
+{
+ size_t i = 0;
+ uint32_t offset;
+ struct pack_window *w_curs = NULL;
+
+ if (allow_ofs_delta)
+ i = write_reused_pack_verbatim(f, &w_curs);
+
+ for (; i < reuse_packfile_bitmap->word_alloc; ++i) {
+ eword_t word = reuse_packfile_bitmap->words[i];
+ size_t pos = (i * BITS_IN_EWORD);
+
+ for (offset = 0; offset < BITS_IN_EWORD; ++offset) {
+ if ((word >> offset) == 0)
+ break;
+
+ offset += ewah_bit_ctz64(word >> offset);
+ write_reused_pack_one(pos + offset, f, &w_curs);
+ display_progress(progress_state, ++written);
+ }
+ }
- close(fd);
- written = reuse_packfile_objects;
- display_progress(progress_state, written);
- return reuse_packfile_offset - sizeof(struct pack_header);
+ unuse_pack(&w_curs);
}
static const char no_split_warning[] = N_(
@@ -861,11 +998,9 @@ static void write_pack_file(void)
offset = write_pack_header(f, nr_remaining);
if (reuse_packfile) {
- off_t packfile_size;
assert(pack_to_stdout);
-
- packfile_size = write_reused_pack(f);
- offset += packfile_size;
+ write_reused_pack(f);
+ offset = hashfile_total(f);
}
nr_written = 0;
@@ -890,7 +1025,8 @@ static void write_pack_file(void)
nr_written, oid.hash, offset);
close(fd);
if (write_bitmap_index) {
- warning(_(no_split_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_split_warning));
write_bitmap_index = 0;
}
}
@@ -962,6 +1098,8 @@ static void write_pack_file(void)
if (written != nr_result)
die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
written, nr_result);
+ trace2_data_intmax("pack-objects", the_repository,
+ "write_pack_file/wrote", nr_result);
}
static int no_try_delta(const char *path)
@@ -970,7 +1108,7 @@ static int no_try_delta(const char *path)
if (!check)
check = attr_check_initl("delta", NULL);
- git_check_attr(&the_index, path, check);
+ git_check_attr(the_repository->index, path, check);
if (ATTR_FALSE(check->items[0].value))
return 1;
return 0;
@@ -987,12 +1125,15 @@ static int no_try_delta(const char *path)
* few lines later when we want to add the new entry.
*/
static int have_duplicate_entry(const struct object_id *oid,
- int exclude,
- uint32_t *index_pos)
+ int exclude)
{
struct object_entry *entry;
- entry = packlist_find(&to_pack, oid->hash, index_pos);
+ if (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid))
+ return 1;
+
+ entry = packlist_find(&to_pack, oid);
if (!entry)
return 0;
@@ -1076,7 +1217,7 @@ static int want_object_in_pack(const struct object_id *oid,
for (m = get_multi_pack_index(the_repository); m; m = m->next) {
struct pack_entry e;
- if (fill_midx_entry(oid, &e, m)) {
+ if (fill_midx_entry(the_repository, oid, &e, m)) {
struct packed_git *p = e.p;
off_t offset;
@@ -1132,13 +1273,12 @@ static void create_object_entry(const struct object_id *oid,
uint32_t hash,
int exclude,
int no_try_delta,
- uint32_t index_pos,
struct packed_git *found_pack,
off_t found_offset)
{
struct object_entry *entry;
- entry = packlist_alloc(&to_pack, oid->hash, index_pos);
+ entry = packlist_alloc(&to_pack, oid);
entry->hash = hash;
oe_set_type(entry, type);
if (exclude)
@@ -1162,17 +1302,17 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
{
struct packed_git *found_pack = NULL;
off_t found_offset = 0;
- uint32_t index_pos;
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, exclude, &index_pos))
+ if (have_duplicate_entry(oid, exclude))
return 0;
if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {
/* The pack is missing an object, so it will not have closure */
if (write_bitmap_index) {
- warning(_(no_closure_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_closure_warning));
write_bitmap_index = 0;
}
return 0;
@@ -1180,7 +1320,7 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
create_object_entry(oid, type, pack_name_hash(name),
exclude, name && no_try_delta(name),
- index_pos, found_pack, found_offset);
+ found_pack, found_offset);
return 1;
}
@@ -1189,17 +1329,15 @@ static int add_object_entry_from_bitmap(const struct object_id *oid,
int flags, uint32_t name_hash,
struct packed_git *pack, off_t offset)
{
- uint32_t index_pos;
-
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, 0, &index_pos))
+ if (have_duplicate_entry(oid, 0))
return 0;
if (!want_object_in_pack(oid, 0, &pack, &offset))
return 0;
- create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);
+ create_object_entry(oid, type, name_hash, 0, 0, pack, offset);
return 1;
}
@@ -1334,7 +1472,7 @@ static void add_pbase_object(struct tree_desc *tree,
if (cmp < 0)
return;
if (name[cmplen] != '/') {
- add_object_entry(entry.oid,
+ add_object_entry(&entry.oid,
object_type(entry.mode),
fullname, 1);
return;
@@ -1345,7 +1483,7 @@ static void add_pbase_object(struct tree_desc *tree,
const char *down = name+cmplen+1;
int downlen = name_cmp_len(down);
- tree = pbase_tree_get(entry.oid);
+ tree = pbase_tree_get(&entry.oid);
if (!tree)
return;
init_tree_desc(&sub, tree->tree_data, tree->tree_size);
@@ -1424,7 +1562,8 @@ static void add_preferred_base(struct object_id *oid)
if (window <= num_preferred_base++)
return;
- data = read_object_with_reference(oid, tree_type, &size, &tree_oid);
+ data = read_object_with_reference(the_repository, oid,
+ tree_type, &size, &tree_oid);
if (!data)
return;
@@ -1480,20 +1619,17 @@ static void cleanup_preferred_base(void)
* deltify other objects against, in order to avoid
* circular deltas.
*/
-static int can_reuse_delta(const unsigned char *base_sha1,
+static int can_reuse_delta(const struct object_id *base_oid,
struct object_entry *delta,
struct object_entry **base_out)
{
struct object_entry *base;
- if (!base_sha1)
- return 0;
-
/*
* First see if we're already sending the base (or it's explicitly in
* our "excluded" list).
*/
- base = packlist_find(&to_pack, base_sha1, NULL);
+ base = packlist_find(&to_pack, base_oid);
if (base) {
if (!in_same_island(&delta->idx.oid, &base->idx.oid))
return 0;
@@ -1506,11 +1642,9 @@ static int can_reuse_delta(const unsigned char *base_sha1,
* even if it was buried too deep in history to make it into the
* packing list.
*/
- if (thin && bitmap_has_sha1_in_uninteresting(bitmap_git, base_sha1)) {
+ if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
if (use_delta_islands) {
- struct object_id base_oid;
- hashcpy(base_oid.hash, base_sha1);
- if (!in_same_island(&delta->idx.oid, &base_oid))
+ if (!in_same_island(&delta->idx.oid, base_oid))
return 0;
}
*base_out = NULL;
@@ -1527,7 +1661,8 @@ static void check_object(struct object_entry *entry)
if (IN_PACK(entry)) {
struct packed_git *p = IN_PACK(entry);
struct pack_window *w_curs = NULL;
- const unsigned char *base_ref = NULL;
+ int have_base = 0;
+ struct object_id base_ref;
struct object_entry *base_entry;
unsigned long used, used_0;
unsigned long avail;
@@ -1568,9 +1703,13 @@ static void check_object(struct object_entry *entry)
unuse_pack(&w_curs);
return;
case OBJ_REF_DELTA:
- if (reuse_delta && !entry->preferred_base)
- base_ref = use_pack(p, &w_curs,
- entry->in_pack_offset + used, NULL);
+ if (reuse_delta && !entry->preferred_base) {
+ oidread(&base_ref,
+ use_pack(p, &w_curs,
+ entry->in_pack_offset + used,
+ NULL));
+ have_base = 1;
+ }
entry->in_pack_header_size = used + the_hash_algo->rawsz;
break;
case OBJ_OFS_DELTA:
@@ -1600,13 +1739,15 @@ static void check_object(struct object_entry *entry)
revidx = find_pack_revindex(p, ofs);
if (!revidx)
goto give_up;
- base_ref = nth_packed_object_sha1(p, revidx->nr);
+ if (!nth_packed_object_id(&base_ref, p, revidx->nr))
+ have_base = 1;
}
entry->in_pack_header_size = used + used_0;
break;
}
- if (can_reuse_delta(base_ref, entry, &base_entry)) {
+ if (have_base &&
+ can_reuse_delta(&base_ref, entry, &base_entry)) {
oe_set_type(entry, entry->in_pack_type);
SET_SIZE(entry, in_pack_size); /* delta size */
SET_DELTA_SIZE(entry, in_pack_size);
@@ -1616,7 +1757,7 @@ static void check_object(struct object_entry *entry)
entry->delta_sibling_idx = base_entry->delta_child_idx;
SET_DELTA_CHILD(base_entry, entry);
} else {
- SET_DELTA_EXT(entry, base_ref);
+ SET_DELTA_EXT(entry, &base_ref);
}
unuse_pack(&w_curs);
@@ -1642,7 +1783,7 @@ static void check_object(struct object_entry *entry)
/*
* No choice but to fall back to the recursive delta walk
- * with sha1_object_info() to find about the object type
+ * with oid_object_info() to find about the object type
* at this point...
*/
give_up:
@@ -1718,7 +1859,7 @@ static void drop_reused_delta(struct object_entry *entry)
if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
/*
* We failed to get the info from this pack for some reason;
- * fall back to sha1_object_info, which may find another copy.
+ * fall back to oid_object_info, which may find another copy.
* And if that fails, the error will be recorded in oe_type(entry)
* and dealt with in prepare_pack().
*/
@@ -1901,10 +2042,10 @@ static int type_size_sort(const void *_a, const void *_b)
{
const struct object_entry *a = *(struct object_entry **)_a;
const struct object_entry *b = *(struct object_entry **)_b;
- enum object_type a_type = oe_type(a);
- enum object_type b_type = oe_type(b);
- unsigned long a_size = SIZE(a);
- unsigned long b_size = SIZE(b);
+ const enum object_type a_type = oe_type(a);
+ const enum object_type b_type = oe_type(b);
+ const unsigned long a_size = SIZE(a);
+ const unsigned long b_size = SIZE(b);
if (a_type > b_type)
return -1;
@@ -1919,7 +2060,7 @@ static int type_size_sort(const void *_a, const void *_b)
if (a->preferred_base < b->preferred_base)
return 1;
if (use_delta_islands) {
- int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
+ const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
if (island_cmp)
return island_cmp;
}
@@ -1953,11 +2094,6 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
return 0;
}
-/* Protect access to object database */
-static pthread_mutex_t read_mutex;
-#define read_lock() pthread_mutex_lock(&read_mutex)
-#define read_unlock() pthread_mutex_unlock(&read_mutex)
-
/* Protect delta_cache_size */
static pthread_mutex_t cache_mutex;
#define cache_lock() pthread_mutex_lock(&cache_mutex)
@@ -1993,11 +2129,11 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
unsigned long used, avail, size;
if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
- read_lock();
+ packing_data_lock(&to_pack);
if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)
die(_("unable to get size of %s"),
oid_to_hex(&e->idx.oid));
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
@@ -2005,7 +2141,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
if (!p)
BUG("when e->type is a delta, it must belong to a pack");
- read_lock();
+ packing_data_lock(&to_pack);
w_curs = NULL;
buf = use_pack(p, &w_curs, e->in_pack_offset, &avail);
used = unpack_object_header_buffer(buf, avail, &type, &size);
@@ -2014,7 +2150,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
oid_to_hex(&e->idx.oid));
unuse_pack(&w_curs);
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
@@ -2076,9 +2212,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
/* Load data if not already done */
if (!trg->data) {
- read_lock();
+ packing_data_lock(&to_pack);
trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!trg->data)
die(_("object %s cannot be read"),
oid_to_hex(&trg_entry->idx.oid));
@@ -2089,9 +2225,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
*mem_usage += sz;
}
if (!src->data) {
- read_lock();
+ packing_data_lock(&to_pack);
src->data = read_object_file(&src_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!src->data) {
if (src_entry->preferred_base) {
static int warned = 0;
@@ -2171,7 +2307,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
struct object_entry *child = DELTA_CHILD(me);
unsigned int m = n;
while (child) {
- unsigned int c = check_delta_limit(child, n + 1);
+ const unsigned int c = check_delta_limit(child, n + 1);
if (m < c)
m = c;
child = DELTA_SIBLING(child);
@@ -2226,7 +2362,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
while (window_memory_limit &&
mem_usage > window_memory_limit &&
count > 1) {
- uint32_t tail = (idx + window - count) % window;
+ const uint32_t tail = (idx + window - count) % window;
mem_usage -= free_unpacked(array + tail);
count--;
}
@@ -2335,15 +2471,6 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
free(array);
}
-static void try_to_free_from_threads(size_t size)
-{
- read_lock();
- release_pack_memory(size);
- read_unlock();
-}
-
-static try_to_free_t old_try_to_free_routine;
-
/*
* The main object list is split into smaller lists, each is handed to
* one worker.
@@ -2381,18 +2508,14 @@ static pthread_cond_t progress_cond;
*/
static void init_threaded_search(void)
{
- init_recursive_mutex(&read_mutex);
pthread_mutex_init(&cache_mutex, NULL);
pthread_mutex_init(&progress_mutex, NULL);
pthread_cond_init(&progress_cond, NULL);
- old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads);
}
static void cleanup_threaded_search(void)
{
- set_try_to_free_routine(old_try_to_free_routine);
pthread_cond_destroy(&progress_cond);
- pthread_mutex_destroy(&read_mutex);
pthread_mutex_destroy(&cache_mutex);
pthread_mutex_destroy(&progress_mutex);
}
@@ -2563,6 +2686,13 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size,
free(p);
}
+static int obj_is_packed(const struct object_id *oid)
+{
+ return packlist_find(&to_pack, oid) ||
+ (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid));
+}
+
static void add_tag_chain(const struct object_id *oid)
{
struct tag *tag;
@@ -2574,7 +2704,7 @@ static void add_tag_chain(const struct object_id *oid)
* it was included via bitmaps, we would not have parsed it
* previously).
*/
- if (packlist_find(&to_pack, oid->hash, NULL))
+ if (obj_is_packed(oid))
return;
tag = lookup_tag(the_repository, oid);
@@ -2598,7 +2728,7 @@ static int add_ref_tag(const char *path, const struct object_id *oid, int flag,
if (starts_with(path, "refs/tags/") && /* is a tag? */
!peel_ref(path, &peeled) && /* peelable? */
- packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */
+ obj_is_packed(&peeled)) /* object packed? */
add_tag_chain(oid);
return 0;
}
@@ -2610,7 +2740,7 @@ static void prepare_pack(int window, int depth)
unsigned n;
if (use_delta_islands)
- resolve_tree_islands(progress, &to_pack);
+ resolve_tree_islands(the_repository, progress, &to_pack);
get_object_details();
@@ -2666,6 +2796,7 @@ static void prepare_pack(int window, int depth)
if (nr_deltas && n > 1) {
unsigned nr_done = 0;
+
if (progress)
progress_state = start_progress(_("Compressing objects"),
nr_deltas);
@@ -2710,6 +2841,10 @@ static int git_pack_config(const char *k, const char *v, void *cb)
use_bitmap_index_default = git_config_bool(k, v);
return 0;
}
+ if (!strcmp(k, "pack.allowpackreuse")) {
+ allow_pack_reuse = git_config_bool(k, v);
+ return 0;
+ }
if (!strcmp(k, "pack.threads")) {
delta_search_threads = git_config_int(k, v);
if (delta_search_threads < 0)
@@ -2794,7 +2929,7 @@ static void show_object(struct object *obj, const char *name, void *data)
for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
depth++;
- ent = packlist_find(&to_pack, obj->oid.hash, NULL);
+ ent = packlist_find(&to_pack, &obj->oid);
if (ent && depth > oe_tree_depth(&to_pack, ent))
oe_set_tree_depth(&to_pack, ent, depth);
}
@@ -2898,7 +3033,7 @@ static int ofscmp(const void *a_, const void *b_)
return oidcmp(&a->object->oid, &b->object->oid);
}
-static void add_objects_in_unpacked_packs(struct rev_info *revs)
+static void add_objects_in_unpacked_packs(void)
{
struct packed_git *p;
struct in_pack in_pack;
@@ -2920,8 +3055,8 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
in_pack.alloc);
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- o = lookup_unknown_object(oid.hash);
+ nth_packed_object_id(&oid, p, i);
+ o = lookup_unknown_object(&oid);
if (!(o->flags & OBJECT_ADDED))
mark_in_pack_object(o, p, &in_pack);
o->flags |= OBJECT_ADDED;
@@ -3010,7 +3145,7 @@ static int loosened_object_can_be_discarded(const struct object_id *oid,
return 1;
}
-static void loosen_unused_packed_objects(struct rev_info *revs)
+static void loosen_unused_packed_objects(void)
{
struct packed_git *p;
uint32_t i;
@@ -3024,8 +3159,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
die(_("cannot open pack index"));
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- if (!packlist_find(&to_pack, oid.hash, NULL) &&
+ nth_packed_object_id(&oid, p, i);
+ if (!packlist_find(&to_pack, &oid) &&
!has_sha1_pack_kept_or_nonlocal(&oid) &&
!loosened_object_can_be_discarded(&oid, p->mtime))
if (force_object_loose(&oid, p->mtime))
@@ -3041,8 +3176,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
*/
static int pack_options_allow_reuse(void)
{
- return pack_to_stdout &&
- allow_ofs_delta &&
+ return allow_pack_reuse &&
+ pack_to_stdout &&
!ignore_packed_keep_on_disk &&
!ignore_packed_keep_in_core &&
(!local || !have_non_local_packs) &&
@@ -3051,7 +3186,7 @@ static int pack_options_allow_reuse(void)
static int get_object_list_from_bitmap(struct rev_info *revs)
{
- if (!(bitmap_git = prepare_bitmap_walk(revs)))
+ if (!(bitmap_git = prepare_bitmap_walk(revs, &filter_options)))
return -1;
if (pack_options_allow_reuse() &&
@@ -3059,13 +3194,14 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
bitmap_git,
&reuse_packfile,
&reuse_packfile_objects,
- &reuse_packfile_offset)) {
+ &reuse_packfile_bitmap)) {
assert(reuse_packfile_objects);
nr_result += reuse_packfile_objects;
display_progress(progress_state, nr_result);
}
- traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
+ traverse_bitmap_commit_list(bitmap_git, revs,
+ &add_object_entry_from_bitmap);
return 0;
}
@@ -3084,14 +3220,16 @@ static void record_recent_commit(struct commit *commit, void *data)
static void get_object_list(int ac, const char **av)
{
struct rev_info revs;
+ struct setup_revision_opt s_r_opt = {
+ .allow_exclude_promisor_objects = 1,
+ };
char line[1000];
int flags = 0;
int save_warning;
repo_init_revisions(the_repository, &revs, NULL);
save_commit_buffer = 0;
- revs.allow_exclude_promisor_objects_opt = 1;
- setup_revisions(ac, av, &revs, NULL);
+ setup_revisions(ac, av, &revs, &s_r_opt);
/* make sure shallows are read */
is_repository_shallow(the_repository);
@@ -3131,11 +3269,11 @@ static void get_object_list(int ac, const char **av)
return;
if (use_delta_islands)
- load_delta_islands();
+ load_delta_islands(the_repository, progress);
if (prepare_revision_walk(&revs))
die(_("revision walk setup failed"));
- mark_edges_uninteresting(&revs, show_edge);
+ mark_edges_uninteresting(&revs, show_edge, sparse);
if (!fn_show_object)
fn_show_object = show_object;
@@ -3155,11 +3293,11 @@ static void get_object_list(int ac, const char **av)
}
if (keep_unreachable)
- add_objects_in_unpacked_packs(&revs);
+ add_objects_in_unpacked_packs();
if (pack_loose_unreachable)
add_unreachable_loose_objects();
if (unpack_unreachable)
- loosen_unused_packed_objects(&revs);
+ loosen_unused_packed_objects();
oid_array_clear(&recent_objects);
}
@@ -3242,9 +3380,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
OPT_BOOL(0, "all-progress-implied",
&all_progress_implied,
N_("similar to --all-progress when progress meter is shown")),
- { OPTION_CALLBACK, 0, "index-version", NULL, N_("<version>[,<offset>]"),
+ OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"),
N_("write the pack index file in the specified idx format version"),
- PARSE_OPT_NONEG, option_parse_index_version },
+ PARSE_OPT_NONEG, option_parse_index_version),
OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,
N_("maximum size of each output pack file")),
OPT_BOOL(0, "local", &local,
@@ -3289,9 +3427,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
N_("keep unreachable objects")),
OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable,
N_("pack loose unreachable objects")),
- { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"),
+ OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"),
N_("unpack unreachable objects newer than <time>"),
- PARSE_OPT_OPTARG, option_parse_unpack_unreachable },
+ PARSE_OPT_OPTARG, option_parse_unpack_unreachable),
+ OPT_BOOL(0, "sparse", &sparse,
+ N_("use the sparse reachability algorithm")),
OPT_BOOL(0, "thin", &thin,
N_("create thin packs")),
OPT_BOOL(0, "shallow", &shallow,
@@ -3306,12 +3446,17 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
N_("do not hide commits by grafts"), 0),
OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,
N_("use a bitmap index if available to speed up counting objects")),
- OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
- N_("write a bitmap index together with the pack index")),
+ OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index,
+ N_("write a bitmap index together with the pack index"),
+ WRITE_BITMAP_TRUE),
+ OPT_SET_INT_F(0, "write-bitmap-index-quiet",
+ &write_bitmap_index,
+ N_("write a bitmap index if possible"),
+ WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
- { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
+ OPT_CALLBACK_F(0, "missing", NULL, N_("action"),
N_("handling for missing objects"), PARSE_OPT_NONEG,
- option_parse_missing_action },
+ option_parse_missing_action),
OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
N_("do not pack objects in promisor packfiles")),
OPT_BOOL(0, "delta-islands", &use_delta_islands,
@@ -3324,6 +3469,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
read_replace_refs = 0;
+ sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1);
+ prepare_repo_settings(the_repository);
+ if (sparse < 0)
+ sparse = the_repository->settings.pack_use_sparse;
+
reset_pack_idx_option(&pack_idx_opts);
git_config(git_pack_config, NULL);
@@ -3415,7 +3565,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (filter_options.choice) {
if (!pack_to_stdout)
die(_("cannot use --filter without --stdout"));
- use_bitmap_index = 0;
}
/*
@@ -3470,7 +3619,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
}
}
- prepare_packing_data(&to_pack);
+ trace2_region_enter("pack-objects", "enumerate-objects",
+ the_repository);
+ prepare_packing_data(the_repository, &to_pack);
if (progress)
progress_state = start_progress(_("Enumerating objects"), 0);
@@ -3484,16 +3635,29 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (include_tag && nr_result)
for_each_ref(add_ref_tag, NULL);
stop_progress(&progress_state);
+ trace2_region_leave("pack-objects", "enumerate-objects",
+ the_repository);
if (non_empty && !nr_result)
return 0;
- if (nr_result)
+ if (nr_result) {
+ trace2_region_enter("pack-objects", "prepare-pack",
+ the_repository);
prepare_pack(window, depth);
+ trace2_region_leave("pack-objects", "prepare-pack",
+ the_repository);
+ }
+
+ trace2_region_enter("pack-objects", "write-pack-file", the_repository);
write_pack_file();
+ trace2_region_leave("pack-objects", "write-pack-file", the_repository);
+
if (progress)
fprintf_ln(stderr,
_("Total %"PRIu32" (delta %"PRIu32"),"
- " reused %"PRIu32" (delta %"PRIu32")"),
- written, written_delta, reused, reused_delta);
+ " reused %"PRIu32" (delta %"PRIu32"),"
+ " pack-reused %"PRIu32),
+ written, written_delta, reused, reused_delta,
+ reuse_packfile_objects);
return 0;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-21 14:21:26 +0000