diff options
Diffstat (limited to 'builtin/pack-objects.c')
| -rw-r--r-- | builtin/pack-objects.c | 628 |
1 files changed, 469 insertions, 159 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 4af92f113c..f672225def 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -23,10 +23,11 @@ #include "reachable.h" #include "sha1-array.h" #include "argv-array.h" +#include "mru.h" static const char *pack_usage[] = { - N_("git pack-objects --stdout [options...] [< ref-list | < object-list]"), - N_("git pack-objects [options...] base-name [< ref-list | < object-list]"), + N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), + N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), NULL }; @@ -43,8 +44,10 @@ static uint32_t nr_result, nr_written; static int non_empty; static int reuse_delta = 1, reuse_object = 1; static int keep_unreachable, unpack_unreachable, include_tag; -static unsigned long unpack_unreachable_expiration; +static timestamp_t unpack_unreachable_expiration; +static int pack_loose_unreachable; static int local; +static int have_non_local_packs; static int incremental; static int ignore_packed_keep; static int allow_ofs_delta; @@ -58,14 +61,13 @@ static int delta_search_threads; static int pack_to_stdout; static int num_preferred_base; static struct progress *progress_state; -static int pack_compression_level = Z_DEFAULT_COMPRESSION; -static int pack_compression_seen; static struct packed_git *reuse_packfile; static uint32_t reuse_packfile_objects; static off_t reuse_packfile_offset; -static int use_bitmap_index = 1; +static int use_bitmap_index_default = 1; +static int use_bitmap_index = -1; static int write_bitmap_index; static uint16_t write_bitmap_options; @@ -104,12 +106,14 @@ static void *get_delta(struct object_entry *entry) void *buf, *base_buf, *delta_buf; enum object_type type; - buf = read_sha1_file(entry->idx.sha1, &type, &size); + buf = read_sha1_file(entry->idx.oid.hash, &type, &size); if (!buf) - die("unable to read %s", sha1_to_hex(entry->idx.sha1)); - base_buf = read_sha1_file(entry->delta->idx.sha1, &type, &base_size); + die("unable to read %s", oid_to_hex(&entry->idx.oid)); + base_buf = read_sha1_file(entry->delta->idx.oid.hash, &type, + &base_size); if (!base_buf) - die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1)); + die("unable to read %s", + oid_to_hex(&entry->delta->idx.oid)); delta_buf = diff_delta(base_buf, base_size, buf, size, &delta_size, 0); if (!delta_buf || delta_size != entry->delta_size) @@ -237,7 +241,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent unsigned long limit, int usable_delta) { unsigned long size, datalen; - unsigned char header[10], dheader[10]; + unsigned char header[MAX_PACK_OBJECT_HEADER], + dheader[MAX_PACK_OBJECT_HEADER]; unsigned hdrlen; enum object_type type; void *buf; @@ -246,12 +251,14 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent if (!usable_delta) { if (entry->type == OBJ_BLOB && entry->size > big_file_threshold && - (st = open_istream(entry->idx.sha1, &type, &size, NULL)) != NULL) + (st = open_istream(entry->idx.oid.hash, &type, &size, NULL)) != NULL) buf = NULL; else { - buf = read_sha1_file(entry->idx.sha1, &type, &size); + buf = read_sha1_file(entry->idx.oid.hash, &type, + &size); if (!buf) - die(_("unable to read %s"), sha1_to_hex(entry->idx.sha1)); + die(_("unable to read %s"), + oid_to_hex(&entry->idx.oid)); } /* * make sure no cached delta data remains from a @@ -284,7 +291,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent * The object header is a byte of 'type' followed by zero or * more bytes of length. */ - hdrlen = encode_in_pack_object_header(type, size, header); + hdrlen = encode_in_pack_object_header(header, sizeof(header), + type, size); if (type == OBJ_OFS_DELTA) { /* @@ -318,7 +326,7 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent return 0; } sha1write(f, header, hdrlen); - sha1write(f, entry->delta->idx.sha1, 20); + sha1write(f, entry->delta->idx.oid.hash, 20); hdrlen += 20; } else { if (limit && hdrlen + datalen + 20 >= limit) { @@ -330,7 +338,7 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent sha1write(f, header, hdrlen); } if (st) { - datalen = write_large_blob_data(st, f, entry->idx.sha1); + datalen = write_large_blob_data(st, f, entry->idx.oid.hash); close_istream(st); } else { sha1write(f, buf, datalen); @@ -341,29 +349,32 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent } /* Return 0 if we will bust the pack-size limit */ -static unsigned long write_reuse_object(struct sha1file *f, struct object_entry *entry, - unsigned long limit, int usable_delta) +static off_t write_reuse_object(struct sha1file *f, struct object_entry *entry, + unsigned long limit, int usable_delta) { struct packed_git *p = entry->in_pack; struct pack_window *w_curs = NULL; struct revindex_entry *revidx; off_t offset; enum object_type type = entry->type; - unsigned long datalen; - unsigned char header[10], dheader[10]; + off_t datalen; + unsigned char header[MAX_PACK_OBJECT_HEADER], + dheader[MAX_PACK_OBJECT_HEADER]; unsigned hdrlen; if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - hdrlen = encode_in_pack_object_header(type, entry->size, header); + hdrlen = encode_in_pack_object_header(header, sizeof(header), + type, entry->size); offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); datalen = revidx[1].offset - offset; if (!pack_to_stdout && p->index_version > 1 && check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { - error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + error("bad packed object CRC for %s", + oid_to_hex(&entry->idx.oid)); unuse_pack(&w_curs); return write_no_reuse_object(f, entry, limit, usable_delta); } @@ -373,7 +384,8 @@ static unsigned long write_reuse_object(struct sha1file *f, struct object_entry if (!pack_to_stdout && p->index_version == 1 && check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { - error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); + error("corrupt packed object for %s", + oid_to_hex(&entry->idx.oid)); unuse_pack(&w_curs); return write_no_reuse_object(f, entry, limit, usable_delta); } @@ -398,7 +410,7 @@ static unsigned long write_reuse_object(struct sha1file *f, struct object_entry return 0; } sha1write(f, header, hdrlen); - sha1write(f, entry->delta->idx.sha1, 20); + sha1write(f, entry->delta->idx.oid.hash, 20); hdrlen += 20; reused_delta++; } else { @@ -415,11 +427,12 @@ static unsigned long write_reuse_object(struct sha1file *f, struct object_entry } /* Return 0 if we will bust the pack-size limit */ -static unsigned long write_object(struct sha1file *f, - struct object_entry *entry, - off_t write_offset) +static off_t write_object(struct sha1file *f, + struct object_entry *entry, + off_t write_offset) { - unsigned long limit, len; + unsigned long limit; + off_t len; int usable_delta, to_reuse; if (!pack_to_stdout) @@ -491,7 +504,7 @@ static enum write_one_status write_one(struct sha1file *f, struct object_entry *e, off_t *offset) { - unsigned long size; + off_t size; int recursing; /* @@ -502,7 +515,7 @@ static enum write_one_status write_one(struct sha1file *f, recursing = (e->idx.offset == 1); if (recursing) { warning("recursive delta detected for object %s", - sha1_to_hex(e->idx.sha1)); + oid_to_hex(&e->idx.oid)); return WRITE_ONE_RECURSIVE; } else if (e->idx.offset || e->preferred_base) { /* offset is non zero if object is written already. */ @@ -540,11 +553,11 @@ static enum write_one_status write_one(struct sha1file *f, return WRITE_ONE_WRITTEN; } -static int mark_tagged(const char *path, const unsigned char *sha1, int flag, +static int mark_tagged(const char *path, const struct object_id *oid, int flag, void *cb_data) { unsigned char peeled[20]; - struct object_entry *entry = packlist_find(&to_pack, sha1, NULL); + struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); if (entry) entry->tagged = 1; @@ -624,7 +637,7 @@ static struct object_entry **compute_write_order(void) { unsigned int i, wo_end, last_untagged; - struct object_entry **wo = xmalloc(to_pack.nr_objects * sizeof(*wo)); + struct object_entry **wo; struct object_entry *objects = to_pack.objects; for (i = 0; i < to_pack.nr_objects; i++) { @@ -657,6 +670,7 @@ static struct object_entry **compute_write_order(void) * Give the objects in the original recency order until * we see a tagged tip. */ + ALLOC_ARRAY(wo, to_pack.nr_objects); for (i = wo_end = 0; i < to_pack.nr_objects; i++) { if (objects[i].tagged) break; @@ -714,7 +728,7 @@ static off_t write_reused_pack(struct sha1file *f) if (!is_pack_valid(reuse_packfile)) die("packfile is invalid: %s", reuse_packfile->pack_name); - fd = git_open_noatime(reuse_packfile->pack_name); + fd = git_open(reuse_packfile->pack_name); if (fd < 0) die_errno("unable to open packfile for reuse: %s", reuse_packfile->pack_name); @@ -758,6 +772,10 @@ static off_t write_reused_pack(struct sha1file *f) return reuse_packfile_offset - sizeof(struct pack_header); } +static const char no_split_warning[] = N_( +"disabling bitmap writing, packs are split due to pack.packSizeLimit" +); + static void write_pack_file(void) { uint32_t i = 0, j; @@ -769,7 +787,7 @@ static void write_pack_file(void) if (progress > pack_to_stdout) progress_state = start_progress(_("Writing objects"), nr_result); - written_list = xmalloc(to_pack.nr_objects * sizeof(*written_list)); + ALLOC_ARRAY(written_list, to_pack.nr_objects); write_order = compute_write_order(); do { @@ -812,7 +830,10 @@ static void write_pack_file(void) fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written, sha1, offset); close(fd); - write_bitmap_index = 0; + if (write_bitmap_index) { + warning(_(no_split_warning)); + write_bitmap_index = 0; + } } if (!pack_to_stdout) { @@ -827,8 +848,7 @@ static void write_pack_file(void) * to preserve this property. */ if (stat(pack_tmp_name, &st) < 0) { - warning("failed to stat %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed to stat %s", pack_tmp_name); } else if (!last_mtime) { last_mtime = st.st_mtime; } else { @@ -836,8 +856,7 @@ static void write_pack_file(void) utb.actime = st.st_atime; utb.modtime = --last_mtime; if (utime(pack_tmp_name, &utb) < 0) - warning("failed utime() on %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed utime() on %s", pack_tmp_name); } strbuf_addf(&tmpname, "%s-", base_name); @@ -885,24 +904,15 @@ static void write_pack_file(void) written, nr_result); } -static void setup_delta_attr_check(struct git_attr_check *check) -{ - static struct git_attr *attr_delta; - - if (!attr_delta) - attr_delta = git_attr("delta"); - - check[0].attr = attr_delta; -} - static int no_try_delta(const char *path) { - struct git_attr_check check[1]; + static struct attr_check *check; - setup_delta_attr_check(check); - if (git_check_attr(path, ARRAY_SIZE(check), check)) + if (!check) + check = attr_check_initl("delta", NULL); + if (git_check_attr(path, check)) return 0; - if (ATTR_FALSE(check->value)) + if (ATTR_FALSE(check->items[0].value)) return 1; return 0; } @@ -936,29 +946,80 @@ static int have_duplicate_entry(const unsigned char *sha1, return 1; } +static int want_found_object(int exclude, struct packed_git *p) +{ + if (exclude) + return 1; + if (incremental) + return 0; + + /* + * When asked to do --local (do not include an object that appears in a + * pack we borrow from elsewhere) or --honor-pack-keep (do not include + * an object that appears in a pack marked with .keep), finding a pack + * that matches the criteria is sufficient for us to decide to omit it. + * However, even if this pack does not satisfy the criteria, we need to + * make sure no copy of this object appears in _any_ pack that makes us + * to omit the object, so we need to check all the packs. + * + * We can however first check whether these options can possible matter; + * if they do not matter we know we want the object in generated pack. + * Otherwise, we signal "-1" at the end to tell the caller that we do + * not know either way, and it needs to check more packs. + */ + if (!ignore_packed_keep && + (!local || !have_non_local_packs)) + return 1; + + if (local && !p->pack_local) + return 0; + if (ignore_packed_keep && p->pack_local && p->pack_keep) + return 0; + + /* we don't know yet; keep looking for more packs */ + return -1; +} + /* * Check whether we want the object in the pack (e.g., we do not want * objects found in non-local stores if the "--local" option was used). * - * As a side effect of this check, we will find the packed version of this - * object, if any. We therefore pass out the pack information to avoid having - * to look it up again later. + * If the caller already knows an existing pack it wants to take the object + * from, that is passed in *found_pack and *found_offset; otherwise this + * function finds if there is any pack that has the object and returns the pack + * and its offset in these variables. */ static int want_object_in_pack(const unsigned char *sha1, int exclude, struct packed_git **found_pack, off_t *found_offset) { - struct packed_git *p; + struct mru_entry *entry; + int want; if (!exclude && local && has_loose_object_nonlocal(sha1)) return 0; - *found_pack = NULL; - *found_offset = 0; + /* + * If we already know the pack object lives in, start checks from that + * pack - in the usual case when neither --local was given nor .keep files + * are present we will determine the answer right now. + */ + if (*found_pack) { + want = want_found_object(exclude, *found_pack); + if (want != -1) + return want; + } + + for (entry = packed_git_mru->head; entry; entry = entry->next) { + struct packed_git *p = entry->item; + off_t offset; + + if (p == *found_pack) + offset = *found_offset; + else + offset = find_pack_entry_one(sha1, p); - for (p = packed_git; p; p = p->next) { - off_t offset = find_pack_entry_one(sha1, p); if (offset) { if (!*found_pack) { if (!is_pack_valid(p)) @@ -966,14 +1027,11 @@ static int want_object_in_pack(const unsigned char *sha1, *found_offset = offset; *found_pack = p; } - if (exclude) - return 1; - if (incremental) - return 0; - if (local && !p->pack_local) - return 0; - if (ignore_packed_keep && p->pack_local && p->pack_keep) - return 0; + want = want_found_object(exclude, p); + if (!exclude && want > 0) + mru_mark(packed_git_mru, entry); + if (want != -1) + return want; } } @@ -1014,8 +1072,8 @@ static const char no_closure_warning[] = N_( static int add_object_entry(const unsigned char *sha1, enum object_type type, const char *name, int exclude) { - struct packed_git *found_pack; - off_t found_offset; + struct packed_git *found_pack = NULL; + off_t found_offset = 0; uint32_t index_pos; if (have_duplicate_entry(sha1, exclude, &index_pos)) @@ -1048,6 +1106,9 @@ static int add_object_entry_from_bitmap(const unsigned char *sha1, if (have_duplicate_entry(sha1, 0, &index_pos)) return 0; + if (!want_object_in_pack(sha1, 0, &pack, &offset)) + return 0; + create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset); display_progress(progress_state, nr_result); @@ -1185,7 +1246,7 @@ static void add_pbase_object(struct tree_desc *tree, if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.sha1, + add_object_entry(entry.oid->hash, object_type(entry.mode), fullname, 1); return; @@ -1196,7 +1257,7 @@ static void add_pbase_object(struct tree_desc *tree, const char *down = name+cmplen+1; int downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.sha1); + tree = pbase_tree_get(entry.oid->hash); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); @@ -1377,7 +1438,7 @@ static void check_object(struct object_entry *entry) ofs += 1; if (!ofs || MSB(ofs, 7)) { error("delta base offset overflow in pack for %s", - sha1_to_hex(entry->idx.sha1)); + oid_to_hex(&entry->idx.oid)); goto give_up; } c = buf[used_0++]; @@ -1386,7 +1447,7 @@ static void check_object(struct object_entry *entry) ofs = entry->in_pack_offset - ofs; if (ofs <= 0 || ofs >= entry->in_pack_offset) { error("delta base offset out of bound for %s", - sha1_to_hex(entry->idx.sha1)); + oid_to_hex(&entry->idx.oid)); goto give_up; } if (reuse_delta && !entry->preferred_base) { @@ -1443,7 +1504,7 @@ static void check_object(struct object_entry *entry) unuse_pack(&w_curs); } - entry->type = sha1_object_info(entry->idx.sha1, &entry->size); + entry->type = sha1_object_info(entry->idx.oid.hash, &entry->size); /* * The error condition is checked in prepare_pack(). This is * to permit a missing preferred base object to be ignored @@ -1459,7 +1520,7 @@ static int pack_offset_sort(const void *_a, const void *_b) /* avoid filesystem trashing with loose objects */ if (!a->in_pack && !b->in_pack) - return hashcmp(a->idx.sha1, b->idx.sha1); + return oidcmp(&a->idx.oid, &b->idx.oid); if (a->in_pack < b->in_pack) return -1; @@ -1469,6 +1530,171 @@ static int pack_offset_sort(const void *_a, const void *_b) (a->in_pack_offset > b->in_pack_offset); } +/* + * Drop an on-disk delta we were planning to reuse. Naively, this would + * just involve blanking out the "delta" field, but we have to deal + * with some extra book-keeping: + * + * 1. Removing ourselves from the delta_sibling linked list. + * + * 2. Updating our size/type to the non-delta representation. These were + * either not recorded initially (size) or overwritten with the delta type + * (type) when check_object() decided to reuse the delta. + * + * 3. Resetting our delta depth, as we are now a base object. + */ +static void drop_reused_delta(struct object_entry *entry) +{ + struct object_entry **p = &entry->delta->delta_child; + struct object_info oi = OBJECT_INFO_INIT; + + while (*p) { + if (*p == entry) + *p = (*p)->delta_sibling; + else + p = &(*p)->delta_sibling; + } + entry->delta = NULL; + entry->depth = 0; + + oi.sizep = &entry->size; + oi.typep = &entry->type; + if (packed_object_info(entry->in_pack, entry->in_pack_offset, &oi) < 0) { + /* + * We failed to get the info from this pack for some reason; + * fall back to sha1_object_info, which may find another copy. + * And if that fails, the error will be recorded in entry->type + * and dealt with in prepare_pack(). + */ + entry->type = sha1_object_info(entry->idx.oid.hash, + &entry->size); + } +} + +/* + * Follow the chain of deltas from this entry onward, throwing away any links + * that cause us to hit a cycle (as determined by the DFS state flags in + * the entries). + * + * We also detect too-long reused chains that would violate our --depth + * limit. + */ +static void break_delta_chains(struct object_entry *entry) +{ + /* + * The actual depth of each object we will write is stored as an int, + * as it cannot exceed our int "depth" limit. But before we break + * changes based no that limit, we may potentially go as deep as the + * number of objects, which is elsewhere bounded to a uint32_t. + */ + uint32_t total_depth; + struct object_entry *cur, *next; + + for (cur = entry, total_depth = 0; + cur; + cur = cur->delta, total_depth++) { + if (cur->dfs_state == DFS_DONE) { + /* + * We've already seen this object and know it isn't + * part of a cycle. We do need to append its depth + * to our count. + */ + total_depth += cur->depth; + break; + } + + /* + * We break cycles before looping, so an ACTIVE state (or any + * other cruft which made its way into the state variable) + * is a bug. + */ + if (cur->dfs_state != DFS_NONE) + die("BUG: confusing delta dfs state in first pass: %d", + cur->dfs_state); + + /* + * Now we know this is the first time we've seen the object. If + * it's not a delta, we're done traversing, but we'll mark it + * done to save time on future traversals. + */ + if (!cur->delta) { + cur->dfs_state = DFS_DONE; + break; + } + + /* + * Mark ourselves as active and see if the next step causes + * us to cycle to another active object. It's important to do + * this _before_ we loop, because it impacts where we make the + * cut, and thus how our total_depth counter works. + * E.g., We may see a partial loop like: + * + * A -> B -> C -> D -> B + * + * Cutting B->C breaks the cycle. But now the depth of A is + * only 1, and our total_depth counter is at 3. The size of the + * error is always one less than the size of the cycle we + * broke. Commits C and D were "lost" from A's chain. + * + * If we instead cut D->B, then the depth of A is correct at 3. + * We keep all commits in the chain that we examined. + */ + cur->dfs_state = DFS_ACTIVE; + if (cur->delta->dfs_state == DFS_ACTIVE) { + drop_reused_delta(cur); + cur->dfs_state = DFS_DONE; + break; + } + } + + /* + * And now that we've gone all the way to the bottom of the chain, we + * need to clear the active flags and set the depth fields as + * appropriate. Unlike the loop above, which can quit when it drops a + * delta, we need to keep going to look for more depth cuts. So we need + * an extra "next" pointer to keep going after we reset cur->delta. + */ + for (cur = entry; cur; cur = next) { + next = cur->delta; + + /* + * We should have a chain of zero or more ACTIVE states down to + * a final DONE. We can quit after the DONE, because either it + * has no bases, or we've already handled them in a previous + * call. + */ + if (cur->dfs_state == DFS_DONE) + break; + else if (cur->dfs_state != DFS_ACTIVE) + die("BUG: confusing delta dfs state in second pass: %d", + cur->dfs_state); + + /* + * If the total_depth is more than depth, then we need to snip + * the chain into two or more smaller chains that don't exceed + * the maximum depth. Most of the resulting chains will contain + * (depth + 1) entries (i.e., depth deltas plus one base), and + * the last chain (i.e., the one containing entry) will contain + * whatever entries are left over, namely + * (total_depth % (depth + 1)) of them. + * + * Since we are iterating towards decreasing depth, we need to + * decrement total_depth as we go, and we need to write to the + * entry what its final depth will be after all of the + * snipping. Since we're snipping into chains of length (depth + * + 1) entries, the final depth of an entry will be its + * original depth modulo (depth + 1). Any time we encounter an + * entry whose final depth is supposed to be zero, we snip it + * from its delta base, thereby making it so. + */ + cur->depth = (total_depth--) % (depth + 1); + if (!cur->depth) + drop_reused_delta(cur); + + cur->dfs_state = DFS_DONE; + } +} + static void get_object_details(void) { uint32_t i; @@ -1477,7 +1703,7 @@ static void get_object_details(void) sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *)); for (i = 0; i < to_pack.nr_objects; i++) sorted_by_offset[i] = to_pack.objects + i; - qsort(sorted_by_offset, to_pack.nr_objects, sizeof(*sorted_by_offset), pack_offset_sort); + QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort); for (i = 0; i < to_pack.nr_objects; i++) { struct object_entry *entry = sorted_by_offset[i]; @@ -1486,6 +1712,13 @@ static void get_object_details(void) entry->no_try_delta = 1; } + /* + * This must happen in a second pass, since we rely on the delta + * information for the whole list being completed. + */ + for (i = 0; i < to_pack.nr_objects; i++) + break_delta_chains(&to_pack.objects[i]); + free(sorted_by_offset); } @@ -1626,26 +1859,29 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, /* Load data if not already done */ if (!trg->data) { read_lock(); - trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz); + trg->data = read_sha1_file(trg_entry->idx.oid.hash, &type, + &sz); read_unlock(); if (!trg->data) die("object %s cannot be read", - sha1_to_hex(trg_entry->idx.sha1)); + oid_to_hex(&trg_entry->idx.oid)); if (sz != trg_size) die("object %s inconsistent object length (%lu vs %lu)", - sha1_to_hex(trg_entry->idx.sha1), sz, trg_size); + oid_to_hex(&trg_entry->idx.oid), sz, + trg_size); *mem_usage += sz; } if (!src->data) { read_lock(); - src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz); + src->data = read_sha1_file(src_entry->idx.oid.hash, &type, + &sz); read_unlock(); if (!src->data) { if (src_entry->preferred_base) { static int warned = 0; if (!warned++) warning("object %s cannot be read", - sha1_to_hex(src_entry->idx.sha1)); + oid_to_hex(&src_entry->idx.oid)); /* * Those objects are not included in the * resulting pack. Be resilient and ignore @@ -1655,11 +1891,12 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, return 0; } die("object %s cannot be read", - sha1_to_hex(src_entry->idx.sha1)); + oid_to_hex(&src_entry->idx.oid)); } if (sz != src_size) die("object %s inconsistent object length (%lu vs %lu)", - sha1_to_hex(src_entry->idx.sha1), sz, src_size); + oid_to_hex(&src_entry->idx.oid), sz, + src_size); *mem_usage += sz; } if (!src->index) { @@ -2097,14 +2334,43 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, #define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p) #endif -static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, void *cb_data) +static void add_tag_chain(const struct object_id *oid) { - unsigned char peeled[20]; + struct tag *tag; + + /* + * We catch duplicates already in add_object_entry(), but we'd + * prefer to do this extra check to avoid having to parse the + * tag at all if we already know that it's being packed (e.g., if + * it was included via bitmaps, we would not have parsed it + * previously). + */ + if (packlist_find(&to_pack, oid->hash, NULL)) + return; + + tag = lookup_tag(oid); + while (1) { + if (!tag || parse_tag(tag) || !tag->tagged) + die("unable to pack objects reachable from tag %s", + oid_to_hex(oid)); + + add_object_entry(tag->object.oid.hash, OBJ_TAG, NULL, 0); + + if (tag->tagged->type != OBJ_TAG) + return; + + tag = (struct tag *)tag->tagged; + } +} + +static int add_ref_tag(const char *path, const struct object_id *oid, int flag, void *cb_data) +{ + struct object_id peeled; if (starts_with(path, "refs/tags/") && /* is a tag? */ - !peel_ref(path, peeled) && /* peelable? */ - packlist_find(&to_pack, peeled, NULL)) /* object packed? */ - add_object_entry(sha1, OBJ_TAG, NULL, 0); + !peel_ref(path, peeled.hash) && /* peelable? */ + packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ + add_tag_chain(oid); return 0; } @@ -2129,7 +2395,7 @@ static void prepare_pack(int window, int depth) if (!to_pack.nr_objects || !window || !depth) return; - delta_list = xmalloc(to_pack.nr_objects * sizeof(*delta_list)); + ALLOC_ARRAY(delta_list, to_pack.nr_objects); nr_deltas = n = 0; for (i = 0; i < to_pack.nr_objects; i++) { @@ -2151,7 +2417,7 @@ static void prepare_pack(int window, int depth) nr_deltas++; if (entry->type < 0) die("unable to get type of object %s", - sha1_to_hex(entry->idx.sha1)); + oid_to_hex(&entry->idx.oid)); } else { if (entry->type < 0) { /* @@ -2170,7 +2436,7 @@ static void prepare_pack(int window, int depth) if (progress) progress_state = start_progress(_("Compressing objects"), nr_deltas); - qsort(delta_list, n, sizeof(*delta_list), type_size_sort); + QSORT(delta_list, n, type_size_sort); ll_find_deltas(delta_list, n, window+1, depth, &nr_done); stop_progress(&progress_state); if (nr_done != nr_deltas) @@ -2193,16 +2459,6 @@ static int git_pack_config(const char *k, const char *v, void *cb) depth = git_config_int(k, v); return 0; } - if (!strcmp(k, "pack.compression")) { - int level = git_config_int(k, v); - if (level == -1) - level = Z_DEFAULT_COMPRESSION; - else if (level < 0 || level > Z_BEST_COMPRESSION) - die("bad pack compression level %d", level); - pack_compression_level = level; - pack_compression_seen = 1; - return 0; - } if (!strcmp(k, "pack.deltacachesize")) { max_delta_cache_size = git_config_int(k, v); return 0; @@ -2218,7 +2474,7 @@ static int git_pack_config(const char *k, const char *v, void *cb) write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE; } if (!strcmp(k, "pack.usebitmaps")) { - use_bitmap_index = git_config_bool(k, v); + use_bitmap_index_default = git_config_bool(k, v); return 0; } if (!strcmp(k, "pack.threads")) { @@ -2227,8 +2483,10 @@ static int git_pack_config(const char *k, const char *v, void *cb) die("invalid number of threads specified (%d)", delta_search_threads); #ifdef NO_PTHREADS - if (delta_search_threads != 1) + if (delta_search_threads != 1) { warning("no threads support, ignoring %s", k); + delta_search_threads = 0; + } #endif return 0; } @@ -2277,7 +2535,7 @@ static void read_object_list_from_stdin(void) static void show_commit(struct commit *commit, void *data) { - add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0); + add_object_entry(commit->object.oid.hash, OBJ_COMMIT, NULL, 0); commit->object.flags |= OBJECT_ADDED; if (write_bitmap_index) @@ -2287,13 +2545,13 @@ static void show_commit(struct commit *commit, void *data) static void show_object(struct object *obj, const char *name, void *data) { add_preferred_base_object(name); - add_object_entry(obj->sha1, obj->type, name, 0); + add_object_entry(obj->oid.hash, obj->type, name, 0); obj->flags |= OBJECT_ADDED; } static void show_edge(struct commit *commit) { - add_preferred_base(commit->object.sha1); + add_preferred_base(commit->object.oid.hash); } struct in_pack_object { @@ -2309,7 +2567,7 @@ struct in_pack { static void mark_in_pack_object(struct object *object, struct packed_git *p, struct in_pack *in_pack) { - in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->sha1, p); + in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->oid.hash, p); in_pack->array[in_pack->nr].object = object; in_pack->nr++; } @@ -2328,7 +2586,7 @@ static int ofscmp(const void *a_, const void *b_) else if (a->offset > b->offset) return 1; else - return hashcmp(a->object->sha1, b->object->sha1); + return oidcmp(&a->object->oid, &b->object->oid); } static void add_objects_in_unpacked_packs(struct rev_info *revs) @@ -2362,16 +2620,41 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs) } if (in_pack.nr) { - qsort(in_pack.array, in_pack.nr, sizeof(in_pack.array[0]), - ofscmp); + QSORT(in_pack.array, in_pack.nr, ofscmp); for (i = 0; i < in_pack.nr; i++) { struct object *o = in_pack.array[i].object; - add_object_entry(o->sha1, o->type, "", 0); + add_object_entry(o->oid.hash, o->type, "", 0); } } free(in_pack.array); } +static int add_loose_object(const struct object_id *oid, const char *path, + void *data) +{ + enum object_type type = sha1_object_info(oid->hash, NULL); + + if (type < 0) { + warning("loose object at %s could not be examined", path); + return 0; + } + + add_object_entry(oid->hash, type, "", 0); + return 0; +} + +/* + * We actually don't even have to worry about reachability here. + * add_object_entry will weed out duplicates, so we just add every + * loose object we find. + */ +static void add_unreachable_loose_objects(void) +{ + for_each_loose_file_in_objdir(get_object_directory(), + add_loose_object, + NULL, NULL, NULL); +} + static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) { static struct packed_git *last_found = (void *)1; @@ -2402,16 +2685,16 @@ static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) * * This is filled by get_object_list. */ -static struct sha1_array recent_objects; +static struct oid_array recent_objects; -static int loosened_object_can_be_discarded(const unsigned char *sha1, - unsigned long mtime) +static int loosened_object_can_be_discarded(const struct object_id *oid, + timestamp_t mtime) { if (!unpack_unreachable_expiration) return 0; if (mtime > unpack_unreachable_expiration) return 0; - if (sha1_array_lookup(&recent_objects, sha1) >= 0) + if (oid_array_lookup(&recent_objects, oid) >= 0) return 0; return 1; } @@ -2420,7 +2703,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs) { struct packed_git *p; uint32_t i; - const unsigned char *sha1; + struct object_id oid; for (p = packed_git; p; p = p->next) { if (!p->pack_local || p->pack_keep) @@ -2430,24 +2713,28 @@ static void loosen_unused_packed_objects(struct rev_info *revs) die("cannot open pack index"); for (i = 0; i < p->num_objects; i++) { - sha1 = nth_packed_object_sha1(p, i); - if (!packlist_find(&to_pack, sha1, NULL) && - !has_sha1_pack_kept_or_nonlocal(sha1) && - !loosened_object_can_be_discarded(sha1, p->mtime)) - if (force_object_loose(sha1, p->mtime)) + nth_packed_object_oid(&oid, p, i); + if (!packlist_find(&to_pack, oid.hash, NULL) && + !has_sha1_pack_kept_or_nonlocal(oid.hash) && + !loosened_object_can_be_discarded(&oid, p->mtime)) + if (force_object_loose(oid.hash, p->mtime)) die("unable to force loose object"); } } } /* - * This tracks any options which a reader of the pack might - * not understand, and which would therefore prevent blind reuse - * of what we have on disk. + * This tracks any options which pack-reuse code expects to be on, or which a + * reader of the pack might not understand, and which would therefore prevent + * blind reuse of what we have on disk. */ static int pack_options_allow_reuse(void) { - return allow_ofs_delta; + return pack_to_stdout && + allow_ofs_delta && + !ignore_packed_keep && + (!local || !have_non_local_packs) && + !incremental; } static int get_object_list_from_bitmap(struct rev_info *revs) @@ -2473,12 +2760,12 @@ static void record_recent_object(struct object *obj, const char *name, void *data) { - sha1_array_append(&recent_objects, obj->sha1); + oid_array_append(&recent_objects, &obj->oid); } static void record_recent_commit(struct commit *commit, void *data) { - sha1_array_append(&recent_objects, commit->object.sha1); + oid_array_append(&recent_objects, &commit->object.oid); } static void get_object_list(int ac, const char **av) @@ -2507,10 +2794,10 @@ static void get_object_list(int ac, const char **av) continue; } if (starts_with(line, "--shallow ")) { - unsigned char sha1[20]; - if (get_sha1_hex(line + 10, sha1)) + struct object_id oid; + if (get_oid_hex(line + 10, &oid)) die("not an SHA-1 '%s'", line + 10); - register_shallow(sha1); + register_shallow(&oid); use_bitmap_index = 0; continue; } @@ -2541,10 +2828,12 @@ static void get_object_list(int ac, const char **av) if (keep_unreachable) add_objects_in_unpacked_packs(&revs); + if (pack_loose_unreachable) + add_unreachable_loose_objects(); if (unpack_unreachable) loosen_unused_packed_objects(&revs); - sha1_array_clear(&recent_objects); + oid_array_clear(&recent_objects); } static int option_parse_index_version(const struct option *opt, @@ -2577,23 +2866,6 @@ static int option_parse_unpack_unreachable(const struct option *opt, return 0; } -static int option_parse_ulong(const struct option *opt, - const char *arg, int unset) -{ - if (unset) - die(_("option %s does not accept negative form"), - opt->long_name); - - if (!git_parse_ulong(arg, opt->value)) - die(_("unable to parse value '%s' for option %s"), - arg, opt->long_name); - return 0; -} - -#define OPT_ULONG(s, l, v, h) \ - { OPTION_CALLBACK, (s), (l), (v), "n", (h), \ - PARSE_OPT_NONEG, option_parse_ulong } - int cmd_pack_objects(int argc, const char **argv, const char *prefix) { int use_internal_rev_list = 0; @@ -2616,16 +2888,16 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) { OPTION_CALLBACK, 0, "index-version", NULL, N_("version[,offset]"), N_("write the pack index file in the specified idx format version"), 0, option_parse_index_version }, - OPT_ULONG(0, "max-pack-size", &pack_size_limit, - N_("maximum size of each output pack file")), + OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit, + N_("maximum size of each output pack file")), OPT_BOOL(0, "local", &local, N_("ignore borrowed objects from alternate object store")), OPT_BOOL(0, "incremental", &incremental, N_("ignore packed objects")), OPT_INTEGER(0, "window", &window, N_("limit pack window by objects")), - OPT_ULONG(0, "window-memory", &window_memory_limit, - N_("limit pack window by memory in addition to object limit")), + OPT_MAGNITUDE(0, "window-memory", &window_memory_limit, + N_("limit pack window by memory in addition to object limit")), OPT_INTEGER(0, "depth", &depth, N_("maximum length of delta chain allowed in the resulting pack")), OPT_BOOL(0, "reuse-delta", &reuse_delta, @@ -2658,6 +2930,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("include tag objects that refer to objects to be packed")), OPT_BOOL(0, "keep-unreachable", &keep_unreachable, N_("keep unreachable objects")), + OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable, + N_("pack loose unreachable objects")), { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"), N_("unpack unreachable objects newer than <time>"), PARSE_OPT_OPTARG, option_parse_unpack_unreachable }, @@ -2682,8 +2956,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); - if (!pack_compression_seen && core_compression_seen) - pack_compression_level = core_compression_level; progress = isatty(2); argc = parse_options(argc, argv, prefix, pack_objects_options, @@ -2753,7 +3025,23 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (!rev_list_all || !rev_list_reflog || !rev_list_index) unpack_unreachable_expiration = 0; - if (!use_internal_rev_list || !pack_to_stdout || is_repository_shallow()) + /* + * "soft" reasons not to use bitmaps - for on-disk repack by default we want + * + * - to produce good pack (with bitmap index not-yet-packed objects are + * packed in suboptimal order). + * + * - to use more robust pack-generation codepath (avoiding possible + * bugs in bitmap code and possible bitmap index corruption). + */ + if (!pack_to_stdout) + use_bitmap_index_default = 0; + + if (use_bitmap_index < 0) + use_bitmap_index = use_bitmap_index_default; + + /* "hard" reasons not to use bitmaps; these just won't work at all */ + if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow()) use_bitmap_index = 0; if (pack_to_stdout || !rev_list_all) @@ -2763,6 +3051,28 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) progress = 2; prepare_packed_git(); + if (ignore_packed_keep) { + struct packed_git *p; + for (p = packed_git; p; p = p->next) + if (p->pack_local && p->pack_keep) + break; + if (!p) /* no keep-able packs found */ + ignore_packed_keep = 0; + } + if (local) { + /* + * unlike ignore_packed_keep above, we do not want to + * unset "local" based on looking at packs, as it + * also covers non-local objects + */ + struct packed_git *p; + for (p = packed_git; p; p = p->next) { + if (!p->pack_local) { + have_non_local_packs = 1; + break; + } + } + } if (progress) progress_state = start_progress(_("Counting objects"), 0); |