diff options
Diffstat (limited to 'builtin/pack-objects.c')
| -rw-r--r-- | builtin/pack-objects.c | 642 |
1 files changed, 492 insertions, 150 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 238b5021eb..84af7c2324 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -20,10 +20,14 @@ #include "streaming.h" #include "thread-utils.h" #include "pack-bitmap.h" +#include "reachable.h" +#include "sha1-array.h" +#include "argv-array.h" +#include "mru.h" static const char *pack_usage[] = { - N_("git pack-objects --stdout [options...] [< ref-list | < object-list]"), - N_("git pack-objects [options...] base-name [< ref-list | < object-list]"), + N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), + N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), NULL }; @@ -41,7 +45,9 @@ static int non_empty; static int reuse_delta = 1, reuse_object = 1; static int keep_unreachable, unpack_unreachable, include_tag; static unsigned long unpack_unreachable_expiration; +static int pack_loose_unreachable; static int local; +static int have_non_local_packs; static int incremental; static int ignore_packed_keep; static int allow_ofs_delta; @@ -55,14 +61,13 @@ static int delta_search_threads; static int pack_to_stdout; static int num_preferred_base; static struct progress *progress_state; -static int pack_compression_level = Z_DEFAULT_COMPRESSION; -static int pack_compression_seen; static struct packed_git *reuse_packfile; static uint32_t reuse_packfile_objects; static off_t reuse_packfile_offset; -static int use_bitmap_index = 1; +static int use_bitmap_index_default = 1; +static int use_bitmap_index = -1; static int write_bitmap_index; static uint16_t write_bitmap_options; @@ -89,8 +94,7 @@ static void index_commit_for_bitmap(struct commit *commit) { if (indexed_commits_nr >= indexed_commits_alloc) { indexed_commits_alloc = (indexed_commits_alloc + 32) * 2; - indexed_commits = xrealloc(indexed_commits, - indexed_commits_alloc * sizeof(struct commit *)); + REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); } indexed_commits[indexed_commits_nr++] = commit; @@ -123,7 +127,6 @@ static unsigned long do_compress(void **pptr, unsigned long size) void *in, *out; unsigned long maxsize; - memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, pack_compression_level); maxsize = git_deflate_bound(&stream, size); @@ -151,7 +154,6 @@ static unsigned long write_large_blob_data(struct git_istream *st, struct sha1fi unsigned char obuf[1024 * 16]; unsigned long olen = 0; - memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, pack_compression_level); for (;;) { @@ -237,7 +239,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent unsigned long limit, int usable_delta) { unsigned long size, datalen; - unsigned char header[10], dheader[10]; + unsigned char header[MAX_PACK_OBJECT_HEADER], + dheader[MAX_PACK_OBJECT_HEADER]; unsigned hdrlen; enum object_type type; void *buf; @@ -284,7 +287,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent * The object header is a byte of 'type' followed by zero or * more bytes of length. */ - hdrlen = encode_in_pack_object_header(type, size, header); + hdrlen = encode_in_pack_object_header(header, sizeof(header), + type, size); if (type == OBJ_OFS_DELTA) { /* @@ -341,22 +345,24 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent } /* Return 0 if we will bust the pack-size limit */ -static unsigned long write_reuse_object(struct sha1file *f, struct object_entry *entry, - unsigned long limit, int usable_delta) +static off_t write_reuse_object(struct sha1file *f, struct object_entry *entry, + unsigned long limit, int usable_delta) { struct packed_git *p = entry->in_pack; struct pack_window *w_curs = NULL; struct revindex_entry *revidx; off_t offset; enum object_type type = entry->type; - unsigned long datalen; - unsigned char header[10], dheader[10]; + off_t datalen; + unsigned char header[MAX_PACK_OBJECT_HEADER], + dheader[MAX_PACK_OBJECT_HEADER]; unsigned hdrlen; if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - hdrlen = encode_in_pack_object_header(type, entry->size, header); + hdrlen = encode_in_pack_object_header(header, sizeof(header), + type, entry->size); offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); @@ -415,11 +421,12 @@ static unsigned long write_reuse_object(struct sha1file *f, struct object_entry } /* Return 0 if we will bust the pack-size limit */ -static unsigned long write_object(struct sha1file *f, - struct object_entry *entry, - off_t write_offset) +static off_t write_object(struct sha1file *f, + struct object_entry *entry, + off_t write_offset) { - unsigned long limit, len; + unsigned long limit; + off_t len; int usable_delta, to_reuse; if (!pack_to_stdout) @@ -491,7 +498,7 @@ static enum write_one_status write_one(struct sha1file *f, struct object_entry *e, off_t *offset) { - unsigned long size; + off_t size; int recursing; /* @@ -540,11 +547,11 @@ static enum write_one_status write_one(struct sha1file *f, return WRITE_ONE_WRITTEN; } -static int mark_tagged(const char *path, const unsigned char *sha1, int flag, +static int mark_tagged(const char *path, const struct object_id *oid, int flag, void *cb_data) { unsigned char peeled[20]; - struct object_entry *entry = packlist_find(&to_pack, sha1, NULL); + struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); if (entry) entry->tagged = 1; @@ -624,7 +631,7 @@ static struct object_entry **compute_write_order(void) { unsigned int i, wo_end, last_untagged; - struct object_entry **wo = xmalloc(to_pack.nr_objects * sizeof(*wo)); + struct object_entry **wo; struct object_entry *objects = to_pack.objects; for (i = 0; i < to_pack.nr_objects; i++) { @@ -657,6 +664,7 @@ static struct object_entry **compute_write_order(void) * Give the objects in the original recency order until * we see a tagged tip. */ + ALLOC_ARRAY(wo, to_pack.nr_objects); for (i = wo_end = 0; i < to_pack.nr_objects; i++) { if (objects[i].tagged) break; @@ -714,7 +722,7 @@ static off_t write_reused_pack(struct sha1file *f) if (!is_pack_valid(reuse_packfile)) die("packfile is invalid: %s", reuse_packfile->pack_name); - fd = git_open_noatime(reuse_packfile->pack_name); + fd = git_open(reuse_packfile->pack_name); if (fd < 0) die_errno("unable to open packfile for reuse: %s", reuse_packfile->pack_name); @@ -758,6 +766,10 @@ static off_t write_reused_pack(struct sha1file *f) return reuse_packfile_offset - sizeof(struct pack_header); } +static const char no_split_warning[] = N_( +"disabling bitmap writing, packs are split due to pack.packSizeLimit" +); + static void write_pack_file(void) { uint32_t i = 0, j; @@ -769,7 +781,7 @@ static void write_pack_file(void) if (progress > pack_to_stdout) progress_state = start_progress(_("Writing objects"), nr_result); - written_list = xmalloc(to_pack.nr_objects * sizeof(*written_list)); + ALLOC_ARRAY(written_list, to_pack.nr_objects); write_order = compute_write_order(); do { @@ -812,6 +824,10 @@ static void write_pack_file(void) fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written, sha1, offset); close(fd); + if (write_bitmap_index) { + warning(_(no_split_warning)); + write_bitmap_index = 0; + } } if (!pack_to_stdout) { @@ -826,8 +842,7 @@ static void write_pack_file(void) * to preserve this property. */ if (stat(pack_tmp_name, &st) < 0) { - warning("failed to stat %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed to stat %s", pack_tmp_name); } else if (!last_mtime) { last_mtime = st.st_mtime; } else { @@ -835,8 +850,7 @@ static void write_pack_file(void) utb.actime = st.st_atime; utb.modtime = --last_mtime; if (utime(pack_tmp_name, &utb) < 0) - warning("failed utime() on %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed utime() on %s", pack_tmp_name); } strbuf_addf(&tmpname, "%s-", base_name); @@ -884,24 +898,15 @@ static void write_pack_file(void) written, nr_result); } -static void setup_delta_attr_check(struct git_attr_check *check) -{ - static struct git_attr *attr_delta; - - if (!attr_delta) - attr_delta = git_attr("delta"); - - check[0].attr = attr_delta; -} - static int no_try_delta(const char *path) { - struct git_attr_check check[1]; + static struct attr_check *check; - setup_delta_attr_check(check); - if (git_check_attr(path, ARRAY_SIZE(check), check)) + if (!check) + check = attr_check_initl("delta", NULL); + if (git_check_attr(path, check)) return 0; - if (ATTR_FALSE(check->value)) + if (ATTR_FALSE(check->items[0].value)) return 1; return 0; } @@ -935,46 +940,92 @@ static int have_duplicate_entry(const unsigned char *sha1, return 1; } +static int want_found_object(int exclude, struct packed_git *p) +{ + if (exclude) + return 1; + if (incremental) + return 0; + + /* + * When asked to do --local (do not include an object that appears in a + * pack we borrow from elsewhere) or --honor-pack-keep (do not include + * an object that appears in a pack marked with .keep), finding a pack + * that matches the criteria is sufficient for us to decide to omit it. + * However, even if this pack does not satisfy the criteria, we need to + * make sure no copy of this object appears in _any_ pack that makes us + * to omit the object, so we need to check all the packs. + * + * We can however first check whether these options can possible matter; + * if they do not matter we know we want the object in generated pack. + * Otherwise, we signal "-1" at the end to tell the caller that we do + * not know either way, and it needs to check more packs. + */ + if (!ignore_packed_keep && + (!local || !have_non_local_packs)) + return 1; + + if (local && !p->pack_local) + return 0; + if (ignore_packed_keep && p->pack_local && p->pack_keep) + return 0; + + /* we don't know yet; keep looking for more packs */ + return -1; +} + /* * Check whether we want the object in the pack (e.g., we do not want * objects found in non-local stores if the "--local" option was used). * - * As a side effect of this check, we will find the packed version of this - * object, if any. We therefore pass out the pack information to avoid having - * to look it up again later. + * If the caller already knows an existing pack it wants to take the object + * from, that is passed in *found_pack and *found_offset; otherwise this + * function finds if there is any pack that has the object and returns the pack + * and its offset in these variables. */ static int want_object_in_pack(const unsigned char *sha1, int exclude, struct packed_git **found_pack, off_t *found_offset) { - struct packed_git *p; + struct mru_entry *entry; + int want; if (!exclude && local && has_loose_object_nonlocal(sha1)) return 0; - *found_pack = NULL; - *found_offset = 0; + /* + * If we already know the pack object lives in, start checks from that + * pack - in the usual case when neither --local was given nor .keep files + * are present we will determine the answer right now. + */ + if (*found_pack) { + want = want_found_object(exclude, *found_pack); + if (want != -1) + return want; + } + + for (entry = packed_git_mru->head; entry; entry = entry->next) { + struct packed_git *p = entry->item; + off_t offset; + + if (p == *found_pack) + offset = *found_offset; + else + offset = find_pack_entry_one(sha1, p); - for (p = packed_git; p; p = p->next) { - off_t offset = find_pack_entry_one(sha1, p); if (offset) { if (!*found_pack) { - if (!is_pack_valid(p)) { - warning("packfile %s cannot be accessed", p->pack_name); + if (!is_pack_valid(p)) continue; - } *found_offset = offset; *found_pack = p; } - if (exclude) - return 1; - if (incremental) - return 0; - if (local && !p->pack_local) - return 0; - if (ignore_packed_keep && p->pack_local && p->pack_keep) - return 0; + want = want_found_object(exclude, p); + if (!exclude && want > 0) + mru_mark(packed_git_mru, entry); + if (want != -1) + return want; } } @@ -1015,8 +1066,8 @@ static const char no_closure_warning[] = N_( static int add_object_entry(const unsigned char *sha1, enum object_type type, const char *name, int exclude) { - struct packed_git *found_pack; - off_t found_offset; + struct packed_git *found_pack = NULL; + off_t found_offset = 0; uint32_t index_pos; if (have_duplicate_entry(sha1, exclude, &index_pos)) @@ -1049,6 +1100,9 @@ static int add_object_entry_from_bitmap(const unsigned char *sha1, if (have_duplicate_entry(sha1, 0, &index_pos)) return 0; + if (!want_object_in_pack(sha1, 0, &pack, &offset)) + return 0; + create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset); display_progress(progress_state, nr_result); @@ -1186,7 +1240,7 @@ static void add_pbase_object(struct tree_desc *tree, if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.sha1, + add_object_entry(entry.oid->hash, object_type(entry.mode), fullname, 1); return; @@ -1197,7 +1251,7 @@ static void add_pbase_object(struct tree_desc *tree, const char *down = name+cmplen+1; int downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.sha1); + tree = pbase_tree_get(entry.oid->hash); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); @@ -1470,6 +1524,170 @@ static int pack_offset_sort(const void *_a, const void *_b) (a->in_pack_offset > b->in_pack_offset); } +/* + * Drop an on-disk delta we were planning to reuse. Naively, this would + * just involve blanking out the "delta" field, but we have to deal + * with some extra book-keeping: + * + * 1. Removing ourselves from the delta_sibling linked list. + * + * 2. Updating our size/type to the non-delta representation. These were + * either not recorded initially (size) or overwritten with the delta type + * (type) when check_object() decided to reuse the delta. + * + * 3. Resetting our delta depth, as we are now a base object. + */ +static void drop_reused_delta(struct object_entry *entry) +{ + struct object_entry **p = &entry->delta->delta_child; + struct object_info oi = OBJECT_INFO_INIT; + + while (*p) { + if (*p == entry) + *p = (*p)->delta_sibling; + else + p = &(*p)->delta_sibling; + } + entry->delta = NULL; + entry->depth = 0; + + oi.sizep = &entry->size; + oi.typep = &entry->type; + if (packed_object_info(entry->in_pack, entry->in_pack_offset, &oi) < 0) { + /* + * We failed to get the info from this pack for some reason; + * fall back to sha1_object_info, which may find another copy. + * And if that fails, the error will be recorded in entry->type + * and dealt with in prepare_pack(). + */ + entry->type = sha1_object_info(entry->idx.sha1, &entry->size); + } +} + +/* + * Follow the chain of deltas from this entry onward, throwing away any links + * that cause us to hit a cycle (as determined by the DFS state flags in + * the entries). + * + * We also detect too-long reused chains that would violate our --depth + * limit. + */ +static void break_delta_chains(struct object_entry *entry) +{ + /* + * The actual depth of each object we will write is stored as an int, + * as it cannot exceed our int "depth" limit. But before we break + * changes based no that limit, we may potentially go as deep as the + * number of objects, which is elsewhere bounded to a uint32_t. + */ + uint32_t total_depth; + struct object_entry *cur, *next; + + for (cur = entry, total_depth = 0; + cur; + cur = cur->delta, total_depth++) { + if (cur->dfs_state == DFS_DONE) { + /* + * We've already seen this object and know it isn't + * part of a cycle. We do need to append its depth + * to our count. + */ + total_depth += cur->depth; + break; + } + + /* + * We break cycles before looping, so an ACTIVE state (or any + * other cruft which made its way into the state variable) + * is a bug. + */ + if (cur->dfs_state != DFS_NONE) + die("BUG: confusing delta dfs state in first pass: %d", + cur->dfs_state); + + /* + * Now we know this is the first time we've seen the object. If + * it's not a delta, we're done traversing, but we'll mark it + * done to save time on future traversals. + */ + if (!cur->delta) { + cur->dfs_state = DFS_DONE; + break; + } + + /* + * Mark ourselves as active and see if the next step causes + * us to cycle to another active object. It's important to do + * this _before_ we loop, because it impacts where we make the + * cut, and thus how our total_depth counter works. + * E.g., We may see a partial loop like: + * + * A -> B -> C -> D -> B + * + * Cutting B->C breaks the cycle. But now the depth of A is + * only 1, and our total_depth counter is at 3. The size of the + * error is always one less than the size of the cycle we + * broke. Commits C and D were "lost" from A's chain. + * + * If we instead cut D->B, then the depth of A is correct at 3. + * We keep all commits in the chain that we examined. + */ + cur->dfs_state = DFS_ACTIVE; + if (cur->delta->dfs_state == DFS_ACTIVE) { + drop_reused_delta(cur); + cur->dfs_state = DFS_DONE; + break; + } + } + + /* + * And now that we've gone all the way to the bottom of the chain, we + * need to clear the active flags and set the depth fields as + * appropriate. Unlike the loop above, which can quit when it drops a + * delta, we need to keep going to look for more depth cuts. So we need + * an extra "next" pointer to keep going after we reset cur->delta. + */ + for (cur = entry; cur; cur = next) { + next = cur->delta; + + /* + * We should have a chain of zero or more ACTIVE states down to + * a final DONE. We can quit after the DONE, because either it + * has no bases, or we've already handled them in a previous + * call. + */ + if (cur->dfs_state == DFS_DONE) + break; + else if (cur->dfs_state != DFS_ACTIVE) + die("BUG: confusing delta dfs state in second pass: %d", + cur->dfs_state); + + /* + * If the total_depth is more than depth, then we need to snip + * the chain into two or more smaller chains that don't exceed + * the maximum depth. Most of the resulting chains will contain + * (depth + 1) entries (i.e., depth deltas plus one base), and + * the last chain (i.e., the one containing entry) will contain + * whatever entries are left over, namely + * (total_depth % (depth + 1)) of them. + * + * Since we are iterating towards decreasing depth, we need to + * decrement total_depth as we go, and we need to write to the + * entry what its final depth will be after all of the + * snipping. Since we're snipping into chains of length (depth + * + 1) entries, the final depth of an entry will be its + * original depth modulo (depth + 1). Any time we encounter an + * entry whose final depth is supposed to be zero, we snip it + * from its delta base, thereby making it so. + */ + cur->depth = (total_depth--) % (depth + 1); + if (!cur->depth) + drop_reused_delta(cur); + + cur->dfs_state = DFS_DONE; + } +} + static void get_object_details(void) { uint32_t i; @@ -1478,7 +1696,7 @@ static void get_object_details(void) sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *)); for (i = 0; i < to_pack.nr_objects; i++) sorted_by_offset[i] = to_pack.objects + i; - qsort(sorted_by_offset, to_pack.nr_objects, sizeof(*sorted_by_offset), pack_offset_sort); + QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort); for (i = 0; i < to_pack.nr_objects; i++) { struct object_entry *entry = sorted_by_offset[i]; @@ -1487,6 +1705,13 @@ static void get_object_details(void) entry->no_try_delta = 1; } + /* + * This must happen in a second pass, since we rely on the delta + * information for the whole list being completed. + */ + for (i = 0; i < to_pack.nr_objects; i++) + break_delta_chains(&to_pack.objects[i]); + free(sorted_by_offset); } @@ -1973,8 +2198,6 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, init_threaded_search(); - if (!delta_search_threads) /* --threads=0 means autodetect */ - delta_search_threads = online_cpus(); if (delta_search_threads <= 1) { find_deltas(list, &list_size, window, depth, processed); cleanup_threaded_search(); @@ -2100,14 +2323,43 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, #define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p) #endif -static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, void *cb_data) +static void add_tag_chain(const struct object_id *oid) { - unsigned char peeled[20]; + struct tag *tag; + + /* + * We catch duplicates already in add_object_entry(), but we'd + * prefer to do this extra check to avoid having to parse the + * tag at all if we already know that it's being packed (e.g., if + * it was included via bitmaps, we would not have parsed it + * previously). + */ + if (packlist_find(&to_pack, oid->hash, NULL)) + return; + + tag = lookup_tag(oid->hash); + while (1) { + if (!tag || parse_tag(tag) || !tag->tagged) + die("unable to pack objects reachable from tag %s", + oid_to_hex(oid)); + + add_object_entry(tag->object.oid.hash, OBJ_TAG, NULL, 0); + + if (tag->tagged->type != OBJ_TAG) + return; + + tag = (struct tag *)tag->tagged; + } +} + +static int add_ref_tag(const char *path, const struct object_id *oid, int flag, void *cb_data) +{ + struct object_id peeled; if (starts_with(path, "refs/tags/") && /* is a tag? */ - !peel_ref(path, peeled) && /* peelable? */ - packlist_find(&to_pack, peeled, NULL)) /* object packed? */ - add_object_entry(sha1, OBJ_TAG, NULL, 0); + !peel_ref(path, peeled.hash) && /* peelable? */ + packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ + add_tag_chain(oid); return 0; } @@ -2132,7 +2384,7 @@ static void prepare_pack(int window, int depth) if (!to_pack.nr_objects || !window || !depth) return; - delta_list = xmalloc(to_pack.nr_objects * sizeof(*delta_list)); + ALLOC_ARRAY(delta_list, to_pack.nr_objects); nr_deltas = n = 0; for (i = 0; i < to_pack.nr_objects; i++) { @@ -2173,7 +2425,7 @@ static void prepare_pack(int window, int depth) if (progress) progress_state = start_progress(_("Compressing objects"), nr_deltas); - qsort(delta_list, n, sizeof(*delta_list), type_size_sort); + QSORT(delta_list, n, type_size_sort); ll_find_deltas(delta_list, n, window+1, depth, &nr_done); stop_progress(&progress_state); if (nr_done != nr_deltas) @@ -2196,16 +2448,6 @@ static int git_pack_config(const char *k, const char *v, void *cb) depth = git_config_int(k, v); return 0; } - if (!strcmp(k, "pack.compression")) { - int level = git_config_int(k, v); - if (level == -1) - level = Z_DEFAULT_COMPRESSION; - else if (level < 0 || level > Z_BEST_COMPRESSION) - die("bad pack compression level %d", level); - pack_compression_level = level; - pack_compression_seen = 1; - return 0; - } if (!strcmp(k, "pack.deltacachesize")) { max_delta_cache_size = git_config_int(k, v); return 0; @@ -2221,7 +2463,7 @@ static int git_pack_config(const char *k, const char *v, void *cb) write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE; } if (!strcmp(k, "pack.usebitmaps")) { - use_bitmap_index = git_config_bool(k, v); + use_bitmap_index_default = git_config_bool(k, v); return 0; } if (!strcmp(k, "pack.threads")) { @@ -2280,33 +2522,23 @@ static void read_object_list_from_stdin(void) static void show_commit(struct commit *commit, void *data) { - add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0); + add_object_entry(commit->object.oid.hash, OBJ_COMMIT, NULL, 0); commit->object.flags |= OBJECT_ADDED; if (write_bitmap_index) index_commit_for_bitmap(commit); } -static void show_object(struct object *obj, - const struct name_path *path, const char *last, - void *data) +static void show_object(struct object *obj, const char *name, void *data) { - char *name = path_name(path, last); - add_preferred_base_object(name); - add_object_entry(obj->sha1, obj->type, name, 0); + add_object_entry(obj->oid.hash, obj->type, name, 0); obj->flags |= OBJECT_ADDED; - - /* - * We will have generated the hash from the name, - * but not saved a pointer to it - we can free it - */ - free((char *)name); } static void show_edge(struct commit *commit) { - add_preferred_base(commit->object.sha1); + add_preferred_base(commit->object.oid.hash); } struct in_pack_object { @@ -2322,7 +2554,7 @@ struct in_pack { static void mark_in_pack_object(struct object *object, struct packed_git *p, struct in_pack *in_pack) { - in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->sha1, p); + in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->oid.hash, p); in_pack->array[in_pack->nr].object = object; in_pack->nr++; } @@ -2341,7 +2573,7 @@ static int ofscmp(const void *a_, const void *b_) else if (a->offset > b->offset) return 1; else - return hashcmp(a->object->sha1, b->object->sha1); + return oidcmp(&a->object->oid, &b->object->oid); } static void add_objects_in_unpacked_packs(struct rev_info *revs) @@ -2375,16 +2607,41 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs) } if (in_pack.nr) { - qsort(in_pack.array, in_pack.nr, sizeof(in_pack.array[0]), - ofscmp); + QSORT(in_pack.array, in_pack.nr, ofscmp); for (i = 0; i < in_pack.nr; i++) { struct object *o = in_pack.array[i].object; - add_object_entry(o->sha1, o->type, "", 0); + add_object_entry(o->oid.hash, o->type, "", 0); } } free(in_pack.array); } +static int add_loose_object(const struct object_id *oid, const char *path, + void *data) +{ + enum object_type type = sha1_object_info(oid->hash, NULL); + + if (type < 0) { + warning("loose object at %s could not be examined", path); + return 0; + } + + add_object_entry(oid->hash, type, "", 0); + return 0; +} + +/* + * We actually don't even have to worry about reachability here. + * add_object_entry will weed out duplicates, so we just add every + * loose object we find. + */ +static void add_unreachable_loose_objects(void) +{ + for_each_loose_file_in_objdir(get_object_directory(), + add_loose_object, + NULL, NULL, NULL); +} + static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) { static struct packed_git *last_found = (void *)1; @@ -2408,6 +2665,27 @@ static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) return 0; } +/* + * Store a list of sha1s that are should not be discarded + * because they are either written too recently, or are + * reachable from another object that was. + * + * This is filled by get_object_list. + */ +static struct sha1_array recent_objects; + +static int loosened_object_can_be_discarded(const unsigned char *sha1, + unsigned long mtime) +{ + if (!unpack_unreachable_expiration) + return 0; + if (mtime > unpack_unreachable_expiration) + return 0; + if (sha1_array_lookup(&recent_objects, sha1) >= 0) + return 0; + return 1; +} + static void loosen_unused_packed_objects(struct rev_info *revs) { struct packed_git *p; @@ -2418,17 +2696,14 @@ static void loosen_unused_packed_objects(struct rev_info *revs) if (!p->pack_local || p->pack_keep) continue; - if (unpack_unreachable_expiration && - p->mtime < unpack_unreachable_expiration) - continue; - if (open_pack_index(p)) die("cannot open pack index"); for (i = 0; i < p->num_objects; i++) { sha1 = nth_packed_object_sha1(p, i); if (!packlist_find(&to_pack, sha1, NULL) && - !has_sha1_pack_kept_or_nonlocal(sha1)) + !has_sha1_pack_kept_or_nonlocal(sha1) && + !loosened_object_can_be_discarded(sha1, p->mtime)) if (force_object_loose(sha1, p->mtime)) die("unable to force loose object"); } @@ -2436,13 +2711,13 @@ static void loosen_unused_packed_objects(struct rev_info *revs) } /* - * This tracks any options which a reader of the pack might - * not understand, and which would therefore prevent blind reuse - * of what we have on disk. + * This tracks any options which pack-reuse code expects to be on, or which a + * reader of the pack might not understand, and which would therefore prevent + * blind reuse of what we have on disk. */ static int pack_options_allow_reuse(void) { - return allow_ofs_delta; + return pack_to_stdout && allow_ofs_delta; } static int get_object_list_from_bitmap(struct rev_info *revs) @@ -2464,6 +2739,18 @@ static int get_object_list_from_bitmap(struct rev_info *revs) return 0; } +static void record_recent_object(struct object *obj, + const char *name, + void *data) +{ + sha1_array_append(&recent_objects, obj->oid.hash); +} + +static void record_recent_commit(struct commit *commit, void *data) +{ + sha1_array_append(&recent_objects, commit->object.oid.hash); +} + static void get_object_list(int ac, const char **av) { struct rev_info revs; @@ -2494,6 +2781,7 @@ static void get_object_list(int ac, const char **av) if (get_sha1_hex(line + 10, sha1)) die("not an SHA-1 '%s'", line + 10); register_shallow(sha1); + use_bitmap_index = 0; continue; } die("not a rev '%s'", line); @@ -2510,10 +2798,25 @@ static void get_object_list(int ac, const char **av) mark_edges_uninteresting(&revs, show_edge); traverse_commit_list(&revs, show_commit, show_object, NULL); + if (unpack_unreachable_expiration) { + revs.ignore_missing_links = 1; + if (add_unseen_recent_objects_to_traversal(&revs, + unpack_unreachable_expiration)) + die("unable to add recent objects"); + if (prepare_revision_walk(&revs)) + die("revision walk setup failed"); + traverse_commit_list(&revs, record_recent_commit, + record_recent_object, NULL); + } + if (keep_unreachable) add_objects_in_unpacked_packs(&revs); + if (pack_loose_unreachable) + add_unreachable_loose_objects(); if (unpack_unreachable) loosen_unused_packed_objects(&revs); + + sha1_array_clear(&recent_objects); } static int option_parse_index_version(const struct option *opt, @@ -2546,31 +2849,15 @@ static int option_parse_unpack_unreachable(const struct option *opt, return 0; } -static int option_parse_ulong(const struct option *opt, - const char *arg, int unset) -{ - if (unset) - die(_("option %s does not accept negative form"), - opt->long_name); - - if (!git_parse_ulong(arg, opt->value)) - die(_("unable to parse value '%s' for option %s"), - arg, opt->long_name); - return 0; -} - -#define OPT_ULONG(s, l, v, h) \ - { OPTION_CALLBACK, (s), (l), (v), "n", (h), \ - PARSE_OPT_NONEG, option_parse_ulong } - int cmd_pack_objects(int argc, const char **argv, const char *prefix) { int use_internal_rev_list = 0; int thin = 0; + int shallow = 0; int all_progress_implied = 0; - const char *rp_av[6]; - int rp_ac = 0; + struct argv_array rp = ARGV_ARRAY_INIT; int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0; + int rev_list_index = 0; struct option pack_objects_options[] = { OPT_SET_INT('q', "quiet", &progress, N_("do not show progress meter"), 0), @@ -2584,16 +2871,16 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) { OPTION_CALLBACK, 0, "index-version", NULL, N_("version[,offset]"), N_("write the pack index file in the specified idx format version"), 0, option_parse_index_version }, - OPT_ULONG(0, "max-pack-size", &pack_size_limit, - N_("maximum size of each output pack file")), + OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit, + N_("maximum size of each output pack file")), OPT_BOOL(0, "local", &local, N_("ignore borrowed objects from alternate object store")), OPT_BOOL(0, "incremental", &incremental, N_("ignore packed objects")), OPT_INTEGER(0, "window", &window, N_("limit pack window by objects")), - OPT_ULONG(0, "window-memory", &window_memory_limit, - N_("limit pack window by memory in addition to object limit")), + OPT_MAGNITUDE(0, "window-memory", &window_memory_limit, + N_("limit pack window by memory in addition to object limit")), OPT_INTEGER(0, "depth", &depth, N_("maximum length of delta chain allowed in the resulting pack")), OPT_BOOL(0, "reuse-delta", &reuse_delta, @@ -2617,17 +2904,24 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) { OPTION_SET_INT, 0, "reflog", &rev_list_reflog, NULL, N_("include objects referred by reflog entries"), PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 }, + { OPTION_SET_INT, 0, "indexed-objects", &rev_list_index, NULL, + N_("include objects referred to by the index"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 }, OPT_BOOL(0, "stdout", &pack_to_stdout, N_("output pack to stdout")), OPT_BOOL(0, "include-tag", &include_tag, N_("include tag objects that refer to objects to be packed")), OPT_BOOL(0, "keep-unreachable", &keep_unreachable, N_("keep unreachable objects")), + OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable, + N_("pack loose unreachable objects")), { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"), N_("unpack unreachable objects newer than <time>"), PARSE_OPT_OPTARG, option_parse_unpack_unreachable }, OPT_BOOL(0, "thin", &thin, N_("create thin packs")), + OPT_BOOL(0, "shallow", &shallow, + N_("create packs suitable for shallow fetches")), OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep, N_("ignore packs that have companion .keep file")), OPT_INTEGER(0, "compression", &pack_compression_level, @@ -2645,8 +2939,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); - if (!pack_compression_seen && core_compression_seen) - pack_compression_level = core_compression_level; progress = isatty(2); argc = parse_options(argc, argv, prefix, pack_objects_options, @@ -2659,24 +2951,30 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (pack_to_stdout != !base_name || argc) usage_with_options(pack_usage, pack_objects_options); - rp_av[rp_ac++] = "pack-objects"; + argv_array_push(&rp, "pack-objects"); if (thin) { use_internal_rev_list = 1; - rp_av[rp_ac++] = "--objects-edge"; + argv_array_push(&rp, shallow + ? "--objects-edge-aggressive" + : "--objects-edge"); } else - rp_av[rp_ac++] = "--objects"; + argv_array_push(&rp, "--objects"); if (rev_list_all) { use_internal_rev_list = 1; - rp_av[rp_ac++] = "--all"; + argv_array_push(&rp, "--all"); } if (rev_list_reflog) { use_internal_rev_list = 1; - rp_av[rp_ac++] = "--reflog"; + argv_array_push(&rp, "--reflog"); + } + if (rev_list_index) { + use_internal_rev_list = 1; + argv_array_push(&rp, "--indexed-objects"); } if (rev_list_unpacked) { use_internal_rev_list = 1; - rp_av[rp_ac++] = "--unpacked"; + argv_array_push(&rp, "--unpacked"); } if (!reuse_object) @@ -2685,6 +2983,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) pack_compression_level = Z_DEFAULT_COMPRESSION; else if (pack_compression_level < 0 || pack_compression_level > Z_BEST_COMPRESSION) die("bad pack compression level %d", pack_compression_level); + + if (!delta_search_threads) /* --threads=0 means autodetect */ + delta_search_threads = online_cpus(); + #ifdef NO_PTHREADS if (delta_search_threads != 1) warning("no threads support, ignoring --threads"); @@ -2703,8 +3005,26 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (keep_unreachable && unpack_unreachable) die("--keep-unreachable and --unpack-unreachable are incompatible."); + if (!rev_list_all || !rev_list_reflog || !rev_list_index) + unpack_unreachable_expiration = 0; + + /* + * "soft" reasons not to use bitmaps - for on-disk repack by default we want + * + * - to produce good pack (with bitmap index not-yet-packed objects are + * packed in suboptimal order). + * + * - to use more robust pack-generation codepath (avoiding possible + * bugs in bitmap code and possible bitmap index corruption). + */ + if (!pack_to_stdout) + use_bitmap_index_default = 0; + + if (use_bitmap_index < 0) + use_bitmap_index = use_bitmap_index_default; - if (!use_internal_rev_list || !pack_to_stdout || is_repository_shallow()) + /* "hard" reasons not to use bitmaps; these just won't work at all */ + if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow()) use_bitmap_index = 0; if (pack_to_stdout || !rev_list_all) @@ -2714,14 +3034,36 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) progress = 2; prepare_packed_git(); + if (ignore_packed_keep) { + struct packed_git *p; + for (p = packed_git; p; p = p->next) + if (p->pack_local && p->pack_keep) + break; + if (!p) /* no keep-able packs found */ + ignore_packed_keep = 0; + } + if (local) { + /* + * unlike ignore_packed_keep above, we do not want to + * unset "local" based on looking at packs, as it + * also covers non-local objects + */ + struct packed_git *p; + for (p = packed_git; p; p = p->next) { + if (!p->pack_local) { + have_non_local_packs = 1; + break; + } + } + } if (progress) progress_state = start_progress(_("Counting objects"), 0); if (!use_internal_rev_list) read_object_list_from_stdin(); else { - rp_av[rp_ac] = NULL; - get_object_list(rp_ac, rp_av); + get_object_list(rp.argc, rp.argv); + argv_array_clear(&rp); } cleanup_preferred_base(); if (include_tag && nr_result) |