diff options
Diffstat (limited to 'commit-graph.c')
-rw-r--r-- | commit-graph.c | 485 |
1 files changed, 410 insertions, 75 deletions
diff --git a/commit-graph.c b/commit-graph.c index c8d521923c..40c855f185 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -7,10 +7,15 @@ #include "packfile.h" #include "commit.h" #include "object.h" +#include "refs.h" #include "revision.h" #include "sha1-lookup.h" #include "commit-graph.h" #include "object-store.h" +#include "alloc.h" +#include "hashmap.h" +#include "replace-object.h" +#include "progress.h" #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */ #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */ @@ -35,10 +40,11 @@ #define GRAPH_LAST_EDGE 0x80000000 +#define GRAPH_HEADER_SIZE 8 #define GRAPH_FANOUT_SIZE (4 * 256) #define GRAPH_CHUNKLOOKUP_WIDTH 12 -#define GRAPH_MIN_SIZE (5 * GRAPH_CHUNKLOOKUP_WIDTH + GRAPH_FANOUT_SIZE + \ - GRAPH_OID_LEN + 8) +#define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \ + + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN) char *get_commit_graph_filename(const char *obj_dir) { @@ -53,6 +59,28 @@ static struct commit_graph *alloc_commit_graph(void) return g; } +extern int read_replace_refs; + +static int commit_graph_compatible(struct repository *r) +{ + if (!r->gitdir) + return 0; + + if (read_replace_refs) { + prepare_replace_object(r); + if (hashmap_get_size(&r->objects->replace_map->map)) + return 0; + } + + prepare_commit_graft(r); + if (r->parsed_objects && r->parsed_objects->grafts_nr) + return 0; + if (is_repository_shallow(r)) + return 0; + + return 1; +} + struct commit_graph *load_commit_graph_one(const char *graph_file) { void *graph_map; @@ -180,53 +208,82 @@ cleanup_fail: exit(1); } -/* global storage */ -static struct commit_graph *commit_graph = NULL; - -static void prepare_commit_graph_one(const char *obj_dir) +static void prepare_commit_graph_one(struct repository *r, const char *obj_dir) { char *graph_name; - if (commit_graph) + if (r->objects->commit_graph) return; graph_name = get_commit_graph_filename(obj_dir); - commit_graph = load_commit_graph_one(graph_name); + r->objects->commit_graph = + load_commit_graph_one(graph_name); FREE_AND_NULL(graph_name); } -static int prepare_commit_graph_run_once = 0; -static void prepare_commit_graph(void) +/* + * Return 1 if commit_graph is non-NULL, and 0 otherwise. + * + * On the first invocation, this function attemps to load the commit + * graph if the_repository is configured to have one. + */ +static int prepare_commit_graph(struct repository *r) { struct alternate_object_database *alt; char *obj_dir; + int config_value; + + if (r->objects->commit_graph_attempted) + return !!r->objects->commit_graph; + r->objects->commit_graph_attempted = 1; + + if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && + (repo_config_get_bool(r, "core.commitgraph", &config_value) || + !config_value)) + /* + * This repository is not configured to use commit graphs, so + * do not load one. (But report commit_graph_attempted anyway + * so that commit graph loading is not attempted again for this + * repository.) + */ + return 0; - if (prepare_commit_graph_run_once) - return; - prepare_commit_graph_run_once = 1; + if (!commit_graph_compatible(r)) + return 0; - obj_dir = get_object_directory(); - prepare_commit_graph_one(obj_dir); - prepare_alt_odb(the_repository); - for (alt = the_repository->objects->alt_odb_list; - !commit_graph && alt; + obj_dir = r->objects->objectdir; + prepare_commit_graph_one(r, obj_dir); + prepare_alt_odb(r); + for (alt = r->objects->alt_odb_list; + !r->objects->commit_graph && alt; alt = alt->next) - prepare_commit_graph_one(alt->path); + prepare_commit_graph_one(r, alt->path); + return !!r->objects->commit_graph; } -static void close_commit_graph(void) +int generation_numbers_enabled(struct repository *r) { - if (!commit_graph) - return; + uint32_t first_generation; + struct commit_graph *g; + if (!prepare_commit_graph(r)) + return 0; - if (commit_graph->graph_fd >= 0) { - munmap((void *)commit_graph->data, commit_graph->data_len); - commit_graph->data = NULL; - close(commit_graph->graph_fd); - } + g = r->objects->commit_graph; + + if (!g->num_commits) + return 0; + + first_generation = get_be32(g->chunk_commit_data + + g->hash_len + 8) >> 2; - FREE_AND_NULL(commit_graph); + return !!first_generation; +} + +void close_commit_graph(struct repository *r) +{ + free_commit_graph(r->objects->commit_graph); + r->objects->commit_graph = NULL; } static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos) @@ -241,8 +298,12 @@ static struct commit_list **insert_parent_or_die(struct commit_graph *g, { struct commit *c; struct object_id oid; + + if (pos >= g->num_commits) + die("invalid parent position %"PRIu64, pos); + hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos); - c = lookup_commit(&oid); + c = lookup_commit(the_repository, &oid); if (!c) die(_("could not find commit %s"), oid_to_hex(&oid)); c->graph_pos = pos; @@ -313,28 +374,33 @@ static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uin } } -int parse_commit_in_graph(struct commit *item) +static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item) { uint32_t pos; - if (!core_commit_graph) - return 0; if (item->object.parsed) return 1; - prepare_commit_graph(); - if (commit_graph && find_commit_in_graph(item, commit_graph, &pos)) - return fill_commit_in_graph(item, commit_graph, pos); + + if (find_commit_in_graph(item, g, &pos)) + return fill_commit_in_graph(item, g, pos); + return 0; } -void load_commit_graph_info(struct commit *item) +int parse_commit_in_graph(struct repository *r, struct commit *item) +{ + if (!prepare_commit_graph(r)) + return 0; + return parse_commit_in_graph_one(r->objects->commit_graph, item); +} + +void load_commit_graph_info(struct repository *r, struct commit *item) { uint32_t pos; - if (!core_commit_graph) + if (!prepare_commit_graph(r)) return; - prepare_commit_graph(); - if (commit_graph && find_commit_in_graph(item, commit_graph, &pos)) - fill_commit_graph_info(item, commit_graph, pos); + if (find_commit_in_graph(item, r->objects->commit_graph, &pos)) + fill_commit_graph_info(item, r->objects->commit_graph, pos); } static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c) @@ -344,19 +410,25 @@ static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit * GRAPH_DATA_WIDTH * (c->graph_pos); hashcpy(oid.hash, commit_data); - c->maybe_tree = lookup_tree(&oid); + c->maybe_tree = lookup_tree(the_repository, &oid); return c->maybe_tree; } -struct tree *get_commit_tree_in_graph(const struct commit *c) +static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g, + const struct commit *c) { if (c->maybe_tree) return c->maybe_tree; if (c->graph_pos == COMMIT_NOT_FROM_GRAPH) - BUG("get_commit_tree_in_graph called from non-commit-graph commit"); + BUG("get_commit_tree_in_graph_one called from non-commit-graph commit"); + + return load_tree_for_commit(g, (struct commit *)c); +} - return load_tree_for_commit(commit_graph, (struct commit *)c); +struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c) +{ + return get_commit_tree_in_graph_one(r->objects->commit_graph, c); } static void write_graph_chunk_fanout(struct hashfile *f, @@ -523,6 +595,8 @@ struct packed_oid_list { struct object_id *list; int nr; int alloc; + struct progress *progress; + int progress_done; }; static int add_packed_commits(const struct object_id *oid, @@ -535,6 +609,9 @@ static int add_packed_commits(const struct object_id *oid, off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT; + if (list->progress) + display_progress(list->progress, ++list->progress_done); + oi.typep = &type; if (packed_object_info(the_repository, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid)); @@ -562,13 +639,19 @@ static void add_missing_parents(struct packed_oid_list *oids, struct commit *com } } -static void close_reachable(struct packed_oid_list *oids) +static void close_reachable(struct packed_oid_list *oids, int report_progress) { int i; struct commit *commit; + struct progress *progress = NULL; + int j = 0; + if (report_progress) + progress = start_delayed_progress( + _("Annotating commits in commit graph"), 0); for (i = 0; i < oids->nr; i++) { - commit = lookup_commit(&oids->list[i]); + display_progress(progress, ++j); + commit = lookup_commit(the_repository, &oids->list[i]); if (commit) commit->object.flags |= UNINTERESTING; } @@ -579,26 +662,36 @@ static void close_reachable(struct packed_oid_list *oids) * closure. */ for (i = 0; i < oids->nr; i++) { - commit = lookup_commit(&oids->list[i]); + display_progress(progress, ++j); + commit = lookup_commit(the_repository, &oids->list[i]); if (commit && !parse_commit(commit)) add_missing_parents(oids, commit); } for (i = 0; i < oids->nr; i++) { - commit = lookup_commit(&oids->list[i]); + display_progress(progress, ++j); + commit = lookup_commit(the_repository, &oids->list[i]); if (commit) commit->object.flags &= ~UNINTERESTING; } + stop_progress(&progress); } -static void compute_generation_numbers(struct packed_commit_list* commits) +static void compute_generation_numbers(struct packed_commit_list* commits, + int report_progress) { int i; struct commit_list *list = NULL; + struct progress *progress = NULL; + if (report_progress) + progress = start_progress( + _("Computing commit graph generation numbers"), + commits->nr); for (i = 0; i < commits->nr; i++) { + display_progress(progress, i + 1); if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY && commits->list[i]->generation != GENERATION_NUMBER_ZERO) continue; @@ -630,14 +723,34 @@ static void compute_generation_numbers(struct packed_commit_list* commits) } } } + stop_progress(&progress); +} + +static int add_ref_to_list(const char *refname, + const struct object_id *oid, + int flags, void *cb_data) +{ + struct string_list *list = (struct string_list *)cb_data; + + string_list_append(list, oid_to_hex(oid)); + return 0; +} + +void write_commit_graph_reachable(const char *obj_dir, int append, + int report_progress) +{ + struct string_list list = STRING_LIST_INIT_DUP; + + for_each_ref(add_ref_to_list, &list); + write_commit_graph(obj_dir, NULL, &list, append, report_progress); + + string_list_clear(&list, 0); } void write_commit_graph(const char *obj_dir, - const char **pack_indexes, - int nr_packs, - const char **commit_hex, - int nr_commits, - int append) + struct string_list *pack_indexes, + struct string_list *commit_hex, + int append, int report_progress) { struct packed_oid_list oids; struct packed_commit_list commits; @@ -650,21 +763,29 @@ void write_commit_graph(const char *obj_dir, int num_chunks; int num_extra_edges; struct commit_list *parent; + struct progress *progress = NULL; + + if (!commit_graph_compatible(the_repository)) + return; oids.nr = 0; - oids.alloc = approximate_object_count() / 4; + oids.alloc = approximate_object_count() / 32; + oids.progress = NULL; + oids.progress_done = 0; if (append) { - prepare_commit_graph_one(obj_dir); - if (commit_graph) - oids.alloc += commit_graph->num_commits; + prepare_commit_graph_one(the_repository, obj_dir); + if (the_repository->objects->commit_graph) + oids.alloc += the_repository->objects->commit_graph->num_commits; } if (oids.alloc < 1024) oids.alloc = 1024; ALLOC_ARRAY(oids.list, oids.alloc); - if (append && commit_graph) { + if (append && the_repository->objects->commit_graph) { + struct commit_graph *commit_graph = + the_repository->objects->commit_graph; for (i = 0; i < commit_graph->num_commits; i++) { const unsigned char *hash = commit_graph->chunk_oid_lookup + commit_graph->hash_len * i; @@ -677,31 +798,44 @@ void write_commit_graph(const char *obj_dir, int dirlen; strbuf_addf(&packname, "%s/pack/", obj_dir); dirlen = packname.len; - for (i = 0; i < nr_packs; i++) { + if (report_progress) { + oids.progress = start_delayed_progress( + _("Finding commits for commit graph"), 0); + oids.progress_done = 0; + } + for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes[i]); + strbuf_addstr(&packname, pack_indexes->items[i].string); p = add_packed_git(packname.buf, packname.len, 1); if (!p) die(_("error adding pack %s"), packname.buf); if (open_pack_index(p)) die(_("error opening index for %s"), packname.buf); - for_each_object_in_pack(p, add_packed_commits, &oids); + for_each_object_in_pack(p, add_packed_commits, &oids, 0); close_pack(p); + free(p); } + stop_progress(&oids.progress); strbuf_release(&packname); } if (commit_hex) { - for (i = 0; i < nr_commits; i++) { + if (report_progress) + progress = start_delayed_progress( + _("Finding commits for commit graph"), + commit_hex->nr); + for (i = 0; i < commit_hex->nr; i++) { const char *end; struct object_id oid; struct commit *result; - if (commit_hex[i] && parse_oid_hex(commit_hex[i], &oid, &end)) + display_progress(progress, i + 1); + if (commit_hex->items[i].string && + parse_oid_hex(commit_hex->items[i].string, &oid, &end)) continue; - result = lookup_commit_reference_gently(&oid, 1); + result = lookup_commit_reference_gently(the_repository, &oid, 1); if (result) { ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc); @@ -709,18 +843,24 @@ void write_commit_graph(const char *obj_dir, oids.nr++; } } + stop_progress(&progress); } - if (!pack_indexes && !commit_hex) + if (!pack_indexes && !commit_hex) { + if (report_progress) + oids.progress = start_delayed_progress( + _("Finding commits for commit graph"), 0); for_each_packed_object(add_packed_commits, &oids, 0); + stop_progress(&oids.progress); + } - close_reachable(&oids); + close_reachable(&oids, report_progress); QSORT(oids.list, oids.nr, commit_compare); count_distinct = 1; for (i = 1; i < oids.nr; i++) { - if (oidcmp(&oids.list[i-1], &oids.list[i])) + if (!oideq(&oids.list[i - 1], &oids.list[i])) count_distinct++; } @@ -734,10 +874,10 @@ void write_commit_graph(const char *obj_dir, num_extra_edges = 0; for (i = 0; i < oids.nr; i++) { int num_parents = 0; - if (i > 0 && !oidcmp(&oids.list[i-1], &oids.list[i])) + if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i])) continue; - commits.list[commits.nr] = lookup_commit(&oids.list[i]); + commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); parse_commit(commits.list[commits.nr]); for (parent = commits.list[commits.nr]->parents; @@ -754,12 +894,14 @@ void write_commit_graph(const char *obj_dir, if (commits.nr >= GRAPH_PARENT_MISSING) die(_("too many commits to write graph")); - compute_generation_numbers(&commits); + compute_generation_numbers(&commits, report_progress); graph_name = get_commit_graph_filename(obj_dir); - if (safe_create_leading_directories(graph_name)) + if (safe_create_leading_directories(graph_name)) { + UNLEAK(graph_name); die_errno(_("unable to create leading directories of %s"), graph_name); + } hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); @@ -800,11 +942,204 @@ void write_commit_graph(const char *obj_dir, write_graph_chunk_data(f, GRAPH_OID_LEN, commits.list, commits.nr); write_graph_chunk_large_edges(f, commits.list, commits.nr); - close_commit_graph(); + close_commit_graph(the_repository); finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); + free(graph_name); + free(commits.list); free(oids.list); - oids.alloc = 0; - oids.nr = 0; +} + +#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 +static int verify_commit_graph_error; + +static void graph_report(const char *fmt, ...) +{ + va_list ap; + + verify_commit_graph_error = 1; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); +} + +#define GENERATION_ZERO_EXISTS 1 +#define GENERATION_NUMBER_EXISTS 2 + +int verify_commit_graph(struct repository *r, struct commit_graph *g) +{ + uint32_t i, cur_fanout_pos = 0; + struct object_id prev_oid, cur_oid, checksum; + int generation_zero = 0; + struct hashfile *f; + int devnull; + struct progress *progress = NULL; + + if (!g) { + graph_report("no commit-graph file loaded"); + return 1; + } + + verify_commit_graph_error = 0; + + if (!g->chunk_oid_fanout) + graph_report("commit-graph is missing the OID Fanout chunk"); + if (!g->chunk_oid_lookup) + graph_report("commit-graph is missing the OID Lookup chunk"); + if (!g->chunk_commit_data) + graph_report("commit-graph is missing the Commit Data chunk"); + + if (verify_commit_graph_error) + return verify_commit_graph_error; + + devnull = open("/dev/null", O_WRONLY); + f = hashfd(devnull, NULL); + hashwrite(f, g->data, g->data_len - g->hash_len); + finalize_hashfile(f, checksum.hash, CSUM_CLOSE); + if (!hasheq(checksum.hash, g->data + g->data_len - g->hash_len)) { + graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt")); + verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH; + } + + for (i = 0; i < g->num_commits; i++) { + struct commit *graph_commit; + + hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); + + if (i && oidcmp(&prev_oid, &cur_oid) >= 0) + graph_report("commit-graph has incorrect OID order: %s then %s", + oid_to_hex(&prev_oid), + oid_to_hex(&cur_oid)); + + oidcpy(&prev_oid, &cur_oid); + + while (cur_oid.hash[0] > cur_fanout_pos) { + uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos); + + if (i != fanout_value) + graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u", + cur_fanout_pos, fanout_value, i); + cur_fanout_pos++; + } + + graph_commit = lookup_commit(r, &cur_oid); + if (!parse_commit_in_graph_one(g, graph_commit)) + graph_report("failed to parse %s from commit-graph", + oid_to_hex(&cur_oid)); + } + + while (cur_fanout_pos < 256) { + uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos); + + if (g->num_commits != fanout_value) + graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u", + cur_fanout_pos, fanout_value, i); + + cur_fanout_pos++; + } + + if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH) + return verify_commit_graph_error; + + progress = start_progress(_("Verifying commits in commit graph"), + g->num_commits); + for (i = 0; i < g->num_commits; i++) { + struct commit *graph_commit, *odb_commit; + struct commit_list *graph_parents, *odb_parents; + uint32_t max_generation = 0; + + display_progress(progress, i + 1); + hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); + + graph_commit = lookup_commit(r, &cur_oid); + odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r)); + if (parse_commit_internal(odb_commit, 0, 0)) { + graph_report("failed to parse %s from object database", + oid_to_hex(&cur_oid)); + continue; + } + + if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, + get_commit_tree_oid(odb_commit))) + graph_report("root tree OID for commit %s in commit-graph is %s != %s", + oid_to_hex(&cur_oid), + oid_to_hex(get_commit_tree_oid(graph_commit)), + oid_to_hex(get_commit_tree_oid(odb_commit))); + + graph_parents = graph_commit->parents; + odb_parents = odb_commit->parents; + + while (graph_parents) { + if (odb_parents == NULL) { + graph_report("commit-graph parent list for commit %s is too long", + oid_to_hex(&cur_oid)); + break; + } + + if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid)) + graph_report("commit-graph parent for %s is %s != %s", + oid_to_hex(&cur_oid), + oid_to_hex(&graph_parents->item->object.oid), + oid_to_hex(&odb_parents->item->object.oid)); + + if (graph_parents->item->generation > max_generation) + max_generation = graph_parents->item->generation; + + graph_parents = graph_parents->next; + odb_parents = odb_parents->next; + } + + if (odb_parents != NULL) + graph_report("commit-graph parent list for commit %s terminates early", + oid_to_hex(&cur_oid)); + + if (!graph_commit->generation) { + if (generation_zero == GENERATION_NUMBER_EXISTS) + graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere", + oid_to_hex(&cur_oid)); + generation_zero = GENERATION_ZERO_EXISTS; + } else if (generation_zero == GENERATION_ZERO_EXISTS) + graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere", + oid_to_hex(&cur_oid)); + + if (generation_zero == GENERATION_ZERO_EXISTS) + continue; + + /* + * If one of our parents has generation GENERATION_NUMBER_MAX, then + * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid + * extra logic in the following condition. + */ + if (max_generation == GENERATION_NUMBER_MAX) + max_generation--; + + if (graph_commit->generation != max_generation + 1) + graph_report("commit-graph generation for commit %s is %u != %u", + oid_to_hex(&cur_oid), + graph_commit->generation, + max_generation + 1); + + if (graph_commit->date != odb_commit->date) + graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime, + oid_to_hex(&cur_oid), + graph_commit->date, + odb_commit->date); + } + stop_progress(&progress); + + return verify_commit_graph_error; +} + +void free_commit_graph(struct commit_graph *g) +{ + if (!g) + return; + if (g->graph_fd >= 0) { + munmap((void *)g->data, g->data_len); + g->data = NULL; + close(g->graph_fd); + } + free(g); } |