summaryrefslogtreecommitdiff
path: root/commit-graph.c
diff options
context:
space:
mode:
Diffstat (limited to 'commit-graph.c')
-rw-r--r--commit-graph.c552
1 files changed, 427 insertions, 125 deletions
diff --git a/commit-graph.c b/commit-graph.c
index f013a84e29..1af68c297d 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -16,13 +16,33 @@
#include "hashmap.h"
#include "replace-object.h"
#include "progress.h"
+#include "bloom.h"
+#include "commit-slab.h"
+#include "shallow.h"
+
+void git_test_write_commit_graph_or_die(void)
+{
+ int flags = 0;
+ if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0))
+ return;
+
+ if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0))
+ flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS;
+
+ if (write_commit_graph_reachable(the_repository->objects->odb,
+ flags, NULL))
+ die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH");
+}
#define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
+#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */
+#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */
#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
+#define MAX_NUM_CHUNKS 7
#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
@@ -44,9 +64,105 @@
/* Remember to update object flag allocation in object.h */
#define REACHABLE (1u<<15)
-char *get_commit_graph_filename(struct object_directory *odb)
+/* Keep track of the order in which commits are added to our list. */
+define_commit_slab(commit_pos, int);
+static struct commit_pos commit_pos = COMMIT_SLAB_INIT(1, commit_pos);
+
+static void set_commit_pos(struct repository *r, const struct object_id *oid)
+{
+ static int32_t max_pos;
+ struct commit *commit = lookup_commit(r, oid);
+
+ if (!commit)
+ return; /* should never happen, but be lenient */
+
+ *commit_pos_at(&commit_pos, commit) = max_pos++;
+}
+
+static int commit_pos_cmp(const void *va, const void *vb)
+{
+ const struct commit *a = *(const struct commit **)va;
+ const struct commit *b = *(const struct commit **)vb;
+ return commit_pos_at(&commit_pos, a) -
+ commit_pos_at(&commit_pos, b);
+}
+
+define_commit_slab(commit_graph_data_slab, struct commit_graph_data);
+static struct commit_graph_data_slab commit_graph_data_slab =
+ COMMIT_SLAB_INIT(1, commit_graph_data_slab);
+
+uint32_t commit_graph_position(const struct commit *c)
+{
+ struct commit_graph_data *data =
+ commit_graph_data_slab_peek(&commit_graph_data_slab, c);
+
+ return data ? data->graph_pos : COMMIT_NOT_FROM_GRAPH;
+}
+
+uint32_t commit_graph_generation(const struct commit *c)
+{
+ struct commit_graph_data *data =
+ commit_graph_data_slab_peek(&commit_graph_data_slab, c);
+
+ if (!data)
+ return GENERATION_NUMBER_INFINITY;
+ else if (data->graph_pos == COMMIT_NOT_FROM_GRAPH)
+ return GENERATION_NUMBER_INFINITY;
+
+ return data->generation;
+}
+
+static struct commit_graph_data *commit_graph_data_at(const struct commit *c)
+{
+ unsigned int i, nth_slab;
+ struct commit_graph_data *data =
+ commit_graph_data_slab_peek(&commit_graph_data_slab, c);
+
+ if (data)
+ return data;
+
+ nth_slab = c->index / commit_graph_data_slab.slab_size;
+ data = commit_graph_data_slab_at(&commit_graph_data_slab, c);
+
+ /*
+ * commit-slab initializes elements with zero, overwrite this with
+ * COMMIT_NOT_FROM_GRAPH for graph_pos.
+ *
+ * We avoid initializing generation with checking if graph position
+ * is not COMMIT_NOT_FROM_GRAPH.
+ */
+ for (i = 0; i < commit_graph_data_slab.slab_size; i++) {
+ commit_graph_data_slab.slab[nth_slab][i].graph_pos =
+ COMMIT_NOT_FROM_GRAPH;
+ }
+
+ return data;
+}
+
+static int commit_gen_cmp(const void *va, const void *vb)
+{
+ const struct commit *a = *(const struct commit **)va;
+ const struct commit *b = *(const struct commit **)vb;
+
+ uint32_t generation_a = commit_graph_generation(a);
+ uint32_t generation_b = commit_graph_generation(b);
+ /* lower generation commits first */
+ if (generation_a < generation_b)
+ return -1;
+ else if (generation_a > generation_b)
+ return 1;
+
+ /* use date as a heuristic when generations are equal */
+ if (a->date < b->date)
+ return -1;
+ else if (a->date > b->date)
+ return 1;
+ return 0;
+}
+
+char *get_commit_graph_filename(struct object_directory *obj_dir)
{
- return xstrfmt("%s/info/commit-graph", odb->path);
+ return xstrfmt("%s/info/commit-graph", obj_dir->path);
}
static char *get_split_graph_filename(struct object_directory *odb,
@@ -69,7 +185,6 @@ static uint8_t oid_version(void)
static struct commit_graph *alloc_commit_graph(void)
{
struct commit_graph *g = xcalloc(1, sizeof(*g));
- g->graph_fd = -1;
return g;
}
@@ -88,7 +203,8 @@ static int commit_graph_compatible(struct repository *r)
}
prepare_commit_graft(r);
- if (r->parsed_objects && r->parsed_objects->grafts_nr)
+ if (r->parsed_objects &&
+ (r->parsed_objects->grafts_nr || r->parsed_objects->substituted_parent))
return 0;
if (is_repository_shallow(r))
return 0;
@@ -123,14 +239,13 @@ struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st,
return NULL;
}
graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
- ret = parse_commit_graph(graph_map, fd, graph_size);
+ close(fd);
+ ret = parse_commit_graph(graph_map, graph_size);
if (ret)
ret->odb = odb;
- else {
+ else
munmap(graph_map, graph_size);
- close(fd);
- }
return ret;
}
@@ -165,8 +280,7 @@ static int verify_commit_graph_lite(struct commit_graph *g)
return 0;
}
-struct commit_graph *parse_commit_graph(void *graph_map, int fd,
- size_t graph_size)
+struct commit_graph *parse_commit_graph(void *graph_map, size_t graph_size)
{
const unsigned char *data, *chunk_lookup;
uint32_t i;
@@ -209,7 +323,6 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
graph->hash_len = the_hash_algo->rawsz;
graph->num_chunks = *(unsigned char*)(data + 6);
- graph->graph_fd = fd;
graph->data = graph_map;
graph->data_len = graph_size;
@@ -224,8 +337,7 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
if (data + graph_size - chunk_lookup <
GRAPH_CHUNKLOOKUP_WIDTH) {
error(_("commit-graph chunk lookup table entry missing; file may be incomplete"));
- free(graph);
- return NULL;
+ goto free_and_return;
}
chunk_id = get_be32(chunk_lookup + 0);
@@ -236,8 +348,7 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
if (chunk_offset > graph_size - the_hash_algo->rawsz) {
error(_("commit-graph improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
(uint32_t)chunk_offset);
- free(graph);
- return NULL;
+ goto free_and_return;
}
switch (chunk_id) {
@@ -274,12 +385,37 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
chunk_repeated = 1;
else
graph->chunk_base_graphs = data + chunk_offset;
+ break;
+
+ case GRAPH_CHUNKID_BLOOMINDEXES:
+ if (graph->chunk_bloom_indexes)
+ chunk_repeated = 1;
+ else
+ graph->chunk_bloom_indexes = data + chunk_offset;
+ break;
+
+ case GRAPH_CHUNKID_BLOOMDATA:
+ if (graph->chunk_bloom_data)
+ chunk_repeated = 1;
+ else {
+ uint32_t hash_version;
+ graph->chunk_bloom_data = data + chunk_offset;
+ hash_version = get_be32(data + chunk_offset);
+
+ if (hash_version != 1)
+ break;
+
+ graph->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
+ graph->bloom_filter_settings->hash_version = hash_version;
+ graph->bloom_filter_settings->num_hashes = get_be32(data + chunk_offset + 4);
+ graph->bloom_filter_settings->bits_per_entry = get_be32(data + chunk_offset + 8);
+ }
+ break;
}
if (chunk_repeated) {
error(_("commit-graph chunk id %08x appears multiple times"), chunk_id);
- free(graph);
- return NULL;
+ goto free_and_return;
}
if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
@@ -292,14 +428,26 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
last_chunk_offset = chunk_offset;
}
+ if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) {
+ init_bloom_filters();
+ } else {
+ /* We need both the bloom chunks to exist together. Else ignore the data */
+ graph->chunk_bloom_indexes = NULL;
+ graph->chunk_bloom_data = NULL;
+ FREE_AND_NULL(graph->bloom_filter_settings);
+ }
+
hashcpy(graph->oid.hash, graph->data + graph->data_len - graph->hash_len);
- if (verify_commit_graph_lite(graph)) {
- free(graph);
- return NULL;
- }
+ if (verify_commit_graph_lite(graph))
+ goto free_and_return;
return graph;
+
+free_and_return:
+ free(graph->bloom_filter_settings);
+ free(graph);
+ return NULL;
}
static struct commit_graph *load_commit_graph_one(const char *graph_file,
@@ -577,13 +725,14 @@ static struct commit_list **insert_parent_or_die(struct repository *r,
c = lookup_commit(r, &oid);
if (!c)
die(_("could not find commit %s"), oid_to_hex(&oid));
- c->graph_pos = pos;
+ commit_graph_data_at(c)->graph_pos = pos;
return &commit_list_insert(c, pptr)->next;
}
static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
{
const unsigned char *commit_data;
+ struct commit_graph_data *graph_data;
uint32_t lex_index;
while (pos < g->num_commits_in_base)
@@ -591,8 +740,10 @@ static void fill_commit_graph_info(struct commit *item, struct commit_graph *g,
lex_index = pos - g->num_commits_in_base;
commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * lex_index;
- item->graph_pos = pos;
- item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
+
+ graph_data = commit_graph_data_at(item);
+ graph_data->graph_pos = pos;
+ graph_data->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
}
static inline void set_commit_tree(struct commit *c, struct tree *t)
@@ -608,6 +759,7 @@ static int fill_commit_in_graph(struct repository *r,
uint32_t *parent_data_ptr;
uint64_t date_low, date_high;
struct commit_list **pptr;
+ struct commit_graph_data *graph_data;
const unsigned char *commit_data;
uint32_t lex_index;
@@ -621,7 +773,8 @@ static int fill_commit_in_graph(struct repository *r,
* Store the "full" position, but then use the
* "local" position for the rest of the calculation.
*/
- item->graph_pos = pos;
+ graph_data = commit_graph_data_at(item);
+ graph_data->graph_pos = pos;
lex_index = pos - g->num_commits_in_base;
commit_data = g->chunk_commit_data + (g->hash_len + 16) * lex_index;
@@ -634,7 +787,7 @@ static int fill_commit_in_graph(struct repository *r,
date_low = get_be32(commit_data + g->hash_len + 12);
item->date = (timestamp_t)((date_high << 32) | date_low);
- item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
+ graph_data->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
pptr = &item->parents;
@@ -666,8 +819,9 @@ static int fill_commit_in_graph(struct repository *r,
static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
{
- if (item->graph_pos != COMMIT_NOT_FROM_GRAPH) {
- *pos = item->graph_pos;
+ uint32_t graph_pos = commit_graph_position(item);
+ if (graph_pos != COMMIT_NOT_FROM_GRAPH) {
+ *pos = graph_pos;
return 1;
} else {
struct commit_graph *cur_g = g;
@@ -722,12 +876,13 @@ static struct tree *load_tree_for_commit(struct repository *r,
{
struct object_id oid;
const unsigned char *commit_data;
+ uint32_t graph_pos = commit_graph_position(c);
- while (c->graph_pos < g->num_commits_in_base)
+ while (graph_pos < g->num_commits_in_base)
g = g->base_graph;
commit_data = g->chunk_commit_data +
- GRAPH_DATA_WIDTH * (c->graph_pos - g->num_commits_in_base);
+ GRAPH_DATA_WIDTH * (graph_pos - g->num_commits_in_base);
hashcpy(oid.hash, commit_data);
set_commit_tree(c, lookup_tree(r, &oid));
@@ -741,7 +896,7 @@ static struct tree *get_commit_tree_in_graph_one(struct repository *r,
{
if (c->maybe_tree)
return c->maybe_tree;
- if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
+ if (commit_graph_position(c) == COMMIT_NOT_FROM_GRAPH)
BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
return load_tree_for_commit(r, g, (struct commit *)c);
@@ -788,9 +943,11 @@ struct write_commit_graph_context {
unsigned append:1,
report_progress:1,
split:1,
- check_oids:1;
+ changed_paths:1,
+ order_by_pack:1;
const struct split_commit_graph_opts *split_opts;
+ size_t total_bloom_filter_data_size;
};
static void write_graph_chunk_fanout(struct hashfile *f,
@@ -866,7 +1023,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
if (edge_value >= 0)
edge_value += ctx->new_num_commits_in_base;
- else {
+ else if (ctx->new_base_graph) {
uint32_t pos;
if (find_commit_in_graph(parent->item,
ctx->new_base_graph,
@@ -897,7 +1054,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
if (edge_value >= 0)
edge_value += ctx->new_num_commits_in_base;
- else {
+ else if (ctx->new_base_graph) {
uint32_t pos;
if (find_commit_in_graph(parent->item,
ctx->new_base_graph,
@@ -925,7 +1082,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
else
packedDate[0] = 0;
- packedDate[0] |= htonl((*list)->generation << 2);
+ packedDate[0] |= htonl(commit_graph_data_at(*list)->generation << 2);
packedDate[1] = htonl((*list)->date);
hashwrite(f, packedDate, 8);
@@ -964,7 +1121,7 @@ static void write_graph_chunk_extra_edges(struct hashfile *f,
if (edge_value >= 0)
edge_value += ctx->new_num_commits_in_base;
- else {
+ else if (ctx->new_base_graph) {
uint32_t pos;
if (find_commit_in_graph(parent->item,
ctx->new_base_graph,
@@ -986,6 +1143,41 @@ static void write_graph_chunk_extra_edges(struct hashfile *f,
}
}
+static void write_graph_chunk_bloom_indexes(struct hashfile *f,
+ struct write_commit_graph_context *ctx)
+{
+ struct commit **list = ctx->commits.list;
+ struct commit **last = ctx->commits.list + ctx->commits.nr;
+ uint32_t cur_pos = 0;
+
+ while (list < last) {
+ struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);
+ cur_pos += filter->len;
+ display_progress(ctx->progress, ++ctx->progress_cnt);
+ hashwrite_be32(f, cur_pos);
+ list++;
+ }
+}
+
+static void write_graph_chunk_bloom_data(struct hashfile *f,
+ struct write_commit_graph_context *ctx,
+ const struct bloom_filter_settings *settings)
+{
+ struct commit **list = ctx->commits.list;
+ struct commit **last = ctx->commits.list + ctx->commits.nr;
+
+ hashwrite_be32(f, settings->hash_version);
+ hashwrite_be32(f, settings->num_hashes);
+ hashwrite_be32(f, settings->bits_per_entry);
+
+ while (list < last) {
+ struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);
+ display_progress(ctx->progress, ++ctx->progress_cnt);
+ hashwrite(f, filter->data, filter->len * sizeof(unsigned char));
+ list++;
+ }
+}
+
static int oid_compare(const void *_a, const void *_b)
{
const struct object_id *a = (const struct object_id *)_a;
@@ -1017,6 +1209,8 @@ static int add_packed_commits(const struct object_id *oid,
oidcpy(&(ctx->oids.list[ctx->oids.nr]), oid);
ctx->oids.nr++;
+ set_commit_pos(ctx->r, oid);
+
return 0;
}
@@ -1037,6 +1231,8 @@ static void close_reachable(struct write_commit_graph_context *ctx)
{
int i;
struct commit *commit;
+ enum commit_graph_split_flags flags = ctx->split_opts ?
+ ctx->split_opts->flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
@@ -1066,8 +1262,9 @@ static void close_reachable(struct write_commit_graph_context *ctx)
if (!commit)
continue;
if (ctx->split) {
- if (!parse_commit(commit) &&
- commit->graph_pos == COMMIT_NOT_FROM_GRAPH)
+ if ((!parse_commit(commit) &&
+ commit_graph_position(commit) == COMMIT_NOT_FROM_GRAPH) ||
+ flags == COMMIT_GRAPH_SPLIT_REPLACE)
add_missing_parents(ctx, commit);
} else if (!parse_commit_no_graph(commit))
add_missing_parents(ctx, commit);
@@ -1098,9 +1295,11 @@ static void compute_generation_numbers(struct write_commit_graph_context *ctx)
_("Computing commit graph generation numbers"),
ctx->commits.nr);
for (i = 0; i < ctx->commits.nr; i++) {
+ uint32_t generation = commit_graph_data_at(ctx->commits.list[i])->generation;
+
display_progress(ctx->progress, i + 1);
- if (ctx->commits.list[i]->generation != GENERATION_NUMBER_INFINITY &&
- ctx->commits.list[i]->generation != GENERATION_NUMBER_ZERO)
+ if (generation != GENERATION_NUMBER_INFINITY &&
+ generation != GENERATION_NUMBER_ZERO)
continue;
commit_list_insert(ctx->commits.list[i], &list);
@@ -1111,35 +1310,83 @@ static void compute_generation_numbers(struct write_commit_graph_context *ctx)
uint32_t max_generation = 0;
for (parent = current->parents; parent; parent = parent->next) {
- if (parent->item->generation == GENERATION_NUMBER_INFINITY ||
- parent->item->generation == GENERATION_NUMBER_ZERO) {
+ generation = commit_graph_data_at(parent->item)->generation;
+
+ if (generation == GENERATION_NUMBER_INFINITY ||
+ generation == GENERATION_NUMBER_ZERO) {
all_parents_computed = 0;
commit_list_insert(parent->item, &list);
break;
- } else if (parent->item->generation > max_generation) {
- max_generation = parent->item->generation;
+ } else if (generation > max_generation) {
+ max_generation = generation;
}
}
if (all_parents_computed) {
- current->generation = max_generation + 1;
+ struct commit_graph_data *data = commit_graph_data_at(current);
+
+ data->generation = max_generation + 1;
pop_commit(&list);
- if (current->generation > GENERATION_NUMBER_MAX)
- current->generation = GENERATION_NUMBER_MAX;
+ if (data->generation > GENERATION_NUMBER_MAX)
+ data->generation = GENERATION_NUMBER_MAX;
}
}
}
stop_progress(&ctx->progress);
}
-static int add_ref_to_list(const char *refname,
- const struct object_id *oid,
- int flags, void *cb_data)
+static void compute_bloom_filters(struct write_commit_graph_context *ctx)
{
- struct string_list *list = (struct string_list *)cb_data;
+ int i;
+ struct progress *progress = NULL;
+ struct commit **sorted_commits;
+
+ init_bloom_filters();
+
+ if (ctx->report_progress)
+ progress = start_delayed_progress(
+ _("Computing commit changed paths Bloom filters"),
+ ctx->commits.nr);
+
+ ALLOC_ARRAY(sorted_commits, ctx->commits.nr);
+ COPY_ARRAY(sorted_commits, ctx->commits.list, ctx->commits.nr);
+
+ if (ctx->order_by_pack)
+ QSORT(sorted_commits, ctx->commits.nr, commit_pos_cmp);
+ else
+ QSORT(sorted_commits, ctx->commits.nr, commit_gen_cmp);
+
+ for (i = 0; i < ctx->commits.nr; i++) {
+ struct commit *c = sorted_commits[i];
+ struct bloom_filter *filter = get_bloom_filter(ctx->r, c, 1);
+ ctx->total_bloom_filter_data_size += sizeof(unsigned char) * filter->len;
+ display_progress(progress, i + 1);
+ }
+
+ free(sorted_commits);
+ stop_progress(&progress);
+}
+
+struct refs_cb_data {
+ struct oidset *commits;
+ struct progress *progress;
+};
+
+static int add_ref_to_set(const char *refname,
+ const struct object_id *oid,
+ int flags, void *cb_data)
+{
+ struct object_id peeled;
+ struct refs_cb_data *data = (struct refs_cb_data *)cb_data;
+
+ if (!peel_ref(refname, &peeled))
+ oid = &peeled;
+ if (oid_object_info(the_repository, oid, NULL) == OBJ_COMMIT)
+ oidset_insert(data->commits, oid);
+
+ display_progress(data->progress, oidset_size(data->commits));
- string_list_append(list, oid_to_hex(oid));
return 0;
}
@@ -1147,14 +1394,24 @@ int write_commit_graph_reachable(struct object_directory *odb,
enum commit_graph_write_flags flags,
const struct split_commit_graph_opts *split_opts)
{
- struct string_list list = STRING_LIST_INIT_DUP;
+ struct oidset commits = OIDSET_INIT;
+ struct refs_cb_data data;
int result;
- for_each_ref(add_ref_to_list, &list);
- result = write_commit_graph(odb, NULL, &list,
+ memset(&data, 0, sizeof(data));
+ data.commits = &commits;
+ if (flags & COMMIT_GRAPH_WRITE_PROGRESS)
+ data.progress = start_delayed_progress(
+ _("Collecting referenced commits"), 0);
+
+ for_each_ref(add_ref_to_set, &data);
+
+ stop_progress(&data.progress);
+
+ result = write_commit_graph(odb, NULL, &commits,
flags, split_opts);
- string_list_clear(&list, 0);
+ oidset_clear(&commits);
return result;
}
@@ -1203,41 +1460,21 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx,
return 0;
}
-static int fill_oids_from_commit_hex(struct write_commit_graph_context *ctx,
- struct string_list *commit_hex)
+static int fill_oids_from_commits(struct write_commit_graph_context *ctx,
+ struct oidset *commits)
{
- uint32_t i;
- struct strbuf progress_title = STRBUF_INIT;
+ struct oidset_iter iter;
+ struct object_id *oid;
- if (ctx->report_progress) {
- strbuf_addf(&progress_title,
- Q_("Finding commits for commit graph from %d ref",
- "Finding commits for commit graph from %d refs",
- commit_hex->nr),
- commit_hex->nr);
- ctx->progress = start_delayed_progress(
- progress_title.buf,
- commit_hex->nr);
- }
- for (i = 0; i < commit_hex->nr; i++) {
- const char *end;
- struct object_id oid;
- struct commit *result;
+ if (!oidset_size(commits))
+ return 0;
- display_progress(ctx->progress, i + 1);
- if (!parse_oid_hex(commit_hex->items[i].string, &oid, &end) &&
- (result = lookup_commit_reference_gently(ctx->r, &oid, 1))) {
- ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc);
- oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid));
- ctx->oids.nr++;
- } else if (ctx->check_oids) {
- error(_("invalid commit object id: %s"),
- commit_hex->items[i].string);
- return -1;
- }
+ oidset_iter_init(commits, &iter);
+ while ((oid = oidset_iter_next(&iter))) {
+ ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc);
+ oidcpy(&ctx->oids.list[ctx->oids.nr], oid);
+ ctx->oids.nr++;
}
- stop_progress(&ctx->progress);
- strbuf_release(&progress_title);
return 0;
}
@@ -1272,7 +1509,7 @@ static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx)
if (ctx->split) {
struct commit *c = lookup_commit(ctx->r, &ctx->oids.list[i]);
- if (!c || c->graph_pos != COMMIT_NOT_FROM_GRAPH)
+ if (!c || commit_graph_position(c) != COMMIT_NOT_FROM_GRAPH)
continue;
}
@@ -1287,6 +1524,8 @@ static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx)
static void copy_oids_to_commits(struct write_commit_graph_context *ctx)
{
uint32_t i;
+ enum commit_graph_split_flags flags = ctx->split_opts ?
+ ctx->split_opts->flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
ctx->num_extra_edges = 0;
if (ctx->report_progress)
@@ -1303,11 +1542,14 @@ static void copy_oids_to_commits(struct write_commit_graph_context *ctx)
ALLOC_GROW(ctx->commits.list, ctx->commits.nr + 1, ctx->commits.alloc);
ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]);
- if (ctx->split &&
- ctx->commits.list[ctx->commits.nr]->graph_pos != COMMIT_NOT_FROM_GRAPH)
+ if (ctx->split && flags != COMMIT_GRAPH_SPLIT_REPLACE &&
+ commit_graph_position(ctx->commits.list[ctx->commits.nr]) != COMMIT_NOT_FROM_GRAPH)
continue;
- parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]);
+ if (ctx->split && flags == COMMIT_GRAPH_SPLIT_REPLACE)
+ parse_commit(ctx->commits.list[ctx->commits.nr]);
+ else
+ parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]);
num_parents = commit_list_count(ctx->commits.list[ctx->commits.nr]->parents);
if (num_parents > 2)
@@ -1350,12 +1592,13 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
int fd;
struct hashfile *f;
struct lock_file lk = LOCK_INIT;
- uint32_t chunk_ids[6];
- uint64_t chunk_offsets[6];
+ uint32_t chunk_ids[MAX_NUM_CHUNKS + 1];
+ uint64_t chunk_offsets[MAX_NUM_CHUNKS + 1];
const unsigned hashsz = the_hash_algo->rawsz;
struct strbuf progress_title = STRBUF_INIT;
int num_chunks = 3;
struct object_id file_hash;
+ const struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS;
if (ctx->split) {
struct strbuf tmp_file = STRBUF_INIT;
@@ -1378,17 +1621,25 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
if (ctx->split) {
char *lock_name = get_chain_filename(ctx->odb);
- hold_lock_file_for_update(&lk, lock_name, LOCK_DIE_ON_ERROR);
+ hold_lock_file_for_update_mode(&lk, lock_name,
+ LOCK_DIE_ON_ERROR, 0444);
fd = git_mkstemp_mode(ctx->graph_name, 0444);
if (fd < 0) {
- error(_("unable to create '%s'"), ctx->graph_name);
+ error(_("unable to create temporary graph layer"));
+ return -1;
+ }
+
+ if (adjust_shared_perm(ctx->graph_name)) {
+ error(_("unable to adjust shared permissions for '%s'"),
+ ctx->graph_name);
return -1;
}
f = hashfd(fd, ctx->graph_name);
} else {
- hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR);
+ hold_lock_file_for_update_mode(&lk, ctx->graph_name,
+ LOCK_DIE_ON_ERROR, 0444);
fd = lk.tempfile->fd;
f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
}
@@ -1400,6 +1651,12 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
chunk_ids[num_chunks] = GRAPH_CHUNKID_EXTRAEDGES;
num_chunks++;
}
+ if (ctx->changed_paths) {
+ chunk_ids[num_chunks] = GRAPH_CHUNKID_BLOOMINDEXES;
+ num_chunks++;
+ chunk_ids[num_chunks] = GRAPH_CHUNKID_BLOOMDATA;
+ num_chunks++;
+ }
if (ctx->num_commit_graphs_after > 1) {
chunk_ids[num_chunks] = GRAPH_CHUNKID_BASE;
num_chunks++;
@@ -1418,6 +1675,15 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
4 * ctx->num_extra_edges;
num_chunks++;
}
+ if (ctx->changed_paths) {
+ chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
+ sizeof(uint32_t) * ctx->commits.nr;
+ num_chunks++;
+
+ chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
+ sizeof(uint32_t) * 3 + ctx->total_bloom_filter_data_size;
+ num_chunks++;
+ }
if (ctx->num_commit_graphs_after > 1) {
chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +
hashsz * (ctx->num_commit_graphs_after - 1);
@@ -1455,6 +1721,10 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
write_graph_chunk_data(f, hashsz, ctx);
if (ctx->num_extra_edges)
write_graph_chunk_extra_edges(f, ctx);
+ if (ctx->changed_paths) {
+ write_graph_chunk_bloom_indexes(f, ctx);
+ write_graph_chunk_bloom_data(f, ctx, &bloom_settings);
+ }
if (ctx->num_commit_graphs_after > 1 &&
write_graph_chunk_base(f, ctx)) {
return -1;
@@ -1488,8 +1758,12 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
}
if (ctx->base_graph_name) {
- const char *dest = ctx->commit_graph_filenames_after[
- ctx->num_commit_graphs_after - 2];
+ const char *dest;
+ int idx = ctx->num_commit_graphs_after - 1;
+ if (ctx->num_commit_graphs_after > 1)
+ idx--;
+
+ dest = ctx->commit_graph_filenames_after[idx];
if (strcmp(ctx->base_graph_name, dest)) {
result = rename(ctx->base_graph_name, dest);
@@ -1529,6 +1803,7 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx)
{
struct commit_graph *g;
uint32_t num_commits;
+ enum commit_graph_split_flags flags = COMMIT_GRAPH_SPLIT_UNSPECIFIED;
uint32_t i;
int max_commits = 0;
@@ -1539,24 +1814,36 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx)
if (ctx->split_opts->size_multiple)
size_mult = ctx->split_opts->size_multiple;
+
+ flags = ctx->split_opts->flags;
}
g = ctx->r->objects->commit_graph;
num_commits = ctx->commits.nr;
- ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
-
- while (g && (g->num_commits <= size_mult * num_commits ||
- (max_commits && num_commits > max_commits))) {
- if (g->odb != ctx->odb)
- break;
+ if (flags == COMMIT_GRAPH_SPLIT_REPLACE)
+ ctx->num_commit_graphs_after = 1;
+ else
+ ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
+
+ if (flags != COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED &&
+ flags != COMMIT_GRAPH_SPLIT_REPLACE) {
+ while (g && (g->num_commits <= size_mult * num_commits ||
+ (max_commits && num_commits > max_commits))) {
+ if (g->odb != ctx->odb)
+ break;
- num_commits += g->num_commits;
- g = g->base_graph;
+ num_commits += g->num_commits;
+ g = g->base_graph;
- ctx->num_commit_graphs_after--;
+ ctx->num_commit_graphs_after--;
+ }
}
- ctx->new_base_graph = g;
+ if (flags != COMMIT_GRAPH_SPLIT_REPLACE)
+ ctx->new_base_graph = g;
+ else if (ctx->num_commit_graphs_after != 1)
+ BUG("split_graph_merge_strategy: num_commit_graphs_after "
+ "should be 1 with --split=replace");
if (ctx->num_commit_graphs_after == 2) {
char *old_graph_name = get_commit_graph_filename(g->odb);
@@ -1570,8 +1857,8 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx)
free(old_graph_name);
}
- ALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after);
- ALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);
+ CALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after);
+ CALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);
for (i = 0; i < ctx->num_commit_graphs_after &&
i < ctx->num_commit_graphs_before; i++)
@@ -1707,7 +1994,7 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx)
timestamp_t expire_time = time(NULL);
if (ctx->split_opts && ctx->split_opts->expire_time)
- expire_time -= ctx->split_opts->expire_time;
+ expire_time = ctx->split_opts->expire_time;
if (!ctx->split) {
char *chain_file_name = get_chain_filename(ctx->odb);
unlink(chain_file_name);
@@ -1756,13 +2043,14 @@ out:
int write_commit_graph(struct object_directory *odb,
struct string_list *pack_indexes,
- struct string_list *commit_hex,
+ struct oidset *commits,
enum commit_graph_write_flags flags,
const struct split_commit_graph_opts *split_opts)
{
struct write_commit_graph_context *ctx;
uint32_t i, count_distinct = 0;
int res = 0;
+ int replace = 0;
if (!commit_graph_compatible(the_repository))
return 0;
@@ -1773,8 +2061,9 @@ int write_commit_graph(struct object_directory *odb,
ctx->append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0;
ctx->report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0;
ctx->split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0;
- ctx->check_oids = flags & COMMIT_GRAPH_WRITE_CHECK_OIDS ? 1 : 0;
ctx->split_opts = split_opts;
+ ctx->changed_paths = flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS ? 1 : 0;
+ ctx->total_bloom_filter_data_size = 0;
if (ctx->split) {
struct commit_graph *g;
@@ -1797,6 +2086,9 @@ int write_commit_graph(struct object_directory *odb,
g = g->base_graph;
}
}
+
+ if (ctx->split_opts)
+ replace = ctx->split_opts->flags & COMMIT_GRAPH_SPLIT_REPLACE;
}
ctx->approx_nr_objects = approximate_object_count();
@@ -1824,17 +2116,20 @@ int write_commit_graph(struct object_directory *odb,
}
if (pack_indexes) {
+ ctx->order_by_pack = 1;
if ((res = fill_oids_from_packs(ctx, pack_indexes)))
goto cleanup;
}
- if (commit_hex) {
- if ((res = fill_oids_from_commit_hex(ctx, commit_hex)))
+ if (commits) {
+ if ((res = fill_oids_from_commits(ctx, commits)))
goto cleanup;
}
- if (!pack_indexes && !commit_hex)
+ if (!pack_indexes && !commits) {
+ ctx->order_by_pack = 1;
fill_oids_from_all_packs(ctx);
+ }
close_reachable(ctx);
@@ -1857,18 +2152,22 @@ int write_commit_graph(struct object_directory *odb,
goto cleanup;
}
- if (!ctx->commits.nr)
+ if (!ctx->commits.nr && !replace)
goto cleanup;
if (ctx->split) {
split_graph_merge_strategy(ctx);
- merge_commit_graphs(ctx);
+ if (!replace)
+ merge_commit_graphs(ctx);
} else
ctx->num_commit_graphs_after = 1;
compute_generation_numbers(ctx);
+ if (ctx->changed_paths)
+ compute_bloom_filters(ctx);
+
res = write_commit_graph_file(ctx);
if (ctx->split)
@@ -1993,6 +2292,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
struct commit *graph_commit, *odb_commit;
struct commit_list *graph_parents, *odb_parents;
uint32_t max_generation = 0;
+ uint32_t generation;
display_progress(progress, i + 1);
hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
@@ -2031,8 +2331,9 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
oid_to_hex(&graph_parents->item->object.oid),
oid_to_hex(&odb_parents->item->object.oid));
- if (graph_parents->item->generation > max_generation)
- max_generation = graph_parents->item->generation;
+ generation = commit_graph_generation(graph_parents->item);
+ if (generation > max_generation)
+ max_generation = generation;
graph_parents = graph_parents->next;
odb_parents = odb_parents->next;
@@ -2042,7 +2343,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
graph_report(_("commit-graph parent list for commit %s terminates early"),
oid_to_hex(&cur_oid));
- if (!graph_commit->generation) {
+ if (!commit_graph_generation(graph_commit)) {
if (generation_zero == GENERATION_NUMBER_EXISTS)
graph_report(_("commit-graph has generation number zero for commit %s, but non-zero elsewhere"),
oid_to_hex(&cur_oid));
@@ -2062,10 +2363,11 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
if (max_generation == GENERATION_NUMBER_MAX)
max_generation--;
- if (graph_commit->generation != max_generation + 1)
+ generation = commit_graph_generation(graph_commit);
+ if (generation != max_generation + 1)
graph_report(_("commit-graph generation for commit %s is %u != %u"),
oid_to_hex(&cur_oid),
- graph_commit->generation,
+ generation,
max_generation + 1);
if (graph_commit->date != odb_commit->date)
@@ -2088,12 +2390,12 @@ void free_commit_graph(struct commit_graph *g)
{
if (!g)
return;
- if (g->graph_fd >= 0) {
+ if (g->data) {
munmap((void *)g->data, g->data_len);
g->data = NULL;
- close(g->graph_fd);
}
free(g->filename);
+ free(g->bloom_filter_settings);
free(g);
}