diff options
Diffstat (limited to 'builtin/fast-export.c')
| -rw-r--r-- | builtin/fast-export.c | 399 |
1 files changed, 364 insertions, 35 deletions
diff --git a/builtin/fast-export.c b/builtin/fast-export.c index b8d8a3aaf9..1e815b5577 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -5,6 +5,7 @@ */ #include "builtin.h" #include "cache.h" +#include "refs.h" #include "commit.h" #include "object.h" #include "tag.h" @@ -17,6 +18,8 @@ #include "utf8.h" #include "parse-options.h" #include "quote.h" +#include "remote.h" +#include "blob.h" static const char *fast_export_usage[] = { N_("git fast-export [rev-list-opts]"), @@ -31,6 +34,9 @@ static int use_done_feature; static int no_data; static int full_tree; static struct string_list extra_refs = STRING_LIST_INIT_NODUP; +static struct refspec *refspecs; +static int refspecs_nr; +static int anonymize; static int parse_opt_signed_tag_mode(const struct option *opt, const char *arg, int unset) @@ -78,6 +84,76 @@ static int has_unshown_parent(struct commit *commit) return 0; } +struct anonymized_entry { + struct hashmap_entry hash; + const char *orig; + size_t orig_len; + const char *anon; + size_t anon_len; +}; + +static int anonymized_entry_cmp(const void *va, const void *vb, + const void *data) +{ + const struct anonymized_entry *a = va, *b = vb; + return a->orig_len != b->orig_len || + memcmp(a->orig, b->orig, a->orig_len); +} + +/* + * Basically keep a cache of X->Y so that we can repeatedly replace + * the same anonymized string with another. The actual generation + * is farmed out to the generate function. + */ +static const void *anonymize_mem(struct hashmap *map, + void *(*generate)(const void *, size_t *), + const void *orig, size_t *len) +{ + struct anonymized_entry key, *ret; + + if (!map->cmpfn) + hashmap_init(map, anonymized_entry_cmp, 0); + + hashmap_entry_init(&key, memhash(orig, *len)); + key.orig = orig; + key.orig_len = *len; + ret = hashmap_get(map, &key, NULL); + + if (!ret) { + ret = xmalloc(sizeof(*ret)); + hashmap_entry_init(&ret->hash, key.hash.hash); + ret->orig = xstrdup(orig); + ret->orig_len = *len; + ret->anon = generate(orig, len); + ret->anon_len = *len; + hashmap_put(map, ret); + } + + *len = ret->anon_len; + return ret->anon; +} + +/* + * We anonymize each component of a path individually, + * so that paths a/b and a/c will share a common root. + * The paths are cached via anonymize_mem so that repeated + * lookups for "a" will yield the same value. + */ +static void anonymize_path(struct strbuf *out, const char *path, + struct hashmap *map, + void *(*generate)(const void *, size_t *)) +{ + while (*path) { + const char *end_of_component = strchrnul(path, '/'); + size_t len = end_of_component - path; + const char *c = anonymize_mem(map, generate, path, &len); + strbuf_add(out, c, len); + path = end_of_component; + if (*path) + strbuf_addch(out, *path++); + } +} + /* Since intptr_t is C99, we do not use it here */ static inline uint32_t *mark_to_ptr(uint32_t mark) { @@ -116,6 +192,26 @@ static void show_progress(void) printf("progress %d objects\n", counter); } +/* + * Ideally we would want some transformation of the blob data here + * that is unreversible, but would still be the same size and have + * the same data relationship to other blobs (so that we get the same + * delta and packing behavior as the original). But the first and last + * requirements there are probably mutually exclusive, so let's take + * the easy way out for now, and just generate arbitrary content. + * + * There's no need to cache this result with anonymize_mem, since + * we already handle blob content caching with marks. + */ +static char *anonymize_blob(unsigned long *size) +{ + static int counter; + struct strbuf out = STRBUF_INIT; + strbuf_addf(&out, "anonymous blob %d", counter++); + *size = out.len; + return strbuf_detach(&out, NULL); +} + static void export_blob(const unsigned char *sha1) { unsigned long size; @@ -134,12 +230,19 @@ static void export_blob(const unsigned char *sha1) if (object && object->flags & SHOWN) return; - buf = read_sha1_file(sha1, &type, &size); - if (!buf) - die ("Could not read blob %s", sha1_to_hex(sha1)); - if (check_sha1_signature(sha1, buf, size, typename(type)) < 0) - die("sha1 mismatch in blob %s", sha1_to_hex(sha1)); - object = parse_object_buffer(sha1, type, size, buf, &eaten); + if (anonymize) { + buf = anonymize_blob(&size); + object = (struct object *)lookup_blob(sha1); + eaten = 0; + } else { + buf = read_sha1_file(sha1, &type, &size); + if (!buf) + die ("Could not read blob %s", sha1_to_hex(sha1)); + if (check_sha1_signature(sha1, buf, size, typename(type)) < 0) + die("sha1 mismatch in blob %s", sha1_to_hex(sha1)); + object = parse_object_buffer(sha1, type, size, buf, &eaten); + } + if (!object) die("Could not read blob %s", sha1_to_hex(sha1)); @@ -187,7 +290,7 @@ static int depth_first(const void *a_, const void *b_) return (a->status == 'R') - (b->status == 'R'); } -static void print_path(const char *path) +static void print_path_1(const char *path) { int need_quote = quote_c_style(path, NULL, NULL, 0); if (need_quote) @@ -198,6 +301,43 @@ static void print_path(const char *path) printf("%s", path); } +static void *anonymize_path_component(const void *path, size_t *len) +{ + static int counter; + struct strbuf out = STRBUF_INIT; + strbuf_addf(&out, "path%d", counter++); + return strbuf_detach(&out, len); +} + +static void print_path(const char *path) +{ + if (!anonymize) + print_path_1(path); + else { + static struct hashmap paths; + static struct strbuf anon = STRBUF_INIT; + + anonymize_path(&anon, path, &paths, anonymize_path_component); + print_path_1(anon.buf); + strbuf_reset(&anon); + } +} + +static void *generate_fake_sha1(const void *old, size_t *len) +{ + static uint32_t counter = 1; /* avoid null sha1 */ + unsigned char *out = xcalloc(20, 1); + put_be32(out + 16, counter++); + return out; +} + +static const unsigned char *anonymize_sha1(const unsigned char *sha1) +{ + static struct hashmap sha1s; + size_t len = 20; + return anonymize_mem(&sha1s, generate_fake_sha1, sha1, &len); +} + static void show_filemodify(struct diff_queue_struct *q, struct diff_options *options, void *data) { @@ -207,7 +347,7 @@ static void show_filemodify(struct diff_queue_struct *q, * Handle files below a directory first, in case they are all deleted * and the directory changes to a file or symlink. */ - qsort(q->queue, q->nr, sizeof(q->queue[0]), depth_first); + QSORT(q->queue, q->nr, depth_first); for (i = 0; i < q->nr; i++) { struct diff_filespec *ospec = q->queue[i]->one; @@ -228,7 +368,7 @@ static void show_filemodify(struct diff_queue_struct *q, print_path(spec->path); putchar('\n'); - if (!hashcmp(ospec->sha1, spec->sha1) && + if (!oidcmp(&ospec->oid, &spec->oid) && ospec->mode == spec->mode) break; /* fallthrough */ @@ -242,9 +382,11 @@ static void show_filemodify(struct diff_queue_struct *q, */ if (no_data || S_ISGITLINK(spec->mode)) printf("M %06o %s ", spec->mode, - sha1_to_hex(spec->sha1)); + sha1_to_hex(anonymize ? + anonymize_sha1(spec->oid.hash) : + spec->oid.hash)); else { - struct object *object = lookup_object(spec->sha1); + struct object *object = lookup_object(spec->oid.hash); printf("M %06o :%d ", spec->mode, get_object_mark(object)); } @@ -276,28 +418,139 @@ static const char *find_encoding(const char *begin, const char *end) return bol; } +static void *anonymize_ref_component(const void *old, size_t *len) +{ + static int counter; + struct strbuf out = STRBUF_INIT; + strbuf_addf(&out, "ref%d", counter++); + return strbuf_detach(&out, len); +} + +static const char *anonymize_refname(const char *refname) +{ + /* + * If any of these prefixes is found, we will leave it intact + * so that tags remain tags and so forth. + */ + static const char *prefixes[] = { + "refs/heads/", + "refs/tags/", + "refs/remotes/", + "refs/" + }; + static struct hashmap refs; + static struct strbuf anon = STRBUF_INIT; + int i; + + /* + * We also leave "master" as a special case, since it does not reveal + * anything interesting. + */ + if (!strcmp(refname, "refs/heads/master")) + return refname; + + strbuf_reset(&anon); + for (i = 0; i < ARRAY_SIZE(prefixes); i++) { + if (skip_prefix(refname, prefixes[i], &refname)) { + strbuf_addstr(&anon, prefixes[i]); + break; + } + } + + anonymize_path(&anon, refname, &refs, anonymize_ref_component); + return anon.buf; +} + +/* + * We do not even bother to cache commit messages, as they are unlikely + * to be repeated verbatim, and it is not that interesting when they are. + */ +static char *anonymize_commit_message(const char *old) +{ + static int counter; + return xstrfmt("subject %d\n\nbody\n", counter++); +} + +static struct hashmap idents; +static void *anonymize_ident(const void *old, size_t *len) +{ + static int counter; + struct strbuf out = STRBUF_INIT; + strbuf_addf(&out, "User %d <user%d@example.com>", counter, counter); + counter++; + return strbuf_detach(&out, len); +} + +/* + * Our strategy here is to anonymize the names and email addresses, + * but keep timestamps intact, as they influence things like traversal + * order (and by themselves should not be too revealing). + */ +static void anonymize_ident_line(const char **beg, const char **end) +{ + static struct strbuf buffers[] = { STRBUF_INIT, STRBUF_INIT }; + static unsigned which_buffer; + + struct strbuf *out; + struct ident_split split; + const char *end_of_header; + + out = &buffers[which_buffer++]; + which_buffer %= ARRAY_SIZE(buffers); + strbuf_reset(out); + + /* skip "committer", "author", "tagger", etc */ + end_of_header = strchr(*beg, ' '); + if (!end_of_header) + die("BUG: malformed line fed to anonymize_ident_line: %.*s", + (int)(*end - *beg), *beg); + end_of_header++; + strbuf_add(out, *beg, end_of_header - *beg); + + if (!split_ident_line(&split, end_of_header, *end - end_of_header) && + split.date_begin) { + const char *ident; + size_t len; + + len = split.mail_end - split.name_begin; + ident = anonymize_mem(&idents, anonymize_ident, + split.name_begin, &len); + strbuf_add(out, ident, len); + strbuf_addch(out, ' '); + strbuf_add(out, split.date_begin, split.tz_end - split.date_begin); + } else { + strbuf_addstr(out, "Malformed Ident <malformed@example.com> 0 -0000"); + } + + *beg = out->buf; + *end = out->buf + out->len; +} + static void handle_commit(struct commit *commit, struct rev_info *rev) { int saved_output_format = rev->diffopt.output_format; + const char *commit_buffer; const char *author, *author_end, *committer, *committer_end; const char *encoding, *message; char *reencoded = NULL; struct commit_list *p; + const char *refname; int i; rev->diffopt.output_format = DIFF_FORMAT_CALLBACK; parse_commit_or_die(commit); - author = strstr(commit->buffer, "\nauthor "); + commit_buffer = get_commit_buffer(commit, NULL); + author = strstr(commit_buffer, "\nauthor "); if (!author) die ("Could not find author in commit %s", - sha1_to_hex(commit->object.sha1)); + oid_to_hex(&commit->object.oid)); author++; author_end = strchrnul(author, '\n'); committer = strstr(author_end, "\ncommitter "); if (!committer) die ("Could not find committer in commit %s", - sha1_to_hex(commit->object.sha1)); + oid_to_hex(&commit->object.oid)); committer++; committer_end = strchrnul(committer, '\n'); message = strstr(committer_end, "\n\n"); @@ -309,25 +562,34 @@ static void handle_commit(struct commit *commit, struct rev_info *rev) get_object_mark(&commit->parents->item->object) != 0 && !full_tree) { parse_commit_or_die(commit->parents->item); - diff_tree_sha1(commit->parents->item->tree->object.sha1, - commit->tree->object.sha1, "", &rev->diffopt); + diff_tree_sha1(commit->parents->item->tree->object.oid.hash, + commit->tree->object.oid.hash, "", &rev->diffopt); } else - diff_root_tree_sha1(commit->tree->object.sha1, + diff_root_tree_sha1(commit->tree->object.oid.hash, "", &rev->diffopt); /* Export the referenced blobs, and remember the marks. */ for (i = 0; i < diff_queued_diff.nr; i++) if (!S_ISGITLINK(diff_queued_diff.queue[i]->two->mode)) - export_blob(diff_queued_diff.queue[i]->two->sha1); + export_blob(diff_queued_diff.queue[i]->two->oid.hash); + + refname = commit->util; + if (anonymize) { + refname = anonymize_refname(refname); + anonymize_ident_line(&committer, &committer_end); + anonymize_ident_line(&author, &author_end); + } mark_next_object(&commit->object); - if (!is_encoding_utf8(encoding)) + if (anonymize) + reencoded = anonymize_commit_message(message); + else if (!is_encoding_utf8(encoding)) reencoded = reencode_string(message, "UTF-8", encoding); if (!commit->parents) - printf("reset %s\n", (const char*)commit->util); + printf("reset %s\n", refname); printf("commit %s\nmark :%"PRIu32"\n%.*s\n%.*s\ndata %u\n%s", - (const char *)commit->util, last_idnum, + refname, last_idnum, (int)(author_end - author), author, (int)(committer_end - committer), committer, (unsigned)(reencoded @@ -335,6 +597,7 @@ static void handle_commit(struct commit *commit, struct rev_info *rev) ? strlen(message) : 0), reencoded ? reencoded : message ? message : ""); free(reencoded); + unuse_commit_buffer(commit, commit_buffer); for (i = 0, p = commit->parents; p; p = p->next) { int mark = get_object_mark(&p->item->object); @@ -357,6 +620,14 @@ static void handle_commit(struct commit *commit, struct rev_info *rev) show_progress(); } +static void *anonymize_tag(const void *old, size_t *len) +{ + static int counter; + struct strbuf out = STRBUF_INIT; + strbuf_addf(&out, "tag message %d", counter++); + return strbuf_detach(&out, len); +} + static void handle_tail(struct object_array *commits, struct rev_info *revs) { struct commit *commit; @@ -390,13 +661,13 @@ static void handle_tag(const char *name, struct tag *tag) } if (tagged->type == OBJ_TREE) { warning("Omitting tag %s,\nsince tags of trees (or tags of tags of trees, etc.) are not supported.", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); return; } - buf = read_sha1_file(tag->object.sha1, &type, &size); + buf = read_sha1_file(tag->object.oid.hash, &type, &size); if (!buf) - die ("Could not read tag %s", sha1_to_hex(tag->object.sha1)); + die ("Could not read tag %s", oid_to_hex(&tag->object.oid)); message = memmem(buf, size, "\n\n", 2); if (message) { message += 2; @@ -413,6 +684,17 @@ static void handle_tag(const char *name, struct tag *tag) } else { tagger++; tagger_end = strchrnul(tagger, '\n'); + if (anonymize) + anonymize_ident_line(&tagger, &tagger_end); + } + + if (anonymize) { + name = anonymize_refname(name); + if (message) { + static struct hashmap tags; + message = anonymize_mem(&tags, anonymize_tag, + message, &message_size); + } } /* handle signed tags */ @@ -424,16 +706,16 @@ static void handle_tag(const char *name, struct tag *tag) case ABORT: die ("Encountered signed tag %s; use " "--signed-tags=<mode> to handle it.", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); case WARN: warning ("Exporting signed tag %s", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); /* fallthru */ case VERBATIM: break; case WARN_STRIP: warning ("Stripping signature from tag %s", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); /* fallthru */ case STRIP: message_size = signature + 1 - message; @@ -449,14 +731,14 @@ static void handle_tag(const char *name, struct tag *tag) case ABORT: die ("Tag %s tags unexported object; use " "--tag-of-filtered-object=<mode> to handle it.", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); case DROP: /* Ignore this tag altogether */ return; case REWRITE: if (tagged->type != OBJ_COMMIT) { die ("Tag %s tags unexported %s!", - sha1_to_hex(tag->object.sha1), + oid_to_hex(&tag->object.oid), typename(tagged->type)); } p = (struct commit *)tagged; @@ -469,7 +751,7 @@ static void handle_tag(const char *name, struct tag *tag) break; if (!p->parents) die ("Can't find replacement commit for tag %s\n", - sha1_to_hex(tag->object.sha1)); + oid_to_hex(&tag->object.oid)); p = p->parents->item; } tagged_mark = get_object_mark(&p->object); @@ -495,7 +777,7 @@ static struct commit *get_commit(struct rev_cmdline_entry *e, char *full_name) /* handle nested tags */ while (tag && tag->object.type == OBJ_TAG) { - parse_object(tag->object.sha1); + parse_object(tag->object.oid.hash); string_list_append(&extra_refs, full_name)->util = tag; tag = (struct tag *)tag->tagged; } @@ -525,6 +807,15 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) if (dwim_ref(e->name, strlen(e->name), sha1, &full_name) != 1) continue; + if (refspecs) { + char *private; + private = apply_refspecs(refspecs, refspecs_nr, full_name); + if (private) { + free(full_name); + full_name = private; + } + } + commit = get_commit(e, full_name); if (!commit) { warning("%s: Unexpected object of type %s, skipping.", @@ -537,7 +828,7 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) case OBJ_COMMIT: break; case OBJ_BLOB: - export_blob(commit->object.sha1); + export_blob(commit->object.oid.hash); continue; default: /* OBJ_TAG (nested tags) is already handled */ warning("Tag points to object of unexpected type %s, skipping.", @@ -569,6 +860,8 @@ static void handle_tags_and_duplicates(void) handle_tag(name, (struct tag *)object); break; case OBJ_COMMIT: + if (anonymize) + name = anonymize_refname(name); /* create refs pointing to already seen commits */ commit = (struct commit *)object; printf("reset %s\nfrom :%d\n\n", name, @@ -587,7 +880,7 @@ static void export_marks(char *file) FILE *f; int e = 0; - f = fopen(file, "w"); + f = fopen_for_writing(file); if (!f) die_errno("Unable to open marks file %s for writing.", file); @@ -595,7 +888,7 @@ static void export_marks(char *file) if (deco->base && deco->base->type == 1) { mark = ptr_to_mark(deco->decoration); if (fprintf(f, ":%"PRIu32" %s\n", mark, - sha1_to_hex(deco->base->sha1)) < 0) { + oid_to_hex(&deco->base->oid)) < 0) { e = 1; break; } @@ -661,6 +954,19 @@ static void import_marks(char *input_file) fclose(f); } +static void handle_deletes(void) +{ + int i; + for (i = 0; i < refspecs_nr; i++) { + struct refspec *refspec = &refspecs[i]; + if (*refspec->src) + continue; + + printf("reset %s\nfrom %s\n\n", + refspec->dst, sha1_to_hex(null_sha1)); + } +} + int cmd_fast_export(int argc, const char **argv, const char *prefix) { struct rev_info revs; @@ -668,6 +974,7 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix) struct commit *commit; char *export_filename = NULL, *import_filename = NULL; uint32_t lastimportid; + struct string_list refspecs_list = STRING_LIST_INIT_NODUP; struct option options[] = { OPT_INTEGER(0, "progress", &progress, N_("show progress after <n> objects")), @@ -688,6 +995,9 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "use-done-feature", &use_done_feature, N_("Use the done feature to terminate the stream")), OPT_BOOL(0, "no-data", &no_data, N_("Skip output of blob data")), + OPT_STRING_LIST(0, "refspec", &refspecs_list, N_("refspec"), + N_("Apply refspec to exported refs")), + OPT_BOOL(0, "anonymize", &anonymize, N_("anonymize output")), OPT_END() }; @@ -701,11 +1011,27 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix) revs.topo_order = 1; revs.show_source = 1; revs.rewrite_parents = 1; + argc = parse_options(argc, argv, prefix, options, fast_export_usage, + PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN); argc = setup_revisions(argc, argv, &revs, NULL); - argc = parse_options(argc, argv, prefix, options, fast_export_usage, 0); if (argc > 1) usage_with_options (fast_export_usage, options); + if (refspecs_list.nr) { + const char **refspecs_str; + int i; + + ALLOC_ARRAY(refspecs_str, refspecs_list.nr); + for (i = 0; i < refspecs_list.nr; i++) + refspecs_str[i] = refspecs_list.items[i].string; + + refspecs_nr = refspecs_list.nr; + refspecs = parse_fetch_refspec(refspecs_nr, refspecs_str); + + string_list_clear(&refspecs_list, 1); + free(refspecs_str); + } + if (use_done_feature) printf("feature done\n"); @@ -733,6 +1059,7 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix) } handle_tags_and_duplicates(); + handle_deletes(); if (export_filename && lastimportid != last_idnum) export_marks(export_filename); @@ -740,5 +1067,7 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix) if (use_done_feature) printf("done\n"); + free_refspec(refspecs_nr, refspecs); + return 0; } |