diff options
| -rw-r--r-- | Documentation/config.txt | 12 | ||||
| -rw-r--r-- | Documentation/git-clone.txt | 5 | ||||
| -rw-r--r-- | builtin/clone.c | 83 | ||||
| -rw-r--r-- | builtin/submodule--helper.c | 136 | ||||
| -rw-r--r-- | cache.h | 1 | ||||
| -rwxr-xr-x | git-submodule.sh | 2 | ||||
| -rw-r--r-- | sha1_file.c | 76 | ||||
| -rwxr-xr-x | t/t7408-submodule-reference.sh | 185 |
8 files changed, 379 insertions, 121 deletions
diff --git a/Documentation/config.txt b/Documentation/config.txt index 8a115b3702..32f065ca6a 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -2853,6 +2853,18 @@ submodule.fetchJobs:: in parallel. A value of 0 will give some reasonable default. If unset, it defaults to 1. +submodule.alternateLocation:: + Specifies how the submodules obtain alternates when submodules are + cloned. Possible values are `no`, `superproject`. + By default `no` is assumed, which doesn't add references. When the + value is set to `superproject` the submodule to be cloned computes + its alternates location relative to the superprojects alternate. + +submodule.alternateErrorStrategy + Specifies how to treat errors with the alternates for a submodule + as computed via `submodule.alternateLocation`. Possible values are + `ignore`, `info`, `die`. Default is `die`. + tag.forceSignAnnotated:: A boolean to specify whether annotated tags created should be GPG signed. If `--annotate` is specified on the command line, it takes diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index ec41d3d698..e316c4bd51 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -90,13 +90,16 @@ If you want to break the dependency of a repository cloned with `-s` on its source repository, you can simply run `git repack -a` to copy all objects from the source repository into a pack in the cloned repository. ---reference <repository>:: +--reference[-if-able] <repository>:: If the reference repository is on the local machine, automatically setup `.git/objects/info/alternates` to obtain objects from the reference repository. Using an already existing repository as an alternate will require fewer objects to be copied from the repository being cloned, reducing network and local storage costs. + When using the `--reference-if-able`, a non existing + directory is skipped with a warning instead of aborting + the clone. + *NOTE*: see the NOTE for the `--shared` option, and also the `--dissociate` option. diff --git a/builtin/clone.c b/builtin/clone.c index f044a8c27f..404c5e8022 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -50,7 +50,8 @@ static int option_verbosity; static int option_progress = -1; static enum transport_family family; static struct string_list option_config = STRING_LIST_INIT_NODUP; -static struct string_list option_reference = STRING_LIST_INIT_NODUP; +static struct string_list option_required_reference = STRING_LIST_INIT_NODUP; +static struct string_list option_optional_reference = STRING_LIST_INIT_NODUP; static int option_dissociate; static int max_jobs = -1; @@ -79,8 +80,10 @@ static struct option builtin_clone_options[] = { N_("number of submodules cloned in parallel")), OPT_STRING(0, "template", &option_template, N_("template-directory"), N_("directory from which templates will be used")), - OPT_STRING_LIST(0, "reference", &option_reference, N_("repo"), + OPT_STRING_LIST(0, "reference", &option_required_reference, N_("repo"), N_("reference repository")), + OPT_STRING_LIST(0, "reference-if-able", &option_optional_reference, + N_("repo"), N_("reference repository")), OPT_BOOL(0, "dissociate", &option_dissociate, N_("use --reference only while cloning")), OPT_STRING('o', "origin", &option_origin, N_("name"), @@ -282,50 +285,37 @@ static void strip_trailing_slashes(char *dir) static int add_one_reference(struct string_list_item *item, void *cb_data) { - char *ref_git; - const char *repo; - struct strbuf alternate = STRBUF_INIT; - - /* Beware: read_gitfile(), real_path() and mkpath() return static buffer */ - ref_git = xstrdup(real_path(item->string)); - - repo = read_gitfile(ref_git); - if (!repo) - repo = read_gitfile(mkpath("%s/.git", ref_git)); - if (repo) { - free(ref_git); - ref_git = xstrdup(repo); - } + struct strbuf err = STRBUF_INIT; + int *required = cb_data; + char *ref_git = compute_alternate_path(item->string, &err); - if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) { - char *ref_git_git = mkpathdup("%s/.git", ref_git); - free(ref_git); - ref_git = ref_git_git; - } else if (!is_directory(mkpath("%s/objects", ref_git))) { + if (!ref_git) { + if (*required) + die("%s", err.buf); + else + fprintf(stderr, + _("info: Could not add alternate for '%s': %s\n"), + item->string, err.buf); + } else { struct strbuf sb = STRBUF_INIT; - if (get_common_dir(&sb, ref_git)) - die(_("reference repository '%s' as a linked checkout is not supported yet."), - item->string); - die(_("reference repository '%s' is not a local repository."), - item->string); + strbuf_addf(&sb, "%s/objects", ref_git); + add_to_alternates_file(sb.buf); + strbuf_release(&sb); } - if (!access(mkpath("%s/shallow", ref_git), F_OK)) - die(_("reference repository '%s' is shallow"), item->string); - - if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) - die(_("reference repository '%s' is grafted"), item->string); - - strbuf_addf(&alternate, "%s/objects", ref_git); - add_to_alternates_file(alternate.buf); - strbuf_release(&alternate); + strbuf_release(&err); free(ref_git); return 0; } static void setup_reference(void) { - for_each_string_list(&option_reference, add_one_reference, NULL); + int required = 1; + for_each_string_list(&option_required_reference, + add_one_reference, &required); + required = 0; + for_each_string_list(&option_optional_reference, + add_one_reference, &required); } static void copy_alternates(struct strbuf *src, struct strbuf *dst, @@ -957,6 +947,25 @@ int cmd_clone(int argc, const char **argv, const char *prefix) else fprintf(stderr, _("Cloning into '%s'...\n"), dir); } + + if (option_recursive) { + if (option_required_reference.nr && + option_optional_reference.nr) + die(_("clone --recursive is not compatible with " + "both --reference and --reference-if-able")); + else if (option_required_reference.nr) { + string_list_append(&option_config, + "submodule.alternateLocation=superproject"); + string_list_append(&option_config, + "submodule.alternateErrorStrategy=die"); + } else if (option_optional_reference.nr) { + string_list_append(&option_config, + "submodule.alternateLocation=superproject"); + string_list_append(&option_config, + "submodule.alternateErrorStrategy=info"); + } + } + init_db(option_template, INIT_DB_QUIET); write_config(&option_config); @@ -977,7 +986,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) git_config_set(key.buf, repo); strbuf_reset(&key); - if (option_reference.nr) + if (option_required_reference.nr || option_optional_reference.nr) setup_reference(); fetch_pattern = value.buf; diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index e79790f0bd..9d79f1994a 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -442,7 +442,7 @@ static int module_name(int argc, const char **argv, const char *prefix) } static int clone_submodule(const char *path, const char *gitdir, const char *url, - const char *depth, const char *reference, int quiet) + const char *depth, struct string_list *reference, int quiet) { struct child_process cp = CHILD_PROCESS_INIT; @@ -452,8 +452,12 @@ static int clone_submodule(const char *path, const char *gitdir, const char *url argv_array_push(&cp.args, "--quiet"); if (depth && *depth) argv_array_pushl(&cp.args, "--depth", depth, NULL); - if (reference && *reference) - argv_array_pushl(&cp.args, "--reference", reference, NULL); + if (reference->nr) { + struct string_list_item *item; + for_each_string_list_item(item, reference) + argv_array_pushl(&cp.args, "--reference", + item->string, NULL); + } if (gitdir && *gitdir) argv_array_pushl(&cp.args, "--separate-git-dir", gitdir, NULL); @@ -467,15 +471,114 @@ static int clone_submodule(const char *path, const char *gitdir, const char *url return run_command(&cp); } +struct submodule_alternate_setup { + const char *submodule_name; + enum SUBMODULE_ALTERNATE_ERROR_MODE { + SUBMODULE_ALTERNATE_ERROR_DIE, + SUBMODULE_ALTERNATE_ERROR_INFO, + SUBMODULE_ALTERNATE_ERROR_IGNORE + } error_mode; + struct string_list *reference; +}; +#define SUBMODULE_ALTERNATE_SETUP_INIT { NULL, \ + SUBMODULE_ALTERNATE_ERROR_IGNORE, NULL } + +static int add_possible_reference_from_superproject( + struct alternate_object_database *alt, void *sas_cb) +{ + struct submodule_alternate_setup *sas = sas_cb; + + /* directory name, minus trailing slash */ + size_t namelen = alt->name - alt->base - 1; + struct strbuf name = STRBUF_INIT; + strbuf_add(&name, alt->base, namelen); + + /* + * If the alternate object store is another repository, try the + * standard layout with .git/modules/<name>/objects + */ + if (ends_with(name.buf, ".git/objects")) { + char *sm_alternate; + struct strbuf sb = STRBUF_INIT; + struct strbuf err = STRBUF_INIT; + strbuf_add(&sb, name.buf, name.len - strlen("objects")); + /* + * We need to end the new path with '/' to mark it as a dir, + * otherwise a submodule name containing '/' will be broken + * as the last part of a missing submodule reference would + * be taken as a file name. + */ + strbuf_addf(&sb, "modules/%s/", sas->submodule_name); + + sm_alternate = compute_alternate_path(sb.buf, &err); + if (sm_alternate) { + string_list_append(sas->reference, xstrdup(sb.buf)); + free(sm_alternate); + } else { + switch (sas->error_mode) { + case SUBMODULE_ALTERNATE_ERROR_DIE: + die(_("submodule '%s' cannot add alternate: %s"), + sas->submodule_name, err.buf); + case SUBMODULE_ALTERNATE_ERROR_INFO: + fprintf(stderr, _("submodule '%s' cannot add alternate: %s"), + sas->submodule_name, err.buf); + case SUBMODULE_ALTERNATE_ERROR_IGNORE: + ; /* nothing */ + } + } + strbuf_release(&sb); + } + + strbuf_release(&name); + return 0; +} + +static void prepare_possible_alternates(const char *sm_name, + struct string_list *reference) +{ + char *sm_alternate = NULL, *error_strategy = NULL; + struct submodule_alternate_setup sas = SUBMODULE_ALTERNATE_SETUP_INIT; + + git_config_get_string("submodule.alternateLocation", &sm_alternate); + if (!sm_alternate) + return; + + git_config_get_string("submodule.alternateErrorStrategy", &error_strategy); + + if (!error_strategy) + error_strategy = xstrdup("die"); + + sas.submodule_name = sm_name; + sas.reference = reference; + if (!strcmp(error_strategy, "die")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_DIE; + else if (!strcmp(error_strategy, "info")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_INFO; + else if (!strcmp(error_strategy, "ignore")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_IGNORE; + else + die(_("Value '%s' for submodule.alternateErrorStrategy is not recognized"), error_strategy); + + if (!strcmp(sm_alternate, "superproject")) + foreach_alt_odb(add_possible_reference_from_superproject, &sas); + else if (!strcmp(sm_alternate, "no")) + ; /* do nothing */ + else + die(_("Value '%s' for submodule.alternateLocation is not recognized"), sm_alternate); + + free(sm_alternate); + free(error_strategy); +} + static int module_clone(int argc, const char **argv, const char *prefix) { - const char *name = NULL, *url = NULL; - const char *reference = NULL, *depth = NULL; + const char *name = NULL, *url = NULL, *depth = NULL; int quiet = 0; FILE *submodule_dot_git; char *p, *path = NULL, *sm_gitdir; struct strbuf rel_path = STRBUF_INIT; struct strbuf sb = STRBUF_INIT; + struct string_list reference = STRING_LIST_INIT_NODUP; struct option module_clone_options[] = { OPT_STRING(0, "prefix", &prefix, @@ -490,8 +593,8 @@ static int module_clone(int argc, const char **argv, const char *prefix) OPT_STRING(0, "url", &url, N_("string"), N_("url where to clone the submodule from")), - OPT_STRING(0, "reference", &reference, - N_("string"), + OPT_STRING_LIST(0, "reference", &reference, + N_("repo"), N_("reference repository")), OPT_STRING(0, "depth", &depth, N_("string"), @@ -527,7 +630,10 @@ static int module_clone(int argc, const char **argv, const char *prefix) if (!file_exists(sm_gitdir)) { if (safe_create_leading_directories_const(sm_gitdir) < 0) die(_("could not create directory '%s'"), sm_gitdir); - if (clone_submodule(path, sm_gitdir, url, depth, reference, quiet)) + + prepare_possible_alternates(name, &reference); + + if (clone_submodule(path, sm_gitdir, url, depth, &reference, quiet)) die(_("clone of '%s' into submodule path '%s' failed"), url, path); } else { @@ -579,7 +685,7 @@ struct submodule_update_clone { /* configuration parameters which are passed on to the children */ int quiet; int recommend_shallow; - const char *reference; + struct string_list references; const char *depth; const char *recursive_prefix; const char *prefix; @@ -595,7 +701,8 @@ struct submodule_update_clone { int failed_clones_nr, failed_clones_alloc; }; #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ - SUBMODULE_UPDATE_STRATEGY_INIT, 0, -1, NULL, NULL, NULL, NULL, \ + SUBMODULE_UPDATE_STRATEGY_INIT, 0, -1, STRING_LIST_INIT_DUP, \ + NULL, NULL, NULL, \ STRING_LIST_INIT_DUP, 0, NULL, 0, 0} @@ -705,8 +812,11 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce, argv_array_pushl(&child->args, "--path", sub->path, NULL); argv_array_pushl(&child->args, "--name", sub->name, NULL); argv_array_pushl(&child->args, "--url", url, NULL); - if (suc->reference) - argv_array_push(&child->args, suc->reference); + if (suc->references.nr) { + struct string_list_item *item; + for_each_string_list_item(item, &suc->references) + argv_array_pushl(&child->args, "--reference", item->string, NULL); + } if (suc->depth) argv_array_push(&child->args, suc->depth); @@ -829,7 +939,7 @@ static int update_clone(int argc, const char **argv, const char *prefix) OPT_STRING(0, "update", &update, N_("string"), N_("rebase, merge, checkout or none")), - OPT_STRING(0, "reference", &suc.reference, N_("repo"), + OPT_STRING_LIST(0, "reference", &suc.references, N_("repo"), N_("reference repository")), OPT_STRING(0, "depth", &suc.depth, "<depth>", N_("Create a shallow clone truncated to the " @@ -1344,6 +1344,7 @@ extern struct alternate_object_database { } *alt_odb_list; extern void prepare_alt_odb(void); extern void read_info_alternates(const char * relative_base, int depth); +extern char *compute_alternate_path(const char *path, struct strbuf *err); extern void add_to_alternates_file(const char *reference); typedef int alt_odb_fn(struct alternate_object_database *, void *); extern int foreach_alt_odb(alt_odb_fn, void*); diff --git a/git-submodule.sh b/git-submodule.sh index b57f87de65..a1cc71b521 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -576,7 +576,7 @@ cmd_update() ${wt_prefix:+--prefix "$wt_prefix"} \ ${prefix:+--recursive-prefix "$prefix"} \ ${update:+--update "$update"} \ - ${reference:+--reference "$reference"} \ + ${reference:+"$reference"} \ ${depth:+--depth "$depth"} \ ${recommend_shallow:+"$recommend_shallow"} \ ${jobs:+$jobs} \ diff --git a/sha1_file.c b/sha1_file.c index a57b71d133..6f38dc6548 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -419,6 +419,82 @@ void add_to_alternates_file(const char *reference) free(alts); } +/* + * Compute the exact path an alternate is at and returns it. In case of + * error NULL is returned and the human readable error is added to `err` + * `path` may be relative and should point to $GITDIR. + * `err` must not be null. + */ +char *compute_alternate_path(const char *path, struct strbuf *err) +{ + char *ref_git = NULL; + const char *repo, *ref_git_s; + int seen_error = 0; + + ref_git_s = real_path_if_valid(path); + if (!ref_git_s) { + seen_error = 1; + strbuf_addf(err, _("path '%s' does not exist"), path); + goto out; + } else + /* + * Beware: read_gitfile(), real_path() and mkpath() + * return static buffer + */ + ref_git = xstrdup(ref_git_s); + + repo = read_gitfile(ref_git); + if (!repo) + repo = read_gitfile(mkpath("%s/.git", ref_git)); + if (repo) { + free(ref_git); + ref_git = xstrdup(repo); + } + + if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) { + char *ref_git_git = mkpathdup("%s/.git", ref_git); + free(ref_git); + ref_git = ref_git_git; + } else if (!is_directory(mkpath("%s/objects", ref_git))) { + struct strbuf sb = STRBUF_INIT; + seen_error = 1; + if (get_common_dir(&sb, ref_git)) { + strbuf_addf(err, + _("reference repository '%s' as a linked " + "checkout is not supported yet."), + path); + goto out; + } + + strbuf_addf(err, _("reference repository '%s' is not a " + "local repository."), path); + goto out; + } + + if (!access(mkpath("%s/shallow", ref_git), F_OK)) { + strbuf_addf(err, _("reference repository '%s' is shallow"), + path); + seen_error = 1; + goto out; + } + + if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) { + strbuf_addf(err, + _("reference repository '%s' is grafted"), + path); + seen_error = 1; + goto out; + } + +out: + if (seen_error) { + free(ref_git); + ref_git = NULL; + } + + return ref_git; +} + int foreach_alt_odb(alt_odb_fn fn, void *cb) { struct alternate_object_database *ent; diff --git a/t/t7408-submodule-reference.sh b/t/t7408-submodule-reference.sh index eaea19b8f2..1c1e289ffd 100755 --- a/t/t7408-submodule-reference.sh +++ b/t/t7408-submodule-reference.sh @@ -8,74 +8,121 @@ test_description='test clone --reference' base_dir=$(pwd) -U=$base_dir/UPLOAD_LOG - -test_expect_success 'preparing first repository' \ -'test_create_repo A && cd A && -echo first > file1 && -git add file1 && -git commit -m A-initial' - -cd "$base_dir" - -test_expect_success 'preparing second repository' \ -'git clone A B && cd B && -echo second > file2 && -git add file2 && -git commit -m B-addition && -git repack -a -d && -git prune' - -cd "$base_dir" - -test_expect_success 'preparing superproject' \ -'test_create_repo super && cd super && -echo file > file && -git add file && -git commit -m B-super-initial' - -cd "$base_dir" - -test_expect_success 'submodule add --reference' \ -'cd super && git submodule add --reference ../B "file://$base_dir/A" sub && -git commit -m B-super-added' - -cd "$base_dir" - -test_expect_success 'after add: existence of info/alternates' \ -'test_line_count = 1 super/.git/modules/sub/objects/info/alternates' - -cd "$base_dir" - -test_expect_success 'that reference gets used with add' \ -'cd super/sub && -echo "0 objects, 0 kilobytes" > expected && -git count-objects > current && -diff expected current' - -cd "$base_dir" - -test_expect_success 'cloning superproject' \ -'git clone super super-clone' - -cd "$base_dir" - -test_expect_success 'update with reference' \ -'cd super-clone && git submodule update --init --reference ../B' - -cd "$base_dir" - -test_expect_success 'after update: existence of info/alternates' \ -'test_line_count = 1 super-clone/.git/modules/sub/objects/info/alternates' - -cd "$base_dir" - -test_expect_success 'that reference gets used with update' \ -'cd super-clone/sub && -echo "0 objects, 0 kilobytes" > expected && -git count-objects > current && -diff expected current' - -cd "$base_dir" +test_alternate_is_used () { + alternates_file="$1" && + working_dir="$2" && + test_line_count = 1 "$alternates_file" && + echo "0 objects, 0 kilobytes" >expect && + git -C "$working_dir" count-objects >actual && + test_cmp expect actual +} + +test_expect_success 'preparing first repository' ' + test_create_repo A && + ( + cd A && + echo first >file1 && + git add file1 && + git commit -m A-initial + ) +' + +test_expect_success 'preparing second repository' ' + git clone A B && + ( + cd B && + echo second >file2 && + git add file2 && + git commit -m B-addition && + git repack -a -d && + git prune + ) +' + +test_expect_success 'preparing superproject' ' + test_create_repo super && + ( + cd super && + echo file >file && + git add file && + git commit -m B-super-initial + ) +' + +test_expect_success 'submodule add --reference uses alternates' ' + ( + cd super && + git submodule add --reference ../B "file://$base_dir/A" sub && + git commit -m B-super-added && + git repack -ad + ) && + test_alternate_is_used super/.git/modules/sub/objects/info/alternates super/sub +' + +test_expect_success 'that reference gets used with add' ' + ( + cd super/sub && + echo "0 objects, 0 kilobytes" >expected && + git count-objects >current && + diff expected current + ) +' + +# The tests up to this point, and repositories created by them +# (A, B, super and super/sub), are about setting up the stage +# for subsequent tests and meant to be kept throughout the +# remainder of the test. +# Tests from here on, if they create their own test repository, +# are expected to clean after themselves. + +test_expect_success 'updating superproject keeps alternates' ' + test_when_finished "rm -rf super-clone" && + git clone super super-clone && + git -C super-clone submodule update --init --reference ../B && + test_alternate_is_used super-clone/.git/modules/sub/objects/info/alternates super-clone/sub +' + +test_expect_success 'submodules use alternates when cloning a superproject' ' + test_when_finished "rm -rf super-clone" && + git clone --reference super --recursive super super-clone && + ( + cd super-clone && + # test superproject has alternates setup correctly + test_alternate_is_used .git/objects/info/alternates . && + # test submodule has correct setup + test_alternate_is_used .git/modules/sub/objects/info/alternates sub + ) +' + +test_expect_success 'missing submodule alternate fails clone and submodule update' ' + test_when_finished "rm -rf super-clone" && + git clone super super2 && + test_must_fail git clone --recursive --reference super2 super2 super-clone && + ( + cd super-clone && + # test superproject has alternates setup correctly + test_alternate_is_used .git/objects/info/alternates . && + # update of the submodule succeeds + test_must_fail git submodule update --init && + # and we have no alternates: + test_must_fail test_alternate_is_used .git/modules/sub/objects/info/alternates sub && + test_must_fail test_path_is_file sub/file1 + ) +' + +test_expect_success 'ignoring missing submodule alternates passes clone and submodule update' ' + test_when_finished "rm -rf super-clone" && + git clone --reference-if-able super2 --recursive super2 super-clone && + ( + cd super-clone && + # test superproject has alternates setup correctly + test_alternate_is_used .git/objects/info/alternates . && + # update of the submodule succeeds + git submodule update --init && + # and we have no alternates: + test_must_fail test_alternate_is_used .git/modules/sub/objects/info/alternates sub && + test_path_is_file sub/file1 + ) +' test_done |