diff options
| -rw-r--r-- | Documentation/fetch-options.txt | 14 | ||||
| -rw-r--r-- | Documentation/git-clone.txt | 8 | ||||
| -rw-r--r-- | Documentation/git-fetch-pack.txt | 14 | ||||
| -rw-r--r-- | Documentation/gitremote-helpers.txt | 11 | ||||
| -rw-r--r-- | Documentation/technical/pack-protocol.txt | 4 | ||||
| -rw-r--r-- | Documentation/technical/protocol-capabilities.txt | 25 | ||||
| -rw-r--r-- | builtin/clone.c | 24 | ||||
| -rw-r--r-- | builtin/fetch-pack.c | 27 | ||||
| -rw-r--r-- | builtin/fetch.c | 50 | ||||
| -rw-r--r-- | commit.h | 2 | ||||
| -rw-r--r-- | fetch-pack.c | 169 | ||||
| -rw-r--r-- | fetch-pack.h | 4 | ||||
| -rw-r--r-- | object.h | 2 | ||||
| -rw-r--r-- | refs.c | 8 | ||||
| -rw-r--r-- | refs.h | 1 | ||||
| -rw-r--r-- | remote-curl.c | 80 | ||||
| -rw-r--r-- | shallow.c | 78 | ||||
| -rwxr-xr-x | t/t5500-fetch-pack.sh | 68 | ||||
| -rwxr-xr-x | t/t5539-fetch-http-shallow.sh | 73 | ||||
| -rw-r--r-- | transport-helper.c | 62 | ||||
| -rw-r--r-- | transport.c | 12 | ||||
| -rw-r--r-- | transport.h | 14 | ||||
| -rw-r--r-- | upload-pack.c | 364 |
23 files changed, 890 insertions, 224 deletions
diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index 9eab1f5fa4..fb6bebbc61 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -14,6 +14,20 @@ linkgit:git-clone[1]), deepen or shorten the history to the specified number of commits. Tags for the deepened commits are not fetched. +--deepen=<depth>:: + Similar to --depth, except it specifies the number of commits + from the current shallow boundary instead of from the tip of + each remote branch history. + +--shallow-since=<date>:: + Deepen or shorten the history of a shallow repository to + include all reachable commits after <date>. + +--shallow-exclude=<revision>:: + Deepen or shorten the history of a shallow repository to + exclude commits reachable from a specified remote branch or tag. + This option can be specified multiple times. + --unshallow:: If the source repository is complete, convert a shallow repository to a complete one, removing all the limitations diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index e316c4bd51..35cc34b2fb 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -197,6 +197,14 @@ objects from the source repository into a pack in the cloned repository. tips of all branches. If you want to clone submodules shallowly, also pass `--shallow-submodules`. +--shallow-since=<date>:: + Create a shallow clone with a history after the specified time. + +--shallow-exclude=<revision>:: + Create a shallow clone with a history, excluding commits + reachable from a specified remote branch or tag. This option + can be specified multiple times. + --[no-]single-branch:: Clone only the history leading to the tip of a single branch, either specified by the `--branch` option or the primary diff --git a/Documentation/git-fetch-pack.txt b/Documentation/git-fetch-pack.txt index 24417ee3a6..d45f6adc69 100644 --- a/Documentation/git-fetch-pack.txt +++ b/Documentation/git-fetch-pack.txt @@ -87,6 +87,20 @@ be in a separate packet, and the list must end with a flush packet. 'git-upload-pack' treats the special depth 2147483647 as infinite even if there is an ancestor-chain that long. +--shallow-since=<date>:: + Deepen or shorten the history of a shallow'repository to + include all reachable commits after <date>. + +--shallow-exclude=<revision>:: + Deepen or shorten the history of a shallow repository to + exclude commits reachable from a specified remote branch or tag. + This option can be specified multiple times. + +--deepen-relative:: + Argument --depth specifies the number of commits from the + current shallow boundary instead of from the tip of each + remote branch history. + --no-progress:: Do not show the progress. diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt index a4de50ad22..9e8681f9e1 100644 --- a/Documentation/gitremote-helpers.txt +++ b/Documentation/gitremote-helpers.txt @@ -415,6 +415,17 @@ set by Git if the remote helper has the 'option' capability. 'option depth' <depth>:: Deepens the history of a shallow repository. +'option deepen-since <timestamp>:: + Deepens the history of a shallow repository based on time. + +'option deepen-not <ref>:: + Deepens the history of a shallow repository excluding ref. + Multiple options add up. + +'option deepen-relative {'true'|'false'}:: + Deepens the history of a shallow repository relative to + current boundary. Only valid when used with "option depth". + 'option followtags' {'true'|'false'}:: If enabled the helper should automatically fetch annotated tag objects if the object the tag points at was transferred diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt index 736f3894a8..c59ac9936a 100644 --- a/Documentation/technical/pack-protocol.txt +++ b/Documentation/technical/pack-protocol.txt @@ -219,7 +219,9 @@ out of what the server said it could do with the first 'want' line. shallow-line = PKT-LINE("shallow" SP obj-id) - depth-request = PKT-LINE("deepen" SP depth) + depth-request = PKT-LINE("deepen" SP depth) / + PKT-LINE("deepen-since" SP timestamp) / + PKT-LINE("deepen-not" SP ref) first-want = PKT-LINE("want" SP obj-id SP capability-list) additional-want = PKT-LINE("want" SP obj-id) diff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/technical/protocol-capabilities.txt index 4c28d3a8ae..26dcc6f502 100644 --- a/Documentation/technical/protocol-capabilities.txt +++ b/Documentation/technical/protocol-capabilities.txt @@ -179,6 +179,31 @@ This capability adds "deepen", "shallow" and "unshallow" commands to the fetch-pack/upload-pack protocol so clients can request shallow clones. +deepen-since +------------ + +This capability adds "deepen-since" command to fetch-pack/upload-pack +protocol so the client can request shallow clones that are cut at a +specific time, instead of depth. Internally it's equivalent of doing +"rev-list --max-age=<timestamp>" on the server side. "deepen-since" +cannot be used with "deepen". + +deepen-not +---------- + +This capability adds "deepen-not" command to fetch-pack/upload-pack +protocol so the client can request shallow clones that are cut at a +specific revision, instead of depth. Internally it's equivalent of +doing "rev-list --not <rev>" on the server side. "deepen-not" +cannot be used with "deepen", but can be used with "deepen-since". + +deepen-relative +--------------- + +If this capability is requested by the client, the semantics of +"deepen" command is changed. The "depth" argument is the depth from +the current shallow boundary, instead of the depth from remote refs. + no-progress ----------- diff --git a/builtin/clone.c b/builtin/clone.c index fb75f7ee64..6c80690adf 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -41,9 +41,11 @@ static const char * const builtin_clone_usage[] = { static int option_no_checkout, option_bare, option_mirror, option_single_branch = -1; static int option_local = -1, option_no_hardlinks, option_shared, option_recursive; static int option_shallow_submodules; -static char *option_template, *option_depth; +static int deepen; +static char *option_template, *option_depth, *option_since; static char *option_origin = NULL; static char *option_branch = NULL; +static struct string_list option_not = STRING_LIST_INIT_NODUP; static const char *real_git_dir; static char *option_upload_pack = "git-upload-pack"; static int option_verbosity; @@ -94,6 +96,10 @@ static struct option builtin_clone_options[] = { N_("path to git-upload-pack on the remote")), OPT_STRING(0, "depth", &option_depth, N_("depth"), N_("create a shallow clone of that depth")), + OPT_STRING(0, "shallow-since", &option_since, N_("time"), + N_("create a shallow clone since a specific time")), + OPT_STRING_LIST(0, "shallow-exclude", &option_not, N_("revision"), + N_("deepen history of shallow clone by excluding rev")), OPT_BOOL(0, "single-branch", &option_single_branch, N_("clone only one branch, HEAD or --branch")), OPT_BOOL(0, "shallow-submodules", &option_shallow_submodules, @@ -861,8 +867,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) usage_msg_opt(_("You must specify a repository to clone."), builtin_clone_usage, builtin_clone_options); + if (option_depth || option_since || option_not.nr) + deepen = 1; if (option_single_branch == -1) - option_single_branch = option_depth ? 1 : 0; + option_single_branch = deepen ? 1 : 0; if (option_mirror) option_bare = 1; @@ -1006,6 +1014,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (is_local) { if (option_depth) warning(_("--depth is ignored in local clones; use file:// instead.")); + if (option_since) + warning(_("--shallow-since is ignored in local clones; use file:// instead.")); + if (option_not.nr) + warning(_("--shallow-exclude is ignored in local clones; use file:// instead.")); if (!access(mkpath("%s/shallow", path), F_OK)) { if (option_local > 0) warning(_("source repository is shallow, ignoring --local")); @@ -1024,6 +1036,12 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_depth) transport_set_option(transport, TRANS_OPT_DEPTH, option_depth); + if (option_since) + transport_set_option(transport, TRANS_OPT_DEEPEN_SINCE, + option_since); + if (option_not.nr) + transport_set_option(transport, TRANS_OPT_DEEPEN_NOT, + (const char *)&option_not); if (option_single_branch) transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1"); @@ -1031,7 +1049,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) transport_set_option(transport, TRANS_OPT_UPLOADPACK, option_upload_pack); - if (transport->smart_options && !option_depth) + if (transport->smart_options && !deepen) transport->smart_options->check_self_contained_and_connected = 1; refs = transport_get_remote_refs(transport); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index bfd0be44a9..cfe9e447c2 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -51,6 +51,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct child_process *conn; struct fetch_pack_args args; struct sha1_array shallow = SHA1_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; packet_trace_identity("fetch-pack"); @@ -60,12 +61,12 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) for (i = 1; i < argc && *argv[i] == '-'; i++) { const char *arg = argv[i]; - if (starts_with(arg, "--upload-pack=")) { - args.uploadpack = arg + 14; + if (skip_prefix(arg, "--upload-pack=", &arg)) { + args.uploadpack = arg; continue; } - if (starts_with(arg, "--exec=")) { - args.uploadpack = arg + 7; + if (skip_prefix(arg, "--exec=", &arg)) { + args.uploadpack = arg; continue; } if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) { @@ -101,8 +102,20 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.verbose = 1; continue; } - if (starts_with(arg, "--depth=")) { - args.depth = strtol(arg + 8, NULL, 0); + if (skip_prefix(arg, "--depth=", &arg)) { + args.depth = strtol(arg, NULL, 0); + continue; + } + if (skip_prefix(arg, "--shallow-since=", &arg)) { + args.deepen_since = xstrdup(arg); + continue; + } + if (skip_prefix(arg, "--shallow-exclude=", &arg)) { + string_list_append(&deepen_not, arg); + continue; + } + if (!strcmp(arg, "--deepen-relative")) { + args.deepen_relative = 1; continue; } if (!strcmp("--no-progress", arg)) { @@ -132,6 +145,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } usage(fetch_pack_usage); } + if (deepen_not.nr) + args.deepen_not = &deepen_not; if (i < argc) dest = argv[i++]; diff --git a/builtin/fetch.c b/builtin/fetch.c index 164623bb6f..d5329f915e 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -35,13 +35,15 @@ static int fetch_prune_config = -1; /* unspecified */ static int prune = -1; /* unspecified */ #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */ -static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity; +static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity, deepen_relative; static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT; -static int tags = TAGS_DEFAULT, unshallow, update_shallow; +static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen; static int max_children = -1; static enum transport_family family; static const char *depth; +static const char *deepen_since; static const char *upload_pack; +static struct string_list deepen_not = STRING_LIST_INIT_NODUP; static struct strbuf default_rla = STRBUF_INIT; static struct transport *gtransport; static struct transport *gsecondary; @@ -117,6 +119,12 @@ static struct option builtin_fetch_options[] = { OPT_BOOL(0, "progress", &progress, N_("force progress reporting")), OPT_STRING(0, "depth", &depth, N_("depth"), N_("deepen history of shallow clone")), + OPT_STRING(0, "shallow-since", &deepen_since, N_("time"), + N_("deepen history of shallow repository based on time")), + OPT_STRING_LIST(0, "shallow-exclude", &deepen_not, N_("revision"), + N_("deepen history of shallow clone by excluding rev")), + OPT_INTEGER(0, "deepen", &deepen_relative, + N_("deepen history of shallow clone")), { OPTION_SET_INT, 0, "unshallow", &unshallow, NULL, N_("convert to a complete repository"), PARSE_OPT_NONEG | PARSE_OPT_NOARG, NULL, 1 }, @@ -875,7 +883,7 @@ static int quickfetch(struct ref *ref_map) * really need to perform. Claiming failure now will ensure * we perform the network exchange to deepen our history. */ - if (depth) + if (deepen) return -1; opt.quiet = 1; return check_connected(iterate_ref_map, &rm, &opt); @@ -983,7 +991,7 @@ static void set_option(struct transport *transport, const char *name, const char name, transport->url); } -static struct transport *prepare_transport(struct remote *remote) +static struct transport *prepare_transport(struct remote *remote, int deepen) { struct transport *transport; transport = transport_get(remote, NULL); @@ -995,6 +1003,13 @@ static struct transport *prepare_transport(struct remote *remote) set_option(transport, TRANS_OPT_KEEP, "yes"); if (depth) set_option(transport, TRANS_OPT_DEPTH, depth); + if (deepen && deepen_since) + set_option(transport, TRANS_OPT_DEEPEN_SINCE, deepen_since); + if (deepen && deepen_not.nr) + set_option(transport, TRANS_OPT_DEEPEN_NOT, + (const char *)&deepen_not); + if (deepen_relative) + set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes"); if (update_shallow) set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); return transport; @@ -1002,13 +1017,25 @@ static struct transport *prepare_transport(struct remote *remote) static void backfill_tags(struct transport *transport, struct ref *ref_map) { - if (transport->cannot_reuse) { - gsecondary = prepare_transport(transport->remote); + int cannot_reuse; + + /* + * Once we have set TRANS_OPT_DEEPEN_SINCE, we can't unset it + * when remote helper is used (setting it to an empty string + * is not unsetting). We could extend the remote helper + * protocol for that, but for now, just force a new connection + * without deepen-since. Similar story for deepen-not. + */ + cannot_reuse = transport->cannot_reuse || + deepen_since || deepen_not.nr; + if (cannot_reuse) { + gsecondary = prepare_transport(transport->remote, 0); transport = gsecondary; } transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL); transport_set_option(transport, TRANS_OPT_DEPTH, "0"); + transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL); fetch_refs(transport, ref_map); if (gsecondary) { @@ -1219,7 +1246,7 @@ static int fetch_one(struct remote *remote, int argc, const char **argv) die(_("No remote repository specified. Please, specify either a URL or a\n" "remote name from which new revisions should be fetched.")); - gtransport = prepare_transport(remote); + gtransport = prepare_transport(remote, 1); if (prune < 0) { /* no command line request */ @@ -1279,6 +1306,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) argc = parse_options(argc, argv, prefix, builtin_fetch_options, builtin_fetch_usage, 0); + if (deepen_relative) { + if (deepen_relative < 0) + die(_("Negative depth in --deepen is not supported")); + if (depth) + die(_("--deepen and --depth are mutually exclusive")); + depth = xstrfmt("%d", deepen_relative); + } if (unshallow) { if (depth) die(_("--depth and --unshallow cannot be used together")); @@ -1291,6 +1325,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) /* no need to be strict, transport_set_option() will validate it again */ if (depth && atoi(depth) < 1) die(_("depth %s is not a positive number"), depth); + if (depth || deepen_since || deepen_not.nr) + deepen = 1; if (recurse_submodules != RECURSE_SUBMODULES_OFF) { if (recurse_submodules_default) { @@ -267,6 +267,8 @@ extern int for_each_commit_graft(each_commit_graft_fn, void *); extern int is_repository_shallow(void); extern struct commit_list *get_shallow_commits(struct object_array *heads, int depth, int shallow_flag, int not_shallow_flag); +extern struct commit_list *get_shallow_commits_by_rev_list( + int ac, const char **av, int shallow_flag, int not_shallow_flag); extern void set_alternate_shallow_file(const char *path, int override); extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol, const struct sha1_array *extra); diff --git a/fetch-pack.c b/fetch-pack.c index 300763fae1..cb45c346ea 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -21,6 +21,8 @@ static int fetch_unpack_limit = -1; static int unpack_limit = 100; static int prefer_ofs_delta = 1; static int no_done; +static int deepen_since_ok; +static int deepen_not_ok; static int fetch_fsck_objects = -1; static int transfer_fsck_objects = -1; static int agent_supported; @@ -50,6 +52,21 @@ static int non_common_revs, multi_ack, use_sideband; #define ALLOW_REACHABLE_SHA1 02 static unsigned int allow_unadvertised_object_request; +__attribute__((format (printf, 2, 3))) +static inline void print_verbose(const struct fetch_pack_args *args, + const char *fmt, ...) +{ + va_list params; + + if (!args->verbose) + return; + + va_start(params, fmt); + vfprintf(stderr, fmt, params); + va_end(params); + fputc('\n', stderr); +} + static void rev_list_push(struct commit *commit, int mark) { if (!(commit->object.flags & mark)) { @@ -182,7 +199,7 @@ enum ack_type { static void consume_shallow_list(struct fetch_pack_args *args, int fd) { - if (args->stateless_rpc && args->depth > 0) { + if (args->stateless_rpc && args->deepen) { /* If we sent a depth we will get back "duplicate" * shallow and unshallow commands every time there * is a block of have lines exchanged. @@ -193,7 +210,7 @@ static void consume_shallow_list(struct fetch_pack_args *args, int fd) continue; if (starts_with(line, "unshallow ")) continue; - die("git fetch-pack: expected shallow list"); + die(_("git fetch-pack: expected shallow list")); } } } @@ -205,7 +222,7 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) const char *arg; if (!len) - die("git fetch-pack: expected ACK/NAK, got EOF"); + die(_("git fetch-pack: expected ACK/NAK, got EOF")); if (!strcmp(line, "NAK")) return NAK; if (skip_prefix(line, "ACK ", &arg)) { @@ -223,7 +240,7 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) return ACK; } } - die("git fetch_pack: expected ACK/NAK, got '%s'", line); + die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line); } static void send_request(struct fetch_pack_args *args, @@ -275,7 +292,7 @@ static int find_common(struct fetch_pack_args *args, size_t state_len = 0; if (args->stateless_rpc && multi_ack == 1) - die("--stateless-rpc requires multi_ack_detailed"); + die(_("--stateless-rpc requires multi_ack_detailed")); if (marked) for_each_ref(clear_marks, NULL); marked = 1; @@ -312,10 +329,13 @@ static int find_common(struct fetch_pack_args *args, if (no_done) strbuf_addstr(&c, " no-done"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); + if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative"); if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack"); if (args->no_progress) strbuf_addstr(&c, " no-progress"); if (args->include_tag) strbuf_addstr(&c, " include-tag"); if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta"); + if (deepen_since_ok) strbuf_addstr(&c, " deepen-since"); + if (deepen_not_ok) strbuf_addstr(&c, " deepen-not"); if (agent_supported) strbuf_addf(&c, " agent=%s", git_user_agent_sanitized()); packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf); @@ -335,10 +355,21 @@ static int find_common(struct fetch_pack_args *args, write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); + if (args->deepen_since) { + unsigned long max_age = approxidate(args->deepen_since); + packet_buf_write(&req_buf, "deepen-since %lu", max_age); + } + if (args->deepen_not) { + int i; + for (i = 0; i < args->deepen_not->nr; i++) { + struct string_list_item *s = args->deepen_not->items + i; + packet_buf_write(&req_buf, "deepen-not %s", s->string); + } + } packet_buf_flush(&req_buf); state_len = req_buf.len; - if (args->depth > 0) { + if (args->deepen) { char *line; const char *arg; unsigned char sha1[20]; @@ -347,23 +378,23 @@ static int find_common(struct fetch_pack_args *args, while ((line = packet_read_line(fd[0], NULL))) { if (skip_prefix(line, "shallow ", &arg)) { if (get_sha1_hex(arg, sha1)) - die("invalid shallow line: %s", line); + die(_("invalid shallow line: %s"), line); register_shallow(sha1); continue; } if (skip_prefix(line, "unshallow ", &arg)) { if (get_sha1_hex(arg, sha1)) - die("invalid unshallow line: %s", line); + die(_("invalid unshallow line: %s"), line); if (!lookup_object(sha1)) - die("object not found: %s", line); + die(_("object not found: %s"), line); /* make sure that it is parsed as shallow */ if (!parse_object(sha1)) - die("error in object: %s", line); + die(_("error in object: %s"), line); if (unregister_shallow(sha1)) - die("no shallow found: %s", line); + die(_("no shallow found: %s"), line); continue; } - die("expected shallow/unshallow, got %s", line); + die(_("expected shallow/unshallow, got %s"), line); } } else if (!args->stateless_rpc) send_request(args, fd[1], &req_buf); @@ -380,8 +411,7 @@ static int find_common(struct fetch_pack_args *args, retval = -1; while ((sha1 = get_rev())) { packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1)); - if (args->verbose) - fprintf(stderr, "have %s\n", sha1_to_hex(sha1)); + print_verbose(args, "have %s", sha1_to_hex(sha1)); in_vain++; if (flush_at <= ++count) { int ack; @@ -402,9 +432,9 @@ static int find_common(struct fetch_pack_args *args, consume_shallow_list(args, fd[0]); do { ack = get_ack(fd[0], result_sha1); - if (args->verbose && ack) - fprintf(stderr, "got ack %d %s\n", ack, - sha1_to_hex(result_sha1)); + if (ack) + print_verbose(args, _("got %s %d %s"), "ack", + ack, sha1_to_hex(result_sha1)); switch (ack) { case ACK: flushes = 0; @@ -417,7 +447,7 @@ static int find_common(struct fetch_pack_args *args, struct commit *commit = lookup_commit(result_sha1); if (!commit) - die("invalid commit %s", sha1_to_hex(result_sha1)); + die(_("invalid commit %s"), sha1_to_hex(result_sha1)); if (args->stateless_rpc && ack == ACK_common && !(commit->object.flags & COMMON)) { @@ -450,8 +480,7 @@ static int find_common(struct fetch_pack_args *args, } while (ack); flushes--; if (got_continue && MAX_IN_VAIN < in_vain) { - if (args->verbose) - fprintf(stderr, "giving up\n"); + print_verbose(args, _("giving up")); break; /* give up */ } } @@ -461,8 +490,7 @@ done: packet_buf_write(&req_buf, "done\n"); send_request(args, fd[1], &req_buf); } - if (args->verbose) - fprintf(stderr, "done\n"); + print_verbose(args, _("done")); if (retval != 0) { multi_ack = 0; flushes++; @@ -474,9 +502,8 @@ done: while (flushes || multi_ack) { int ack = get_ack(fd[0], result_sha1); if (ack) { - if (args->verbose) - fprintf(stderr, "got ack (%d) %s\n", ack, - sha1_to_hex(result_sha1)); + print_verbose(args, _("got %s (%d) %s"), "ack", + ack, sha1_to_hex(result_sha1)); if (ack == ACK) return 0; multi_ack = 1; @@ -521,9 +548,8 @@ static void mark_recent_complete_commits(struct fetch_pack_args *args, unsigned long cutoff) { while (complete && cutoff <= complete->item->date) { - if (args->verbose) - fprintf(stderr, "Marking %s as complete\n", - oid_to_hex(&complete->item->object.oid)); + print_verbose(args, _("Marking %s as complete"), + oid_to_hex(&complete->item->object.oid)); pop_most_recent_commit(&complete, COMPLETE); } } @@ -559,7 +585,7 @@ static void filter_refs(struct fetch_pack_args *args, } if (!keep && args->fetch_all && - (!args->depth || !starts_with(ref->name, "refs/tags/"))) + (!args->deepen || !starts_with(ref->name, "refs/tags/"))) keep = 1; if (keep) { @@ -629,7 +655,7 @@ static int everything_local(struct fetch_pack_args *args, } } - if (!args->depth) { + if (!args->deepen) { for_each_ref(mark_complete_oid, NULL); for_each_alternate_ref(mark_alternate_complete, NULL); commit_list_sort_by_date(&complete); @@ -664,18 +690,12 @@ static int everything_local(struct fetch_pack_args *args, o = lookup_object(remote); if (!o || !(o->flags & COMPLETE)) { retval = 0; - if (!args->verbose) - continue; - fprintf(stderr, - "want %s (%s)\n", sha1_to_hex(remote), - ref->name); + print_verbose(args, "want %s (%s)", sha1_to_hex(remote), + ref->name); continue; } - if (!args->verbose) - continue; - fprintf(stderr, - "already have %s (%s)\n", sha1_to_hex(remote), - ref->name); + print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote), + ref->name); } return retval; } @@ -712,8 +732,7 @@ static int get_pack(struct fetch_pack_args *args, demux.out = -1; demux.isolate_sigpipe = 1; if (start_async(&demux)) - die("fetch-pack: unable to fork off sideband" - " demultiplexer"); + die(_("fetch-pack: unable to fork off sideband demultiplexer")); } else demux.out = xd[0]; @@ -721,7 +740,7 @@ static int get_pack(struct fetch_pack_args *args, if (!args->keep_pack && unpack_limit) { if (read_pack_header(demux.out, &header)) - die("protocol error: bad pack header"); + die(_("protocol error: bad pack header")); pass_header = 1; if (ntohl(header.hdr_entries) < unpack_limit) do_keep = 0; @@ -777,7 +796,7 @@ static int get_pack(struct fetch_pack_args *args, cmd.in = demux.out; cmd.git_cmd = 1; if (start_command(&cmd)) - die("fetch-pack: unable to fork off %s", cmd_name); + die(_("fetch-pack: unable to fork off %s"), cmd_name); if (do_keep && pack_lockfile) { *pack_lockfile = index_pack_lockfile(cmd.out); close(cmd.out); @@ -793,9 +812,9 @@ static int get_pack(struct fetch_pack_args *args, args->check_self_contained_and_connected && ret == 0; else - die("%s failed", cmd_name); + die(_("%s failed"), cmd_name); if (use_sideband && finish_async(&demux)) - die("error in sideband demultiplexer"); + die(_("error in sideband demultiplexer")); return 0; } @@ -822,41 +841,36 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, QSORT(sought, nr_sought, cmp_ref_by_name); if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow")) - die("Server does not support shallow clients"); + die(_("Server does not support shallow clients")); + if (args->depth > 0 || args->deepen_since || args->deepen_not) + args->deepen = 1; if (server_supports("multi_ack_detailed")) { - if (args->verbose) - fprintf(stderr, "Server supports multi_ack_detailed\n"); + print_verbose(args, _("Server supports multi_ack_detailed")); multi_ack = 2; if (server_supports("no-done")) { - if (args->verbose) - fprintf(stderr, "Server supports no-done\n"); + print_verbose(args, _("Server supports no-done")); if (args->stateless_rpc) no_done = 1; } } else if (server_supports("multi_ack")) { - if (args->verbose) - fprintf(stderr, "Server supports multi_ack\n"); + print_verbose(args, _("Server supports multi_ack")); multi_ack = 1; } if (server_supports("side-band-64k")) { - if (args->verbose) - fprintf(stderr, "Server supports side-band-64k\n"); + print_verbose(args, _("Server supports side-band-64k")); use_sideband = 2; } else if (server_supports("side-band")) { - if (args->verbose) - fprintf(stderr, "Server supports side-band\n"); + print_verbose(args, _("Server supports side-band")); use_sideband = 1; } if (server_supports("allow-tip-sha1-in-want")) { - if (args->verbose) - fprintf(stderr, "Server supports allow-tip-sha1-in-want\n"); + print_verbose(args, _("Server supports allow-tip-sha1-in-want")); allow_unadvertised_object_request |= ALLOW_TIP_SHA1; } if (server_supports("allow-reachable-sha1-in-want")) { - if (args->verbose) - fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n"); + print_verbose(args, _("Server supports allow-reachable-sha1-in-want")); allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; } if (!server_supports("thin-pack")) @@ -865,18 +879,27 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, args->no_progress = 0; if (!server_supports("include-tag")) args->include_tag = 0; - if (server_supports("ofs-delta")) { - if (args->verbose) - fprintf(stderr, "Server supports ofs-delta\n"); - } else + if (server_supports("ofs-delta")) + print_verbose(args, _("Server supports ofs-delta")); + else prefer_ofs_delta = 0; if ((agent_feature = server_feature_value("agent", &agent_len))) { agent_supported = 1; - if (args->verbose && agent_len) - fprintf(stderr, "Server version is %.*s\n", - agent_len, agent_feature); + if (agent_len) + print_verbose(args, _("Server version is %.*s"), + agent_len, agent_feature); } + if (server_supports("deepen-since")) + deepen_since_ok = 1; + else if (args->deepen_since) + die(_("Server does not support --shallow-since")); + if (server_supports("deepen-not")) + deepen_not_ok = 1; + else if (args->deepen_not) + die(_("Server does not support --shallow-exclude")); + if (!server_supports("deepen-relative") && args->deepen_relative) + die(_("Server does not support --deepen")); if (everything_local(args, &ref, sought, nr_sought)) { packet_flush(fd[1]); @@ -887,11 +910,11 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, /* When cloning, it is not unusual to have * no common commit. */ - warning("no common commits"); + warning(_("no common commits")); if (args->stateless_rpc) packet_flush(fd[1]); - if (args->depth > 0) + if (args->deepen) setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); else if (si->nr_ours || si->nr_theirs) @@ -899,7 +922,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, else alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) - die("git fetch-pack: fetch failed."); + die(_("git fetch-pack: fetch failed.")); all_done: return ref; @@ -958,7 +981,7 @@ static void update_shallow(struct fetch_pack_args *args, int *status; int i; - if (args->depth > 0 && alternate_shallow_file) { + if (args->deepen && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ unlink_or_warn(git_path_shallow()); rollback_lock_file(&shallow_lock); @@ -1061,7 +1084,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, if (!ref) { packet_flush(fd[1]); - die("no matching remote head"); + die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, diff --git a/fetch-pack.h b/fetch-pack.h index bb7fd76e59..c912e3d321 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -10,6 +10,9 @@ struct fetch_pack_args { const char *uploadpack; int unpacklimit; int depth; + const char *deepen_since; + const struct string_list *deepen_not; + unsigned deepen_relative:1; unsigned quiet:1; unsigned keep_pack:1; unsigned lock_pack:1; @@ -25,6 +28,7 @@ struct fetch_pack_args { unsigned self_contained_and_connected:1; unsigned cloning:1; unsigned update_shallow:1; + unsigned deepen:1; }; /* @@ -31,7 +31,7 @@ struct object_array { * revision.h: 0---------10 26 * fetch-pack.c: 0---4 * walker.c: 0-2 - * upload-pack.c: 11----------------19 + * upload-pack.c: 4 11----------------19 * builtin/blame.c: 12-13 * bisect.c: 16 * bundle.c: 16 @@ -419,6 +419,13 @@ static char *substitute_branch_name(const char **string, int *len) int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref) { char *last_branch = substitute_branch_name(&str, &len); + int refs_found = expand_ref(str, len, sha1, ref); + free(last_branch); + return refs_found; +} + +int expand_ref(const char *str, int len, unsigned char *sha1, char **ref) +{ const char **p, *r; int refs_found = 0; @@ -444,7 +451,6 @@ int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref) warning("ignoring broken ref %s.", fullref); } } - free(last_branch); return refs_found; } @@ -94,6 +94,7 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, */ int refname_match(const char *abbrev_name, const char *full_name); +int expand_ref(const char *str, int len, unsigned char *sha1, char **ref); int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref); int dwim_log(const char *str, int len, unsigned char *sha1, char **ref); diff --git a/remote-curl.c b/remote-curl.c index 6b83b7783e..f14c41f4c0 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -20,6 +20,8 @@ static struct strbuf url = STRBUF_INIT; struct options { int verbosity; unsigned long depth; + char *deepen_since; + struct string_list deepen_not; unsigned progress : 1, check_self_contained_and_connected : 1, cloning : 1, @@ -28,7 +30,8 @@ struct options { dry_run : 1, thin : 1, /* One of the SEND_PACK_PUSH_CERT_* constants. */ - push_cert : 2; + push_cert : 2, + deepen_relative : 1; }; static struct options options; static struct string_list cas_options = STRING_LIST_INIT_DUP; @@ -60,6 +63,23 @@ static int set_option(const char *name, const char *value) options.depth = v; return 0; } + else if (!strcmp(name, "deepen-since")) { + options.deepen_since = xstrdup(value); + return 0; + } + else if (!strcmp(name, "deepen-not")) { + string_list_append(&options.deepen_not, value); + return 0; + } + else if (!strcmp(name, "deepen-relative")) { + if (!strcmp(value, "true")) + options.deepen_relative = 1; + else if (!strcmp(value, "false")) + options.deepen_relative = 0; + else + return -1; + return 0; + } else if (!strcmp(name, "followtags")) { if (!strcmp(value, "true")) options.followtags = 1; @@ -725,8 +745,8 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) int ret, i; ALLOC_ARRAY(targets, nr_heads); - if (options.depth) - die("dumb http transport does not support --depth"); + if (options.depth || options.deepen_since) + die("dumb http transport does not support shallow capabilities"); for (i = 0; i < nr_heads; i++) targets[i] = xstrdup(oid_to_hex(&to_fetch[i]->old_oid)); @@ -751,38 +771,35 @@ static int fetch_git(struct discovery *heads, { struct rpc_state rpc; struct strbuf preamble = STRBUF_INIT; - char *depth_arg = NULL; - int argc = 0, i, err; - const char *argv[17]; - - argv[argc++] = "fetch-pack"; - argv[argc++] = "--stateless-rpc"; - argv[argc++] = "--stdin"; - argv[argc++] = "--lock-pack"; + int i, err; + struct argv_array args = ARGV_ARRAY_INIT; + + argv_array_pushl(&args, "fetch-pack", "--stateless-rpc", + "--stdin", "--lock-pack", NULL); if (options.followtags) - argv[argc++] = "--include-tag"; + argv_array_push(&args, "--include-tag"); if (options.thin) - argv[argc++] = "--thin"; - if (options.verbosity >= 3) { - argv[argc++] = "-v"; - argv[argc++] = "-v"; - } + argv_array_push(&args, "--thin"); + if (options.verbosity >= 3) + argv_array_pushl(&args, "-v", "-v", NULL); if (options.check_self_contained_and_connected) - argv[argc++] = "--check-self-contained-and-connected"; + argv_array_push(&args, "--check-self-contained-and-connected"); if (options.cloning) - argv[argc++] = "--cloning"; + argv_array_push(&args, "--cloning"); if (options.update_shallow) - argv[argc++] = "--update-shallow"; + argv_array_push(&args, "--update-shallow"); if (!options.progress) - argv[argc++] = "--no-progress"; - if (options.depth) { - struct strbuf buf = STRBUF_INIT; - strbuf_addf(&buf, "--depth=%lu", options.depth); - depth_arg = strbuf_detach(&buf, NULL); - argv[argc++] = depth_arg; - } - argv[argc++] = url.buf; - argv[argc++] = NULL; + argv_array_push(&args, "--no-progress"); + if (options.depth) + argv_array_pushf(&args, "--depth=%lu", options.depth); + if (options.deepen_since) + argv_array_pushf(&args, "--shallow-since=%s", options.deepen_since); + for (i = 0; i < options.deepen_not.nr; i++) + argv_array_pushf(&args, "--shallow-exclude=%s", + options.deepen_not.items[i].string); + if (options.deepen_relative && options.depth) + argv_array_push(&args, "--deepen-relative"); + argv_array_push(&args, url.buf); for (i = 0; i < nr_heads; i++) { struct ref *ref = to_fetch[i]; @@ -795,7 +812,7 @@ static int fetch_git(struct discovery *heads, memset(&rpc, 0, sizeof(rpc)); rpc.service_name = "git-upload-pack", - rpc.argv = argv; + rpc.argv = args.argv; rpc.stdin_preamble = &preamble; rpc.gzip_request = 1; @@ -804,7 +821,7 @@ static int fetch_git(struct discovery *heads, write_or_die(1, rpc.result.buf, rpc.result.len); strbuf_release(&rpc.result); strbuf_release(&preamble); - free(depth_arg); + argv_array_clear(&args); return err; } @@ -998,6 +1015,7 @@ int cmd_main(int argc, const char **argv) options.verbosity = 1; options.progress = !!isatty(2); options.thin = 1; + string_list_init(&options.deepen_not, 1); remote = remote_get(argv[1]); @@ -10,6 +10,8 @@ #include "diff.h" #include "revision.h" #include "commit-slab.h" +#include "revision.h" +#include "list-objects.h" static int is_shallow = -1; static struct stat_validity shallow_stat; @@ -137,6 +139,82 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, return result; } +static void show_commit(struct commit *commit, void *data) +{ + commit_list_insert(commit, data); +} + +/* + * Given rev-list arguments, run rev-list. All reachable commits + * except border ones are marked with not_shallow_flag. Border commits + * are marked with shallow_flag. The list of border/shallow commits + * are also returned. + */ +struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av, + int shallow_flag, + int not_shallow_flag) +{ + struct commit_list *result = NULL, *p; + struct commit_list *not_shallow_list = NULL; + struct rev_info revs; + int both_flags = shallow_flag | not_shallow_flag; + + /* + * SHALLOW (excluded) and NOT_SHALLOW (included) should not be + * set at this point. But better be safe than sorry. + */ + clear_object_flags(both_flags); + + is_repository_shallow(); /* make sure shallows are read */ + + init_revisions(&revs, NULL); + save_commit_buffer = 0; + setup_revisions(ac, av, &revs, NULL); + + if (prepare_revision_walk(&revs)) + die("revision walk setup failed"); + traverse_commit_list(&revs, show_commit, NULL, ¬_shallow_list); + + /* Mark all reachable commits as NOT_SHALLOW */ + for (p = not_shallow_list; p; p = p->next) + p->item->object.flags |= not_shallow_flag; + + /* + * mark border commits SHALLOW + NOT_SHALLOW. + * We cannot clear NOT_SHALLOW right now. Imagine border + * commit A is processed first, then commit B, whose parent is + * A, later. If NOT_SHALLOW on A is cleared at step 1, B + * itself is considered border at step 2, which is incorrect. + */ + for (p = not_shallow_list; p; p = p->next) { + struct commit *c = p->item; + struct commit_list *parent; + + if (parse_commit(c)) + die("unable to parse commit %s", + oid_to_hex(&c->object.oid)); + + for (parent = c->parents; parent; parent = parent->next) + if (!(parent->item->object.flags & not_shallow_flag)) { + c->object.flags |= shallow_flag; + commit_list_insert(c, &result); + break; + } + } + free_commit_list(not_shallow_list); + + /* + * Now we can clean up NOT_SHALLOW on border commits. Having + * both flags set can confuse the caller. + */ + for (p = result; p; p = p->next) { + struct object *o = &p->item->object; + if ((o->flags & both_flags) == both_flags) + o->flags &= ~not_shallow_flag; + } + return result; +} + static void check_shallow_file_for_update(void) { if (is_shallow == -1) diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index 82d913a6a8..505e1b4a7f 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -652,4 +652,72 @@ test_expect_success MINGW 'fetch-pack --diag-url c:repo' ' check_prot_path c:repo file c:repo ' +test_expect_success 'clone shallow since ...' ' + test_create_repo shallow-since && + ( + cd shallow-since && + GIT_COMMITTER_DATE="100000000 +0700" git commit --allow-empty -m one && + GIT_COMMITTER_DATE="200000000 +0700" git commit --allow-empty -m two && + GIT_COMMITTER_DATE="300000000 +0700" git commit --allow-empty -m three && + git clone --shallow-since "300000000 +0700" "file://$(pwd)/." ../shallow11 && + git -C ../shallow11 log --pretty=tformat:%s HEAD >actual && + echo three >expected && + test_cmp expected actual + ) +' + +test_expect_success 'fetch shallow since ...' ' + git -C shallow11 fetch --shallow-since "200000000 +0700" origin && + git -C shallow11 log --pretty=tformat:%s origin/master >actual && + cat >expected <<-\EOF && + three + two + EOF + test_cmp expected actual +' + +test_expect_success 'shallow clone exclude tag two' ' + test_create_repo shallow-exclude && + ( + cd shallow-exclude && + test_commit one && + test_commit two && + test_commit three && + git clone --shallow-exclude two "file://$(pwd)/." ../shallow12 && + git -C ../shallow12 log --pretty=tformat:%s HEAD >actual && + echo three >expected && + test_cmp expected actual + ) +' + +test_expect_success 'fetch exclude tag one' ' + git -C shallow12 fetch --shallow-exclude one origin && + git -C shallow12 log --pretty=tformat:%s origin/master >actual && + test_write_lines three two >expected && + test_cmp expected actual +' + +test_expect_success 'fetching deepen' ' + test_create_repo shallow-deepen && + ( + cd shallow-deepen && + test_commit one && + test_commit two && + test_commit three && + git clone --depth 1 "file://$(pwd)/." deepen && + test_commit four && + git -C deepen log --pretty=tformat:%s master >actual && + echo three >expected && + test_cmp expected actual && + git -C deepen fetch --deepen=1 && + git -C deepen log --pretty=tformat:%s origin/master >actual && + cat >expected <<-\EOF && + four + three + two + EOF + test_cmp expected actual + ) +' + test_done diff --git a/t/t5539-fetch-http-shallow.sh b/t/t5539-fetch-http-shallow.sh index 37a433504e..5fbf67c446 100755 --- a/t/t5539-fetch-http-shallow.sh +++ b/t/t5539-fetch-http-shallow.sh @@ -73,5 +73,78 @@ test_expect_success 'no shallow lines after receiving ACK ready' ' ) ' +test_expect_success 'clone shallow since ...' ' + test_create_repo shallow-since && + ( + cd shallow-since && + GIT_COMMITTER_DATE="100000000 +0700" git commit --allow-empty -m one && + GIT_COMMITTER_DATE="200000000 +0700" git commit --allow-empty -m two && + GIT_COMMITTER_DATE="300000000 +0700" git commit --allow-empty -m three && + mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-since.git" && + git clone --shallow-since "300000000 +0700" $HTTPD_URL/smart/shallow-since.git ../shallow11 && + git -C ../shallow11 log --pretty=tformat:%s HEAD >actual && + echo three >expected && + test_cmp expected actual + ) +' + +test_expect_success 'fetch shallow since ...' ' + git -C shallow11 fetch --shallow-since "200000000 +0700" origin && + git -C shallow11 log --pretty=tformat:%s origin/master >actual && + cat >expected <<-\EOF && + three + two + EOF + test_cmp expected actual +' + +test_expect_success 'shallow clone exclude tag two' ' + test_create_repo shallow-exclude && + ( + cd shallow-exclude && + test_commit one && + test_commit two && + test_commit three && + mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-exclude.git" && + git clone --shallow-exclude two $HTTPD_URL/smart/shallow-exclude.git ../shallow12 && + git -C ../shallow12 log --pretty=tformat:%s HEAD >actual && + echo three >expected && + test_cmp expected actual + ) +' + +test_expect_success 'fetch exclude tag one' ' + git -C shallow12 fetch --shallow-exclude one origin && + git -C shallow12 log --pretty=tformat:%s origin/master >actual && + test_write_lines three two >expected && + test_cmp expected actual +' + +test_expect_success 'fetching deepen' ' + test_create_repo shallow-deepen && + ( + cd shallow-deepen && + test_commit one && + test_commit two && + test_commit three && + mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" && + git clone --depth 1 $HTTPD_URL/smart/shallow-deepen.git deepen && + mv "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" .git && + test_commit four && + git -C deepen log --pretty=tformat:%s master >actual && + echo three >expected && + test_cmp expected actual && + mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" && + git -C deepen fetch --deepen=1 && + git -C deepen log --pretty=tformat:%s origin/master >actual && + cat >expected <<-\EOF && + four + three + two + EOF + test_cmp expected actual + ) +' + stop_httpd test_done diff --git a/transport-helper.c b/transport-helper.c index db2f930c74..91aed35ebb 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -258,8 +258,51 @@ static const char *boolean_options[] = { TRANS_OPT_THIN, TRANS_OPT_KEEP, TRANS_OPT_FOLLOWTAGS, + TRANS_OPT_DEEPEN_RELATIVE }; +static int strbuf_set_helper_option(struct helper_data *data, + struct strbuf *buf) +{ + int ret; + + sendline(data, buf); + if (recvline(data, buf)) + exit(128); + + if (!strcmp(buf->buf, "ok")) + ret = 0; + else if (starts_with(buf->buf, "error")) + ret = -1; + else if (!strcmp(buf->buf, "unsupported")) + ret = 1; + else { + warning("%s unexpectedly said: '%s'", data->name, buf->buf); + ret = 1; + } + return ret; +} + +static int string_list_set_helper_option(struct helper_data *data, + const char *name, + struct string_list *list) +{ + struct strbuf buf = STRBUF_INIT; + int i, ret = 0; + + for (i = 0; i < list->nr; i++) { + strbuf_addf(&buf, "option %s ", name); + quote_c_style(list->items[i].string, &buf, NULL, 0); + strbuf_addch(&buf, '\n'); + + if ((ret = strbuf_set_helper_option(data, &buf))) + break; + strbuf_reset(&buf); + } + strbuf_release(&buf); + return ret; +} + static int set_helper_option(struct transport *transport, const char *name, const char *value) { @@ -272,6 +315,10 @@ static int set_helper_option(struct transport *transport, if (!data->option) return 1; + if (!strcmp(name, "deepen-not")) + return string_list_set_helper_option(data, name, + (struct string_list *)value); + for (i = 0; i < ARRAY_SIZE(unsupported_options); i++) { if (!strcmp(name, unsupported_options[i])) return 1; @@ -291,20 +338,7 @@ static int set_helper_option(struct transport *transport, quote_c_style(value, &buf, NULL, 0); strbuf_addch(&buf, '\n'); - sendline(data, &buf); - if (recvline(data, &buf)) - exit(128); - - if (!strcmp(buf.buf, "ok")) - ret = 0; - else if (starts_with(buf.buf, "error")) { - ret = -1; - } else if (!strcmp(buf.buf, "unsupported")) - ret = 1; - else { - warning("%s unexpectedly said: '%s'", data->name, buf.buf); - ret = 1; - } + ret = strbuf_set_helper_option(data, &buf); strbuf_release(&buf); return ret; } diff --git a/transport.c b/transport.c index 94d6dc3725..a85801042b 100644 --- a/transport.c +++ b/transport.c @@ -151,6 +151,15 @@ static int set_git_option(struct git_transport_options *opts, die(_("transport: invalid depth option '%s'"), value); } return 0; + } else if (!strcmp(name, TRANS_OPT_DEEPEN_SINCE)) { + opts->deepen_since = value; + return 0; + } else if (!strcmp(name, TRANS_OPT_DEEPEN_NOT)) { + opts->deepen_not = (const struct string_list *)value; + return 0; + } else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) { + opts->deepen_relative = !!value; + return 0; } return 1; } @@ -211,6 +220,9 @@ static int fetch_refs_via_pack(struct transport *transport, args.quiet = (transport->verbose < 0); args.no_progress = !transport->progress; args.depth = data->options.depth; + args.deepen_since = data->options.deepen_since; + args.deepen_not = data->options.deepen_not; + args.deepen_relative = data->options.deepen_relative; args.check_self_contained_and_connected = data->options.check_self_contained_and_connected; args.cloning = transport->cloning; diff --git a/transport.h b/transport.h index 6fe3485325..68669f14d0 100644 --- a/transport.h +++ b/transport.h @@ -5,6 +5,8 @@ #include "run-command.h" #include "remote.h" +struct string_list; + struct git_transport_options { unsigned thin : 1; unsigned keep : 1; @@ -12,7 +14,10 @@ struct git_transport_options { unsigned check_self_contained_and_connected : 1; unsigned self_contained_and_connected : 1; unsigned update_shallow : 1; + unsigned deepen_relative : 1; int depth; + const char *deepen_since; + const struct string_list *deepen_not; const char *uploadpack; const char *receivepack; struct push_cas_option *cas; @@ -186,6 +191,15 @@ int transport_restrict_protocols(void); /* Limit the depth of the fetch if not null */ #define TRANS_OPT_DEPTH "depth" +/* Limit the depth of the fetch based on time if not null */ +#define TRANS_OPT_DEEPEN_SINCE "deepen-since" + +/* Limit the depth of the fetch based on revs if not null */ +#define TRANS_OPT_DEEPEN_NOT "deepen-not" + +/* Limit the deepen of the fetch if not null */ +#define TRANS_OPT_DEEPEN_RELATIVE "deepen-relative" + /* Aggressively fetch annotated tags if possible */ #define TRANS_OPT_FOLLOWTAGS "followtags" diff --git a/upload-pack.c b/upload-pack.c index ca7f941780..5ec21e61d9 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -15,6 +15,7 @@ #include "version.h" #include "string-list.h" #include "parse-options.h" +#include "argv-array.h" static const char * const upload_pack_usage[] = { N_("git upload-pack [<options>] <dir>"), @@ -35,6 +36,7 @@ static const char * const upload_pack_usage[] = { static unsigned long oldest_have; +static int deepen_relative; static int multi_ack; static int no_done; static int use_thin_pack, use_ofs_delta, use_include_tag; @@ -281,7 +283,7 @@ static void create_pack_file(void) die("git upload-pack: %s", abort_msg); } -static int got_sha1(char *hex, unsigned char *sha1) +static int got_sha1(const char *hex, unsigned char *sha1) { struct object *o; int we_knew_they_have = 0; @@ -387,6 +389,8 @@ static int get_common_commits(void) for (;;) { char *line = packet_read_line(0, NULL); + const char *arg; + reset_timeout(); if (!line) { @@ -408,8 +412,8 @@ static int get_common_commits(void) got_other = 0; continue; } - if (starts_with(line, "have ")) { - switch (got_sha1(line+5, sha1)) { + if (skip_prefix(line, "have ", &arg)) { + switch (got_sha1(arg, sha1)) { case -1: /* they have what we do not */ got_other = 1; if (multi_ack && ok_to_give_up()) { @@ -454,73 +458,136 @@ static int is_our_ref(struct object *o) return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF); } -static void check_non_tip(void) +/* + * on successful case, it's up to the caller to close cmd->out + */ +static int do_reachable_revlist(struct child_process *cmd, + struct object_array *src, + struct object_array *reachable) { static const char *argv[] = { "rev-list", "--stdin", NULL, }; - static struct child_process cmd = CHILD_PROCESS_INIT; struct object *o; char namebuf[42]; /* ^ + SHA-1 + LF */ int i; - /* - * In the normal in-process case without - * uploadpack.allowReachableSHA1InWant, - * non-tip requests can never happen. - */ - if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1)) - goto error; - - cmd.argv = argv; - cmd.git_cmd = 1; - cmd.no_stderr = 1; - cmd.in = -1; - cmd.out = -1; - - if (start_command(&cmd)) - goto error; + cmd->argv = argv; + cmd->git_cmd = 1; + cmd->no_stderr = 1; + cmd->in = -1; + cmd->out = -1; /* - * If rev-list --stdin encounters an unknown commit, it - * terminates, which will cause SIGPIPE in the write loop + * If the next rev-list --stdin encounters an unknown commit, + * it terminates, which will cause SIGPIPE in the write loop * below. */ sigchain_push(SIGPIPE, SIG_IGN); + if (start_command(cmd)) + goto error; + namebuf[0] = '^'; namebuf[41] = '\n'; for (i = get_max_object_index(); 0 < i; ) { o = get_indexed_object(--i); if (!o) continue; + if (reachable && o->type == OBJ_COMMIT) + o->flags &= ~TMP_MARK; if (!is_our_ref(o)) continue; memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 42) < 0) + if (write_in_full(cmd->in, namebuf, 42) < 0) goto error; } namebuf[40] = '\n'; - for (i = 0; i < want_obj.nr; i++) { - o = want_obj.objects[i].item; - if (is_our_ref(o)) + for (i = 0; i < src->nr; i++) { + o = src->objects[i].item; + if (is_our_ref(o)) { + if (reachable) + add_object_array(o, NULL, reachable); continue; + } + if (reachable && o->type == OBJ_COMMIT) + o->flags |= TMP_MARK; memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 41) < 0) + if (write_in_full(cmd->in, namebuf, 41) < 0) goto error; } - close(cmd.in); + close(cmd->in); + cmd->in = -1; + sigchain_pop(SIGPIPE); + return 0; + +error: sigchain_pop(SIGPIPE); + if (cmd->in >= 0) + close(cmd->in); + if (cmd->out >= 0) + close(cmd->out); + return -1; +} + +static int get_reachable_list(struct object_array *src, + struct object_array *reachable) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + int i; + struct object *o; + char namebuf[42]; /* ^ + SHA-1 + LF */ + + if (do_reachable_revlist(&cmd, src, reachable) < 0) + return -1; + + while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) { + struct object_id sha1; + + if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1)) + break; + + o = lookup_object(sha1.hash); + if (o && o->type == OBJ_COMMIT) { + o->flags &= ~TMP_MARK; + } + } + for (i = get_max_object_index(); 0 < i; i--) { + o = get_indexed_object(i - 1); + if (o && o->type == OBJ_COMMIT && + (o->flags & TMP_MARK)) { + add_object_array(o, NULL, reachable); + o->flags &= ~TMP_MARK; + } + } + close(cmd.out); + + if (finish_command(&cmd)) + return -1; + + return 0; +} + +static int has_unreachable(struct object_array *src) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + char buf[1]; + int i; + + if (do_reachable_revlist(&cmd, src, NULL) < 0) + return 1; + /* * The commits out of the rev-list are not ancestors of * our ref. */ - i = read_in_full(cmd.out, namebuf, 1); + i = read_in_full(cmd.out, buf, 1); if (i) goto error; close(cmd.out); + cmd.out = -1; /* * rev-list may have died by encountering a bad commit @@ -531,23 +598,142 @@ static void check_non_tip(void) goto error; /* All the non-tip ones are ancestors of what we advertised */ - return; + return 0; + +error: + sigchain_pop(SIGPIPE); + if (cmd.out >= 0) + close(cmd.out); + return 1; +} + +static void check_non_tip(void) +{ + int i; + + /* + * In the normal in-process case without + * uploadpack.allowReachableSHA1InWant, + * non-tip requests can never happen. + */ + if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1)) + goto error; + if (!has_unreachable(&want_obj)) + /* All the non-tip ones are ancestors of what we advertised */ + return; error: /* Pick one of them (we know there at least is one) */ for (i = 0; i < want_obj.nr; i++) { - o = want_obj.objects[i].item; + struct object *o = want_obj.objects[i].item; if (!is_our_ref(o)) die("git upload-pack: not our ref %s", oid_to_hex(&o->oid)); } } +static void send_shallow(struct commit_list *result) +{ + while (result) { + struct object *object = &result->item->object; + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { + packet_write(1, "shallow %s", + oid_to_hex(&object->oid)); + register_shallow(object->oid.hash); + shallow_nr++; + } + result = result->next; + } +} + +static void send_unshallow(const struct object_array *shallows) +{ + int i; + + for (i = 0; i < shallows->nr; i++) { + struct object *object = shallows->objects[i].item; + if (object->flags & NOT_SHALLOW) { + struct commit_list *parents; + packet_write(1, "unshallow %s", + oid_to_hex(&object->oid)); + object->flags &= ~CLIENT_SHALLOW; + /* + * We want to _register_ "object" as shallow, but we + * also need to traverse object's parents to deepen a + * shallow clone. Unregister it for now so we can + * parse and add the parents to the want list, then + * re-register it. + */ + unregister_shallow(object->oid.hash); + object->parsed = 0; + parse_commit_or_die((struct commit *)object); + parents = ((struct commit *)object)->parents; + while (parents) { + add_object_array(&parents->item->object, + NULL, &want_obj); + parents = parents->next; + } + add_object_array(object, NULL, &extra_edge_obj); + } + /* make sure commit traversal conforms to client */ + register_shallow(object->oid.hash); + } +} + +static void deepen(int depth, int deepen_relative, + struct object_array *shallows) +{ + if (depth == INFINITE_DEPTH && !is_repository_shallow()) { + int i; + + for (i = 0; i < shallows->nr; i++) { + struct object *object = shallows->objects[i].item; + object->flags |= NOT_SHALLOW; + } + } else if (deepen_relative) { + struct object_array reachable_shallows = OBJECT_ARRAY_INIT; + struct commit_list *result; + + get_reachable_list(shallows, &reachable_shallows); + result = get_shallow_commits(&reachable_shallows, + depth + 1, + SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + object_array_clear(&reachable_shallows); + } else { + struct commit_list *result; + + result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + } + + send_unshallow(shallows); + packet_flush(1); +} + +static void deepen_by_rev_list(int ac, const char **av, + struct object_array *shallows) +{ + struct commit_list *result; + + result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + send_unshallow(shallows); + packet_flush(1); +} + static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; int depth = 0; int has_non_tip = 0; + unsigned long deepen_since = 0; + int deepen_rev_list = 0; shallow_nr = 0; for (;;) { @@ -555,14 +741,16 @@ static void receive_needs(void) const char *features; unsigned char sha1_buf[20]; char *line = packet_read_line(0, NULL); + const char *arg; + reset_timeout(); if (!line) break; - if (starts_with(line, "shallow ")) { + if (skip_prefix(line, "shallow ", &arg)) { unsigned char sha1[20]; struct object *object; - if (get_sha1_hex(line + 8, sha1)) + if (get_sha1_hex(arg, sha1)) die("invalid shallow line: %s", line); object = parse_object(sha1); if (!object) @@ -575,20 +763,42 @@ static void receive_needs(void) } continue; } - if (starts_with(line, "deepen ")) { - char *end; - depth = strtol(line + 7, &end, 0); - if (end == line + 7 || depth <= 0) + if (skip_prefix(line, "deepen ", &arg)) { + char *end = NULL; + depth = strtol(arg, &end, 0); + if (!end || *end || depth <= 0) die("Invalid deepen: %s", line); continue; } - if (!starts_with(line, "want ") || - get_sha1_hex(line+5, sha1_buf)) + if (skip_prefix(line, "deepen-since ", &arg)) { + char *end = NULL; + deepen_since = strtoul(arg, &end, 0); + if (!end || *end || !deepen_since || + /* revisions.c's max_age -1 is special */ + deepen_since == -1) + die("Invalid deepen-since: %s", line); + deepen_rev_list = 1; + continue; + } + if (skip_prefix(line, "deepen-not ", &arg)) { + char *ref = NULL; + unsigned char sha1[20]; + if (expand_ref(arg, strlen(arg), sha1, &ref) != 1) + die("git upload-pack: ambiguous deepen-not: %s", line); + string_list_append(&deepen_not, ref); + free(ref); + deepen_rev_list = 1; + continue; + } + if (!skip_prefix(line, "want ", &arg) || + get_sha1_hex(arg, sha1_buf)) die("git upload-pack: protocol error, " "expected to get sha, not '%s'", line); - features = line + 45; + features = arg + 40; + if (parse_feature_request(features, "deepen-relative")) + deepen_relative = 1; if (parse_feature_request(features, "multi_ack_detailed")) multi_ack = 2; else if (parse_feature_request(features, "multi_ack")) @@ -633,55 +843,35 @@ static void receive_needs(void) if (!use_sideband && daemon_mode) no_progress = 1; - if (depth == 0 && shallows.nr == 0) + if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; - if (depth > 0) { - struct commit_list *result = NULL, *backup = NULL; + if (depth > 0 && deepen_rev_list) + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); + if (depth > 0) + deepen(depth, deepen_relative, &shallows); + else if (deepen_rev_list) { + struct argv_array av = ARGV_ARRAY_INIT; int i; - if (depth == INFINITE_DEPTH && !is_repository_shallow()) - for (i = 0; i < shallows.nr; i++) { - struct object *object = shallows.objects[i].item; - object->flags |= NOT_SHALLOW; - } - else - backup = result = - get_shallow_commits(&want_obj, depth, - SHALLOW, NOT_SHALLOW); - while (result) { - struct object *object = &result->item->object; - if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { - packet_write(1, "shallow %s", - oid_to_hex(&object->oid)); - register_shallow(object->oid.hash); - shallow_nr++; + + argv_array_push(&av, "rev-list"); + if (deepen_since) + argv_array_pushf(&av, "--max-age=%lu", deepen_since); + if (deepen_not.nr) { + argv_array_push(&av, "--not"); + for (i = 0; i < deepen_not.nr; i++) { + struct string_list_item *s = deepen_not.items + i; + argv_array_push(&av, s->string); } - result = result->next; + argv_array_push(&av, "--not"); } - free_commit_list(backup); - for (i = 0; i < shallows.nr; i++) { - struct object *object = shallows.objects[i].item; - if (object->flags & NOT_SHALLOW) { - struct commit_list *parents; - packet_write(1, "unshallow %s", - oid_to_hex(&object->oid)); - object->flags &= ~CLIENT_SHALLOW; - /* make sure the real parents are parsed */ - unregister_shallow(object->oid.hash); - object->parsed = 0; - parse_commit_or_die((struct commit *)object); - parents = ((struct commit *)object)->parents; - while (parents) { - add_object_array(&parents->item->object, - NULL, &want_obj); - parents = parents->next; - } - add_object_array(object, NULL, &extra_edge_obj); - } - /* make sure commit traversal conforms to client */ - register_shallow(object->oid.hash); + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); } - packet_flush(1); - } else + deepen_by_rev_list(av.argc, av.argv, &shallows); + argv_array_clear(&av); + } + else if (shallows.nr > 0) { int i; for (i = 0; i < shallows.nr; i++) @@ -729,8 +919,8 @@ static int send_ref(const char *refname, const struct object_id *oid, int flag, void *cb_data) { static const char *capabilities = "multi_ack thin-pack side-band" - " side-band-64k ofs-delta shallow no-progress" - " include-tag multi_ack_detailed"; + " side-band-64k ofs-delta shallow deepen-since deepen-not" + " deepen-relative no-progress include-tag multi_ack_detailed"; const char *refname_nons = strip_namespace(refname); struct object_id peeled; |