From 81b50f3ce40bfdd66e5d967bf82be001039a9a98 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Mon, 22 Feb 2010 08:42:18 -0800 Subject: Move 'builtin-*' into a 'builtin/' subdirectory This shrinks the top-level directory a bit, and makes it much more pleasant to use auto-completion on the thing. Instead of [torvalds@nehalem git]$ em buil Display all 180 possibilities? (y or n) [torvalds@nehalem git]$ em builtin-sh builtin-shortlog.c builtin-show-branch.c builtin-show-ref.c builtin-shortlog.o builtin-show-branch.o builtin-show-ref.o [torvalds@nehalem git]$ em builtin-shor builtin-shortlog.c builtin-shortlog.o [torvalds@nehalem git]$ em builtin-shortlog.c you get [torvalds@nehalem git]$ em buil [type] builtin/ builtin.h [torvalds@nehalem git]$ em builtin [auto-completes to] [torvalds@nehalem git]$ em builtin/sh [type] shortlog.c shortlog.o show-branch.c show-branch.o show-ref.c show-ref.o [torvalds@nehalem git]$ em builtin/sho [auto-completes to] [torvalds@nehalem git]$ em builtin/shor [type] shortlog.c shortlog.o [torvalds@nehalem git]$ em builtin/shortlog.c which doesn't seem all that different, but not having that annoying break in "Display all 180 possibilities?" is quite a relief. NOTE! If you do this in a clean tree (no object files etc), or using an editor that has auto-completion rules that ignores '*.o' files, you won't see that annoying 'Display all 180 possibilities?' message - it will just show the choices instead. I think bash has some cut-off around 100 choices or something. So the reason I see this is that I'm using an odd editory, and thus don't have the rules to cut down on auto-completion. But you can simulate that by using 'ls' instead, or something similar. Signed-off-by: Linus Torvalds Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 976 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 976 insertions(+) create mode 100644 builtin/fetch-pack.c (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c new file mode 100644 index 0000000000..dbd8b7bcc8 --- /dev/null +++ b/builtin/fetch-pack.c @@ -0,0 +1,976 @@ +#include "cache.h" +#include "refs.h" +#include "pkt-line.h" +#include "commit.h" +#include "tag.h" +#include "exec_cmd.h" +#include "pack.h" +#include "sideband.h" +#include "fetch-pack.h" +#include "remote.h" +#include "run-command.h" + +static int transfer_unpack_limit = -1; +static int fetch_unpack_limit = -1; +static int unpack_limit = 100; +static int prefer_ofs_delta = 1; +static struct fetch_pack_args args = { + /* .uploadpack = */ "git-upload-pack", +}; + +static const char fetch_pack_usage[] = +"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=] [--depth=] [--no-progress] [-v] [:] [...]"; + +#define COMPLETE (1U << 0) +#define COMMON (1U << 1) +#define COMMON_REF (1U << 2) +#define SEEN (1U << 3) +#define POPPED (1U << 4) + +static int marked; + +/* + * After sending this many "have"s if we do not get any new ACK , we + * give up traversing our history. + */ +#define MAX_IN_VAIN 256 + +static struct commit_list *rev_list; +static int non_common_revs, multi_ack, use_sideband; + +static void rev_list_push(struct commit *commit, int mark) +{ + if (!(commit->object.flags & mark)) { + commit->object.flags |= mark; + + if (!(commit->object.parsed)) + if (parse_commit(commit)) + return; + + insert_by_date(commit, &rev_list); + + if (!(commit->object.flags & COMMON)) + non_common_revs++; + } +} + +static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + struct object *o = deref_tag(parse_object(sha1), path, 0); + + if (o && o->type == OBJ_COMMIT) + rev_list_push((struct commit *)o, SEEN); + + return 0; +} + +static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + struct object *o = deref_tag(parse_object(sha1), path, 0); + + if (o && o->type == OBJ_COMMIT) + clear_commit_marks((struct commit *)o, + COMMON | COMMON_REF | SEEN | POPPED); + return 0; +} + +/* + This function marks a rev and its ancestors as common. + In some cases, it is desirable to mark only the ancestors (for example + when only the server does not yet know that they are common). +*/ + +static void mark_common(struct commit *commit, + int ancestors_only, int dont_parse) +{ + if (commit != NULL && !(commit->object.flags & COMMON)) { + struct object *o = (struct object *)commit; + + if (!ancestors_only) + o->flags |= COMMON; + + if (!(o->flags & SEEN)) + rev_list_push(commit, SEEN); + else { + struct commit_list *parents; + + if (!ancestors_only && !(o->flags & POPPED)) + non_common_revs--; + if (!o->parsed && !dont_parse) + if (parse_commit(commit)) + return; + + for (parents = commit->parents; + parents; + parents = parents->next) + mark_common(parents->item, 0, dont_parse); + } + } +} + +/* + Get the next rev to send, ignoring the common. +*/ + +static const unsigned char *get_rev(void) +{ + struct commit *commit = NULL; + + while (commit == NULL) { + unsigned int mark; + struct commit_list *parents; + + if (rev_list == NULL || non_common_revs == 0) + return NULL; + + commit = rev_list->item; + if (!commit->object.parsed) + parse_commit(commit); + parents = commit->parents; + + commit->object.flags |= POPPED; + if (!(commit->object.flags & COMMON)) + non_common_revs--; + + if (commit->object.flags & COMMON) { + /* do not send "have", and ignore ancestors */ + commit = NULL; + mark = COMMON | SEEN; + } else if (commit->object.flags & COMMON_REF) + /* send "have", and ignore ancestors */ + mark = COMMON | SEEN; + else + /* send "have", also for its ancestors */ + mark = SEEN; + + while (parents) { + if (!(parents->item->object.flags & SEEN)) + rev_list_push(parents->item, mark); + if (mark & COMMON) + mark_common(parents->item, 1, 0); + parents = parents->next; + } + + rev_list = rev_list->next; + } + + return commit->object.sha1; +} + +enum ack_type { + NAK = 0, + ACK, + ACK_continue, + ACK_common, + ACK_ready +}; + +static void consume_shallow_list(int fd) +{ + if (args.stateless_rpc && args.depth > 0) { + /* If we sent a depth we will get back "duplicate" + * shallow and unshallow commands every time there + * is a block of have lines exchanged. + */ + char line[1000]; + while (packet_read_line(fd, line, sizeof(line))) { + if (!prefixcmp(line, "shallow ")) + continue; + if (!prefixcmp(line, "unshallow ")) + continue; + die("git fetch-pack: expected shallow list"); + } + } +} + +static enum ack_type get_ack(int fd, unsigned char *result_sha1) +{ + static char line[1000]; + int len = packet_read_line(fd, line, sizeof(line)); + + if (!len) + die("git fetch-pack: expected ACK/NAK, got EOF"); + if (line[len-1] == '\n') + line[--len] = 0; + if (!strcmp(line, "NAK")) + return NAK; + if (!prefixcmp(line, "ACK ")) { + if (!get_sha1_hex(line+4, result_sha1)) { + if (strstr(line+45, "continue")) + return ACK_continue; + if (strstr(line+45, "common")) + return ACK_common; + if (strstr(line+45, "ready")) + return ACK_ready; + return ACK; + } + } + die("git fetch_pack: expected ACK/NAK, got '%s'", line); +} + +static void send_request(int fd, struct strbuf *buf) +{ + if (args.stateless_rpc) { + send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX); + packet_flush(fd); + } else + safe_write(fd, buf->buf, buf->len); +} + +static int find_common(int fd[2], unsigned char *result_sha1, + struct ref *refs) +{ + int fetching; + int count = 0, flushes = 0, retval; + const unsigned char *sha1; + unsigned in_vain = 0; + int got_continue = 0; + struct strbuf req_buf = STRBUF_INIT; + size_t state_len = 0; + + if (args.stateless_rpc && multi_ack == 1) + die("--stateless-rpc requires multi_ack_detailed"); + if (marked) + for_each_ref(clear_marks, NULL); + marked = 1; + + for_each_ref(rev_list_insert_ref, NULL); + + fetching = 0; + for ( ; refs ; refs = refs->next) { + unsigned char *remote = refs->old_sha1; + const char *remote_hex; + struct object *o; + + /* + * If that object is complete (i.e. it is an ancestor of a + * local ref), we tell them we have it but do not have to + * tell them about its ancestors, which they already know + * about. + * + * We use lookup_object here because we are only + * interested in the case we *know* the object is + * reachable and we have already scanned it. + */ + if (((o = lookup_object(remote)) != NULL) && + (o->flags & COMPLETE)) { + continue; + } + + remote_hex = sha1_to_hex(remote); + if (!fetching) { + struct strbuf c = STRBUF_INIT; + if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); + if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); + if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); + if (use_sideband == 1) strbuf_addstr(&c, " side-band"); + if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); + if (args.no_progress) strbuf_addstr(&c, " no-progress"); + if (args.include_tag) strbuf_addstr(&c, " include-tag"); + if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta"); + packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf); + strbuf_release(&c); + } else + packet_buf_write(&req_buf, "want %s\n", remote_hex); + fetching++; + } + + if (!fetching) { + strbuf_release(&req_buf); + packet_flush(fd[1]); + return 1; + } + + if (is_repository_shallow()) + write_shallow_commits(&req_buf, 1); + if (args.depth > 0) + packet_buf_write(&req_buf, "deepen %d", args.depth); + packet_buf_flush(&req_buf); + state_len = req_buf.len; + + if (args.depth > 0) { + char line[1024]; + unsigned char sha1[20]; + + send_request(fd[1], &req_buf); + while (packet_read_line(fd[0], line, sizeof(line))) { + if (!prefixcmp(line, "shallow ")) { + if (get_sha1_hex(line + 8, sha1)) + die("invalid shallow line: %s", line); + register_shallow(sha1); + continue; + } + if (!prefixcmp(line, "unshallow ")) { + if (get_sha1_hex(line + 10, sha1)) + die("invalid unshallow line: %s", line); + if (!lookup_object(sha1)) + die("object not found: %s", line); + /* make sure that it is parsed as shallow */ + if (!parse_object(sha1)) + die("error in object: %s", line); + if (unregister_shallow(sha1)) + die("no shallow found: %s", line); + continue; + } + die("expected shallow/unshallow, got %s", line); + } + } else if (!args.stateless_rpc) + send_request(fd[1], &req_buf); + + if (!args.stateless_rpc) { + /* If we aren't using the stateless-rpc interface + * we don't need to retain the headers. + */ + strbuf_setlen(&req_buf, 0); + state_len = 0; + } + + flushes = 0; + retval = -1; + while ((sha1 = get_rev())) { + packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1)); + if (args.verbose) + fprintf(stderr, "have %s\n", sha1_to_hex(sha1)); + in_vain++; + if (!(31 & ++count)) { + int ack; + + packet_buf_flush(&req_buf); + send_request(fd[1], &req_buf); + strbuf_setlen(&req_buf, state_len); + flushes++; + + /* + * We keep one window "ahead" of the other side, and + * will wait for an ACK only on the next one + */ + if (!args.stateless_rpc && count == 32) + continue; + + consume_shallow_list(fd[0]); + do { + ack = get_ack(fd[0], result_sha1); + if (args.verbose && ack) + fprintf(stderr, "got ack %d %s\n", ack, + sha1_to_hex(result_sha1)); + switch (ack) { + case ACK: + flushes = 0; + multi_ack = 0; + retval = 0; + goto done; + case ACK_common: + case ACK_ready: + case ACK_continue: { + struct commit *commit = + lookup_commit(result_sha1); + if (args.stateless_rpc + && ack == ACK_common + && !(commit->object.flags & COMMON)) { + /* We need to replay the have for this object + * on the next RPC request so the peer knows + * it is in common with us. + */ + const char *hex = sha1_to_hex(result_sha1); + packet_buf_write(&req_buf, "have %s\n", hex); + state_len = req_buf.len; + } + mark_common(commit, 0, 1); + retval = 0; + in_vain = 0; + got_continue = 1; + break; + } + } + } while (ack); + flushes--; + if (got_continue && MAX_IN_VAIN < in_vain) { + if (args.verbose) + fprintf(stderr, "giving up\n"); + break; /* give up */ + } + } + } +done: + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); + if (args.verbose) + fprintf(stderr, "done\n"); + if (retval != 0) { + multi_ack = 0; + flushes++; + } + strbuf_release(&req_buf); + + consume_shallow_list(fd[0]); + while (flushes || multi_ack) { + int ack = get_ack(fd[0], result_sha1); + if (ack) { + if (args.verbose) + fprintf(stderr, "got ack (%d) %s\n", ack, + sha1_to_hex(result_sha1)); + if (ack == ACK) + return 0; + multi_ack = 1; + continue; + } + flushes--; + } + /* it is no error to fetch into a completely empty repo */ + return count ? retval : 0; +} + +static struct commit_list *complete; + +static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + struct object *o = parse_object(sha1); + + while (o && o->type == OBJ_TAG) { + struct tag *t = (struct tag *) o; + if (!t->tagged) + break; /* broken repository */ + o->flags |= COMPLETE; + o = parse_object(t->tagged->sha1); + } + if (o && o->type == OBJ_COMMIT) { + struct commit *commit = (struct commit *)o; + commit->object.flags |= COMPLETE; + insert_by_date(commit, &complete); + } + return 0; +} + +static void mark_recent_complete_commits(unsigned long cutoff) +{ + while (complete && cutoff <= complete->item->date) { + if (args.verbose) + fprintf(stderr, "Marking %s as complete\n", + sha1_to_hex(complete->item->object.sha1)); + pop_most_recent_commit(&complete, COMPLETE); + } +} + +static void filter_refs(struct ref **refs, int nr_match, char **match) +{ + struct ref **return_refs; + struct ref *newlist = NULL; + struct ref **newtail = &newlist; + struct ref *ref, *next; + struct ref *fastarray[32]; + + if (nr_match && !args.fetch_all) { + if (ARRAY_SIZE(fastarray) < nr_match) + return_refs = xcalloc(nr_match, sizeof(struct ref *)); + else { + return_refs = fastarray; + memset(return_refs, 0, sizeof(struct ref *) * nr_match); + } + } + else + return_refs = NULL; + + for (ref = *refs; ref; ref = next) { + next = ref->next; + if (!memcmp(ref->name, "refs/", 5) && + check_ref_format(ref->name + 5)) + ; /* trash */ + else if (args.fetch_all && + (!args.depth || prefixcmp(ref->name, "refs/tags/") )) { + *newtail = ref; + ref->next = NULL; + newtail = &ref->next; + continue; + } + else { + int order = path_match(ref->name, nr_match, match); + if (order) { + return_refs[order-1] = ref; + continue; /* we will link it later */ + } + } + free(ref); + } + + if (!args.fetch_all) { + int i; + for (i = 0; i < nr_match; i++) { + ref = return_refs[i]; + if (ref) { + *newtail = ref; + ref->next = NULL; + newtail = &ref->next; + } + } + if (return_refs != fastarray) + free(return_refs); + } + *refs = newlist; +} + +static int everything_local(struct ref **refs, int nr_match, char **match) +{ + struct ref *ref; + int retval; + unsigned long cutoff = 0; + + save_commit_buffer = 0; + + for (ref = *refs; ref; ref = ref->next) { + struct object *o; + + o = parse_object(ref->old_sha1); + if (!o) + continue; + + /* We already have it -- which may mean that we were + * in sync with the other side at some time after + * that (it is OK if we guess wrong here). + */ + if (o->type == OBJ_COMMIT) { + struct commit *commit = (struct commit *)o; + if (!cutoff || cutoff < commit->date) + cutoff = commit->date; + } + } + + if (!args.depth) { + for_each_ref(mark_complete, NULL); + if (cutoff) + mark_recent_complete_commits(cutoff); + } + + /* + * Mark all complete remote refs as common refs. + * Don't mark them common yet; the server has to be told so first. + */ + for (ref = *refs; ref; ref = ref->next) { + struct object *o = deref_tag(lookup_object(ref->old_sha1), + NULL, 0); + + if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) + continue; + + if (!(o->flags & SEEN)) { + rev_list_push((struct commit *)o, COMMON_REF | SEEN); + + mark_common((struct commit *)o, 1, 1); + } + } + + filter_refs(refs, nr_match, match); + + for (retval = 1, ref = *refs; ref ; ref = ref->next) { + const unsigned char *remote = ref->old_sha1; + unsigned char local[20]; + struct object *o; + + o = lookup_object(remote); + if (!o || !(o->flags & COMPLETE)) { + retval = 0; + if (!args.verbose) + continue; + fprintf(stderr, + "want %s (%s)\n", sha1_to_hex(remote), + ref->name); + continue; + } + + hashcpy(ref->new_sha1, local); + if (!args.verbose) + continue; + fprintf(stderr, + "already have %s (%s)\n", sha1_to_hex(remote), + ref->name); + } + return retval; +} + +static int sideband_demux(int in, int out, void *data) +{ + int *xd = data; + + int ret = recv_sideband("fetch-pack", xd[0], out); + close(out); + return ret; +} + +static int get_pack(int xd[2], char **pack_lockfile) +{ + struct async demux; + const char *argv[20]; + char keep_arg[256]; + char hdr_arg[256]; + const char **av; + int do_keep = args.keep_pack; + struct child_process cmd; + + memset(&demux, 0, sizeof(demux)); + if (use_sideband) { + /* xd[] is talking with upload-pack; subprocess reads from + * xd[0], spits out band#2 to stderr, and feeds us band#1 + * through demux->out. + */ + demux.proc = sideband_demux; + demux.data = xd; + demux.out = -1; + if (start_async(&demux)) + die("fetch-pack: unable to fork off sideband" + " demultiplexer"); + } + else + demux.out = xd[0]; + + memset(&cmd, 0, sizeof(cmd)); + cmd.argv = argv; + av = argv; + *hdr_arg = 0; + if (!args.keep_pack && unpack_limit) { + struct pack_header header; + + if (read_pack_header(demux.out, &header)) + die("protocol error: bad pack header"); + snprintf(hdr_arg, sizeof(hdr_arg), + "--pack_header=%"PRIu32",%"PRIu32, + ntohl(header.hdr_version), ntohl(header.hdr_entries)); + if (ntohl(header.hdr_entries) < unpack_limit) + do_keep = 0; + else + do_keep = 1; + } + + if (do_keep) { + if (pack_lockfile) + cmd.out = -1; + *av++ = "index-pack"; + *av++ = "--stdin"; + if (!args.quiet && !args.no_progress) + *av++ = "-v"; + if (args.use_thin_pack) + *av++ = "--fix-thin"; + if (args.lock_pack || unpack_limit) { + int s = sprintf(keep_arg, + "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid()); + if (gethostname(keep_arg + s, sizeof(keep_arg) - s)) + strcpy(keep_arg + s, "localhost"); + *av++ = keep_arg; + } + } + else { + *av++ = "unpack-objects"; + if (args.quiet) + *av++ = "-q"; + } + if (*hdr_arg) + *av++ = hdr_arg; + *av++ = NULL; + + cmd.in = demux.out; + cmd.git_cmd = 1; + if (start_command(&cmd)) + die("fetch-pack: unable to fork off %s", argv[0]); + if (do_keep && pack_lockfile) { + *pack_lockfile = index_pack_lockfile(cmd.out); + close(cmd.out); + } + + if (finish_command(&cmd)) + die("%s failed", argv[0]); + if (use_sideband && finish_async(&demux)) + die("error in sideband demultiplexer"); + return 0; +} + +static struct ref *do_fetch_pack(int fd[2], + const struct ref *orig_ref, + int nr_match, + char **match, + char **pack_lockfile) +{ + struct ref *ref = copy_ref_list(orig_ref); + unsigned char sha1[20]; + + if (is_repository_shallow() && !server_supports("shallow")) + die("Server does not support shallow clients"); + if (server_supports("multi_ack_detailed")) { + if (args.verbose) + fprintf(stderr, "Server supports multi_ack_detailed\n"); + multi_ack = 2; + } + else if (server_supports("multi_ack")) { + if (args.verbose) + fprintf(stderr, "Server supports multi_ack\n"); + multi_ack = 1; + } + if (server_supports("side-band-64k")) { + if (args.verbose) + fprintf(stderr, "Server supports side-band-64k\n"); + use_sideband = 2; + } + else if (server_supports("side-band")) { + if (args.verbose) + fprintf(stderr, "Server supports side-band\n"); + use_sideband = 1; + } + if (server_supports("ofs-delta")) { + if (args.verbose) + fprintf(stderr, "Server supports ofs-delta\n"); + } else + prefer_ofs_delta = 0; + if (everything_local(&ref, nr_match, match)) { + packet_flush(fd[1]); + goto all_done; + } + if (find_common(fd, sha1, ref) < 0) + if (!args.keep_pack) + /* When cloning, it is not unusual to have + * no common commit. + */ + warning("no common commits"); + + if (args.stateless_rpc) + packet_flush(fd[1]); + if (get_pack(fd, pack_lockfile)) + die("git fetch-pack: fetch failed."); + + all_done: + return ref; +} + +static int remove_duplicates(int nr_heads, char **heads) +{ + int src, dst; + + for (src = dst = 0; src < nr_heads; src++) { + /* If heads[src] is different from any of + * heads[0..dst], push it in. + */ + int i; + for (i = 0; i < dst; i++) { + if (!strcmp(heads[i], heads[src])) + break; + } + if (i < dst) + continue; + if (src != dst) + heads[dst] = heads[src]; + dst++; + } + return dst; +} + +static int fetch_pack_config(const char *var, const char *value, void *cb) +{ + if (strcmp(var, "fetch.unpacklimit") == 0) { + fetch_unpack_limit = git_config_int(var, value); + return 0; + } + + if (strcmp(var, "transfer.unpacklimit") == 0) { + transfer_unpack_limit = git_config_int(var, value); + return 0; + } + + if (strcmp(var, "repack.usedeltabaseoffset") == 0) { + prefer_ofs_delta = git_config_bool(var, value); + return 0; + } + + return git_default_config(var, value, cb); +} + +static struct lock_file lock; + +static void fetch_pack_setup(void) +{ + static int did_setup; + if (did_setup) + return; + git_config(fetch_pack_config, NULL); + if (0 <= transfer_unpack_limit) + unpack_limit = transfer_unpack_limit; + else if (0 <= fetch_unpack_limit) + unpack_limit = fetch_unpack_limit; + did_setup = 1; +} + +int cmd_fetch_pack(int argc, const char **argv, const char *prefix) +{ + int i, ret, nr_heads; + struct ref *ref = NULL; + char *dest = NULL, **heads; + int fd[2]; + char *pack_lockfile = NULL; + char **pack_lockfile_ptr = NULL; + struct child_process *conn; + + nr_heads = 0; + heads = NULL; + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + + if (*arg == '-') { + if (!prefixcmp(arg, "--upload-pack=")) { + args.uploadpack = arg + 14; + continue; + } + if (!prefixcmp(arg, "--exec=")) { + args.uploadpack = arg + 7; + continue; + } + if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) { + args.quiet = 1; + continue; + } + if (!strcmp("--keep", arg) || !strcmp("-k", arg)) { + args.lock_pack = args.keep_pack; + args.keep_pack = 1; + continue; + } + if (!strcmp("--thin", arg)) { + args.use_thin_pack = 1; + continue; + } + if (!strcmp("--include-tag", arg)) { + args.include_tag = 1; + continue; + } + if (!strcmp("--all", arg)) { + args.fetch_all = 1; + continue; + } + if (!strcmp("-v", arg)) { + args.verbose = 1; + continue; + } + if (!prefixcmp(arg, "--depth=")) { + args.depth = strtol(arg + 8, NULL, 0); + continue; + } + if (!strcmp("--no-progress", arg)) { + args.no_progress = 1; + continue; + } + if (!strcmp("--stateless-rpc", arg)) { + args.stateless_rpc = 1; + continue; + } + if (!strcmp("--lock-pack", arg)) { + args.lock_pack = 1; + pack_lockfile_ptr = &pack_lockfile; + continue; + } + usage(fetch_pack_usage); + } + dest = (char *)arg; + heads = (char **)(argv + i + 1); + nr_heads = argc - i - 1; + break; + } + if (!dest) + usage(fetch_pack_usage); + + if (args.stateless_rpc) { + conn = NULL; + fd[0] = 0; + fd[1] = 1; + } else { + conn = git_connect(fd, (char *)dest, args.uploadpack, + args.verbose ? CONNECT_VERBOSE : 0); + } + + get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL); + + ref = fetch_pack(&args, fd, conn, ref, dest, + nr_heads, heads, pack_lockfile_ptr); + if (pack_lockfile) { + printf("lock %s\n", pack_lockfile); + fflush(stdout); + } + close(fd[0]); + close(fd[1]); + if (finish_connect(conn)) + ref = NULL; + ret = !ref; + + if (!ret && nr_heads) { + /* If the heads to pull were given, we should have + * consumed all of them by matching the remote. + * Otherwise, 'git fetch remote no-such-ref' would + * silently succeed without issuing an error. + */ + for (i = 0; i < nr_heads; i++) + if (heads[i] && heads[i][0]) { + error("no such remote ref %s", heads[i]); + ret = 1; + } + } + while (ref) { + printf("%s %s\n", + sha1_to_hex(ref->old_sha1), ref->name); + ref = ref->next; + } + + return ret; +} + +struct ref *fetch_pack(struct fetch_pack_args *my_args, + int fd[], struct child_process *conn, + const struct ref *ref, + const char *dest, + int nr_heads, + char **heads, + char **pack_lockfile) +{ + struct stat st; + struct ref *ref_cpy; + + fetch_pack_setup(); + if (&args != my_args) + memcpy(&args, my_args, sizeof(args)); + if (args.depth > 0) { + if (stat(git_path("shallow"), &st)) + st.st_mtime = 0; + } + + if (heads && nr_heads) + nr_heads = remove_duplicates(nr_heads, heads); + if (!ref) { + packet_flush(fd[1]); + die("no matching remote head"); + } + ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile); + + if (args.depth > 0) { + struct cache_time mtime; + struct strbuf sb = STRBUF_INIT; + char *shallow = git_path("shallow"); + int fd; + + mtime.sec = st.st_mtime; + mtime.nsec = ST_MTIME_NSEC(st); + if (stat(shallow, &st)) { + if (mtime.sec) + die("shallow file was removed during fetch"); + } else if (st.st_mtime != mtime.sec +#ifdef USE_NSEC + || ST_MTIME_NSEC(st) != mtime.nsec +#endif + ) + die("shallow file was changed during fetch"); + + fd = hold_lock_file_for_update(&lock, shallow, + LOCK_DIE_ON_ERROR); + if (!write_shallow_commits(&sb, 0) + || write_in_full(fd, sb.buf, sb.len) != sb.len) { + unlink_or_warn(shallow); + rollback_lock_file(&lock); + } else { + commit_lock_file(&lock); + } + strbuf_release(&sb); + } + + reprepare_packed_git(); + return ref_cpy; +} -- cgit v1.2.3 From 47e44ed1dc17d3a94ec4bf8dd29810ab7882041c Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Fri, 26 Nov 2010 23:58:14 -0200 Subject: commit: Add commit_list prefix in two function names. Add commit_list prefix to insert_by_date function and to sort_by_date, so it's clear that these functions refer to commit_list structure. Signed-off-by: Thiago Farina Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index dbd8b7bcc8..b999413934 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -47,7 +47,7 @@ static void rev_list_push(struct commit *commit, int mark) if (parse_commit(commit)) return; - insert_by_date(commit, &rev_list); + commit_list_insert_by_date(commit, &rev_list); if (!(commit->object.flags & COMMON)) non_common_revs++; @@ -436,7 +436,7 @@ static int mark_complete(const char *path, const unsigned char *sha1, int flag, if (o && o->type == OBJ_COMMIT) { struct commit *commit = (struct commit *)o; commit->object.flags |= COMPLETE; - insert_by_date(commit, &complete); + commit_list_insert_by_date(commit, &complete); } return 0; } -- cgit v1.2.3 From bbc30f996380eacd71ca061675d5d0c5f21c45d2 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 24 Feb 2011 09:30:19 -0500 Subject: add packet tracing debug code This shows a trace of all packets coming in or out of a given program. This can help with debugging object negotiation or other protocol issues. To keep the code changes simple, we operate at the lowest level, meaning we don't necessarily understand what's in the packets. The one exception is a packet starting with "PACK", which causes us to skip that packet and turn off tracing (since the gigantic pack data will not be interesting to read, at least not in the trace format). We show both written and read packets. In the local case, this may mean you will see packets twice (written by the sender and read by the receiver). However, for cases where the other end is remote, this allows you to see the full conversation. Packet tracing can be enabled with GIT_TRACE_PACKET=, where takes the same arguments as GIT_TRACE. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b999413934..272bc383d6 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -804,6 +804,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) char **pack_lockfile_ptr = NULL; struct child_process *conn; + packet_trace_identity("fetch-pack"); + nr_heads = 0; heads = NULL; for (i = 1; i < argc; i++) { -- cgit v1.2.3 From f2cba9299be45f8e027f7b45c6e4a3cae55576c6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 14 Mar 2011 16:48:38 -0700 Subject: fetch-pack: Finish negotation if remote replies "ACK %s ready" If multi_ack_detailed was selected in the protocol capabilities (both client and server are >= Git 1.6.6) the upload-pack side will send "ACK %s ready" when it knows how to safely cut the graph and produce a reasonable pack for the want list that was already sent on the connection. Upon receiving "ACK %s ready" there is no point in looking at the remaining commits inside of rev_list. Sending additional "have %s" lines to the remote will not construct a smaller pack. It is unlikely a commit older than the current cut point will have a better delta base than the cut point itself has. The original design of this code had fetch-pack empty rev_list by marking a commit and its transitive ancestors COMMON whenever the remote side said "ACK %s {continue,common}" and skipping over any already COMMON commits during get_rev(). This approach does not work when most of rev_list is actually COMMON_REF, commits that are pointed to by a reference on the remote, which exist locally, and which have not yet been sent to the remote as a "have %s" line. Most of the common references are tags in the ref/tags namespace, using points in the commit graph that are more than 1 commit apart. In git.git itself, this is currently 340 tags, 339 of which point to commits in the commit graph. fetch-pack pushes all of these into rev_list, but is unable to mark them COMMON and discard during a remote's "ACK %s {continue,common}" because it does not parse through the entire parent chain. Not parsing the entire parent chain is an optimization to avoid walking back to the roots of the repository. Assuming the client is only following the remote (and does not make its own local commits), the client needs 11 rounds to spin through the entire list of tags (32 commits per round, ceil(339/32) == 11). Unfortunately the server knows on the first "have %s" line that it can produce a good pack, and does not need to see the remaining 320 tags in the other 10 rounds. Over git:// and ssh:// this isn't as bad as it sounds, the client is only transmitting an extra 16,000 bytes that it doesn't need to send. Over smart HTTP, the client must do an additional 10 HTTP POST requests, each of which incurs round-trip latency, and must upload the entire state vector of all known common objects. On the final POST request, this is 16 KiB worth of data. Fix all of this by clearing rev_list as soon as the remote side says it can construct a pack. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b999413934..5173dc9aad 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -379,6 +379,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; + if (ack == ACK_ready) + rev_list = NULL; break; } } -- cgit v1.2.3 From 761ecf0bc7b6cddf311f00877c59e6381cdbdeea Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 14 Mar 2011 17:59:39 -0700 Subject: fetch-pack: Implement no-done capability If enabled on the connection "multi_ack_detailed no-done" as a pair allows the remote upload-pack process to send a PACK down to the client as soon as a "ACK %s ready" message was also sent. Over git:// and ssh:// where a bi-directional stream is in place this has very little difference over the classical version that waits for the client to send a "done\n" line by itself. It does slightly reduce the latency involved to start the pack stream as there is one less round-trip from client->server required. Over smart HTTP this avoids needing to send a final RPC that has all of the prior common objects. Instead the server is able to return a pack as soon as its ready to. For many common users the smart HTTP fetch is now just 2 requests: GET .../info/refs, and a POST .../git-upload-pack to not only negotiate but also receive the pack stream. Only users who have more than 32 local unshared commits with the remote will need additional requests to negotiate a common merge base. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 5173dc9aad..59fbda522b 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -14,6 +14,7 @@ static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; static int unpack_limit = 100; static int prefer_ofs_delta = 1; +static int no_done = 0; static struct fetch_pack_args args = { /* .uploadpack = */ "git-upload-pack", }; @@ -225,6 +226,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; + int got_ready = 0; struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; @@ -262,6 +264,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, struct strbuf c = STRBUF_INIT; if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); + if (no_done) strbuf_addstr(&c, " no-done"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); @@ -379,8 +382,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; - if (ack == ACK_ready) + if (ack == ACK_ready) { rev_list = NULL; + got_ready = 1; + } break; } } @@ -394,8 +399,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - packet_buf_write(&req_buf, "done\n"); - send_request(fd[1], &req_buf); + if (!got_ready || !no_done) { + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); + } if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -698,6 +705,11 @@ static struct ref *do_fetch_pack(int fd[2], if (args.verbose) fprintf(stderr, "Server supports multi_ack_detailed\n"); multi_ack = 2; + if (server_supports("no-done")) { + if (args.verbose) + fprintf(stderr, "Server supports no-done\n"); + no_done = 1; + } } else if (server_supports("multi_ack")) { if (args.verbose) -- cgit v1.2.3 From e52d719266a06a8553043cb5616d9b4ce4abd27a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 11 Mar 2011 11:53:52 -0800 Subject: fetch-pack: objects in our alternates are available to us Use the helper function split from the receiving end of "git push" to allow the same optimization on the receiving end of "git fetch". Signed-off-by: Junio C Hamano Acked-by: Shawn O. Pearce --- builtin/fetch-pack.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b999413934..4c25968e16 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -9,6 +9,7 @@ #include "fetch-pack.h" #include "remote.h" #include "run-command.h" +#include "transport.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -217,6 +218,16 @@ static void send_request(int fd, struct strbuf *buf) safe_write(fd, buf->buf, buf->len); } +static void insert_one_alternate_ref(const struct ref *ref, void *unused) +{ + rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL); +} + +static void insert_alternate_refs(void) +{ + foreach_alt_odb(refs_from_alternate_cb, insert_one_alternate_ref); +} + static int find_common(int fd[2], unsigned char *result_sha1, struct ref *refs) { @@ -235,6 +246,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, marked = 1; for_each_ref(rev_list_insert_ref, NULL); + insert_alternate_refs(); fetching = 0; for ( ; refs ; refs = refs->next) { -- cgit v1.2.3 From c12f5917e4f528b056a8b9ca625397aee97ae1e4 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 20 Mar 2011 21:52:40 -0700 Subject: fetch-pack: factor out hardcoded handshake window size The "git fetch" client presents the most recent 32 commits it has to the server and gives a chance to the server to say "ok, we heard enough", and continues reporting what it has in chunks of 32 commits, digging its history down to older commits. Move the hardcoded size of the handshake window outside the code, so that we can tweak it more easily. Signed-off-by: Junio C Hamano Acked-by: Shawn Pearce --- builtin/fetch-pack.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 59fbda522b..1abe624dc8 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -218,11 +218,18 @@ static void send_request(int fd, struct strbuf *buf) safe_write(fd, buf->buf, buf->len); } +#define INITIAL_FLUSH 32 + +static int next_flush(int count) +{ + return INITIAL_FLUSH + count; +} + static int find_common(int fd[2], unsigned char *result_sha1, struct ref *refs) { int fetching; - int count = 0, flushes = 0, retval; + int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval; const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; @@ -335,19 +342,20 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (args.verbose) fprintf(stderr, "have %s\n", sha1_to_hex(sha1)); in_vain++; - if (!(31 & ++count)) { + if (flush_at <= ++count) { int ack; packet_buf_flush(&req_buf); send_request(fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); flushes++; + flush_at = next_flush(count); /* * We keep one window "ahead" of the other side, and * will wait for an ACK only on the next one */ - if (!args.stateless_rpc && count == 32) + if (!args.stateless_rpc && count == INITIAL_FLUSH) continue; consume_shallow_list(fd[0]); -- cgit v1.2.3 From 6afca450c3f2f05385900a7b8d3a0d47286f983f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 20 Mar 2011 21:52:44 -0700 Subject: fetch-pack: progressively use larger handshake windows The client has to dig the history deeper when more recent parts of its history do not have any overlap with the server it is fetching from. Make the handshake window exponentially larger as we dig deeper, with a reasonable upper cap. Signed-off-by: Junio C Hamano Acked-by: Shawn Pearce --- builtin/fetch-pack.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 1abe624dc8..b4f34a2cf9 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -219,10 +219,15 @@ static void send_request(int fd, struct strbuf *buf) } #define INITIAL_FLUSH 32 +#define LARGE_FLUSH 1024 static int next_flush(int count) { - return INITIAL_FLUSH + count; + if (count < LARGE_FLUSH) + count <<= 1; + else + count += LARGE_FLUSH; + return count; } static int find_common(int fd[2], unsigned char *result_sha1, -- cgit v1.2.3 From 066bf4c2e43c7fb40405843e49f809431314865d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 20 Mar 2011 21:52:45 -0700 Subject: fetch-pack: use smaller handshake window for initial request Start the initial request small by halving the INITIAL_FLUSH (we will try to stay one window ahead of the server, so we would end up giving twice as many "have" in flight at the very beginning). We may want to tweak these values even more, taking MTU into account. Signed-off-by: Junio C Hamano Acked-by: Shawn Pearce --- builtin/fetch-pack.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b4f34a2cf9..3c2c9406c4 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -218,12 +218,14 @@ static void send_request(int fd, struct strbuf *buf) safe_write(fd, buf->buf, buf->len); } -#define INITIAL_FLUSH 32 +#define INITIAL_FLUSH 16 #define LARGE_FLUSH 1024 static int next_flush(int count) { - if (count < LARGE_FLUSH) + if (count < INITIAL_FLUSH * 2) + count += INITIAL_FLUSH; + else if (count < LARGE_FLUSH) count <<= 1; else count += LARGE_FLUSH; -- cgit v1.2.3 From c2e86addb86689306b992065328ec52aa2479658 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Tue, 22 Mar 2011 00:51:05 -0700 Subject: Fix sparse warnings Fix warnings from 'make check'. - These files don't include 'builtin.h' causing sparse to complain that cmd_* isn't declared: builtin/clone.c:364, builtin/fetch-pack.c:797, builtin/fmt-merge-msg.c:34, builtin/hash-object.c:78, builtin/merge-index.c:69, builtin/merge-recursive.c:22 builtin/merge-tree.c:341, builtin/mktag.c:156, builtin/notes.c:426 builtin/notes.c:822, builtin/pack-redundant.c:596, builtin/pack-refs.c:10, builtin/patch-id.c:60, builtin/patch-id.c:149, builtin/remote.c:1512, builtin/remote-ext.c:240, builtin/remote-fd.c:53, builtin/reset.c:236, builtin/send-pack.c:384, builtin/unpack-file.c:25, builtin/var.c:75 - These files have symbols which should be marked static since they're only file scope: submodule.c:12, diff.c:631, replace_object.c:92, submodule.c:13, submodule.c:14, trace.c:78, transport.c:195, transport-helper.c:79, unpack-trees.c:19, url.c:3, url.c:18, url.c:104, url.c:117, url.c:123, url.c:129, url.c:136, thread-utils.c:21, thread-utils.c:48 - These files redeclare symbols to be different types: builtin/index-pack.c:210, parse-options.c:564, parse-options.c:571, usage.c:49, usage.c:58, usage.c:63, usage.c:72 - These files use a literal integer 0 when they really should use a NULL pointer: daemon.c:663, fast-import.c:2942, imap-send.c:1072, notes-merge.c:362 While we're in the area, clean up some unused #includes in builtin files (mostly exec_cmd.h). Signed-off-by: Stephen Boyd Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 272bc383d6..ef398620af 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -1,4 +1,4 @@ -#include "cache.h" +#include "builtin.h" #include "refs.h" #include "pkt-line.h" #include "commit.h" -- cgit v1.2.3 From 07514c83c2b7e6de926ef758905e3f43b4de6bfa Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 28 Mar 2011 23:35:39 -0700 Subject: Revert "fetch-pack: Implement no-done capability" This reverts commit 761ecf0bc7b6cddf311f00877c59e6381cdbdeea. --- builtin/fetch-pack.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index bf9990ce15..1724b76052 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -15,7 +15,6 @@ static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; static int unpack_limit = 100; static int prefer_ofs_delta = 1; -static int no_done = 0; static struct fetch_pack_args args = { /* .uploadpack = */ "git-upload-pack", }; @@ -251,7 +250,6 @@ static int find_common(int fd[2], unsigned char *result_sha1, const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; - int got_ready = 0; struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; @@ -290,7 +288,6 @@ static int find_common(int fd[2], unsigned char *result_sha1, struct strbuf c = STRBUF_INIT; if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); - if (no_done) strbuf_addstr(&c, " no-done"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); @@ -409,10 +406,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; - if (ack == ACK_ready) { + if (ack == ACK_ready) rev_list = NULL; - got_ready = 1; - } break; } } @@ -426,10 +421,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - if (!got_ready || !no_done) { - packet_buf_write(&req_buf, "done\n"); - send_request(fd[1], &req_buf); - } + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -732,11 +725,6 @@ static struct ref *do_fetch_pack(int fd[2], if (args.verbose) fprintf(stderr, "Server supports multi_ack_detailed\n"); multi_ack = 2; - if (server_supports("no-done")) { - if (args.verbose) - fprintf(stderr, "Server supports no-done\n"); - no_done = 1; - } } else if (server_supports("multi_ack")) { if (args.verbose) -- cgit v1.2.3 From 8e9182e09127b20b8d8ce19655037991feac798d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 29 Mar 2011 10:16:29 -0700 Subject: enable "no-done" extension only when fetching over smart-http When 'no-done' protocol extension is used, the upload-pack (i.e. the server side) process stops listening to the fetch-pack after issuing the final NAK, and starts sending the generated pack data back, but there may be more "have" send by the latter in flight that the fetch-pack is expecting to be responded with ACK/NAK. This will typically result in a deadlock (both will block on write that the other end never reads) or SIGPIPE on the fetch-pack end (upload-pack will finish writing a small pack and goes away). Disable it unless fetch-pack is running under smart-http, where there is no such streaming issue. Signed-off-by: Junio C Hamano Acked-by: Shawn O. Pearce --- builtin/fetch-pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 59fbda522b..52707a80ad 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -708,7 +708,8 @@ static struct ref *do_fetch_pack(int fd[2], if (server_supports("no-done")) { if (args.verbose) fprintf(stderr, "Server supports no-done\n"); - no_done = 1; + if (args.stateless_rpc) + no_done = 1; } } else if (server_supports("multi_ack")) { -- cgit v1.2.3 From 44d8dc54e73e8010c4bdf57a422fc8d5ce709029 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 29 Mar 2011 10:06:19 -0700 Subject: Fix potential local deadlock during fetch-pack The fetch-pack/upload-pack protocol relies on the underlying transport (local pipe or TCP socket) to have enough slack to allow one window worth of data in flight without blocking the writer. Traditionally we always relied on being able to have two windows of 32 "have"s in flight (roughly 3k bytes) to stream. The recent "progressive-stride" change allows "fetch-pack" to send up to 1024 "have"s without reading any response from "upload-pack". The outgoing pipe of "upload-pack" can be clogged with many ACK and NAK that are unread, while "fetch-pack" is still stuffing its outgoing pipe with more "have"s, leading to a deadlock. Revert the change unless we are in stateless rpc (aka smart-http) mode, as using a large window full of "have"s is still a good way to help reduce the number of back-and-forth, and there is no buffering issue there (it is strictly "ping-pong" without an overlap). Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 3c2c9406c4..147d67dca4 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -219,16 +219,17 @@ static void send_request(int fd, struct strbuf *buf) } #define INITIAL_FLUSH 16 +#define PIPESAFE_FLUSH 32 #define LARGE_FLUSH 1024 static int next_flush(int count) { - if (count < INITIAL_FLUSH * 2) - count += INITIAL_FLUSH; - else if (count < LARGE_FLUSH) + int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH; + + if (count < flush_limit) count <<= 1; else - count += LARGE_FLUSH; + count += flush_limit; return count; } -- cgit v1.2.3 From 4e10cf9a17467c08754b36683c240fbab69156de Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 29 Mar 2011 12:29:10 -0700 Subject: Revert two "no-done" reverts Last night I had to make these two emergency reverts, but now we have a better understanding of which part of the topic was broken, let's get rid of the revert to fix it correctly. Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'builtin/fetch-pack.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 1724b76052..bf9990ce15 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -15,6 +15,7 @@ static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; static int unpack_limit = 100; static int prefer_ofs_delta = 1; +static int no_done = 0; static struct fetch_pack_args args = { /* .uploadpack = */ "git-upload-pack", }; @@ -250,6 +251,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; + int got_ready = 0; struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; @@ -288,6 +290,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, struct strbuf c = STRBUF_INIT; if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); + if (no_done) strbuf_addstr(&c, " no-done"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); @@ -406,8 +409,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; - if (ack == ACK_ready) + if (ack == ACK_ready) { rev_list = NULL; + got_ready = 1; + } break; } } @@ -421,8 +426,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - packet_buf_write(&req_buf, "done\n"); - send_request(fd[1], &req_buf); + if (!got_ready || !no_done) { + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); + } if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -725,6 +732,11 @@ static struct ref *do_fetch_pack(int fd[2], if (args.verbose) fprintf(stderr, "Server supports multi_ack_detailed\n"); multi_ack = 2; + if (server_supports("no-done")) { + if (args.verbose) + fprintf(stderr, "Server supports no-done\n"); + no_done = 1; + } } else if (server_supports("multi_ack")) { if (args.verbose) -- cgit v1.2.3