From edace6f02eeae6f4a06ed1e4f6308703523d8535 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:23 -0700 Subject: fetch-pack: Use a strbuf to compose the want list This change is being offered as a refactoring to make later commits in the smart HTTP series easier. By changing the enabled capabilities to be formatted in a strbuf it is easier to add a new capability to the set of supported capabilities. By formatting the want portion of the request into a strbuf and writing it as a whole block we can later decide to hold onto the req_buf (instead of releasing it) to recycle in stateless communications. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-fetch-pack.c | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) (limited to 'builtin-fetch-pack.c') diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 629735f547..783c2b0834 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -165,6 +165,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; + struct strbuf req_buf = STRBUF_INIT; if (marked) for_each_ref(clear_marks, NULL); @@ -175,6 +176,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, fetching = 0; for ( ; refs ; refs = refs->next) { unsigned char *remote = refs->old_sha1; + const char *remote_hex; struct object *o; /* @@ -192,27 +194,36 @@ static int find_common(int fd[2], unsigned char *result_sha1, continue; } - if (!fetching) - packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n", - sha1_to_hex(remote), - (multi_ack ? " multi_ack" : ""), - (use_sideband == 2 ? " side-band-64k" : ""), - (use_sideband == 1 ? " side-band" : ""), - (args.use_thin_pack ? " thin-pack" : ""), - (args.no_progress ? " no-progress" : ""), - (args.include_tag ? " include-tag" : ""), - (prefer_ofs_delta ? " ofs-delta" : "")); - else - packet_write(fd[1], "want %s\n", sha1_to_hex(remote)); + remote_hex = sha1_to_hex(remote); + if (!fetching) { + struct strbuf c = STRBUF_INIT; + if (multi_ack) strbuf_addstr(&c, " multi_ack"); + if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); + if (use_sideband == 1) strbuf_addstr(&c, " side-band"); + if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); + if (args.no_progress) strbuf_addstr(&c, " no-progress"); + if (args.include_tag) strbuf_addstr(&c, " include-tag"); + if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta"); + packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf); + strbuf_release(&c); + } else + packet_buf_write(&req_buf, "want %s\n", remote_hex); fetching++; } + + if (!fetching) { + strbuf_release(&req_buf); + packet_flush(fd[1]); + return 1; + } + if (is_repository_shallow()) - write_shallow_commits(fd[1], 1); + write_shallow_commits(&req_buf, 1); if (args.depth > 0) - packet_write(fd[1], "deepen %d", args.depth); - packet_flush(fd[1]); - if (!fetching) - return 1; + packet_buf_write(&req_buf, "deepen %d", args.depth); + packet_buf_flush(&req_buf); + + safe_write(fd[1], req_buf.buf, req_buf.len); if (args.depth > 0) { char line[1024]; @@ -296,6 +307,8 @@ done: multi_ack = 0; flushes++; } + strbuf_release(&req_buf); + while (flushes || multi_ack) { int ack = get_ack(fd[0], result_sha1); if (ack) { @@ -809,6 +822,7 @@ struct ref *fetch_pack(struct fetch_pack_args *my_args, if (args.depth > 0) { struct cache_time mtime; + struct strbuf sb = STRBUF_INIT; char *shallow = git_path("shallow"); int fd; @@ -826,12 +840,14 @@ struct ref *fetch_pack(struct fetch_pack_args *my_args, fd = hold_lock_file_for_update(&lock, shallow, LOCK_DIE_ON_ERROR); - if (!write_shallow_commits(fd, 0)) { + if (!write_shallow_commits(&sb, 0) + || write_in_full(fd, sb.buf, sb.len) != sb.len) { unlink_or_warn(shallow); rollback_lock_file(&lock); } else { commit_lock_file(&lock); } + strbuf_release(&sb); } reprepare_packed_git(); -- cgit v1.2.3 From 28754ab5f0ce9b4f6ca1641c3e10e2c68bd9b3fc Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:24 -0700 Subject: Move "get_ack()" back to fetch-pack In 41cb7488 Linus moved this function to connect.c for reuse inside of the git-clone-pack command. That was 2005, but in 2006 Junio retired git-clone-pack in commit efc7fa53. Since then the only caller has been fetch-pack. Since this ACK/NAK exchange is only used by the fetch-pack/upload-pack protocol we should move it back to be a private detail of fetch-pack. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-fetch-pack.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'builtin-fetch-pack.c') diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 783c2b0834..7c09d466c8 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -157,6 +157,27 @@ static const unsigned char *get_rev(void) return commit->object.sha1; } +static int get_ack(int fd, unsigned char *result_sha1) +{ + static char line[1000]; + int len = packet_read_line(fd, line, sizeof(line)); + + if (!len) + die("git fetch-pack: expected ACK/NAK, got EOF"); + if (line[len-1] == '\n') + line[--len] = 0; + if (!strcmp(line, "NAK")) + return 0; + if (!prefixcmp(line, "ACK ")) { + if (!get_sha1_hex(line+4, result_sha1)) { + if (strstr(line+45, "continue")) + return 2; + return 1; + } + } + die("git fetch_pack: expected ACK/NAK, got '%s'", line); +} + static int find_common(int fd[2], unsigned char *result_sha1, struct ref *refs) { -- cgit v1.2.3 From 78affc49ded26700450d25168534af02b29a6c61 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:25 -0700 Subject: Add multi_ack_detailed capability to fetch-pack/upload-pack When multi_ack_detailed is enabled the ACK continue messages returned by the remote upload-pack are broken out to describe the different states within the peer. This permits the client to better understand the server's in-memory state. The fetch-pack/upload-pack protocol now looks like: NAK --------------------------------- Always sent in response to "done" if there was no common base selected from the "have" lines (or no have lines were sent). * no multi_ack or multi_ack_detailed: Sent when the client has sent a pkt-line flush ("0000") and the server has not yet found a common base object. * either multi_ack or multi_ack_detailed: Always sent in response to a pkt-line flush. ACK %s ----------------------------------- * no multi_ack or multi_ack_detailed: Sent in response to "have" when the object exists on the remote side and is therefore an object in common between the peers. The argument is the SHA-1 of the common object. * either multi_ack or multi_ack_detailed: Sent in response to "done" if there are common objects. The argument is the last SHA-1 determined to be common. ACK %s continue ----------------------------------- * multi_ack only: Sent in response to "have". The remote side wants the client to consider this object as common, and immediately stop transmitting additional "have" lines for objects that are reachable from it. The reason the client should stop is not given, but is one of the two cases below available under multi_ack_detailed. ACK %s common ----------------------------------- * multi_ack_detailed only: Sent in response to "have". Both sides have this object. Like with "ACK %s continue" above the client should stop sending have lines reachable for objects from the argument. ACK %s ready ----------------------------------- * multi_ack_detailed only: Sent in response to "have". The client should stop transmitting objects which are reachable from the argument, and send "done" soon to get the objects. If the remote side has the specified object, it should first send an "ACK %s common" message prior to sending "ACK %s ready". Clients may still submit additional "have" lines if there are more side branches for the client to explore that might be added to the common set and reduce the number of objects to transfer. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-fetch-pack.c | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'builtin-fetch-pack.c') diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 7c09d466c8..615f54974e 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -157,7 +157,15 @@ static const unsigned char *get_rev(void) return commit->object.sha1; } -static int get_ack(int fd, unsigned char *result_sha1) +enum ack_type { + NAK = 0, + ACK, + ACK_continue, + ACK_common, + ACK_ready +}; + +static enum ack_type get_ack(int fd, unsigned char *result_sha1) { static char line[1000]; int len = packet_read_line(fd, line, sizeof(line)); @@ -167,12 +175,16 @@ static int get_ack(int fd, unsigned char *result_sha1) if (line[len-1] == '\n') line[--len] = 0; if (!strcmp(line, "NAK")) - return 0; + return NAK; if (!prefixcmp(line, "ACK ")) { if (!get_sha1_hex(line+4, result_sha1)) { if (strstr(line+45, "continue")) - return 2; - return 1; + return ACK_continue; + if (strstr(line+45, "common")) + return ACK_common; + if (strstr(line+45, "ready")) + return ACK_ready; + return ACK; } } die("git fetch_pack: expected ACK/NAK, got '%s'", line); @@ -218,7 +230,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, remote_hex = sha1_to_hex(remote); if (!fetching) { struct strbuf c = STRBUF_INIT; - if (multi_ack) strbuf_addstr(&c, " multi_ack"); + if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); + if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); @@ -298,18 +311,23 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (args.verbose && ack) fprintf(stderr, "got ack %d %s\n", ack, sha1_to_hex(result_sha1)); - if (ack == 1) { + switch (ack) { + case ACK: flushes = 0; multi_ack = 0; retval = 0; goto done; - } else if (ack == 2) { + case ACK_common: + case ACK_ready: + case ACK_continue: { struct commit *commit = lookup_commit(result_sha1); mark_common(commit, 0, 1); retval = 0; in_vain = 0; got_continue = 1; + break; + } } } while (ack); flushes--; @@ -336,7 +354,7 @@ done: if (args.verbose) fprintf(stderr, "got ack (%d) %s\n", ack, sha1_to_hex(result_sha1)); - if (ack == 1) + if (ack == ACK) return 0; multi_ack = 1; continue; @@ -618,7 +636,12 @@ static struct ref *do_fetch_pack(int fd[2], if (is_repository_shallow() && !server_supports("shallow")) die("Server does not support shallow clients"); - if (server_supports("multi_ack")) { + if (server_supports("multi_ack_detailed")) { + if (args.verbose) + fprintf(stderr, "Server supports multi_ack_detailed\n"); + multi_ack = 2; + } + else if (server_supports("multi_ack")) { if (args.verbose) fprintf(stderr, "Server supports multi_ack\n"); multi_ack = 1; -- cgit v1.2.3 From 249b2004d8c9c58ed1ea1665dfd376af0312ed7e Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:42 -0700 Subject: Smart fetch over HTTP: client side The git-remote-curl backend detects if the remote server supports the git-upload-pack service, and if so, runs git-fetch-pack locally in a pipe to generate the want/have commands. The advertisements from the server that were obtained during the discovery are passed into git-fetch-pack before the POST request starts, permitting server capability discovery and enablement. Common objects that are discovered are appended onto the request as have lines and are sent again on the next request. This allows the remote side to reinitialize its in-memory list of common objects during the next request. Because all requests are relatively short, below git-remote-curl's 1 MiB buffer limit, requests will use the standard Content-Length header and be valid HTTP/1.0 POST requests. This makes the fetch client more tolerant of proxy servers which don't support HTTP/1.1 or the chunked transfer encoding. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- builtin-fetch-pack.c | 110 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 17 deletions(-) (limited to 'builtin-fetch-pack.c') diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 615f54974e..8ed4a6feaa 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -165,6 +165,24 @@ enum ack_type { ACK_ready }; +static void consume_shallow_list(int fd) +{ + if (args.stateless_rpc && args.depth > 0) { + /* If we sent a depth we will get back "duplicate" + * shallow and unshallow commands every time there + * is a block of have lines exchanged. + */ + char line[1000]; + while (packet_read_line(fd, line, sizeof(line))) { + if (!prefixcmp(line, "shallow ")) + continue; + if (!prefixcmp(line, "unshallow ")) + continue; + die("git fetch-pack: expected shallow list"); + } + } +} + static enum ack_type get_ack(int fd, unsigned char *result_sha1) { static char line[1000]; @@ -190,6 +208,15 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) die("git fetch_pack: expected ACK/NAK, got '%s'", line); } +static void send_request(int fd, struct strbuf *buf) +{ + if (args.stateless_rpc) { + send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX); + packet_flush(fd); + } else + safe_write(fd, buf->buf, buf->len); +} + static int find_common(int fd[2], unsigned char *result_sha1, struct ref *refs) { @@ -199,7 +226,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, unsigned in_vain = 0; int got_continue = 0; struct strbuf req_buf = STRBUF_INIT; + size_t state_len = 0; + if (args.stateless_rpc && multi_ack == 1) + die("--stateless-rpc requires multi_ack_detailed"); if (marked) for_each_ref(clear_marks, NULL); marked = 1; @@ -256,13 +286,13 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (args.depth > 0) packet_buf_write(&req_buf, "deepen %d", args.depth); packet_buf_flush(&req_buf); - - safe_write(fd[1], req_buf.buf, req_buf.len); + state_len = req_buf.len; if (args.depth > 0) { char line[1024]; unsigned char sha1[20]; + send_request(fd[1], &req_buf); while (packet_read_line(fd[0], line, sizeof(line))) { if (!prefixcmp(line, "shallow ")) { if (get_sha1_hex(line + 8, sha1)) @@ -284,28 +314,40 @@ static int find_common(int fd[2], unsigned char *result_sha1, } die("expected shallow/unshallow, got %s", line); } + } else if (!args.stateless_rpc) + send_request(fd[1], &req_buf); + + if (!args.stateless_rpc) { + /* If we aren't using the stateless-rpc interface + * we don't need to retain the headers. + */ + strbuf_setlen(&req_buf, 0); + state_len = 0; } flushes = 0; retval = -1; while ((sha1 = get_rev())) { - packet_write(fd[1], "have %s\n", sha1_to_hex(sha1)); + packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1)); if (args.verbose) fprintf(stderr, "have %s\n", sha1_to_hex(sha1)); in_vain++; if (!(31 & ++count)) { int ack; - packet_flush(fd[1]); + packet_buf_flush(&req_buf); + send_request(fd[1], &req_buf); + strbuf_setlen(&req_buf, state_len); flushes++; /* * We keep one window "ahead" of the other side, and * will wait for an ACK only on the next one */ - if (count == 32) + if (!args.stateless_rpc && count == 32) continue; + consume_shallow_list(fd[0]); do { ack = get_ack(fd[0], result_sha1); if (args.verbose && ack) @@ -322,6 +364,17 @@ static int find_common(int fd[2], unsigned char *result_sha1, case ACK_continue: { struct commit *commit = lookup_commit(result_sha1); + if (args.stateless_rpc + && ack == ACK_common + && !(commit->object.flags & COMMON)) { + /* We need to replay the have for this object + * on the next RPC request so the peer knows + * it is in common with us. + */ + const char *hex = sha1_to_hex(result_sha1); + packet_buf_write(&req_buf, "have %s\n", hex); + state_len = req_buf.len; + } mark_common(commit, 0, 1); retval = 0; in_vain = 0; @@ -339,7 +392,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - packet_write(fd[1], "done\n"); + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -348,6 +402,7 @@ done: } strbuf_release(&req_buf); + consume_shallow_list(fd[0]); while (flushes || multi_ack) { int ack = get_ack(fd[0], result_sha1); if (ack) { @@ -672,6 +727,8 @@ static struct ref *do_fetch_pack(int fd[2], */ warning("no common commits"); + if (args.stateless_rpc) + packet_flush(fd[1]); if (get_pack(fd, pack_lockfile)) die("git fetch-pack: fetch failed."); @@ -742,6 +799,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct ref *ref = NULL; char *dest = NULL, **heads; int fd[2]; + char *pack_lockfile = NULL; + char **pack_lockfile_ptr = NULL; struct child_process *conn; nr_heads = 0; @@ -791,6 +850,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.no_progress = 1; continue; } + if (!strcmp("--stateless-rpc", arg)) { + args.stateless_rpc = 1; + continue; + } + if (!strcmp("--lock-pack", arg)) { + args.lock_pack = 1; + pack_lockfile_ptr = &pack_lockfile; + continue; + } usage(fetch_pack_usage); } dest = (char *)arg; @@ -801,19 +869,27 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!dest) usage(fetch_pack_usage); - conn = git_connect(fd, (char *)dest, args.uploadpack, - args.verbose ? CONNECT_VERBOSE : 0); - if (conn) { - get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL); - - ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL); - close(fd[0]); - close(fd[1]); - if (finish_connect(conn)) - ref = NULL; + if (args.stateless_rpc) { + conn = NULL; + fd[0] = 0; + fd[1] = 1; } else { - ref = NULL; + conn = git_connect(fd, (char *)dest, args.uploadpack, + args.verbose ? CONNECT_VERBOSE : 0); + } + + get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL); + + ref = fetch_pack(&args, fd, conn, ref, dest, + nr_heads, heads, pack_lockfile_ptr); + if (pack_lockfile) { + printf("lock %s\n", pack_lockfile); + fflush(stdout); } + close(fd[0]); + close(fd[1]); + if (finish_connect(conn)) + ref = NULL; ret = !ref; if (!ret && nr_heads) { -- cgit v1.2.3