From 2153d478b74cfef58ee49ee0305cccf5e8a77b4f Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:38 -0700 Subject: pkt-line: introduce packet_read_with_status The current pkt-line API encodes the status of a pkt-line read in the length of the read content. An error is indicated with '-1', a flush with '0' (which can be confusing since a return value of '0' can also indicate an empty pkt-line), and a positive integer for the length of the read content otherwise. This doesn't leave much room for allowing the addition of additional special packets in the future. To solve this introduce 'packet_read_with_status()' which reads a packet and returns the status of the read encoded as an 'enum packet_status' type. This allows for easily identifying between special and normal packets as well as errors. It also enables easily adding a new special packet in the future. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- pkt-line.c | 51 +++++++++++++++++++++++++++++++++++++-------------- pkt-line.h | 16 ++++++++++++++++ 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/pkt-line.c b/pkt-line.c index 2827ca772a..db2fb29ac3 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -280,28 +280,39 @@ static int packet_length(const char *linelen) return (val < 0) ? val : (val << 8) | hex2chr(linelen + 2); } -int packet_read(int fd, char **src_buf, size_t *src_len, - char *buffer, unsigned size, int options) +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options) { - int len, ret; + int len; char linelen[4]; - ret = get_packet_data(fd, src_buf, src_len, linelen, 4, options); - if (ret < 0) - return ret; + if (get_packet_data(fd, src_buffer, src_len, linelen, 4, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } + len = packet_length(linelen); - if (len < 0) + + if (len < 0) { die("protocol error: bad line length character: %.4s", linelen); - if (!len) { + } else if (!len) { packet_trace("0000", 4, 0); - return 0; + *pktlen = 0; + return PACKET_READ_FLUSH; + } else if (len < 4) { + die("protocol error: bad line length %d", len); } + len -= 4; - if (len >= size) + if ((unsigned)len >= size) die("protocol error: bad line length %d", len); - ret = get_packet_data(fd, src_buf, src_len, buffer, len, options); - if (ret < 0) - return ret; + + if (get_packet_data(fd, src_buffer, src_len, buffer, len, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } if ((options & PACKET_READ_CHOMP_NEWLINE) && len && buffer[len-1] == '\n') @@ -309,7 +320,19 @@ int packet_read(int fd, char **src_buf, size_t *src_len, buffer[len] = 0; packet_trace(buffer, len, 0); - return len; + *pktlen = len; + return PACKET_READ_NORMAL; +} + +int packet_read(int fd, char **src_buffer, size_t *src_len, + char *buffer, unsigned size, int options) +{ + int pktlen = -1; + + packet_read_with_status(fd, src_buffer, src_len, buffer, size, + &pktlen, options); + + return pktlen; } static char *packet_read_line_generic(int fd, diff --git a/pkt-line.h b/pkt-line.h index 3dad583e2d..099b26b95f 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -65,6 +65,22 @@ int write_packetized_from_buf(const char *src_in, size_t len, int fd_out); int packet_read(int fd, char **src_buffer, size_t *src_len, char *buffer, unsigned size, int options); +/* + * Read a packetized line into a buffer like the 'packet_read()' function but + * returns an 'enum packet_read_status' which indicates the status of the read. + * The number of bytes read will be assigined to *pktlen if the status of the + * read was 'PACKET_READ_NORMAL'. + */ +enum packet_read_status { + PACKET_READ_EOF, + PACKET_READ_NORMAL, + PACKET_READ_FLUSH, +}; +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options); + /* * Convenience wrapper for packet_read that is not gentle, and sets the * CHOMP_NEWLINE option. The return value is NULL for a flush packet, -- cgit v1.2.3 From 77dabc14c497d2498b814967bf21a3ed1618207f Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:39 -0700 Subject: pkt-line: allow peeking a packet line without consuming it Sometimes it is advantageous to be able to peek the next packet line without consuming it (e.g. to be able to determine the protocol version a server is speaking). In order to do that introduce 'struct packet_reader' which is an abstraction around the normal packet reading logic. This enables a caller to be able to peek a single line at a time using 'packet_reader_peek()' and having a caller consume a line by calling 'packet_reader_read()'. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- pkt-line.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ pkt-line.h | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/pkt-line.c b/pkt-line.c index db2fb29ac3..1881dc8813 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -400,3 +400,53 @@ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out) } return sb_out->len - orig_len; } + +/* Packet Reader Functions */ +void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options) +{ + memset(reader, 0, sizeof(*reader)); + + reader->fd = fd; + reader->src_buffer = src_buffer; + reader->src_len = src_len; + reader->buffer = packet_buffer; + reader->buffer_size = sizeof(packet_buffer); + reader->options = options; +} + +enum packet_read_status packet_reader_read(struct packet_reader *reader) +{ + if (reader->line_peeked) { + reader->line_peeked = 0; + return reader->status; + } + + reader->status = packet_read_with_status(reader->fd, + &reader->src_buffer, + &reader->src_len, + reader->buffer, + reader->buffer_size, + &reader->pktlen, + reader->options); + + if (reader->status == PACKET_READ_NORMAL) + reader->line = reader->buffer; + else + reader->line = NULL; + + return reader->status; +} + +enum packet_read_status packet_reader_peek(struct packet_reader *reader) +{ + /* Only allow peeking a single line */ + if (reader->line_peeked) + return reader->status; + + /* Peek a line by reading it and setting peeked flag */ + packet_reader_read(reader); + reader->line_peeked = 1; + return reader->status; +} diff --git a/pkt-line.h b/pkt-line.h index 099b26b95f..11b04f026f 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -112,6 +112,64 @@ char *packet_read_line_buf(char **src_buf, size_t *src_len, int *size); */ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out); +struct packet_reader { + /* source file descriptor */ + int fd; + + /* source buffer and its size */ + char *src_buffer; + size_t src_len; + + /* buffer that pkt-lines are read into and its size */ + char *buffer; + unsigned buffer_size; + + /* options to be used during reads */ + int options; + + /* status of the last read */ + enum packet_read_status status; + + /* length of data read during the last read */ + int pktlen; + + /* the last line read */ + const char *line; + + /* indicates if a line has been peeked */ + int line_peeked; +}; + +/* + * Initialize a 'struct packet_reader' object which is an + * abstraction around the 'packet_read_with_status()' function. + */ +extern void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options); + +/* + * Perform a packet read and return the status of the read. + * The values of 'pktlen' and 'line' are updated based on the status of the + * read as follows: + * + * PACKET_READ_ERROR: 'pktlen' is set to '-1' and 'line' is set to NULL + * PACKET_READ_NORMAL: 'pktlen' is set to the number of bytes read + * 'line' is set to point at the read line + * PACKET_READ_FLUSH: 'pktlen' is set to '0' and 'line' is set to NULL + */ +extern enum packet_read_status packet_reader_read(struct packet_reader *reader); + +/* + * Peek the next packet line without consuming it and return the status. + * The next call to 'packet_reader_read()' will perform a read of the same line + * that was peeked, consuming the line. + * + * Peeking multiple times without calling 'packet_reader_read()' will return + * the same result. + */ +extern enum packet_read_status packet_reader_peek(struct packet_reader *reader); + #define DEFAULT_PACKET_MAX 1000 #define LARGE_PACKET_MAX 65520 #define LARGE_PACKET_DATA_MAX (LARGE_PACKET_MAX - 4) -- cgit v1.2.3 From a4cfd41c7be81c10b925532979e81e4d07f6b376 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:40 -0700 Subject: pkt-line: add delim packet support One of the design goals of protocol-v2 is to improve the semantics of flush packets. Currently in protocol-v1, flush packets are used both to indicate a break in a list of packet lines as well as an indication that one side has finished speaking. This makes it particularly difficult to implement proxies as a proxy would need to completely understand git protocol instead of simply looking for a flush packet. To do this, introduce the special deliminator packet '0001'. A delim packet can then be used as a deliminator between lists of packet lines while flush packets can be reserved to indicate the end of a response. Documentation for how this packet will be used in protocol v2 will included in a future patch. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- pkt-line.c | 16 ++++++++++++++++ pkt-line.h | 3 +++ 2 files changed, 19 insertions(+) diff --git a/pkt-line.c b/pkt-line.c index 1881dc8813..7296731cf3 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -91,6 +91,12 @@ void packet_flush(int fd) write_or_die(fd, "0000", 4); } +void packet_delim(int fd) +{ + packet_trace("0001", 4, 1); + write_or_die(fd, "0001", 4); +} + int packet_flush_gently(int fd) { packet_trace("0000", 4, 1); @@ -105,6 +111,12 @@ void packet_buf_flush(struct strbuf *buf) strbuf_add(buf, "0000", 4); } +void packet_buf_delim(struct strbuf *buf) +{ + packet_trace("0001", 4, 1); + strbuf_add(buf, "0001", 4); +} + static void set_packet_header(char *buf, const int size) { static char hexchar[] = "0123456789abcdef"; @@ -301,6 +313,10 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, packet_trace("0000", 4, 0); *pktlen = 0; return PACKET_READ_FLUSH; + } else if (len == 1) { + packet_trace("0001", 4, 0); + *pktlen = 0; + return PACKET_READ_DELIM; } else if (len < 4) { die("protocol error: bad line length %d", len); } diff --git a/pkt-line.h b/pkt-line.h index 11b04f026f..9570bd7a0a 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -20,8 +20,10 @@ * side can't, we stay with pure read/write interfaces. */ void packet_flush(int fd); +void packet_delim(int fd); void packet_write_fmt(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); void packet_buf_flush(struct strbuf *buf); +void packet_buf_delim(struct strbuf *buf); void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int packet_flush_gently(int fd); @@ -75,6 +77,7 @@ enum packet_read_status { PACKET_READ_EOF, PACKET_READ_NORMAL, PACKET_READ_FLUSH, + PACKET_READ_DELIM, }; enum packet_read_status packet_read_with_status(int fd, char **src_buffer, size_t *src_len, char *buffer, -- cgit v1.2.3 From a3d6b53e92528d20fdaca504614f260c5b457b29 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:41 -0700 Subject: upload-pack: convert to a builtin In order to allow for code sharing with the server-side of fetch in protocol-v2 convert upload-pack to be a builtin. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- Makefile | 3 +- builtin.h | 1 + builtin/upload-pack.c | 67 +++++++++++++++++++++++++++++++ git.c | 1 + upload-pack.c | 107 ++++++++++++-------------------------------------- upload-pack.h | 13 ++++++ 6 files changed, 109 insertions(+), 83 deletions(-) create mode 100644 builtin/upload-pack.c create mode 100644 upload-pack.h diff --git a/Makefile b/Makefile index 2a81ae22e9..91a59a6d50 100644 --- a/Makefile +++ b/Makefile @@ -636,7 +636,6 @@ PROGRAM_OBJS += imap-send.o PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o PROGRAM_OBJS += show-index.o -PROGRAM_OBJS += upload-pack.o PROGRAM_OBJS += remote-testsvn.o # Binary suffix, set to .exe for Windows builds @@ -904,6 +903,7 @@ LIB_OBJS += tree-diff.o LIB_OBJS += tree.o LIB_OBJS += tree-walk.o LIB_OBJS += unpack-trees.o +LIB_OBJS += upload-pack.o LIB_OBJS += url.o LIB_OBJS += urlmatch.o LIB_OBJS += usage.o @@ -1021,6 +1021,7 @@ BUILTIN_OBJS += builtin/update-index.o BUILTIN_OBJS += builtin/update-ref.o BUILTIN_OBJS += builtin/update-server-info.o BUILTIN_OBJS += builtin/upload-archive.o +BUILTIN_OBJS += builtin/upload-pack.o BUILTIN_OBJS += builtin/var.o BUILTIN_OBJS += builtin/verify-commit.o BUILTIN_OBJS += builtin/verify-pack.o diff --git a/builtin.h b/builtin.h index 42378f3aa4..f332a12574 100644 --- a/builtin.h +++ b/builtin.h @@ -231,6 +231,7 @@ extern int cmd_update_ref(int argc, const char **argv, const char *prefix); extern int cmd_update_server_info(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix); +extern int cmd_upload_pack(int argc, const char **argv, const char *prefix); extern int cmd_var(int argc, const char **argv, const char *prefix); extern int cmd_verify_commit(int argc, const char **argv, const char *prefix); extern int cmd_verify_tag(int argc, const char **argv, const char *prefix); diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c new file mode 100644 index 0000000000..2cb5cb35b0 --- /dev/null +++ b/builtin/upload-pack.c @@ -0,0 +1,67 @@ +#include "cache.h" +#include "builtin.h" +#include "exec_cmd.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "protocol.h" +#include "upload-pack.h" + +static const char * const upload_pack_usage[] = { + N_("git upload-pack [] "), + NULL +}; + +int cmd_upload_pack(int argc, const char **argv, const char *prefix) +{ + const char *dir; + int strict = 0; + struct upload_pack_options opts = { 0 }; + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-refs", &opts.advertise_refs, + N_("exit immediately after initial ref advertisement")), + OPT_BOOL(0, "strict", &strict, + N_("do not try /.git/ if is no Git directory")), + OPT_INTEGER(0, "timeout", &opts.timeout, + N_("interrupt transfer after seconds of inactivity")), + OPT_END() + }; + + packet_trace_identity("upload-pack"); + check_replace_refs = 0; + + argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + + if (argc != 1) + usage_with_options(upload_pack_usage, options); + + if (opts.timeout) + opts.daemon_mode = 1; + + setup_path(); + + dir = argv[0]; + + if (!enter_repo(dir, strict)) + die("'%s' does not appear to be a git repository", dir); + + switch (determine_protocol_version_server()) { + case protocol_v1: + /* + * v1 is just the original protocol with a version string, + * so just fall through after writing the version string. + */ + if (opts.advertise_refs || !opts.stateless_rpc) + packet_write_fmt(1, "version 1\n"); + + /* fallthrough */ + case protocol_v0: + upload_pack(&opts); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return 0; +} diff --git a/git.c b/git.c index c870b9719c..f71073dc8d 100644 --- a/git.c +++ b/git.c @@ -478,6 +478,7 @@ static struct cmd_struct commands[] = { { "update-server-info", cmd_update_server_info, RUN_SETUP }, { "upload-archive", cmd_upload_archive }, { "upload-archive--writer", cmd_upload_archive_writer }, + { "upload-pack", cmd_upload_pack }, { "var", cmd_var, RUN_SETUP_GENTLY }, { "verify-commit", cmd_verify_commit, RUN_SETUP }, { "verify-pack", cmd_verify_pack }, diff --git a/upload-pack.c b/upload-pack.c index d5de18127c..2ad73a98b1 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -6,7 +6,6 @@ #include "tag.h" #include "object.h" #include "commit.h" -#include "exec_cmd.h" #include "diff.h" #include "revision.h" #include "list-objects.h" @@ -15,15 +14,10 @@ #include "sigchain.h" #include "version.h" #include "string-list.h" -#include "parse-options.h" #include "argv-array.h" #include "prio-queue.h" #include "protocol.h" - -static const char * const upload_pack_usage[] = { - N_("git upload-pack [] "), - NULL -}; +#include "upload-pack.h" /* Remember to update object flag allocation in object.h */ #define THEY_HAVE (1u << 11) @@ -61,7 +55,6 @@ static int keepalive = 5; * otherwise maximum packet size (up to 65520 bytes). */ static int use_sideband; -static int advertise_refs; static int stateless_rpc; static const char *pack_objects_hook; @@ -977,33 +970,6 @@ static int find_symref(const char *refname, const struct object_id *oid, return 0; } -static void upload_pack(void) -{ - struct string_list symref = STRING_LIST_INIT_DUP; - - head_ref_namespaced(find_symref, &symref); - - if (advertise_refs || !stateless_rpc) { - reset_timeout(); - head_ref_namespaced(send_ref, &symref); - for_each_namespaced_ref(send_ref, &symref); - advertise_shallow_grafts(1); - packet_flush(1); - } else { - head_ref_namespaced(check_ref, NULL); - for_each_namespaced_ref(check_ref, NULL); - } - string_list_clear(&symref, 1); - if (advertise_refs) - return; - - receive_needs(); - if (want_obj.nr) { - get_common_commits(); - create_pack_file(); - } -} - static int upload_pack_config(const char *var, const char *value, void *unused) { if (!strcmp("uploadpack.allowtipsha1inwant", var)) { @@ -1032,58 +998,35 @@ static int upload_pack_config(const char *var, const char *value, void *unused) return parse_hide_refs_config(var, value, "uploadpack"); } -int cmd_main(int argc, const char **argv) +void upload_pack(struct upload_pack_options *options) { - const char *dir; - int strict = 0; - struct option options[] = { - OPT_BOOL(0, "stateless-rpc", &stateless_rpc, - N_("quit after a single request/response exchange")), - OPT_BOOL(0, "advertise-refs", &advertise_refs, - N_("exit immediately after initial ref advertisement")), - OPT_BOOL(0, "strict", &strict, - N_("do not try /.git/ if is no Git directory")), - OPT_INTEGER(0, "timeout", &timeout, - N_("interrupt transfer after seconds of inactivity")), - OPT_END() - }; - - packet_trace_identity("upload-pack"); - check_replace_refs = 0; - - argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); - - if (argc != 1) - usage_with_options(upload_pack_usage, options); - - if (timeout) - daemon_mode = 1; - - setup_path(); - - dir = argv[0]; + struct string_list symref = STRING_LIST_INIT_DUP; - if (!enter_repo(dir, strict)) - die("'%s' does not appear to be a git repository", dir); + stateless_rpc = options->stateless_rpc; + timeout = options->timeout; + daemon_mode = options->daemon_mode; git_config(upload_pack_config, NULL); - switch (determine_protocol_version_server()) { - case protocol_v1: - /* - * v1 is just the original protocol with a version string, - * so just fall through after writing the version string. - */ - if (advertise_refs || !stateless_rpc) - packet_write_fmt(1, "version 1\n"); - - /* fallthrough */ - case protocol_v0: - upload_pack(); - break; - case protocol_unknown_version: - BUG("unknown protocol version"); + head_ref_namespaced(find_symref, &symref); + + if (options->advertise_refs || !stateless_rpc) { + reset_timeout(); + head_ref_namespaced(send_ref, &symref); + for_each_namespaced_ref(send_ref, &symref); + advertise_shallow_grafts(1); + packet_flush(1); + } else { + head_ref_namespaced(check_ref, NULL); + for_each_namespaced_ref(check_ref, NULL); } + string_list_clear(&symref, 1); + if (options->advertise_refs) + return; - return 0; + receive_needs(); + if (want_obj.nr) { + get_common_commits(); + create_pack_file(); + } } diff --git a/upload-pack.h b/upload-pack.h new file mode 100644 index 0000000000..a71e4dc7e2 --- /dev/null +++ b/upload-pack.h @@ -0,0 +1,13 @@ +#ifndef UPLOAD_PACK_H +#define UPLOAD_PACK_H + +struct upload_pack_options { + int stateless_rpc; + int advertise_refs; + unsigned int timeout; + int daemon_mode; +}; + +void upload_pack(struct upload_pack_options *options); + +#endif /* UPLOAD_PACK_H */ -- cgit v1.2.3 From ae2948f30c7de56e2805d26f78ea8205f6c98682 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:42 -0700 Subject: upload-pack: factor out processing lines Factor out the logic for processing shallow, deepen, deepen_since, and deepen_not lines into their own functions to simplify the 'receive_needs()' function in addition to making it easier to reuse some of this logic when implementing protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- upload-pack.c | 113 ++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 39 deletions(-) diff --git a/upload-pack.c b/upload-pack.c index 2ad73a98b1..1e8a9e1caf 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -724,6 +724,75 @@ static void deepen_by_rev_list(int ac, const char **av, packet_flush(1); } +static int process_shallow(const char *line, struct object_array *shallows) +{ + const char *arg; + if (skip_prefix(line, "shallow ", &arg)) { + struct object_id oid; + struct object *object; + if (get_oid_hex(arg, &oid)) + die("invalid shallow line: %s", line); + object = parse_object(&oid); + if (!object) + return 1; + if (object->type != OBJ_COMMIT) + die("invalid shallow object %s", oid_to_hex(&oid)); + if (!(object->flags & CLIENT_SHALLOW)) { + object->flags |= CLIENT_SHALLOW; + add_object_array(object, NULL, shallows); + } + return 1; + } + + return 0; +} + +static int process_deepen(const char *line, int *depth) +{ + const char *arg; + if (skip_prefix(line, "deepen ", &arg)) { + char *end = NULL; + *depth = (int)strtol(arg, &end, 0); + if (!end || *end || *depth <= 0) + die("Invalid deepen: %s", line); + return 1; + } + + return 0; +} + +static int process_deepen_since(const char *line, timestamp_t *deepen_since, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-since ", &arg)) { + char *end = NULL; + *deepen_since = parse_timestamp(arg, &end, 0); + if (!end || *end || !deepen_since || + /* revisions.c's max_age -1 is special */ + *deepen_since == -1) + die("Invalid deepen-since: %s", line); + *deepen_rev_list = 1; + return 1; + } + return 0; +} + +static int process_deepen_not(const char *line, struct string_list *deepen_not, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-not ", &arg)) { + char *ref = NULL; + struct object_id oid; + if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) + die("git upload-pack: ambiguous deepen-not: %s", line); + string_list_append(deepen_not, ref); + free(ref); + *deepen_rev_list = 1; + return 1; + } + return 0; +} + static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; @@ -745,49 +814,15 @@ static void receive_needs(void) if (!line) break; - if (skip_prefix(line, "shallow ", &arg)) { - struct object_id oid; - struct object *object; - if (get_oid_hex(arg, &oid)) - die("invalid shallow line: %s", line); - object = parse_object(&oid); - if (!object) - continue; - if (object->type != OBJ_COMMIT) - die("invalid shallow object %s", oid_to_hex(&oid)); - if (!(object->flags & CLIENT_SHALLOW)) { - object->flags |= CLIENT_SHALLOW; - add_object_array(object, NULL, &shallows); - } + if (process_shallow(line, &shallows)) continue; - } - if (skip_prefix(line, "deepen ", &arg)) { - char *end = NULL; - depth = strtol(arg, &end, 0); - if (!end || *end || depth <= 0) - die("Invalid deepen: %s", line); + if (process_deepen(line, &depth)) continue; - } - if (skip_prefix(line, "deepen-since ", &arg)) { - char *end = NULL; - deepen_since = parse_timestamp(arg, &end, 0); - if (!end || *end || !deepen_since || - /* revisions.c's max_age -1 is special */ - deepen_since == -1) - die("Invalid deepen-since: %s", line); - deepen_rev_list = 1; + if (process_deepen_since(line, &deepen_since, &deepen_rev_list)) continue; - } - if (skip_prefix(line, "deepen-not ", &arg)) { - char *ref = NULL; - struct object_id oid; - if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) - die("git upload-pack: ambiguous deepen-not: %s", line); - string_list_append(&deepen_not, ref); - free(ref); - deepen_rev_list = 1; + if (process_deepen_not(line, &deepen_not, &deepen_rev_list)) continue; - } + if (!skip_prefix(line, "want ", &arg) || get_oid_hex(arg, &oid_buf)) die("git upload-pack: protocol error, " -- cgit v1.2.3 From 635365eb2f67bc17482c5dbe076f20832cf92ae2 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:43 -0700 Subject: transport: use get_refs_via_connect to get refs Remove code duplication and use the existing 'get_refs_via_connect()' function to retrieve a remote's heads in 'fetch_refs_via_pack()' and 'git_transport_push()'. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- transport.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/transport.c b/transport.c index fc802260f6..8e87790962 100644 --- a/transport.c +++ b/transport.c @@ -230,12 +230,8 @@ static int fetch_refs_via_pack(struct transport *transport, args.cloning = transport->cloning; args.update_shallow = data->options.update_shallow; - if (!data->got_remote_heads) { - connect_setup(transport, 0); - get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, - NULL, &data->shallow); - data->got_remote_heads = 1; - } + if (!data->got_remote_heads) + refs_tmp = get_refs_via_connect(transport, 0); refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, @@ -541,14 +537,8 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re struct send_pack_args args; int ret; - if (!data->got_remote_heads) { - struct ref *tmp_refs; - connect_setup(transport, 1); - - get_remote_heads(data->fd[0], NULL, 0, &tmp_refs, REF_NORMAL, - NULL, &data->shallow); - data->got_remote_heads = 1; - } + if (!data->got_remote_heads) + get_refs_via_connect(transport, 1); memset(&args, 0, sizeof(args)); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); -- cgit v1.2.3 From 7e3e479b90fd618fb8eb8222738f7cc53ab288fa Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:44 -0700 Subject: connect: convert get_remote_heads to use struct packet_reader In order to allow for better control flow when protocol_v2 is introduced convert 'get_remote_heads()' to use 'struct packet_reader' to read packet lines. This enables a client to be able to peek the first line of a server's response (without consuming it) in order to determine the protocol version its speaking and then passing control to the appropriate handler. This is needed because the initial response from a server speaking protocol_v0 includes the first ref, while subsequent protocol versions respond with a version line. We want to be able to read this first line without consuming the first ref sent in the protocol_v0 case so that the protocol version the server is speaking can be determined outside of 'get_remote_heads()' in a future patch. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- connect.c | 173 ++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/connect.c b/connect.c index c3a014c5ba..c82c90b7c3 100644 --- a/connect.c +++ b/connect.c @@ -48,6 +48,12 @@ int check_ref_type(const struct ref *ref, int flags) static void die_initial_contact(int unexpected) { + /* + * A hang-up after seeing some response from the other end + * means that it is unexpected, as we know the other end is + * willing to talk to us. A hang-up before seeing any + * response does not necessarily mean an ACL problem, though. + */ if (unexpected) die(_("The remote end hung up upon initial contact")); else @@ -56,6 +62,40 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } +static enum protocol_version discover_version(struct packet_reader *reader) +{ + enum protocol_version version = protocol_unknown_version; + + /* + * Peek the first line of the server's response to + * determine the protocol version the server is speaking. + */ + switch (packet_reader_peek(reader)) { + case PACKET_READ_EOF: + die_initial_contact(0); + case PACKET_READ_FLUSH: + case PACKET_READ_DELIM: + version = protocol_v0; + break; + case PACKET_READ_NORMAL: + version = determine_protocol_version_client(reader->line); + break; + } + + switch (version) { + case protocol_v1: + /* Read the peeked version line */ + packet_reader_read(reader); + break; + case protocol_v0: + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return version; +} + static void parse_one_symref_info(struct string_list *symref, const char *val, int len) { char *sym, *target; @@ -109,60 +149,21 @@ static void annotate_refs_with_symref_info(struct ref *ref) string_list_clear(&symref, 0); } -/* - * Read one line of a server's ref advertisement into packet_buffer. - */ -static int read_remote_ref(int in, char **src_buf, size_t *src_len, - int *responded) +static void process_capabilities(const char *line, int *len) { - int len = packet_read(in, src_buf, src_len, - packet_buffer, sizeof(packet_buffer), - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); - const char *arg; - if (len < 0) - die_initial_contact(*responded); - if (len > 4 && skip_prefix(packet_buffer, "ERR ", &arg)) - die("remote error: %s", arg); - - *responded = 1; - - return len; -} - -#define EXPECTING_PROTOCOL_VERSION 0 -#define EXPECTING_FIRST_REF 1 -#define EXPECTING_REF 2 -#define EXPECTING_SHALLOW 3 - -/* Returns 1 if packet_buffer is a protocol version pkt-line, 0 otherwise. */ -static int process_protocol_version(void) -{ - switch (determine_protocol_version_client(packet_buffer)) { - case protocol_v1: - return 1; - case protocol_v0: - return 0; - default: - die("server is speaking an unknown protocol"); - } -} - -static void process_capabilities(int *len) -{ - int nul_location = strlen(packet_buffer); + int nul_location = strlen(line); if (nul_location == *len) return; - server_capabilities = xstrdup(packet_buffer + nul_location + 1); + server_capabilities = xstrdup(line + nul_location + 1); *len = nul_location; } -static int process_dummy_ref(void) +static int process_dummy_ref(const char *line) { struct object_id oid; const char *name; - if (parse_oid_hex(packet_buffer, &oid, &name)) + if (parse_oid_hex(line, &oid, &name)) return 0; if (*name != ' ') return 0; @@ -171,20 +172,20 @@ static int process_dummy_ref(void) return !oidcmp(&null_oid, &oid) && !strcmp(name, "capabilities^{}"); } -static void check_no_capabilities(int len) +static void check_no_capabilities(const char *line, int len) { - if (strlen(packet_buffer) != len) + if (strlen(line) != len) warning("Ignoring capabilities after first line '%s'", - packet_buffer + strlen(packet_buffer)); + line + strlen(line)); } -static int process_ref(int len, struct ref ***list, unsigned int flags, - struct oid_array *extra_have) +static int process_ref(const char *line, int len, struct ref ***list, + unsigned int flags, struct oid_array *extra_have) { struct object_id old_oid; const char *name; - if (parse_oid_hex(packet_buffer, &old_oid, &name)) + if (parse_oid_hex(line, &old_oid, &name)) return 0; if (*name != ' ') return 0; @@ -200,16 +201,17 @@ static int process_ref(int len, struct ref ***list, unsigned int flags, **list = ref; *list = &ref->next; } - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } -static int process_shallow(int len, struct oid_array *shallow_points) +static int process_shallow(const char *line, int len, + struct oid_array *shallow_points) { const char *arg; struct object_id old_oid; - if (!skip_prefix(packet_buffer, "shallow ", &arg)) + if (!skip_prefix(line, "shallow ", &arg)) return 0; if (get_oid_hex(arg, &old_oid)) @@ -217,10 +219,17 @@ static int process_shallow(int len, struct oid_array *shallow_points) if (!shallow_points) die("repository on the other end cannot be shallow"); oid_array_append(shallow_points, &old_oid); - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } +enum get_remote_heads_state { + EXPECTING_FIRST_REF = 0, + EXPECTING_REF, + EXPECTING_SHALLOW, + EXPECTING_DONE, +}; + /* * Read all the refs from the other end */ @@ -230,47 +239,55 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, struct oid_array *shallow_points) { struct ref **orig_list = list; + int len = 0; + enum get_remote_heads_state state = EXPECTING_FIRST_REF; + struct packet_reader reader; + const char *arg; - /* - * A hang-up after seeing some response from the other end - * means that it is unexpected, as we know the other end is - * willing to talk to us. A hang-up before seeing any - * response does not necessarily mean an ACL problem, though. - */ - int responded = 0; - int len; - int state = EXPECTING_PROTOCOL_VERSION; + packet_reader_init(&reader, in, src_buf, src_len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + discover_version(&reader); *list = NULL; - while ((len = read_remote_ref(in, &src_buf, &src_len, &responded))) { + while (state != EXPECTING_DONE) { + switch (packet_reader_read(&reader)) { + case PACKET_READ_EOF: + die_initial_contact(1); + case PACKET_READ_NORMAL: + len = reader.pktlen; + if (len > 4 && skip_prefix(reader.line, "ERR ", &arg)) + die("remote error: %s", arg); + break; + case PACKET_READ_FLUSH: + state = EXPECTING_DONE; + break; + case PACKET_READ_DELIM: + die("invalid packet"); + } + switch (state) { - case EXPECTING_PROTOCOL_VERSION: - if (process_protocol_version()) { - state = EXPECTING_FIRST_REF; - break; - } - state = EXPECTING_FIRST_REF; - /* fallthrough */ case EXPECTING_FIRST_REF: - process_capabilities(&len); - if (process_dummy_ref()) { + process_capabilities(reader.line, &len); + if (process_dummy_ref(reader.line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(len, &list, flags, extra_have)) + if (process_ref(reader.line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(len, shallow_points)) + if (process_shallow(reader.line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", packet_buffer); - default: - die("unexpected state %d", state); + die("protocol error: unexpected '%s'", reader.line); + case EXPECTING_DONE: + break; } } -- cgit v1.2.3 From ad6ac1244fd175d08bcee62060a9a0b7975930fb Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:45 -0700 Subject: connect: discover protocol version outside of get_remote_heads In order to prepare for the addition of protocol_v2 push the protocol version discovery outside of 'get_remote_heads()'. This will allow for keeping the logic for processing the reference advertisement for protocol_v1 and protocol_v0 separate from the logic for protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 16 +++++++++++++++- builtin/send-pack.c | 17 +++++++++++++++-- connect.c | 27 ++++++++++----------------- connect.h | 3 +++ remote-curl.c | 20 ++++++++++++++++++-- remote.h | 5 +++-- transport.c | 24 +++++++++++++++++++----- 7 files changed, 83 insertions(+), 29 deletions(-) diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 366b9d13f9..85d4faf76c 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -4,6 +4,7 @@ #include "remote.h" #include "connect.h" #include "sha1-array.h" +#include "protocol.h" static const char fetch_pack_usage[] = "git fetch-pack [--all] [--stdin] [--quiet | -q] [--keep | -k] [--thin] " @@ -52,6 +53,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct packet_reader reader; packet_trace_identity("fetch-pack"); @@ -193,7 +195,19 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!conn) return args.diag_url ? 0 : 1; } - get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow); + + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &ref, 0, NULL, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, &shallow, pack_lockfile_ptr); diff --git a/builtin/send-pack.c b/builtin/send-pack.c index fc4f0bb5fb..83cb125a68 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -14,6 +14,7 @@ #include "sha1-array.h" #include "gpg-interface.h" #include "gettext.h" +#include "protocol.h" static const char * const send_pack_usage[] = { N_("git send-pack [--all | --mirror] [--dry-run] [--force] " @@ -154,6 +155,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) int progress = -1; int from_stdin = 0; struct push_cas_option cas = {0}; + struct packet_reader reader; struct option options[] = { OPT__VERBOSITY(&verbose), @@ -256,8 +258,19 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.verbose ? CONNECT_VERBOSE : 0); } - get_remote_heads(fd[0], NULL, 0, &remote_refs, REF_NORMAL, - &extra_have, &shallow); + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &remote_refs, REF_NORMAL, + &extra_have, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } transport_verify_remote_names(nr_refspecs, refspecs); diff --git a/connect.c b/connect.c index c82c90b7c3..0b111e62d7 100644 --- a/connect.c +++ b/connect.c @@ -62,7 +62,7 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } -static enum protocol_version discover_version(struct packet_reader *reader) +enum protocol_version discover_version(struct packet_reader *reader) { enum protocol_version version = protocol_unknown_version; @@ -233,7 +233,7 @@ enum get_remote_heads_state { /* * Read all the refs from the other end */ -struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points) @@ -241,24 +241,17 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, struct ref **orig_list = list; int len = 0; enum get_remote_heads_state state = EXPECTING_FIRST_REF; - struct packet_reader reader; const char *arg; - packet_reader_init(&reader, in, src_buf, src_len, - PACKET_READ_CHOMP_NEWLINE | - PACKET_READ_GENTLE_ON_EOF); - - discover_version(&reader); - *list = NULL; while (state != EXPECTING_DONE) { - switch (packet_reader_read(&reader)) { + switch (packet_reader_read(reader)) { case PACKET_READ_EOF: die_initial_contact(1); case PACKET_READ_NORMAL: - len = reader.pktlen; - if (len > 4 && skip_prefix(reader.line, "ERR ", &arg)) + len = reader->pktlen; + if (len > 4 && skip_prefix(reader->line, "ERR ", &arg)) die("remote error: %s", arg); break; case PACKET_READ_FLUSH: @@ -270,22 +263,22 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, switch (state) { case EXPECTING_FIRST_REF: - process_capabilities(reader.line, &len); - if (process_dummy_ref(reader.line)) { + process_capabilities(reader->line, &len); + if (process_dummy_ref(reader->line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(reader.line, len, &list, flags, extra_have)) + if (process_ref(reader->line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(reader.line, len, shallow_points)) + if (process_shallow(reader->line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", reader.line); + die("protocol error: unexpected '%s'", reader->line); case EXPECTING_DONE: break; } diff --git a/connect.h b/connect.h index 01f14cdf3f..cdb8979dce 100644 --- a/connect.h +++ b/connect.h @@ -13,4 +13,7 @@ extern int parse_feature_request(const char *features, const char *feature); extern const char *server_feature_value(const char *feature, int *len_ret); extern int url_is_local_not_ssh(const char *url); +struct packet_reader; +extern enum protocol_version discover_version(struct packet_reader *reader); + #endif diff --git a/remote-curl.c b/remote-curl.c index 0053b09549..9f6d07683d 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,6 +1,7 @@ #include "cache.h" #include "config.h" #include "remote.h" +#include "connect.h" #include "strbuf.h" #include "walker.h" #include "http.h" @@ -13,6 +14,7 @@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" +#include "protocol.h" static struct remote *remote; /* always ends with a trailing slash */ @@ -176,8 +178,22 @@ static struct discovery *last_discovery; static struct ref *parse_git_refs(struct discovery *heads, int for_push) { struct ref *list = NULL; - get_remote_heads(-1, heads->buf, heads->len, &list, - for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + struct packet_reader reader; + + packet_reader_init(&reader, -1, heads->buf, heads->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, + NULL, &heads->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + return list; } diff --git a/remote.h b/remote.h index 1f6611be21..2016461df9 100644 --- a/remote.h +++ b/remote.h @@ -150,10 +150,11 @@ int check_ref_type(const struct ref *ref, int flags); void free_refs(struct ref *ref); struct oid_array; -extern struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct packet_reader; +extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, - struct oid_array *shallow); + struct oid_array *shallow_points); int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); diff --git a/transport.c b/transport.c index 8e87790962..63c3dbab94 100644 --- a/transport.c +++ b/transport.c @@ -18,6 +18,7 @@ #include "sha1-array.h" #include "sigchain.h" #include "transport-internal.h" +#include "protocol.h" static void set_upstreams(struct transport *transport, struct ref *refs, int pretend) @@ -190,13 +191,26 @@ static int connect_setup(struct transport *transport, int for_push) static struct ref *get_refs_via_connect(struct transport *transport, int for_push) { struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; + struct packet_reader reader; connect_setup(transport, for_push); - get_remote_heads(data->fd[0], NULL, 0, &refs, - for_push ? REF_NORMAL : 0, - &data->extra_have, - &data->shallow); + + packet_reader_init(&reader, data->fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &refs, + for_push ? REF_NORMAL : 0, + &data->extra_have, + &data->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } data->got_remote_heads = 1; return refs; -- cgit v1.2.3 From 432e95651031025e421ae2f521f7c0d9f60c527c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:46 -0700 Subject: transport: store protocol version Once protocol_v2 is introduced requesting a fetch or a push will need to be handled differently depending on the protocol version. Store the protocol version the server is speaking in 'struct git_transport_data' and use it to determine what to do in the case of a fetch or a push. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- transport.c | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/transport.c b/transport.c index 63c3dbab94..2378dcb38c 100644 --- a/transport.c +++ b/transport.c @@ -118,6 +118,7 @@ struct git_transport_data { struct child_process *conn; int fd[2]; unsigned got_remote_heads : 1; + enum protocol_version version; struct oid_array extra_have; struct oid_array shallow; }; @@ -200,7 +201,8 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus PACKET_READ_CHOMP_NEWLINE | PACKET_READ_GENTLE_ON_EOF); - switch (discover_version(&reader)) { + data->version = discover_version(&reader); + switch (data->version) { case protocol_v1: case protocol_v0: get_remote_heads(&reader, &refs, @@ -221,7 +223,7 @@ static int fetch_refs_via_pack(struct transport *transport, { int ret = 0; struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; char *dest = xstrdup(transport->url); struct fetch_pack_args args; struct ref *refs_tmp = NULL; @@ -247,10 +249,18 @@ static int fetch_refs_via_pack(struct transport *transport, if (!data->got_remote_heads) refs_tmp = get_refs_via_connect(transport, 0); - refs = fetch_pack(&args, data->fd, data->conn, - refs_tmp ? refs_tmp : transport->remote_refs, - dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); + switch (data->version) { + case protocol_v1: + case protocol_v0: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + close(data->fd[0]); close(data->fd[1]); if (finish_connect(data->conn)) @@ -549,7 +559,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re { struct git_transport_data *data = transport->data; struct send_pack_args args; - int ret; + int ret = 0; if (!data->got_remote_heads) get_refs_via_connect(transport, 1); @@ -574,8 +584,15 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re else args.push_cert = SEND_PACK_PUSH_CERT_NEVER; - ret = send_pack(&args, data->fd, data->conn, remote_refs, - &data->extra_have); + switch (data->version) { + case protocol_v1: + case protocol_v0: + ret = send_pack(&args, data->fd, data->conn, remote_refs, + &data->extra_have); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } close(data->fd[1]); close(data->fd[0]); -- cgit v1.2.3 From 8f6982b4e16c60bba713a3b6592b2ff5c7476974 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:47 -0700 Subject: protocol: introduce enum protocol_version value protocol_v2 Introduce protocol_v2, a new value for 'enum protocol_version'. Subsequent patches will fill in the implementation of protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 ++ builtin/receive-pack.c | 6 ++++++ builtin/send-pack.c | 3 +++ builtin/upload-pack.c | 7 +++++++ connect.c | 3 +++ protocol.c | 2 ++ protocol.h | 1 + remote-curl.c | 3 +++ transport.c | 9 +++++++++ 9 files changed, 36 insertions(+) diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 85d4faf76c..b2374ddbbf 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -201,6 +201,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); case protocol_v1: case protocol_v0: get_remote_heads(&reader, &ref, 0, NULL, &shallow); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index b7ce7c7f52..3656e94fdb 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1963,6 +1963,12 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) unpack_limit = receive_unpack_limit; switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * push support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 83cb125a68..b5427f75e3 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -263,6 +263,9 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &remote_refs, REF_NORMAL, diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 2cb5cb35b0..8d53e9794b 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -47,6 +47,13 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) die("'%s' does not appear to be a git repository", dir); switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * fetch support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + upload_pack(&opts); + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/connect.c b/connect.c index 0b111e62d7..4b89b984c4 100644 --- a/connect.c +++ b/connect.c @@ -83,6 +83,9 @@ enum protocol_version discover_version(struct packet_reader *reader) } switch (version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: /* Read the peeked version line */ packet_reader_read(reader); diff --git a/protocol.c b/protocol.c index 43012b7eb6..5e636785d1 100644 --- a/protocol.c +++ b/protocol.c @@ -8,6 +8,8 @@ static enum protocol_version parse_protocol_version(const char *value) return protocol_v0; else if (!strcmp(value, "1")) return protocol_v1; + else if (!strcmp(value, "2")) + return protocol_v2; else return protocol_unknown_version; } diff --git a/protocol.h b/protocol.h index 1b2bc94a8d..2ad35e433c 100644 --- a/protocol.h +++ b/protocol.h @@ -5,6 +5,7 @@ enum protocol_version { protocol_unknown_version = -1, protocol_v0 = 0, protocol_v1 = 1, + protocol_v2 = 2, }; /* diff --git a/remote-curl.c b/remote-curl.c index 9f6d07683d..dae8a4a48d 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -185,6 +185,9 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, diff --git a/transport.c b/transport.c index 2378dcb38c..83d9dd1df6 100644 --- a/transport.c +++ b/transport.c @@ -203,6 +203,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &refs, @@ -250,6 +253,9 @@ static int fetch_refs_via_pack(struct transport *transport, refs_tmp = get_refs_via_connect(transport, 0); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, @@ -585,6 +591,9 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re args.push_cert = SEND_PACK_PUSH_CERT_NEVER; switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: ret = send_pack(&args, data->fd, data->conn, remote_refs, -- cgit v1.2.3 From 74e70029615ead3e7203fed715abf0dc31bcb211 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:48 -0700 Subject: test-pkt-line: introduce a packet-line test helper Introduce a packet-line test helper which can either pack or unpack an input stream into packet-lines and writes out the result to stdout. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- Makefile | 1 + t/helper/test-pkt-line.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 t/helper/test-pkt-line.c diff --git a/Makefile b/Makefile index 91a59a6d50..c26fb93b3f 100644 --- a/Makefile +++ b/Makefile @@ -666,6 +666,7 @@ TEST_PROGRAMS_NEED_X += test-mktemp TEST_PROGRAMS_NEED_X += test-online-cpus TEST_PROGRAMS_NEED_X += test-parse-options TEST_PROGRAMS_NEED_X += test-path-utils +TEST_PROGRAMS_NEED_X += test-pkt-line TEST_PROGRAMS_NEED_X += test-prio-queue TEST_PROGRAMS_NEED_X += test-read-cache TEST_PROGRAMS_NEED_X += test-write-cache diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c new file mode 100644 index 0000000000..0f19e53c75 --- /dev/null +++ b/t/helper/test-pkt-line.c @@ -0,0 +1,64 @@ +#include "pkt-line.h" + +static void pack_line(const char *line) +{ + if (!strcmp(line, "0000") || !strcmp(line, "0000\n")) + packet_flush(1); + else if (!strcmp(line, "0001") || !strcmp(line, "0001\n")) + packet_delim(1); + else + packet_write_fmt(1, "%s", line); +} + +static void pack(int argc, const char **argv) +{ + if (argc) { /* read from argv */ + int i; + for (i = 0; i < argc; i++) + pack_line(argv[i]); + } else { /* read from stdin */ + char line[LARGE_PACKET_MAX]; + while (fgets(line, sizeof(line), stdin)) { + pack_line(line); + } + } +} + +static void unpack(void) +{ + struct packet_reader reader; + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_CHOMP_NEWLINE); + + while (packet_reader_read(&reader) != PACKET_READ_EOF) { + switch (reader.status) { + case PACKET_READ_EOF: + break; + case PACKET_READ_NORMAL: + printf("%s\n", reader.line); + break; + case PACKET_READ_FLUSH: + printf("0000\n"); + break; + case PACKET_READ_DELIM: + printf("0001\n"); + break; + } + } +} + +int cmd_main(int argc, const char **argv) +{ + if (argc < 2) + die("too few arguments"); + + if (!strcmp(argv[1], "pack")) + pack(argc - 2, argv + 2); + else if (!strcmp(argv[1], "unpack")) + unpack(); + else + die("invalid argument '%s'", argv[1]); + + return 0; +} -- cgit v1.2.3 From ed10cb952d315ae47d2f7f6f29333a9ee227bf6b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:19 -0700 Subject: serve: introduce git-serve Introduce git-serve, the base server for protocol version 2. Protocol version 2 is intended to be a replacement for Git's current wire protocol. The intention is that it will be a simpler, less wasteful protocol which can evolve over time. Protocol version 2 improves upon version 1 by eliminating the initial ref advertisement. In its place a server will export a list of capabilities and commands which it supports in a capability advertisement. A client can then request that a particular command be executed by providing a number of capabilities and command specific parameters. At the completion of a command, a client can request that another command be executed or can terminate the connection by sending a flush packet. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- .gitignore | 1 + Documentation/Makefile | 1 + Documentation/technical/protocol-v2.txt | 170 ++++++++++++++++++++++ Makefile | 2 + builtin.h | 1 + builtin/serve.c | 30 ++++ git.c | 1 + serve.c | 247 ++++++++++++++++++++++++++++++++ serve.h | 15 ++ t/t5701-git-serve.sh | 60 ++++++++ 10 files changed, 528 insertions(+) create mode 100644 Documentation/technical/protocol-v2.txt create mode 100644 builtin/serve.c create mode 100644 serve.c create mode 100644 serve.h create mode 100755 t/t5701-git-serve.sh diff --git a/.gitignore b/.gitignore index 833ef3b0b7..2d0450c262 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ /git-rm /git-send-email /git-send-pack +/git-serve /git-sh-i18n /git-sh-i18n--envsubst /git-sh-setup diff --git a/Documentation/Makefile b/Documentation/Makefile index 4ae9ba5c86..b105775acd 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -77,6 +77,7 @@ TECH_DOCS += technical/pack-heuristics TECH_DOCS += technical/pack-protocol TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common +TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt new file mode 100644 index 0000000000..270b28f364 --- /dev/null +++ b/Documentation/technical/protocol-v2.txt @@ -0,0 +1,170 @@ + Git Wire Protocol, Version 2 +============================== + +This document presents a specification for a version 2 of Git's wire +protocol. Protocol v2 will improve upon v1 in the following ways: + + * Instead of multiple service names, multiple commands will be + supported by a single service + * Easily extendable as capabilities are moved into their own section + of the protocol, no longer being hidden behind a NUL byte and + limited by the size of a pkt-line + * Separate out other information hidden behind NUL bytes (e.g. agent + string as a capability and symrefs can be requested using 'ls-refs') + * Reference advertisement will be omitted unless explicitly requested + * ls-refs command to explicitly request some refs + * Designed with http and stateless-rpc in mind. With clear flush + semantics the http remote helper can simply act as a proxy + +In protocol v2 communication is command oriented. When first contacting a +server a list of capabilities will advertised. Some of these capabilities +will be commands which a client can request be executed. Once a command +has completed, a client can reuse the connection and request that other +commands be executed. + + Packet-Line Framing +--------------------- + +All communication is done using packet-line framing, just as in v1. See +`Documentation/technical/pack-protocol.txt` and +`Documentation/technical/protocol-common.txt` for more information. + +In protocol v2 these special packets will have the following semantics: + + * '0000' Flush Packet (flush-pkt) - indicates the end of a message + * '0001' Delimiter Packet (delim-pkt) - separates sections of a message + + Initial Client Request +------------------------ + +In general a client can request to speak protocol v2 by sending +`version=2` through the respective side-channel for the transport being +used which inevitably sets `GIT_PROTOCOL`. More information can be +found in `pack-protocol.txt` and `http-protocol.txt`. In all cases the +response from the server is the capability advertisement. + + Git Transport +~~~~~~~~~~~~~~~ + +When using the git:// transport, you can request to use protocol v2 by +sending "version=2" as an extra parameter: + + 003egit-upload-pack /project.git\0host=myserver.com\0\0version=2\0 + + SSH and File Transport +~~~~~~~~~~~~~~~~~~~~~~~~ + +When using either the ssh:// or file:// transport, the GIT_PROTOCOL +environment variable must be set explicitly to include "version=2". + + HTTP Transport +~~~~~~~~~~~~~~~~ + +When using the http:// or https:// transport a client makes a "smart" +info/refs request as described in `http-protocol.txt` and requests that +v2 be used by supplying "version=2" in the `Git-Protocol` header. + + C: Git-Protocol: version=2 + C: + C: GET $GIT_URL/info/refs?service=git-upload-pack HTTP/1.0 + +A v2 server would reply: + + S: 200 OK + S: + S: ... + S: + S: 000eversion 2\n + S: + +Subsequent requests are then made directly to the service +`$GIT_URL/git-upload-pack`. (This works the same for git-receive-pack). + + Capability Advertisement +-------------------------- + +A server which decides to communicate (based on a request from a client) +using protocol version 2, notifies the client by sending a version string +in its initial response followed by an advertisement of its capabilities. +Each capability is a key with an optional value. Clients must ignore all +unknown keys. Semantics of unknown values are left to the definition of +each key. Some capabilities will describe commands which can be requested +to be executed by the client. + + capability-advertisement = protocol-version + capability-list + flush-pkt + + protocol-version = PKT-LINE("version 2" LF) + capability-list = *capability + capability = PKT-LINE(key[=value] LF) + + key = 1*(ALPHA | DIGIT | "-_") + value = 1*(ALPHA | DIGIT | " -_.,?\/{}[]()<>!@#$%^&*+=:;") + + Command Request +----------------- + +After receiving the capability advertisement, a client can then issue a +request to select the command it wants with any particular capabilities +or arguments. There is then an optional section where the client can +provide any command specific parameters or queries. Only a single +command can be requested at a time. + + request = empty-request | command-request + empty-request = flush-pkt + command-request = command + capability-list + [command-args] + flush-pkt + command = PKT-LINE("command=" key LF) + command-args = delim-pkt + *command-specific-arg + + command-specific-args are packet lin