summaryrefslogtreecommitdiff
path: root/fetch-pack.c
diff options
context:
space:
mode:
Diffstat (limited to 'fetch-pack.c')
-rw-r--r--fetch-pack.c493
1 files changed, 457 insertions, 36 deletions
diff --git a/fetch-pack.c b/fetch-pack.c
index 9f6b07ad91..a320ce9872 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -1,11 +1,12 @@
#include "cache.h"
+#include "repository.h"
#include "config.h"
#include "lockfile.h"
#include "refs.h"
#include "pkt-line.h"
#include "commit.h"
#include "tag.h"
-#include "exec_cmd.h"
+#include "exec-cmd.h"
#include "pack.h"
#include "sideband.h"
#include "fetch-pack.h"
@@ -29,6 +30,7 @@ static int deepen_not_ok;
static int fetch_fsck_objects = -1;
static int transfer_fsck_objects = -1;
static int agent_supported;
+static int server_supports_filtering;
static struct lock_file shallow_lock;
static const char *alternate_shallow_file;
@@ -260,8 +262,8 @@ static enum ack_type get_ack(int fd, struct object_id *result_oid)
char *line = packet_read_line(fd, &len);
const char *arg;
- if (!len)
- die(_("git fetch-pack: expected ACK/NAK, got EOF"));
+ if (!line)
+ die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
if (!strcmp(line, "NAK"))
return NAK;
if (skip_prefix(line, "ACK ", &arg)) {
@@ -303,9 +305,9 @@ static void insert_one_alternate_object(struct object *obj)
#define PIPESAFE_FLUSH 32
#define LARGE_FLUSH 16384
-static int next_flush(struct fetch_pack_args *args, int count)
+static int next_flush(int stateless_rpc, int count)
{
- if (args->stateless_rpc) {
+ if (stateless_rpc) {
if (count < LARGE_FLUSH)
count <<= 1;
else
@@ -379,6 +381,8 @@ static int find_common(struct fetch_pack_args *args,
if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
if (agent_supported) strbuf_addf(&c, " agent=%s",
git_user_agent_sanitized());
+ if (args->filter_options.choice)
+ strbuf_addstr(&c, " filter");
packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
strbuf_release(&c);
} else
@@ -407,6 +411,9 @@ static int find_common(struct fetch_pack_args *args,
packet_buf_write(&req_buf, "deepen-not %s", s->string);
}
}
+ if (server_supports_filtering && args->filter_options.choice)
+ packet_buf_write(&req_buf, "filter %s",
+ args->filter_options.filter_spec);
packet_buf_flush(&req_buf);
state_len = req_buf.len;
@@ -450,6 +457,8 @@ static int find_common(struct fetch_pack_args *args,
flushes = 0;
retval = -1;
+ if (args->no_dependents)
+ goto done;
while ((oid = get_rev())) {
packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
print_verbose(args, "have %s", oid_to_hex(oid));
@@ -461,7 +470,7 @@ static int find_common(struct fetch_pack_args *args,
send_request(args, fd[1], &req_buf);
strbuf_setlen(&req_buf, state_len);
flushes++;
- flush_at = next_flush(args, count);
+ flush_at = next_flush(args->stateless_rpc, count);
/*
* We keep one window "ahead" of the other side, and
@@ -703,23 +712,61 @@ static void mark_alternate_complete(struct object *obj)
mark_complete(&obj->oid);
}
+struct loose_object_iter {
+ struct oidset *loose_object_set;
+ struct ref *refs;
+};
+
+/*
+ * If the number of refs is not larger than the number of loose objects,
+ * this function stops inserting.
+ */
+static int add_loose_objects_to_set(const struct object_id *oid,
+ const char *path,
+ void *data)
+{
+ struct loose_object_iter *iter = data;
+ oidset_insert(iter->loose_object_set, oid);
+ if (iter->refs == NULL)
+ return 1;
+
+ iter->refs = iter->refs->next;
+ return 0;
+}
+
static int everything_local(struct fetch_pack_args *args,
struct ref **refs,
struct ref **sought, int nr_sought)
{
struct ref *ref;
int retval;
+ int old_save_commit_buffer = save_commit_buffer;
timestamp_t cutoff = 0;
+ struct oidset loose_oid_set = OIDSET_INIT;
+ int use_oidset = 0;
+ struct loose_object_iter iter = {&loose_oid_set, *refs};
+
+ /* Enumerate all loose objects or know refs are not so many. */
+ use_oidset = !for_each_loose_object(add_loose_objects_to_set,
+ &iter, 0);
save_commit_buffer = 0;
for (ref = *refs; ref; ref = ref->next) {
struct object *o;
+ unsigned int flags = OBJECT_INFO_QUICK;
- if (!has_object_file_with_flags(&ref->old_oid,
- OBJECT_INFO_QUICK))
- continue;
+ if (use_oidset &&
+ !oidset_contains(&loose_oid_set, &ref->old_oid)) {
+ /*
+ * I know this does not exist in the loose form,
+ * so check if it exists in a non-loose form.
+ */
+ flags |= OBJECT_INFO_IGNORE_LOOSE;
+ }
+ if (!has_object_file_with_flags(&ref->old_oid, flags))
+ continue;
o = parse_object(&ref->old_oid);
if (!o)
continue;
@@ -735,29 +782,33 @@ static int everything_local(struct fetch_pack_args *args,
}
}
- if (!args->deepen) {
- for_each_ref(mark_complete_oid, NULL);
- for_each_cached_alternate(mark_alternate_complete);
- commit_list_sort_by_date(&complete);
- if (cutoff)
- mark_recent_complete_commits(args, cutoff);
- }
+ oidset_clear(&loose_oid_set);
- /*
- * Mark all complete remote refs as common refs.
- * Don't mark them common yet; the server has to be told so first.
- */
- for (ref = *refs; ref; ref = ref->next) {
- struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
- NULL, 0);
+ if (!args->no_dependents) {
+ if (!args->deepen) {
+ for_each_ref(mark_complete_oid, NULL);
+ for_each_cached_alternate(mark_alternate_complete);
+ commit_list_sort_by_date(&complete);
+ if (cutoff)
+ mark_recent_complete_commits(args, cutoff);
+ }
- if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
- continue;
+ /*
+ * Mark all complete remote refs as common refs.
+ * Don't mark them common yet; the server has to be told so first.
+ */
+ for (ref = *refs; ref; ref = ref->next) {
+ struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
+ NULL, 0);
- if (!(o->flags & SEEN)) {
- rev_list_push((struct commit *)o, COMMON_REF | SEEN);
+ if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
+ continue;
+
+ if (!(o->flags & SEEN)) {
+ rev_list_push((struct commit *)o, COMMON_REF | SEEN);
- mark_common((struct commit *)o, 1, 1);
+ mark_common((struct commit *)o, 1, 1);
+ }
}
}
@@ -777,6 +828,9 @@ static int everything_local(struct fetch_pack_args *args,
print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
ref->name);
}
+
+ save_commit_buffer = old_save_commit_buffer;
+
return retval;
}
@@ -833,7 +887,7 @@ static int get_pack(struct fetch_pack_args *args,
argv_array_push(&cmd.args, alternate_shallow_file);
}
- if (do_keep) {
+ if (do_keep || args->from_promisor) {
if (pack_lockfile)
cmd.out = -1;
cmd_name = "index-pack";
@@ -843,7 +897,7 @@ static int get_pack(struct fetch_pack_args *args,
argv_array_push(&cmd.args, "-v");
if (args->use_thin_pack)
argv_array_push(&cmd.args, "--fix-thin");
- if (args->lock_pack || unpack_limit) {
+ if (do_keep && (args->lock_pack || unpack_limit)) {
char hostname[HOST_NAME_MAX + 1];
if (xgethostname(hostname, sizeof(hostname)))
xsnprintf(hostname, sizeof(hostname), "localhost");
@@ -853,6 +907,8 @@ static int get_pack(struct fetch_pack_args *args,
}
if (args->check_self_contained_and_connected)
argv_array_push(&cmd.args, "--check-self-contained-and-connected");
+ if (args->from_promisor)
+ argv_array_push(&cmd.args, "--promisor");
}
else {
cmd_name = "unpack-objects";
@@ -870,8 +926,17 @@ static int get_pack(struct fetch_pack_args *args,
? fetch_fsck_objects
: transfer_fsck_objects >= 0
? transfer_fsck_objects
- : 0)
- argv_array_push(&cmd.args, "--strict");
+ : 0) {
+ if (args->from_promisor)
+ /*
+ * We cannot use --strict in index-pack because it
+ * checks both broken objects and links, but we only
+ * want to check for broken objects.
+ */
+ argv_array_push(&cmd.args, "--fsck-objects");
+ else
+ argv_array_push(&cmd.args, "--strict");
+ }
cmd.in = demux.out;
cmd.git_cmd = 1;
@@ -964,6 +1029,13 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
else
prefer_ofs_delta = 0;
+ if (server_supports("filter")) {
+ server_supports_filtering = 1;
+ print_verbose(args, _("Server supports filter"));
+ } else if (args->filter_options.choice) {
+ warning("filtering not recognized by server, ignoring");
+ }
+
if ((agent_feature = server_feature_value("agent", &agent_len))) {
agent_supported = 1;
if (agent_len)
@@ -1008,6 +1080,350 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
return ref;
}
+static void add_shallow_requests(struct strbuf *req_buf,
+ const struct fetch_pack_args *args)
+{
+ if (is_repository_shallow())
+ write_shallow_commits(req_buf, 1, NULL);
+ if (args->depth > 0)
+ packet_buf_write(req_buf, "deepen %d", args->depth);
+ if (args->deepen_since) {
+ timestamp_t max_age = approxidate(args->deepen_since);
+ packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
+ }
+ if (args->deepen_not) {
+ int i;
+ for (i = 0; i < args->deepen_not->nr; i++) {
+ struct string_list_item *s = args->deepen_not->items + i;
+ packet_buf_write(req_buf, "deepen-not %s", s->string);
+ }
+ }
+}
+
+static void add_wants(const struct ref *wants, struct strbuf *req_buf)
+{
+ for ( ; wants ; wants = wants->next) {
+ const struct object_id *remote = &wants->old_oid;
+ const char *remote_hex;
+ struct object *o;
+
+ /*
+ * If that object is complete (i.e. it is an ancestor of a
+ * local ref), we tell them we have it but do not have to
+ * tell them about its ancestors, which they already know
+ * about.
+ *
+ * We use lookup_object here because we are only
+ * interested in the case we *know* the object is
+ * reachable and we have already scanned it.
+ */
+ if (((o = lookup_object(remote->hash)) != NULL) &&
+ (o->flags & COMPLETE)) {
+ continue;
+ }
+
+ remote_hex = oid_to_hex(remote);
+ packet_buf_write(req_buf, "want %s\n", remote_hex);
+ }
+}
+
+static void add_common(struct strbuf *req_buf, struct oidset *common)
+{
+ struct oidset_iter iter;
+ const struct object_id *oid;
+ oidset_iter_init(common, &iter);
+
+ while ((oid = oidset_iter_next(&iter))) {
+ packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
+ }
+}
+
+static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
+{
+ int ret = 0;
+ int haves_added = 0;
+ const struct object_id *oid;
+
+ while ((oid = get_rev())) {
+ packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
+ if (++haves_added >= *haves_to_send)
+ break;
+ }
+
+ *in_vain += haves_added;
+ if (!haves_added || *in_vain >= MAX_IN_VAIN) {
+ /* Send Done */
+ packet_buf_write(req_buf, "done\n");
+ ret = 1;
+ }
+
+ /* Increase haves to send on next round */
+ *haves_to_send = next_flush(1, *haves_to_send);
+
+ return ret;
+}
+
+static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
+ const struct ref *wants, struct oidset *common,
+ int *haves_to_send, int *in_vain)
+{
+ int ret = 0;
+ struct strbuf req_buf = STRBUF_INIT;
+
+ if (server_supports_v2("fetch", 1))
+ packet_buf_write(&req_buf, "command=fetch");
+ if (server_supports_v2("agent", 0))
+ packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
+ if (args->server_options && args->server_options->nr &&
+ server_supports_v2("server-option", 1)) {
+ int i;
+ for (i = 0; i < args->server_options->nr; i++)
+ packet_write_fmt(fd_out, "server-option=%s",
+ args->server_options->items[i].string);
+ }
+
+ packet_buf_delim(&req_buf);
+ if (args->use_thin_pack)
+ packet_buf_write(&req_buf, "thin-pack");
+ if (args->no_progress)
+ packet_buf_write(&req_buf, "no-progress");
+ if (args->include_tag)
+ packet_buf_write(&req_buf, "include-tag");
+ if (prefer_ofs_delta)
+ packet_buf_write(&req_buf, "ofs-delta");
+
+ /* Add shallow-info and deepen request */
+ if (server_supports_feature("fetch", "shallow", 0))
+ add_shallow_requests(&req_buf, args);
+ else if (is_repository_shallow() || args->deepen)
+ die(_("Server does not support shallow requests"));
+
+ /* Add filter */
+ if (server_supports_feature("fetch", "filter", 0) &&
+ args->filter_options.choice) {
+ print_verbose(args, _("Server supports filter"));
+ packet_buf_write(&req_buf, "filter %s",
+ args->filter_options.filter_spec);
+ } else if (args->filter_options.choice) {
+ warning("filtering not recognized by server, ignoring");
+ }
+
+ /* add wants */
+ add_wants(wants, &req_buf);
+
+ if (args->no_dependents) {
+ packet_buf_write(&req_buf, "done");
+ ret = 1;
+ } else {
+ /* Add all of the common commits we've found in previous rounds */
+ add_common(&req_buf, common);
+
+ /* Add initial haves */
+ ret = add_haves(&req_buf, haves_to_send, in_vain);
+ }
+
+ /* Send request */
+ packet_buf_flush(&req_buf);
+ write_or_die(fd_out, req_buf.buf, req_buf.len);
+
+ strbuf_release(&req_buf);
+ return ret;
+}
+
+/*
+ * Processes a section header in a server's response and checks if it matches
+ * `section`. If the value of `peek` is 1, the header line will be peeked (and
+ * not consumed); if 0, the line will be consumed and the function will die if
+ * the section header doesn't match what was expected.
+ */
+static int process_section_header(struct packet_reader *reader,
+ const char *section, int peek)
+{
+ int ret;
+
+ if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
+ die("error reading section header '%s'", section);
+
+ ret = !strcmp(reader->line, section);
+
+ if (!peek) {
+ if (!ret)
+ die("expected '%s', received '%s'",
+ section, reader->line);
+ packet_reader_read(reader);
+ }
+
+ return ret;
+}
+
+static int process_acks(struct packet_reader *reader, struct oidset *common)
+{
+ /* received */
+ int received_ready = 0;
+ int received_ack = 0;
+
+ process_section_header(reader, "acknowledgments", 0);
+ while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
+ const char *arg;
+
+ if (!strcmp(reader->line, "NAK"))
+ continue;
+
+ if (skip_prefix(reader->line, "ACK ", &arg)) {
+ struct object_id oid;
+ if (!get_oid_hex(arg, &oid)) {
+ struct commit *commit;
+ oidset_insert(common, &oid);
+ commit = lookup_commit(&oid);
+ mark_common(commit, 0, 1);
+ }
+ continue;
+ }
+
+ if (!strcmp(reader->line, "ready")) {
+ clear_prio_queue(&rev_list);
+ received_ready = 1;
+ continue;
+ }
+
+ die("unexpected acknowledgment line: '%s'", reader->line);
+ }
+
+ if (reader->status != PACKET_READ_FLUSH &&
+ reader->status != PACKET_READ_DELIM)
+ die("error processing acks: %d", reader->status);
+
+ /* return 0 if no common, 1 if there are common, or 2 if ready */
+ return received_ready ? 2 : (received_ack ? 1 : 0);
+}
+
+static void receive_shallow_info(struct fetch_pack_args *args,
+ struct packet_reader *reader)
+{
+ process_section_header(reader, "shallow-info", 0);
+ while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
+ const char *arg;
+ struct object_id oid;
+
+ if (skip_prefix(reader->line, "shallow ", &arg)) {
+ if (get_oid_hex(arg, &oid))
+ die(_("invalid shallow line: %s"), reader->line);
+ register_shallow(&oid);
+ continue;
+ }
+ if (skip_prefix(reader->line, "unshallow ", &arg)) {
+ if (get_oid_hex(arg, &oid))
+ die(_("invalid unshallow line: %s"), reader->line);
+ if (!lookup_object(oid.hash))
+ die(_("object not found: %s"), reader->line);
+ /* make sure that it is parsed as shallow */
+ if (!parse_object(&oid))
+ die(_("error in object: %s"), reader->line);
+ if (unregister_shallow(&oid))
+ die(_("no shallow found: %s"), reader->line);
+ continue;
+ }
+ die(_("expected shallow/unshallow, got %s"), reader->line);
+ }
+
+ if (reader->status != PACKET_READ_FLUSH &&
+ reader->status != PACKET_READ_DELIM)
+ die("error processing shallow info: %d", reader->status);
+
+ setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
+ args->deepen = 1;
+}
+
+enum fetch_state {
+ FETCH_CHECK_LOCAL = 0,
+ FETCH_SEND_REQUEST,
+ FETCH_PROCESS_ACKS,
+ FETCH_GET_PACK,
+ FETCH_DONE,
+};
+
+static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
+ int fd[2],
+ const struct ref *orig_ref,
+ struct ref **sought, int nr_sought,
+ char **pack_lockfile)
+{
+ struct ref *ref = copy_ref_list(orig_ref);
+ enum fetch_state state = FETCH_CHECK_LOCAL;
+ struct oidset common = OIDSET_INIT;
+ struct packet_reader reader;
+ int in_vain = 0;
+ int haves_to_send = INITIAL_FLUSH;
+ packet_reader_init(&reader, fd[0], NULL, 0,
+ PACKET_READ_CHOMP_NEWLINE);
+
+ while (state != FETCH_DONE) {
+ switch (state) {
+ case FETCH_CHECK_LOCAL:
+ sort_ref_list(&ref, ref_compare_name);
+ QSORT(sought, nr_sought, cmp_ref_by_name);
+
+ /* v2 supports these by default */
+ allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
+ use_sideband = 2;
+ if (args->depth > 0 || args->deepen_since || args->deepen_not)
+ args->deepen = 1;
+
+ if (marked)
+ for_each_ref(clear_marks, NULL);
+ marked = 1;
+
+ for_each_ref(rev_list_insert_ref_oid, NULL);
+ for_each_cached_alternate(insert_one_alternate_object);
+
+ /* Filter 'ref' by 'sought' and those that aren't local */
+ if (everything_local(args, &ref, sought, nr_sought))
+ state = FETCH_DONE;
+ else
+ state = FETCH_SEND_REQUEST;
+ break;
+ case FETCH_SEND_REQUEST:
+ if (send_fetch_request(fd[1], args, ref, &common,
+ &haves_to_send, &in_vain))
+ state = FETCH_GET_PACK;
+ else
+ state = FETCH_PROCESS_ACKS;
+ break;
+ case FETCH_PROCESS_ACKS:
+ /* Process ACKs/NAKs */
+ switch (process_acks(&reader, &common)) {
+ case 2:
+ state = FETCH_GET_PACK;
+ break;
+ case 1:
+ in_vain = 0;
+ /* fallthrough */
+ default:
+ state = FETCH_SEND_REQUEST;
+ break;
+ }
+ break;
+ case FETCH_GET_PACK:
+ /* Check for shallow-info section */
+ if (process_section_header(&reader, "shallow-info", 1))
+ receive_shallow_info(args, &reader);
+
+ /* get the pack */
+ process_section_header(&reader, "packfile", 0);
+ if (get_pack(args, fd, pack_lockfile))
+ die(_("git fetch-pack: fetch failed."));
+
+ state = FETCH_DONE;
+ break;
+ case FETCH_DONE:
+ continue;
+ }
+ }
+
+ oidset_clear(&common);
+ return ref;
+}
+
static void fetch_pack_config(void)
{
git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
@@ -1153,7 +1569,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
const char *dest,
struct ref **sought, int nr_sought,
struct oid_array *shallow,
- char **pack_lockfile)
+ char **pack_lockfile,
+ enum protocol_version version)
{
struct ref *ref_cpy;
struct shallow_info si;
@@ -1167,9 +1584,13 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
die(_("no matching remote head"));
}
prepare_shallow_info(&si, shallow);
- ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
- &si, pack_lockfile);
- reprepare_packed_git();
+ if (version == protocol_v2)
+ ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
+ pack_lockfile);
+ else
+ ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
+ &si, pack_lockfile);
+ reprepare_packed_git(the_repository);
update_shallow(args, sought, nr_sought, &si);
clear_shallow_info(&si);
return ref_cpy;
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-23 02:39:01 +0000