diff options
| -rw-r--r-- | http.c | 35 | ||||
| -rw-r--r-- | midx.c | 36 | ||||
| -rw-r--r-- | midx.h | 2 | ||||
| -rw-r--r-- | pack-bitmap.c | 3 | ||||
| -rw-r--r-- | pack-revindex.c | 13 | ||||
| -rw-r--r-- | pack-revindex.h | 2 | ||||
| -rw-r--r-- | packfile.c | 18 | ||||
| -rw-r--r-- | packfile.h | 88 | ||||
| -rw-r--r-- | server-info.c | 57 | ||||
| -rwxr-xr-x | t/t5319-multi-pack-index.sh | 29 | ||||
| -rwxr-xr-x | t/t5570-git-daemon.sh | 1 |
11 files changed, 173 insertions, 111 deletions
@@ -2153,11 +2153,11 @@ add_pack: int http_get_info_packs(const char *base_url, struct packed_git **packs_head) { struct http_get_options options = {0}; - int ret = 0, i = 0; - char *url, *data; + int ret = 0; + char *url; + const char *data; struct strbuf buf = STRBUF_INIT; - unsigned char hash[GIT_MAX_RAWSZ]; - const unsigned hexsz = the_hash_algo->hexsz; + struct object_id oid; end_url_with_slash(&buf, base_url); strbuf_addstr(&buf, "objects/info/packs"); @@ -2169,24 +2169,17 @@ int http_get_info_packs(const char *base_url, struct packed_git **packs_head) goto cleanup; data = buf.buf; - while (i < buf.len) { - switch (data[i]) { - case 'P': - i++; - if (i + hexsz + 12 <= buf.len && - starts_with(data + i, " pack-") && - starts_with(data + i + hexsz + 6, ".pack\n")) { - get_sha1_hex(data + i + 6, hash); - fetch_and_setup_pack_index(packs_head, hash, - base_url); - i += hexsz + 11; - break; - } - default: - while (i < buf.len && data[i] != '\n') - i++; + while (*data) { + if (skip_prefix(data, "P pack-", &data) && + !parse_oid_hex(data, &oid, &data) && + skip_prefix(data, ".pack", &data) && + (*data == '\n' || *data == '\0')) { + fetch_and_setup_pack_index(packs_head, oid.hash, base_url); + } else { + data = strchrnul(data, '\n'); } - i++; + if (*data) + data++; /* skip past newline */ } cleanup: @@ -311,7 +311,39 @@ int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct mu return nth_midxed_pack_entry(m, e, pos); } -int midx_contains_pack(struct multi_pack_index *m, const char *idx_name) +/* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */ +static int cmp_idx_or_pack_name(const char *idx_or_pack_name, + const char *idx_name) +{ + /* Skip past any initial matching prefix. */ + while (*idx_name && *idx_name == *idx_or_pack_name) { + idx_name++; + idx_or_pack_name++; + } + + /* + * If we didn't match completely, we may have matched "pack-1234." and + * be left with "idx" and "pack" respectively, which is also OK. We do + * not have to check for "idx" and "idx", because that would have been + * a complete match (and in that case these strcmps will be false, but + * we'll correctly return 0 from the final strcmp() below. + * + * Technically this matches "fooidx" and "foopack", but we'd never have + * such names in the first place. + */ + if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack")) + return 0; + + /* + * This not only checks for a complete match, but also orders based on + * the first non-identical character, which means our ordering will + * match a raw strcmp(). That makes it OK to use this to binary search + * a naively-sorted list. + */ + return strcmp(idx_or_pack_name, idx_name); +} + +int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) { uint32_t first = 0, last = m->num_packs; @@ -321,7 +353,7 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_name) int cmp; current = m->pack_names[mid]; - cmp = strcmp(idx_name, current); + cmp = cmp_idx_or_pack_name(idx_or_pack_name, current); if (!cmp) return 1; if (cmp > 0) { @@ -43,7 +43,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); -int midx_contains_pack(struct multi_pack_index *m, const char *idx_name); +int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local); int write_midx_file(const char *object_dir); diff --git a/pack-bitmap.c b/pack-bitmap.c index 4695aaf6b4..3960ad94c8 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -308,7 +308,8 @@ static int load_pack_bitmap(struct bitmap_index *bitmap_git) bitmap_git->bitmaps = kh_init_sha1(); bitmap_git->ext_index.positions = kh_init_sha1_pos(); - load_pack_revindex(bitmap_git->pack); + if (load_pack_revindex(bitmap_git->pack)) + goto failed; if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) || !(bitmap_git->trees = read_bitmap_1(bitmap_git)) || diff --git a/pack-revindex.c b/pack-revindex.c index 50891f77a2..d28a7e43d0 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -1,6 +1,7 @@ #include "cache.h" #include "pack-revindex.h" #include "object-store.h" +#include "packfile.h" /* * Pack index for existing packs give us easy access to the offsets into @@ -158,10 +159,14 @@ static void create_pack_revindex(struct packed_git *p) sort_revindex(p->revindex, num_ent, p->pack_size); } -void load_pack_revindex(struct packed_git *p) +int load_pack_revindex(struct packed_git *p) { - if (!p->revindex) + if (!p->revindex) { + if (open_pack_index(p)) + return -1; create_pack_revindex(p); + } + return 0; } int find_revindex_position(struct packed_git *p, off_t ofs) @@ -188,7 +193,9 @@ struct revindex_entry *find_pack_revindex(struct packed_git *p, off_t ofs) { int pos; - load_pack_revindex(p); + if (load_pack_revindex(p)) + return NULL; + pos = find_revindex_position(p, ofs); if (pos < 0) diff --git a/pack-revindex.h b/pack-revindex.h index e262f3efe8..848331d5d6 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -8,7 +8,7 @@ struct revindex_entry { unsigned int nr; }; -void load_pack_revindex(struct packed_git *p); +int load_pack_revindex(struct packed_git *p); int find_revindex_position(struct packed_git *p, off_t ofs); struct revindex_entry *find_pack_revindex(struct packed_git *p, off_t ofs); diff --git a/packfile.c b/packfile.c index d2bcb2f860..b5ec0eeddd 100644 --- a/packfile.c +++ b/packfile.c @@ -466,6 +466,16 @@ static unsigned int get_max_fd_limit(void) #endif } +const char *pack_basename(struct packed_git *p) +{ + const char *ret = strrchr(p->pack_name, '/'); + if (ret) + ret = ret + 1; /* skip past slash */ + else + ret = p->pack_name; /* we only have a base */ + return ret; +} + /* * Do not call this directly as this leaks p->pack_fd on error return; * call open_packed_git() instead. @@ -482,7 +492,7 @@ static int open_packed_git_1(struct packed_git *p) if (!p->index_data) { struct multi_pack_index *m; - const char *pack_name = strrchr(p->pack_name, '/'); + const char *pack_name = pack_basename(p); for (m = the_repository->objects->multi_pack_index; m; m = m->next) { @@ -2023,8 +2033,10 @@ int for_each_object_in_pack(struct packed_git *p, uint32_t i; int r = 0; - if (flags & FOR_EACH_OBJECT_PACK_ORDER) - load_pack_revindex(p); + if (flags & FOR_EACH_OBJECT_PACK_ORDER) { + if (load_pack_revindex(p)) + return -1; + } for (i = 0; i < p->num_objects; i++) { uint32_t pos; diff --git a/packfile.h b/packfile.h index b1c18504eb..12baa6118a 100644 --- a/packfile.h +++ b/packfile.h @@ -15,23 +15,29 @@ struct object_info; * * Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx" */ -extern char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext); +char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext); /* * Return the name of the (local) packfile with the specified sha1 in * its name. The return value is a pointer to memory that is * overwritten each time this function is called. */ -extern char *sha1_pack_name(const unsigned char *sha1); +char *sha1_pack_name(const unsigned char *sha1); /* * Return the name of the (local) pack index file with the specified * sha1 in its name. The return value is a pointer to memory that is * overwritten each time this function is called. */ -extern char *sha1_pack_index_name(const unsigned char *sha1); +char *sha1_pack_index_name(const unsigned char *sha1); -extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path); +/* + * Return the basename of the packfile, omitting any containing directory + * (e.g., "pack-1234abcd[...].pack"). + */ +const char *pack_basename(struct packed_git *p); + +struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path); typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len, const char *file_pach, void *data); @@ -45,8 +51,8 @@ void for_each_file_in_pack_dir(const char *objdir, #define PACKDIR_FILE_GARBAGE 4 extern void (*report_garbage)(unsigned seen_bits, const char *path); -extern void reprepare_packed_git(struct repository *r); -extern void install_packed_git(struct repository *r, struct packed_git *pack); +void reprepare_packed_git(struct repository *r); +void install_packed_git(struct repository *r, struct packed_git *pack); struct packed_git *get_packed_git(struct repository *r); struct list_head *get_packed_git_mru(struct repository *r); @@ -59,34 +65,34 @@ struct packed_git *get_all_packs(struct repository *r); */ unsigned long approximate_object_count(void); -extern struct packed_git *find_sha1_pack(const unsigned char *sha1, - struct packed_git *packs); +struct packed_git *find_sha1_pack(const unsigned char *sha1, + struct packed_git *packs); -extern void pack_report(void); +void pack_report(void); /* * mmap the index file for the specified packfile (if it is not * already mmapped). Return 0 on success. */ -extern int open_pack_index(struct packed_git *); +int open_pack_index(struct packed_git *); /* * munmap the index file for the specified packfile (if it is * currently mmapped). */ -extern void close_pack_index(struct packed_git *); +void close_pack_index(struct packed_git *); int close_pack_fd(struct packed_git *p); -extern uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); +uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); -extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); -extern void close_pack_windows(struct packed_git *); -extern void close_pack(struct packed_git *); -extern void close_all_packs(struct raw_object_store *o); -extern void unuse_pack(struct pack_window **); -extern void clear_delta_base_cache(void); -extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local); +unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); +void close_pack_windows(struct packed_git *); +void close_pack(struct packed_git *); +void close_all_packs(struct raw_object_store *o); +void unuse_pack(struct pack_window **); +void clear_delta_base_cache(void); +struct packed_git *add_packed_git(const char *path, size_t path_len, int local); /* * Make sure that a pointer access into an mmap'd index file is within bounds, @@ -96,7 +102,7 @@ extern struct packed_git *add_packed_git(const char *path, size_t path_len, int * (like the 64-bit extended offset table), as we compare the size to the * fixed-length parts when we open the file. */ -extern void check_pack_index_ptr(const struct packed_git *p, const void *ptr); +void check_pack_index_ptr(const struct packed_git *p, const void *ptr); /* * Perform binary search on a pack-index for a given oid. Packfile is expected to @@ -112,59 +118,59 @@ int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32 * at the SHA-1 within the mmapped index. Return NULL if there is an * error. */ -extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n); +const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n); /* * Like nth_packed_object_sha1, but write the data into the object specified by * the the first argument. Returns the first argument on success, and NULL on * error. */ -extern const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n); +const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n); /* * Return the offset of the nth object within the specified packfile. * The index must already be opened. */ -extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); +off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); /* * If the object named sha1 is present in the specified packfile, * return its offset within the packfile; otherwise, return 0. */ -extern off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *); +off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *); -extern int is_pack_valid(struct packed_git *); -extern void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *); -extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); -extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); -extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *); +int is_pack_valid(struct packed_git *); +void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *); +unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); +unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); +int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *); -extern void release_pack_memory(size_t); +void release_pack_memory(size_t); /* global flag to enable extra checks when accessing packed objects */ extern int do_check_packed_object_crc; -extern int packed_object_info(struct repository *r, - struct packed_git *pack, - off_t offset, struct object_info *); +int packed_object_info(struct repository *r, + struct packed_git *pack, + off_t offset, struct object_info *); -extern void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1); -extern const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1); +void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1); +const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1); /* * Iff a pack file in the given repository contains the object named by sha1, * return true and store its location to e. */ -extern int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e); +int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e); -extern int has_object_pack(const struct object_id *oid); +int has_object_pack(const struct object_id *oid); -extern int has_pack_index(const unsigned char *sha1); +int has_pack_index(const unsigned char *sha1); /* * Return 1 if an object in a promisor packfile is or refers to the given * object, 0 otherwise. */ -extern int is_promisor_object(const struct object_id *oid); +int is_promisor_object(const struct object_id *oid); /* * Expose a function for fuzz testing. @@ -176,7 +182,7 @@ extern int is_promisor_object(const struct object_id *oid); * have a convenient entry-point for fuzz testing. For real uses, you should * probably use open_pack_index() or parse_pack_index() instead. */ -extern int load_idx(const char *path, const unsigned int hashsz, void *idx_map, - size_t idx_size, struct packed_git *p); +int load_idx(const char *path, const unsigned int hashsz, void *idx_map, + size_t idx_size, struct packed_git *p); #endif diff --git a/server-info.c b/server-info.c index e2b2d6a27a..41274d098b 100644 --- a/server-info.c +++ b/server-info.c @@ -78,7 +78,7 @@ static int generate_info_refs(FILE *fp) return for_each_ref(add_info_ref, fp); } -static int update_info_refs(int force) +static int update_info_refs(void) { char *path = git_pathdup("info/refs"); int ret = update_info_file(path, generate_info_refs); @@ -91,19 +91,15 @@ static struct pack_info { struct packed_git *p; int old_num; int new_num; - int nr_alloc; } **info; static int num_pack; -static const char *objdir; -static int objdirlen; static struct pack_info *find_pack_by_name(const char *name) { int i; for (i = 0; i < num_pack; i++) { struct packed_git *p = info[i]->p; - /* skip "/pack/" after ".git/objects" */ - if (!strcmp(p->pack_name + objdirlen + 6, name)) + if (!strcmp(pack_basename(p), name)) return info[i]; } return NULL; @@ -112,9 +108,9 @@ static struct pack_info *find_pack_by_name(const char *name) /* Returns non-zero when we detect that the info in the * old file is useless. */ -static int parse_pack_def(const char *line, int old_cnt) +static int parse_pack_def(const char *packname, int old_cnt) { - struct pack_info *i = find_pack_by_name(line + 2); + struct pack_info *i = find_pack_by_name(packname); if (i) { i->old_num = old_cnt; return 0; @@ -131,39 +127,40 @@ static int parse_pack_def(const char *line, int old_cnt) static int read_pack_info_file(const char *infofile) { FILE *fp; - char line[1000]; + struct strbuf line = STRBUF_INIT; int old_cnt = 0; + int stale = 1; fp = fopen_or_warn(infofile, "r"); if (!fp) return 1; /* nonexistent is not an error. */ - while (fgets(line, sizeof(line), fp)) { - int len = strlen(line); - if (len && line[len-1] == '\n') - line[--len] = 0; + while (strbuf_getline(&line, fp) != EOF) { + const char *arg; - if (!len) + if (!line.len) continue; - switch (line[0]) { - case 'P': /* P name */ - if (parse_pack_def(line, old_cnt++)) + if (skip_prefix(line.buf, "P ", &arg)) { + /* P name */ + if (parse_pack_def(arg, old_cnt++)) goto out_stale; - break; - case 'D': /* we used to emit D but that was misguided. */ - case 'T': /* we used to emit T but nobody uses it. */ + } else if (line.buf[0] == 'D') { + /* we used to emit D but that was misguided. */ goto out_stale; - default: - error("unrecognized: %s", line); - break; + } else if (line.buf[0] == 'T') { + /* we used to emit T but nobody uses it. */ + goto out_stale; + } else { + error("unrecognized: %s", line.buf); } } - fclose(fp); - return 0; + stale = 0; + out_stale: + strbuf_release(&line); fclose(fp); - return 1; + return stale; } static int compare_info(const void *a_, const void *b_) @@ -196,9 +193,6 @@ static void init_pack_info(const char *infofile, int force) int stale; int i = 0; - objdir = get_object_directory(); - objdirlen = strlen(objdir); - for (p = get_all_packs(the_repository); p; p = p->next) { /* we ignore things on alternate path since they are * not available to the pullers in general. @@ -212,6 +206,7 @@ static void init_pack_info(const char *infofile, int force) for (i = 0, p = get_all_packs(the_repository); p; p = p->next) { if (!p->pack_local) continue; + assert(i < num_pack); info[i] = xcalloc(1, sizeof(struct pack_info)); info[i]->p = p; info[i]->old_num = -1; @@ -245,7 +240,7 @@ static int write_pack_info_file(FILE *fp) { int i; for (i = 0; i < num_pack; i++) { - if (fprintf(fp, "P %s\n", info[i]->p->pack_name + objdirlen + 6) < 0) + if (fprintf(fp, "P %s\n", pack_basename(info[i]->p)) < 0) return -1; } if (fputc('\n', fp) == EOF) @@ -274,7 +269,7 @@ int update_server_info(int force) */ int errs = 0; - errs = errs | update_info_refs(force); + errs = errs | update_info_refs(); errs = errs | update_info_packs(force); /* remove leftover rev-cache file if there is any */ diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 70926b5bc0..1ebf19ec3c 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -86,13 +86,14 @@ test_expect_success 'write midx with one v1 pack' ' ' midx_git_two_modes () { + git -c core.multiPackIndex=false $1 >expect && + git -c core.multiPackIndex=true $1 >actual && if [ "$2" = "sorted" ] then - git -c core.multiPackIndex=false $1 | sort >expect && - git -c core.multiPackIndex=true $1 | sort >actual - else - git -c core.multiPackIndex=false $1 >expect && - git -c core.multiPackIndex=true $1 >actual + sort <expect >expect.sorted && + mv expect.sorted expect && + sort <actual >actual.sorted && + mv actual.sorted actual fi && test_cmp expect actual } @@ -103,8 +104,8 @@ compare_results_with_midx () { midx_git_two_modes "rev-list --objects --all" && midx_git_two_modes "log --raw" && midx_git_two_modes "count-objects --verbose" && - midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check" && - midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check --unsorted" sorted + midx_git_two_modes "cat-file --batch-all-objects --batch-check" && + midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted ' } @@ -116,6 +117,20 @@ test_expect_success 'write midx with one v2 pack' ' compare_results_with_midx "one v2 pack" +test_expect_success 'corrupt idx not opened' ' + idx=$(test-tool read-midx $objdir | grep "\.idx\$") && + mv $objdir/pack/$idx backup-$idx && + test_when_finished "mv backup-\$idx \$objdir/pack/\$idx" && + + # This is the minimum size for a sha-1 based .idx; this lets + # us pass perfunctory tests, but anything that actually opens and reads + # the idx file will complain. + test_copy_bytes 1064 <backup-$idx >$objdir/pack/$idx && + + git -c core.multiPackIndex=true rev-list --objects --all 2>err && + test_must_be_empty err +' + test_expect_success 'add more objects' ' for i in $(test_seq 6 10) do diff --git a/t/t5570-git-daemon.sh b/t/t5570-git-daemon.sh index 00fc612cac..34487bbb8c 100755 --- a/t/t5570-git-daemon.sh +++ b/t/t5570-git-daemon.sh @@ -90,6 +90,7 @@ test_expect_success 'fetch notices corrupt pack' ' test_expect_success 'fetch notices corrupt idx' ' cp -R "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_pack.git "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git && (cd "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git && + rm -f objects/pack/multi-pack-index && p=$(ls objects/pack/pack-*.idx) && chmod u+w $p && printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc |