diff options
Diffstat (limited to 'sha1_file.c')
| -rw-r--r-- | sha1_file.c | 1543 |
1 files changed, 1108 insertions, 435 deletions
diff --git a/sha1_file.c b/sha1_file.c index 4f06a0e450..88f06bac92 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -7,6 +7,8 @@ * creation etc. */ #include "cache.h" +#include "string-list.h" +#include "lockfile.h" #include "delta.h" #include "pack.h" #include "blob.h" @@ -19,6 +21,8 @@ #include "pack-revindex.h" #include "sha1-lookup.h" #include "bulk-checkin.h" +#include "streaming.h" +#include "dir.h" #ifndef O_NOATIME #if defined(__linux__) && (defined(__i386__) || defined(__PPC__)) @@ -54,6 +58,12 @@ static struct cached_object empty_tree = { 0 }; +/* + * A pointer to the last packed_git in which an object was found. + * When an object is sought, we look in this packfile first, because + * objects that are looked up at similar times are often in the same + * packfile as one another. + */ static struct packed_git *last_found_pack; static struct cached_object *find_cached_object(const unsigned char *sha1) @@ -99,45 +109,63 @@ int mkdir_in_gitdir(const char *path) return adjust_shared_perm(path); } -int safe_create_leading_directories(char *path) +enum scld_error safe_create_leading_directories(char *path) { - char *pos = path + offset_1st_component(path); - struct stat st; + char *next_component = path + offset_1st_component(path); + enum scld_error ret = SCLD_OK; + + while (ret == SCLD_OK && next_component) { + struct stat st; + char *slash = next_component, slash_character; + + while (*slash && !is_dir_sep(*slash)) + slash++; - while (pos) { - pos = strchr(pos, '/'); - if (!pos) + if (!*slash) break; - while (*++pos == '/') - ; - if (!*pos) + + next_component = slash + 1; + while (is_dir_sep(*next_component)) + next_component++; + if (!*next_component) break; - *--pos = '\0'; + + slash_character = *slash; + *slash = '\0'; if (!stat(path, &st)) { /* path exists */ - if (!S_ISDIR(st.st_mode)) { - *pos = '/'; - return -3; - } - } - else if (mkdir(path, 0777)) { - *pos = '/'; - return -1; - } - else if (adjust_shared_perm(path)) { - *pos = '/'; - return -2; + if (!S_ISDIR(st.st_mode)) + ret = SCLD_EXISTS; + } else if (mkdir(path, 0777)) { + if (errno == EEXIST && + !stat(path, &st) && S_ISDIR(st.st_mode)) + ; /* somebody created it since we checked */ + else if (errno == ENOENT) + /* + * Either mkdir() failed because + * somebody just pruned the containing + * directory, or stat() failed because + * the file that was in our way was + * just removed. Either way, inform + * the caller that it might be worth + * trying again: + */ + ret = SCLD_VANISHED; + else + ret = SCLD_FAILED; + } else if (adjust_shared_perm(path)) { + ret = SCLD_PERMS; } - *pos++ = '/'; + *slash = slash_character; } - return 0; + return ret; } -int safe_create_leading_directories_const(const char *path) +enum scld_error safe_create_leading_directories_const(const char *path) { /* path points to cache entries, so xstrdup before messing with it */ char *buf = xstrdup(path); - int result = safe_create_leading_directories(buf); + enum scld_error result = safe_create_leading_directories(buf); free(buf); return result; } @@ -154,17 +182,7 @@ static void fill_sha1_path(char *pathbuf, const unsigned char *sha1) } } -/* - * NOTE! This returns a statically allocated buffer, so you have to be - * careful about using it. Do an "xstrdup()" if you need to save the - * filename. - * - * Also note that this returns the location for creating. Reading - * SHA1 file can happen from any alternate directory listed in the - * DB_ENVIRONMENT environment variable if it is not found in - * the primary object database. - */ -char *sha1_file_name(const unsigned char *sha1) +const char *sha1_file_name(const unsigned char *sha1) { static char buf[PATH_MAX]; const char *objdir; @@ -184,6 +202,11 @@ char *sha1_file_name(const unsigned char *sha1) return buf; } +/* + * Return the name of the pack or index file with the specified sha1 + * in its filename. *base and *name are scratch space that must be + * provided by the caller. which should be "pack" or "idx". + */ static char *sha1_get_pack_name(const unsigned char *sha1, char **name, char **base, const char *which) { @@ -228,9 +251,6 @@ char *sha1_pack_index_name(const unsigned char *sha1) struct alternate_object_database *alt_odb_list; static struct alternate_object_database **alt_odb_tail; -static void read_info_alternates(const char * alternates, int depth); -static int git_open_noatime(const char *name); - /* * Prepare alternate object database registry. * @@ -246,9 +266,9 @@ static int git_open_noatime(const char *name); * SHA1, an extra slash for the first level indirection, and the * terminating NUL. */ -static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth) +static int link_alt_odb_entry(const char *entry, const char *relative_base, + int depth, const char *normalized_objdir) { - const char *objdir = get_object_directory(); struct alternate_object_database *ent; struct alternate_object_database *alt; int pfxlen, entlen; @@ -258,7 +278,7 @@ static int link_alt_odb_entry(const char * entry, int len, const char * relative strbuf_addstr(&pathbuf, real_path(relative_base)); strbuf_addch(&pathbuf, '/'); } - strbuf_add(&pathbuf, entry, len); + strbuf_addstr(&pathbuf, entry); normalize_path_copy(pathbuf.buf, pathbuf.buf); @@ -293,12 +313,13 @@ static int link_alt_odb_entry(const char * entry, int len, const char * relative * thing twice, or object directory itself. */ for (alt = alt_odb_list; alt; alt = alt->next) { - if (!memcmp(ent->base, alt->base, pfxlen)) { + if (pfxlen == alt->name - alt->base - 1 && + !memcmp(ent->base, alt->base, pfxlen)) { free(ent); return -1; } } - if (!memcmp(ent->base, objdir, pfxlen)) { + if (!strcmp_icase(ent->base, normalized_objdir)) { free(ent); return -1; } @@ -316,10 +337,13 @@ static int link_alt_odb_entry(const char * entry, int len, const char * relative return 0; } -static void link_alt_odb_entries(const char *alt, const char *ep, int sep, +static void link_alt_odb_entries(const char *alt, int len, int sep, const char *relative_base, int depth) { - const char *cp, *last; + struct string_list entries = STRING_LIST_INIT_NODUP; + char *alt_copy; + int i; + struct strbuf objdirbuf = STRBUF_INIT; if (depth > 5) { error("%s: ignoring alternate object stores, nesting too deep.", @@ -327,33 +351,28 @@ static void link_alt_odb_entries(const char *alt, const char *ep, int sep, return; } - last = alt; - while (last < ep) { - cp = last; - if (cp < ep && *cp == '#') { - while (cp < ep && *cp != sep) - cp++; - last = cp + 1; + strbuf_add_absolute_path(&objdirbuf, get_object_directory()); + normalize_path_copy(objdirbuf.buf, objdirbuf.buf); + + alt_copy = xmemdupz(alt, len); + string_list_split_in_place(&entries, alt_copy, sep, -1); + for (i = 0; i < entries.nr; i++) { + const char *entry = entries.items[i].string; + if (entry[0] == '\0' || entry[0] == '#') continue; + if (!is_absolute_path(entry) && depth) { + error("%s: ignoring relative alternate object store %s", + relative_base, entry); + } else { + link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf); } - while (cp < ep && *cp != sep) - cp++; - if (last != cp) { - if (!is_absolute_path(last) && depth) { - error("%s: ignoring relative alternate object store %s", - relative_base, last); - } else { - link_alt_odb_entry(last, cp - last, - relative_base, depth); - } - } - while (cp < ep && *cp == sep) - cp++; - last = cp; } + string_list_clear(&entries, 0); + free(alt_copy); + strbuf_release(&objdirbuf); } -static void read_info_alternates(const char * relative_base, int depth) +void read_info_alternates(const char * relative_base, int depth) { char *map; size_t mapsz; @@ -377,7 +396,7 @@ static void read_info_alternates(const char * relative_base, int depth) map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - link_alt_odb_entries(map, map + mapsz, '\n', relative_base, depth); + link_alt_odb_entries(map, mapsz, '\n', relative_base, depth); munmap(map, mapsz); } @@ -391,17 +410,21 @@ void add_to_alternates_file(const char *reference) if (commit_lock_file(lock)) die("could not close alternates file"); if (alt_odb_tail) - link_alt_odb_entries(alt, alt + strlen(alt), '\n', NULL, 0); + link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0); } -void foreach_alt_odb(alt_odb_fn fn, void *cb) +int foreach_alt_odb(alt_odb_fn fn, void *cb) { struct alternate_object_database *ent; + int r = 0; prepare_alt_odb(); - for (ent = alt_odb_list; ent; ent = ent->next) - if (fn(ent, cb)) - return; + for (ent = alt_odb_list; ent; ent = ent->next) { + r = fn(ent, cb); + if (r) + break; + } + return r; } void prepare_alt_odb(void) @@ -415,33 +438,58 @@ void prepare_alt_odb(void) if (!alt) alt = ""; alt_odb_tail = &alt_odb_list; - link_alt_odb_entries(alt, alt + strlen(alt), PATH_SEP, NULL, 0); + link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0); read_info_alternates(get_object_directory(), 0); } -static int has_loose_object_local(const unsigned char *sha1) +static int freshen_file(const char *fn) { - char *name = sha1_file_name(sha1); - return !access(name, F_OK); + struct utimbuf t; + t.actime = t.modtime = time(NULL); + return !utime(fn, &t); } -int has_loose_object_nonlocal(const unsigned char *sha1) +static int check_and_freshen_file(const char *fn, int freshen) +{ + if (access(fn, F_OK)) + return 0; + if (freshen && freshen_file(fn)) + return 0; + return 1; +} + +static int check_and_freshen_local(const unsigned char *sha1, int freshen) +{ + return check_and_freshen_file(sha1_file_name(sha1), freshen); +} + +static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen) { struct alternate_object_database *alt; prepare_alt_odb(); for (alt = alt_odb_list; alt; alt = alt->next) { fill_sha1_path(alt->name, sha1); - if (!access(alt->base, F_OK)) + if (check_and_freshen_file(alt->base, freshen)) return 1; } return 0; } +static int check_and_freshen(const unsigned char *sha1, int freshen) +{ + return check_and_freshen_local(sha1, freshen) || + check_and_freshen_nonlocal(sha1, freshen); +} + +int has_loose_object_nonlocal(const unsigned char *sha1) +{ + return check_and_freshen_nonlocal(sha1, 0); +} + static int has_loose_object(const unsigned char *sha1) { - return has_loose_object_local(sha1) || - has_loose_object_nonlocal(sha1); + return check_and_freshen(sha1, 0); } static unsigned int pack_used_ctr; @@ -475,7 +523,12 @@ void pack_report(void) sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped)); } -static int check_packed_git_idx(const char *path, struct packed_git *p) +/* + * Open and mmap the index file at path, perform a couple of + * consistency checks, then record its information to p. Return 0 on + * success. + */ +static int check_packed_git_idx(const char *path, struct packed_git *p) { void *idx_map; struct pack_idx_header *hdr; @@ -611,7 +664,7 @@ static void scan_windows(struct packed_git *p, } } -static int unuse_one_window(struct packed_git *current, int keep_fd) +static int unuse_one_window(struct packed_git *current) { struct packed_git *p, *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; @@ -625,15 +678,8 @@ static int unuse_one_window(struct packed_git *current, int keep_fd) pack_mapped -= lru_w->len; if (lru_l) lru_l->next = lru_w->next; - else { + else lru_p->windows = lru_w->next; - if (!lru_p->windows && lru_p->pack_fd != -1 - && lru_p->pack_fd != keep_fd) { - close(lru_p->pack_fd); - pack_open_fds--; - lru_p->pack_fd = -1; - } - } free(lru_w); pack_open_windows--; return 1; @@ -641,21 +687,37 @@ static int unuse_one_window(struct packed_git *current, int keep_fd) return 0; } -void release_pack_memory(size_t need, int fd) +void release_pack_memory(size_t need) { size_t cur = pack_mapped; - while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd)) + while (need >= (cur - pack_mapped) && unuse_one_window(NULL)) ; /* nothing */ } +static void mmap_limit_check(size_t length) +{ + static size_t limit = 0; + if (!limit) { + limit = git_env_ulong("GIT_MMAP_LIMIT", 0); + if (!limit) + limit = SIZE_MAX; + } + if (length > limit) + die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX, + (uintmax_t)length, (uintmax_t)limit); +} + void *xmmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { - void *ret = mmap(start, length, prot, flags, fd, offset); + void *ret; + + mmap_limit_check(length); + ret = mmap(start, length, prot, flags, fd, offset); if (ret == MAP_FAILED) { if (!length) return NULL; - release_pack_memory(length, fd); + release_pack_memory(length); ret = mmap(start, length, prot, flags, fd, offset); if (ret == MAP_FAILED) die_errno("Out of memory? mmap failed"); @@ -679,6 +741,83 @@ void close_pack_windows(struct packed_git *p) } } +/* + * The LRU pack is the one with the oldest MRU window, preferring packs + * with no used windows, or the oldest mtime if it has no windows allocated. + */ +static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse) +{ + struct pack_window *w, *this_mru_w; + int has_windows_inuse = 0; + + /* + * Reject this pack if it has windows and the previously selected + * one does not. If this pack does not have windows, reject + * it if the pack file is newer than the previously selected one. + */ + if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime)) + return; + + for (w = this_mru_w = p->windows; w; w = w->next) { + /* + * Reject this pack if any of its windows are in use, + * but the previously selected pack did not have any + * inuse windows. Otherwise, record that this pack + * has windows in use. + */ + if (w->inuse_cnt) { + if (*accept_windows_inuse) + has_windows_inuse = 1; + else + return; + } + + if (w->last_used > this_mru_w->last_used) + this_mru_w = w; + + /* + * Reject this pack if it has windows that have been + * used more recently than the previously selected pack. + * If the previously selected pack had windows inuse and + * we have not encountered a window in this pack that is + * inuse, skip this check since we prefer a pack with no + * inuse windows to one that has inuse windows. + */ + if (*mru_w && *accept_windows_inuse == has_windows_inuse && + this_mru_w->last_used > (*mru_w)->last_used) + return; + } + + /* + * Select this pack. + */ + *mru_w = this_mru_w; + *lru_p = p; + *accept_windows_inuse = has_windows_inuse; +} + +static int close_one_pack(void) +{ + struct packed_git *p, *lru_p = NULL; + struct pack_window *mru_w = NULL; + int accept_windows_inuse = 1; + + for (p = packed_git; p; p = p->next) { + if (p->pack_fd == -1) + continue; + find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); + } + + if (lru_p) { + close(lru_p->pack_fd); + pack_open_fds--; + lru_p->pack_fd = -1; + return 1; + } + + return 0; +} + void unuse_pack(struct pack_window **w_cursor) { struct pack_window *w = *w_cursor; @@ -701,7 +840,7 @@ void close_pack_index(struct packed_git *p) * contain the same set of objects as an existing one. In that case * the resulting file might be different even if its name would be the * same. It is best to close any reference to the old pack before it is - * replaced on disk. Of course no index pointers nor windows for given pack + * replaced on disk. Of course no index pointers or windows for given pack * must subsist at this point. If ever objects from this pack are requested * again, the new version of the pack will be reinitialized through * reprepare_packed_git(). @@ -731,6 +870,47 @@ void free_pack_by_name(const char *pack_name) } } +static unsigned int get_max_fd_limit(void) +{ +#ifdef RLIMIT_NOFILE + { + struct rlimit lim; + + if (!getrlimit(RLIMIT_NOFILE, &lim)) + return lim.rlim_cur; + } +#endif + +#ifdef _SC_OPEN_MAX + { + long open_max = sysconf(_SC_OPEN_MAX); + if (0 < open_max) + return open_max; + /* + * Otherwise, we got -1 for one of the two + * reasons: + * + * (1) sysconf() did not understand _SC_OPEN_MAX + * and signaled an error with -1; or + * (2) sysconf() said there is no limit. + * + * We _could_ clear errno before calling sysconf() to + * tell these two cases apart and return a huge number + * in the latter case to let the caller cap it to a + * value that is not so selfish, but letting the + * fallback OPEN_MAX codepath take care of these cases + * is a lot simpler. + */ + } +#endif + +#ifdef OPEN_MAX + return OPEN_MAX; +#else + return 1; /* see the caller ;-) */ +#endif +} + /* * Do not call this directly as this leaks p->pack_fd on error return; * call open_packed_git() instead. @@ -747,13 +927,7 @@ static int open_packed_git_1(struct packed_git *p) return error("packfile %s index unavailable", p->pack_name); if (!pack_max_fds) { - struct rlimit lim; - unsigned int max_fds; - - if (getrlimit(RLIMIT_NOFILE, &lim)) - die_errno("cannot get RLIMIT_NOFILE"); - - max_fds = lim.rlim_cur; + unsigned int max_fds = get_max_fd_limit(); /* Save 3 for stdin/stdout/stderr, 22 for work */ if (25 < max_fds) @@ -762,7 +936,7 @@ static int open_packed_git_1(struct packed_git *p) pack_max_fds = 1; } - while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1)) + while (pack_max_fds <= pack_open_fds && close_one_pack()) ; /* nothing */ p->pack_fd = git_open_noatime(p->pack_name); @@ -878,7 +1052,7 @@ unsigned char *use_pack(struct packed_git *p, win->len = (size_t)len; pack_mapped += win->len; while (packed_git_limit < pack_mapped - && unuse_one_window(p, p->pack_fd)) + && unuse_one_window(p)) ; /* nothing */ win->base = xmmap(NULL, win->len, PROT_READ, MAP_PRIVATE, @@ -924,7 +1098,7 @@ static struct packed_git *alloc_packed_git(int extra) static void try_to_free_pack_memory(size_t size) { - release_pack_memory(size, -1); + release_pack_memory(size); } struct packed_git *add_packed_git(const char *path, int path_len, int local) @@ -994,53 +1168,127 @@ void install_packed_git(struct packed_git *pack) packed_git = pack; } +void (*report_garbage)(const char *desc, const char *path); + +static void report_helper(const struct string_list *list, + int seen_bits, int first, int last) +{ + const char *msg; + switch (seen_bits) { + case 0: + msg = "no corresponding .idx or .pack"; + break; + case 1: + msg = "no corresponding .idx"; + break; + case 2: + msg = "no corresponding .pack"; + break; + default: + return; + } + for (; first < last; first++) + report_garbage(msg, list->items[first].string); +} + +static void report_pack_garbage(struct string_list *list) +{ + int i, baselen = -1, first = 0, seen_bits = 0; + + if (!report_garbage) + return; + + string_list_sort(list); + + for (i = 0; i < list->nr; i++) { + const char *path = list->items[i].string; + if (baselen != -1 && + strncmp(path, list->items[first].string, baselen)) { + report_helper(list, seen_bits, first, i); + baselen = -1; + seen_bits = 0; + } + if (baselen == -1) { + const char *dot = strrchr(path, '.'); + if (!dot) { + report_garbage("garbage found", path); + continue; + } + baselen = dot - path + 1; + first = i; + } + if (!strcmp(path + baselen, "pack")) + seen_bits |= 1; + else if (!strcmp(path + baselen, "idx")) + seen_bits |= 2; + } + report_helper(list, seen_bits, first, list->nr); +} + static void prepare_packed_git_one(char *objdir, int local) { - /* Ensure that this buffer is large enough so that we can - append "/pack/" without clobbering the stack even if - strlen(objdir) were PATH_MAX. */ - char path[PATH_MAX + 1 + 4 + 1 + 1]; - int len; + struct strbuf path = STRBUF_INIT; + size_t dirnamelen; DIR *dir; struct dirent *de; + struct string_list garbage = STRING_LIST_INIT_DUP; - sprintf(path, "%s/pack", objdir); - len = strlen(path); - dir = opendir(path); + strbuf_addstr(&path, objdir); + strbuf_addstr(&path, "/pack"); + dir = opendir(path.buf); if (!dir) { if (errno != ENOENT) error("unable to open object pack directory: %s: %s", - path, strerror(errno)); + path.buf, strerror(errno)); + strbuf_release(&path); return; } - path[len++] = '/'; + strbuf_addch(&path, '/'); + dirnamelen = path.len; while ((de = readdir(dir)) != NULL) { - int namelen = strlen(de->d_name); struct packed_git *p; + size_t base_len; - if (!has_extension(de->d_name, ".idx")) - continue; - - if (len + namelen + 1 > sizeof(path)) + if (is_dot_or_dotdot(de->d_name)) continue; - /* Don't reopen a pack we already have. */ - strcpy(path + len, de->d_name); - for (p = packed_git; p; p = p->next) { - if (!memcmp(path, p->pack_name, len + namelen - 4)) - break; + strbuf_setlen(&path, dirnamelen); + strbuf_addstr(&path, de->d_name); + + base_len = path.len; + if (strip_suffix_mem(path.buf, &base_len, ".idx")) { + /* Don't reopen a pack we already have. */ + for (p = packed_git; p; p = p->next) { + size_t len; + if (strip_suffix(p->pack_name, ".pack", &len) && + len == base_len && + !memcmp(p->pack_name, path.buf, len)) + break; + } + if (p == NULL && + /* + * See if it really is a valid .idx file with + * corresponding .pack file that we can map. + */ + (p = add_packed_git(path.buf, path.len, local)) != NULL) + install_packed_git(p); } - if (p) - continue; - /* See if it really is a valid .idx file with corresponding - * .pack file that we can map. - */ - p = add_packed_git(path, len + namelen, local); - if (!p) + + if (!report_garbage) continue; - install_packed_git(p); + + if (ends_with(de->d_name, ".idx") || + ends_with(de->d_name, ".pack") || + ends_with(de->d_name, ".bitmap") || + ends_with(de->d_name, ".keep")) + string_list_append(&garbage, path.buf); + else + report_garbage("garbage found", path.buf); } closedir(dir); + report_pack_garbage(&garbage); + string_list_clear(&garbage, 0); + strbuf_release(&path); } static int sort_pack(const void *a_, const void *b_) @@ -1117,7 +1365,6 @@ void prepare_packed_git(void) void reprepare_packed_git(void) { - discard_revindex(); prepare_packed_git_run_once = 0; prepare_packed_git(); } @@ -1146,14 +1393,55 @@ static const struct packed_git *has_packed_and_bad(const unsigned char *sha1) return NULL; } -int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type) +/* + * With an in-core object data in "map", rehash it to make sure the + * object name actually matches "sha1" to detect object corruption. + * With "map" == NULL, try reading the object named with "sha1" using + * the streaming interface and rehash it to do the same. + */ +int check_sha1_signature(const unsigned char *sha1, void *map, + unsigned long size, const char *type) { unsigned char real_sha1[20]; - hash_sha1_file(map, size, type, real_sha1); + enum object_type obj_type; + struct git_istream *st; + git_SHA_CTX c; + char hdr[32]; + int hdrlen; + + if (map) { + hash_sha1_file(map, size, type, real_sha1); + return hashcmp(sha1, real_sha1) ? -1 : 0; + } + + st = open_istream(sha1, &obj_type, &size, NULL); + if (!st) + return -1; + + /* Generate the header */ + hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1; + + /* Sha1.. */ + git_SHA1_Init(&c); + git_SHA1_Update(&c, hdr, hdrlen); + for (;;) { + char buf[1024 * 16]; + ssize_t readlen = read_istream(st, buf, sizeof(buf)); + + if (readlen < 0) { + close_istream(st); + return -1; + } + if (!readlen) + break; + git_SHA1_Update(&c, buf, readlen); + } + git_SHA1_Final(real_sha1, &c); + close_istream(st); return hashcmp(sha1, real_sha1) ? -1 : 0; } -static int git_open_noatime(const char *name) +int git_open_noatime(const char *name) { static int sha1_file_open_flag = O_NOATIME; @@ -1172,25 +1460,45 @@ static int git_open_noatime(const char *name) } } +static int stat_sha1_file(const unsigned char *sha1, struct stat *st) +{ + struct alternate_object_database *alt; + + if (!lstat(sha1_file_name(sha1), st)) + return 0; + + prepare_alt_odb(); + errno = ENOENT; + for (alt = alt_odb_list; alt; alt = alt->next) { + fill_sha1_path(alt->name, sha1); + if (!lstat(alt->base, st)) + return 0; + } + + return -1; +} + static int open_sha1_file(const unsigned char *sha1) { int fd; - char *name = sha1_file_name(sha1); struct alternate_object_database *alt; + int most_interesting_errno; - fd = git_open_noatime(name); + fd = git_open_noatime(sha1_file_name(sha1)); if (fd >= 0) return fd; + most_interesting_errno = errno; prepare_alt_odb(); - errno = ENOENT; for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); + fill_sha1_path(alt->name, sha1); fd = git_open_noatime(alt->base); if (fd >= 0) return fd; + if (most_interesting_errno == ENOENT) + most_interesting_errno = errno; } + errno = most_interesting_errno; return -1; } @@ -1218,51 +1526,6 @@ void *map_sha1_file(const unsigned char *sha1, unsigned long *size) return map; } -/* - * There used to be a second loose object header format which - * was meant to mimic the in-pack format, allowing for direct - * copy of the object data. This format turned up not to be - * really worth it and we no longer write loose objects in that - * format. - */ -static int experimental_loose_object(unsigned char *map) -{ - unsigned int word; - - /* - * We must determine if the buffer contains the standard - * zlib-deflated stream or the experimental format based - * on the in-pack object format. Compare the header byte - * for each format: - * - * RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7 - * Experimental pack-based : Stttssss : ttt = 1,2,3,4 - * - * If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be - * in standard loose-object format, UNLESS it is a Git-pack - * format object *exactly* 8 bytes in size when inflated. - * - * However, RFC1950 also specifies that the 1st 16-bit word - * must be divisible by 31 - this checksum tells us our buffer - * is in the standard format, giving a false positive only if - * the 1st word of the Git-pack format object happens to be - * divisible by 31, ie: - * ((byte0 * 256) + byte1) % 31 = 0 - * => 0ttt10000www1000 % 31 = 0 - * - * As it happens, this case can only arise for www=3 & ttt=1 - * - ie, a Commit object, which would have to be 8 bytes in - * size. As no Commit can be that small, we find that the - * combination of these two criteria (bitmask & checksum) - * can always correctly determine the buffer format. - */ - word = (map[0] << 8) + map[1]; - if ((map[0] & 0x8F) == 0x08 && !(word % 31)) - return 0; - else - return 1; -} - unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) { @@ -1290,14 +1553,6 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf, int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { - unsigned long size, used; - static const char valid_loose_object_type[8] = { - 0, /* OBJ_EXT */ - 1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */ - 0, /* "delta" and others are invalid in a loose object */ - }; - enum object_type type; - /* Get the data stream */ memset(stream, 0, sizeof(*stream)); stream->next_in = map; @@ -1305,27 +1560,6 @@ int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long ma stream->next_out = buffer; stream->avail_out = bufsiz; - if (experimental_loose_object(map)) { - /* - * The old experimental format we no longer produce; - * we can still read it. - */ - used = unpack_object_header_buffer(map, mapsize, &type, &size); - if (!used || !valid_loose_object_type[type]) - return -1; - map += used; - mapsize -= used; - - /* Set up the stream for the rest.. */ - stream->next_in = map; - stream->avail_in = mapsize; - git_inflate_init(stream); - - /* And generate the fake traditional header */ - stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", - typename(type), size); - return 0; - } git_inflate_init(stream); return git_inflate(stream, 0); } @@ -1517,48 +1751,36 @@ static off_t get_delta_base(struct packed_git *p, return base_offset; } -/* forward declaration for a mutually recursive function */ -static int packed_object_info(struct packed_git *p, off_t offset, - unsigned long *sizep, int *rtype); +/* + * Like get_delta_base above, but we return the sha1 instead of the pack + * offset. This means it is cheaper for REF deltas (we do not have to do + * the final object lookup), but more expensive for OFS deltas (we + * have to load the revidx to convert the offset back into a sha1). + */ +static const unsigned char *get_delta_base_sha1(struct packed_git *p, + struct pack_window **w_curs, + off_t curpos, + enum object_type type, + off_t delta_obj_offset) +{ + if (type == OBJ_REF_DELTA) { + unsigned char *base = use_pack(p, w_curs, curpos, NULL); + return base; + } else if (type == OBJ_OFS_DELTA) { + struct revindex_entry *revidx; + off_t base_offset = get_delta_base(p, w_curs, &curpos, + type, delta_obj_offset); -static int packed_delta_info(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - enum object_type type, - off_t obj_offset, - unsigned long *sizep) -{ - off_t base_offset; + if (!base_offset) + return NULL; - base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); - if (!base_offset) - return OBJ_BAD; - type = packed_object_info(p, base_offset, NULL, NULL); - if (type <= OBJ_NONE) { - struct revindex_entry *revidx; - const unsigned char *base_sha1; revidx = find_pack_revindex(p, base_offset); if (!revidx) - return OBJ_BAD; - base_sha1 = nth_packed_object_sha1(p, revidx->nr); - mark_bad_packed_object(p, base_sha1); - type = sha1_object_info(base_sha1, NULL); - if (type <= OBJ_NONE) - return OBJ_BAD; - } - - /* We choose to only get the type of the base object and - * ignore potentially corrupt pack file that expects the delta - * based on a base with a wrong size. This saves tons of - * inflate() calls. - */ - if (sizep) { - *sizep = get_size_from_delta(p, w_curs, curpos); - if (*sizep == 0) - type = OBJ_BAD; - } + return NULL; - return type; + return nth_packed_object_sha1(p, revidx->nr); + } else + return NULL; } int unpack_object_header(struct packed_git *p, @@ -1587,36 +1809,154 @@ int unpack_object_header(struct packed_git *p, return type; } -static int packed_object_info(struct packed_git *p, off_t obj_offset, - unsigned long *sizep, int *rtype) +static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset) { - struct pack_window *w_curs = NULL; - unsigned long size; - off_t curpos = obj_offset; - enum object_type type; + int type; + struct revindex_entry *revidx; + const unsigned char *sha1; + revidx = find_pack_revindex(p, obj_offset); + if (!revidx) + return OBJ_BAD; + sha1 = nth_packed_object_sha1(p, revidx->nr); + mark_bad_packed_object(p, sha1); + type = sha1_object_info(sha1, NULL); + if (type <= OBJ_NONE) + return OBJ_BAD; + return type; +} - type = unpack_object_header(p, &w_curs, &curpos, &size); - if (rtype) - *rtype = type; /* representation type */ +#define POI_STACK_PREALLOC 64 + +static enum object_type packed_to_object_type(struct packed_git *p, + off_t obj_offset, + enum object_type type, + struct pack_window **w_curs, + off_t curpos) +{ + off_t small_poi_stack[POI_STACK_PREALLOC]; + off_t *poi_stack = small_poi_stack; + int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC; + + while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + off_t base_offset; + unsigned long size; + /* Push the object we're going to leave behind */ + if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) { + poi_stack_alloc = alloc_nr(poi_stack_nr); + poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc); + memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr); + } else { + ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc); + } + poi_stack[poi_stack_nr++] = obj_offset; + /* If parsing the base offset fails, just unwind */ + base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); + if (!base_offset) + goto unwind; + curpos = obj_offset = base_offset; + type = unpack_object_header(p, w_curs, &curpos, &size); + if (type <= OBJ_NONE) { + /* If getting the base itself fails, we first + * retry the base, otherwise unwind */ + type = retry_bad_packed_offset(p, base_offset); + if (type > OBJ_NONE) + goto out; + goto unwind; + } + } switch (type) { - case OBJ_OFS_DELTA: - case OBJ_REF_DELTA: - type = packed_delta_info(p, &w_curs, curpos, - type, obj_offset, sizep); - break; + case OBJ_BAD: case OBJ_COMMIT: case OBJ_TREE: case OBJ_BLOB: case OBJ_TAG: - if (sizep) - *sizep = size; break; default: error("unknown object type %i at offset %"PRIuMAX" in %s", type, (uintmax_t)obj_offset, p->pack_name); type = OBJ_BAD; } + +out: + if (poi_stack != small_poi_stack) + free(poi_stack); + return type; + +unwind: + while (poi_stack_nr) { + obj_offset = poi_stack[--poi_stack_nr]; + type = retry_bad_packed_offset(p, obj_offset); + if (type > OBJ_NONE) + goto out; + } + type = OBJ_BAD; + goto out; +} + +static int packed_object_info(struct packed_git *p, off_t obj_offset, + struct object_info *oi) +{ + struct pack_window *w_curs = NULL; + unsigned long size; + off_t curpos = obj_offset; + enum object_type type; + + /* + * We always get the representation type, but only convert it to + * a "real" type later if the caller is interested. + */ + type = unpack_object_header(p, &w_curs, &curpos, &size); + + if (oi->sizep) { + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + off_t tmp_pos = curpos; + off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos, + type, obj_offset); + if (!base_offset) { + type = OBJ_BAD; + goto out; + } + *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos); + if (*oi->sizep == 0) { + type = OBJ_BAD; + goto out; + } + } else { + *oi->sizep = size; + } + } + + if (oi->disk_sizep) { + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); + *oi->disk_sizep = revidx[1].offset - obj_offset; + } + + if (oi->typep) { + *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos); + if (*oi->typep < 0) { + type = OBJ_BAD; + goto out; + } + } + + if (oi->delta_base_sha1) { + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + const unsigned char *base; + + base = get_delta_base_sha1(p, &w_curs, curpos, + type, obj_offset); + if (!base) { + type = OBJ_BAD; + goto out; + } + + hashcpy(oi->delta_base_sha1, base); + } else + hashclr(oi->delta_base_sha1); + } + +out: unuse_pack(&w_curs); return type; } @@ -1630,7 +1970,9 @@ static void *unpack_compressed_entry(struct packed_git *p, git_zstream stream; unsigned char *buffer, *in; - buffer = xmallocz(size); + buffer = xmallocz_gently(size); + if (!buffer) + return NULL; memset(&stream, 0, sizeof(stream)); stream.next_out = buffer; stream.avail_out = size + 1; @@ -1680,32 +2022,51 @@ static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset) return hash % MAX_DELTA_CACHE; } -static int in_delta_base_cache(struct packed_git *p, off_t base_offset) +static struct delta_base_cache_entry * +get_delta_base_cache_entry(struct packed_git *p, off_t base_offset) { unsigned long hash = pack_entry_hash(p, base_offset); - struct delta_base_cache_entry *ent = delta_base_cache + hash; + return delta_base_cache + hash; +} + +static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent, + struct packed_git *p, off_t base_offset) +{ return (ent->data && ent->p == p && ent->base_offset == base_offset); } +static int in_delta_base_cache(struct packed_git *p, off_t base_offset) +{ + struct delta_base_cache_entry *ent; + ent = get_delta_base_cache_entry(p, base_offset); + return eq_delta_base_cache_entry(ent, p, base_offset); +} + +static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent) +{ + ent->data = NULL; + ent->lru.next->prev = ent->lru.prev; + ent->lru.prev->next = ent->lru.next; + delta_base_cached -= ent->size; +} + static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset, unsigned long *base_size, enum object_type *type, int keep_cache) { + struct delta_base_cache_entry *ent; void *ret; - unsigned long hash = pack_entry_hash(p, base_offset); - struct delta_base_cache_entry *ent = delta_base_cache + hash; - ret = ent->data; - if (!ret || ent->p != p || ent->base_offset != base_offset) + ent = get_delta_base_cache_entry(p, base_offset); + + if (!eq_delta_base_cache_entry(ent, p, base_offset)) return unpack_entry(p, base_offset, type, base_size); - if (!keep_cache) { - ent->data = NULL; - ent->lru.next->prev = ent->lru.prev; - ent->lru.prev->next = ent->lru.next; - delta_base_cached -= ent->size; - } else { + ret = ent->data; + + if (!keep_cache) + clear_delta_base_cache_entry(ent); + else ret = xmemdupz(ent->data, ent->size); - } *type = ent->type; *base_size = ent->size; return ret; @@ -1769,131 +2130,200 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset, static void *read_object(const unsigned char *sha1, enum object_type *type, unsigned long *size); -static void *unpack_delta_entry(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - unsigned long delta_size, - off_t obj_offset, - enum object_type *type, - unsigned long *sizep) -{ - void *delta_data, *result, *base; - unsigned long base_size; - off_t base_offset; - - base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset); - if (!base_offset) { - error("failed to validate delta base reference " - "at offset %"PRIuMAX" from %s", - (uintmax_t)curpos, p->pack_name); - return NULL; - } - unuse_pack(w_curs); - base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0); - if (!base) { - /* - * We're probably in deep shit, but let's try to fetch - * the required base anyway from another pack or loose. - * This is costly but should happen only in the presence - * of a corrupted pack, and is better than failing outright. - */ - struct revindex_entry *revidx; - const unsigned char *base_sha1; - revidx = find_pack_revindex(p, base_offset); - if (!revidx) - return NULL; - base_sha1 = nth_packed_object_sha1(p, revidx->nr); - error("failed to read delta base object %s" - " at offset %"PRIuMAX" from %s", - sha1_to_hex(base_sha1), (uintmax_t)base_offset, - p->pack_name); - mark_bad_packed_object(p, base_sha1); - base = read_object(base_sha1, type, &base_size); - if (!base) - return NULL; - } - - delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size); - if (!delta_data) { - error("failed to unpack compressed delta " - "at offset %"PRIuMAX" from %s", - (uintmax_t)curpos, p->pack_name); - free(base); - return NULL; - } - result = patch_delta(base, base_size, - delta_data, delta_size, - sizep); - if (!result) - die("failed to apply delta"); - free(delta_data); - add_delta_base_cache(p, base_offset, base, base_size, *type); - return result; -} - static void write_pack_access_log(struct packed_git *p, off_t obj_offset) { - static FILE *log_file; - - if (!log_file) { - log_file = fopen(log_pack_access, "w"); - if (!log_file) { - error("cannot open pack access log '%s' for writing: %s", - log_pack_access, strerror(errno)); - log_pack_access = NULL; - return; - } - } - fprintf(log_file, "%s %"PRIuMAX"\n", - p->pack_name, (uintmax_t)obj_offset); - fflush(log_file); + static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS); + trace_printf_key(&pack_access, "%s %"PRIuMAX"\n", + p->pack_name, (uintmax_t)obj_offset); } int do_check_packed_object_crc; +#define UNPACK_ENTRY_STACK_PREALLOC 64 +struct unpack_entry_stack_ent { + off_t obj_offset; + off_t curpos; + unsigned long size; +}; + void *unpack_entry(struct packed_git *p, off_t obj_offset, - enum object_type *type, unsigned long *sizep) + enum object_type *final_type, unsigned long *final_size) { struct pack_window *w_curs = NULL; off_t curpos = obj_offset; - void *data; + void *data = NULL; + unsigned long size; + enum object_type type; + struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC]; + struct unpack_entry_stack_ent *delta_stack = small_delta_stack; + int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; + int base_from_cache = 0; - if (log_pack_access) - write_pack_access_log(p, obj_offset); + write_pack_access_log(p, obj_offset); - if (do_check_packed_object_crc && p->index_version > 1) { - struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); - unsigned long len = revidx[1].offset - obj_offset; - if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) { - const unsigned char *sha1 = - nth_packed_object_sha1(p, revidx->nr); - error("bad packed object CRC for %s", - sha1_to_hex(sha1)); - mark_bad_packed_object(p, sha1); - unuse_pack(&w_curs); - return NULL; + /* PHASE 1: drill down to the innermost base object */ + for (;;) { + off_t base_offset; + int i; + struct delta_base_cache_entry *ent; + + ent = get_delta_base_cache_entry(p, curpos); + if (eq_delta_base_cache_entry(ent, p, curpos)) { + type = ent->type; + data = ent->data; + size = ent->size; + clear_delta_base_cache_entry(ent); + base_from_cache = 1; + break; + } + + if (do_check_packed_object_crc && p->index_version > 1) { + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); + unsigned long len = revidx[1].offset - obj_offset; + if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) { + const unsigned char *sha1 = + nth_packed_object_sha1(p, revidx->nr); + error("bad packed object CRC for %s", + sha1_to_hex(sha1)); + mark_bad_packed_object(p, sha1); + unuse_pack(&w_curs); + return NULL; + } } + + type = unpack_object_header(p, &w_curs, &curpos, &size); + if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA) + break; + + base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset); + if (!base_offset) { + error("failed to validate delta base reference " + "at offset %"PRIuMAX" from %s", + (uintmax_t)curpos, p->pack_name); + /* bail to phase 2, in hopes of recovery */ + data = NULL; + break; + } + + /* push object, proceed to base */ + if (delta_stack_nr >= delta_stack_alloc + && delta_stack == small_delta_stack) { + delta_stack_alloc = alloc_nr(delta_stack_nr); + delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc); + memcpy(delta_stack, small_delta_stack, + sizeof(*delta_stack)*delta_stack_nr); + } else { + ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc); + } + i = delta_stack_nr++; + delta_stack[i].obj_offset = obj_offset; + delta_stack[i].curpos = curpos; + delta_stack[i].size = size; + + curpos = obj_offset = base_offset; } - *type = unpack_object_header(p, &w_curs, &curpos, sizep); - switch (*type) { + /* PHASE 2: handle the base */ + switch (type) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - data = unpack_delta_entry(p, &w_curs, curpos, *sizep, - obj_offset, type, sizep); + if (data) + die("BUG in unpack_entry: left loop at a valid delta"); break; case OBJ_COMMIT: case OBJ_TREE: case OBJ_BLOB: case OBJ_TAG: - data = unpack_compressed_entry(p, &w_curs, curpos, *sizep); + if (!base_from_cache) + data = unpack_compressed_entry(p, &w_curs, curpos, size); break; default: data = NULL; error("unknown object type %i at offset %"PRIuMAX" in %s", - *type, (uintmax_t)obj_offset, p->pack_name); + type, (uintmax_t)obj_offset, p->pack_name); + } + + /* PHASE 3: apply deltas in order */ + + /* invariants: + * 'data' holds the base data, or NULL if there was corruption + */ + while (delta_stack_nr) { + void *delta_data; + void *base = data; + unsigned long delta_size, base_size = size; + int i; + + data = NULL; + + if (base) + add_delta_base_cache(p, obj_offset, base, base_size, type); + + if (!base) { + /* + * We're probably in deep shit, but let's try to fetch + * the required base anyway from another pack or loose. + * This is costly but should happen only in the presence + * of a corrupted pack, and is better than failing outright. + */ + struct revindex_entry *revidx; + const unsigned char *base_sha1; + revidx = find_pack_revindex(p, obj_offset); + if (revidx) { + base_sha1 = nth_packed_object_sha1(p, revidx->nr); + error("failed to read delta base object %s" + " at offset %"PRIuMAX" from %s", + sha1_to_hex(base_sha1), (uintmax_t)obj_offset, + p->pack_name); + mark_bad_packed_object(p, base_sha1); + base = read_object(base_sha1, &type, &base_size); + } + } + + i = --delta_stack_nr; + obj_offset = delta_stack[i].obj_offset; + curpos = delta_stack[i].curpos; + delta_size = delta_stack[i].size; + + if (!base) + continue; + + delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size); + + if (!delta_data) { + error("failed to unpack compressed delta " + "at offset %"PRIuMAX" from %s", + (uintmax_t)curpos, p->pack_name); + data = NULL; + continue; + } + + data = patch_delta(base, base_size, + delta_data, delta_size, + &size); + + /* + * We could not apply the delta; warn the user, but keep going. + * Our failure will be noticed either in the next iteration of + * the loop, or if this is the final delta, in the caller when + * we return NULL. Those code paths will take care of making + * a more explicit warning and retrying with another copy of + * the object. + */ + if (!data) + error("failed to apply delta"); + + free(delta_data); } + + *final_type = type; + *final_size = size; + unuse_pack(&w_curs); + + if (delta_stack != small_delta_stack) + free(delta_stack); + return data; } @@ -2053,6 +2483,10 @@ static int fill_pack_entry(const unsigned char *sha1, return 1; } +/* + * Iff a pack file contains the object named by sha1, return true and + * store its location to e. + */ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) { struct packed_git *p; @@ -2065,11 +2499,13 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) return 1; for (p = packed_git; p; p = p->next) { - if (p == last_found_pack || !fill_pack_entry(sha1, e, p)) - continue; + if (p == last_found_pack) + continue; /* we already checked this one */ - last_found_pack = p; - return 1; + if (fill_pack_entry(sha1, e, p)) { + last_found_pack = p; + return 1; + } } return 0; } @@ -2087,7 +2523,8 @@ struct packed_git *find_sha1_pack(const unsigned char *sha1, } -static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep) +static int sha1_loose_object_info(const unsigned char *sha1, + struct object_info *oi) { int status; unsigned long mapsize, size; @@ -2095,54 +2532,83 @@ static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *size git_zstream stream; char hdr[32]; + if (oi->delta_base_sha1) + hashclr(oi->delta_base_sha1); + + /* + * If we don't care about type or size, then we don't + * need to look inside the object at all. Note that we + * do not optimize out the stat call, even if the + * caller doesn't care about the disk-size, since our + * return value implicitly indicates whether the + * object even exists. + */ + if (!oi->typep && !oi->sizep) { + struct stat st; + if (stat_sha1_file(sha1, &st) < 0) + return -1; + if (oi->disk_sizep) + *oi->disk_sizep = st.st_size; + return 0; + } + map = map_sha1_file(sha1, &mapsize); if (!map) - return error("unable to find %s", sha1_to_hex(sha1)); + return -1; + if (oi->disk_sizep) + *oi->disk_sizep = mapsize; if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) status = error("unable to unpack %s header", sha1_to_hex(sha1)); else if ((status = parse_sha1_header(hdr, &size)) < 0) status = error("unable to parse %s header", sha1_to_hex(sha1)); - else if (sizep) - *sizep = size; + else if (oi->sizep) + *oi->sizep = size; git_inflate_end(&stream); munmap(map, mapsize); - return status; + if (oi->typep) + *oi->typep = status; + return 0; } -/* returns enum object_type or negative */ -int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi) +int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags) { struct cached_object *co; struct pack_entry e; - int status, rtype; + int rtype; + const unsigned char *real = lookup_replace_object_extended(sha1, flags); - co = find_cached_object(sha1); + co = find_cached_object(real); if (co) { + if (oi->typep) + *(oi->typep) = co->type; if (oi->sizep) *(oi->sizep) = co->size; + if (oi->disk_sizep) + *(oi->disk_sizep) = 0; + if (oi->delta_base_sha1) + hashclr(oi->delta_base_sha1); oi->whence = OI_CACHED; - return co->type; + return 0; } - if (!find_pack_entry(sha1, &e)) { + if (!find_pack_entry(real, &e)) { /* Most likely it's a loose object. */ - status = sha1_loose_object_info(sha1, oi->sizep); - if (status >= 0) { + if (!sha1_loose_object_info(real, oi)) { oi->whence = OI_LOOSE; - return status; + return 0; } /* Not a loose object; someone else may have just packed it. */ reprepare_packed_git(); - if (!find_pack_entry(sha1, &e)) - return status; + if (!find_pack_entry(real, &e)) + return -1; } - status = packed_object_info(e.p, e.offset, oi->sizep, &rtype); - if (status < 0) { - mark_bad_packed_object(e.p, sha1); - status = sha1_object_info_extended(sha1, oi); + rtype = packed_object_info(e.p, e.offset, oi); + if (rtype < 0) { + mark_bad_packed_object(e.p, real); + return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; } else { @@ -2153,15 +2619,20 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi) rtype == OBJ_OFS_DELTA); } - return status; + return 0; } +/* returns enum object_type or negative */ int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) { - struct object_info oi; + enum object_type type; + struct object_info oi = {NULL}; + oi.typep = &type; oi.sizep = sizep; - return sha1_object_info_extended(sha1, &oi); + if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0) + return -1; + return type; } static void *read_packed_sha1(const unsigned char *sha1, @@ -2196,12 +2667,7 @@ int pretend_sha1_file(void *buf, unsigned long len, enum object_type type, hash_sha1_file(buf, len, typename(type), sha1); if (has_sha1_file(sha1) || find_cached_object(sha1)) return 0; - if (cached_object_alloc <= cached_object_nr) { - cached_object_alloc = alloc_nr(cached_object_alloc); - cached_objects = xrealloc(cached_objects, - sizeof(*cached_objects) * - cached_object_alloc); - } + ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc); co = &cached_objects[cached_object_nr++]; co->size = len; co->type = type; @@ -2249,10 +2715,8 @@ void *read_sha1_file_extended(const unsigned char *sha1, unsigned flag) { void *data; - char *path; const struct packed_git *p; - const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE) - ? lookup_replace_object(sha1) : sha1; + const unsigned char *repl = lookup_replace_object_extended(sha1, flag); errno = 0; data = read_object(repl, type, size); @@ -2268,7 +2732,8 @@ void *read_sha1_file_extended(const unsigned char *sha1, sha1_to_hex(repl), sha1_to_hex(sha1)); if (has_loose_object(repl)) { - path = sha1_file_name(sha1); + const char *path = sha1_file_name(sha1); + die("loose object %s (stored in %s) is corrupt", sha1_to_hex(repl), path); } @@ -2379,7 +2844,7 @@ int move_temp_to_file(const char *tmpfile, const char *filename) unlink_or_warn(tmpfile); if (ret) { if (ret != EEXIST) { - return error("unable to write sha1 filename %s: %s\n", filename, strerror(ret)); + return error("unable to write sha1 filename %s: %s", filename, strerror(ret)); } /* FIXME!!! Collision check here ? */ } @@ -2446,7 +2911,9 @@ static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename) /* Make sure the directory exists */ memcpy(buffer, filename, dirlen); buffer[dirlen-1] = 0; - if (mkdir(buffer, 0777) || adjust_shared_perm(buffer)) + if (mkdir(buffer, 0777) && errno != EEXIST) + return -1; + if (adjust_shared_perm(buffer)) return -1; /* Try again */ @@ -2464,20 +2931,18 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; - char *filename; static char tmp_file[PATH_MAX]; + const char *filename = sha1_file_name(sha1); - filename = sha1_file_name(sha1); fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename); if (fd < 0) { if (errno == EACCES) - return error("insufficient permission for adding an object to repository database %s\n", get_object_directory()); + return error("insufficient permission for adding an object to repository database %s", get_object_directory()); else - return error("unable to create temporary sha1 filename %s: %s\n", tmp_file, strerror(errno)); + return error("unable to create temporary file: %s", strerror(errno)); } /* Set it up */ - memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); stream.next_out = compressed; stream.avail_out = sizeof(compressed); @@ -2526,6 +2991,17 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, return move_temp_to_file(tmp_file, filename); } +static int freshen_loose_object(const unsigned char *sha1) +{ + return check_and_freshen(sha1, 1); +} + +static int freshen_packed_object(const unsigned char *sha1) +{ + struct pack_entry e; + return find_pack_entry(sha1, &e) && freshen_file(e.p->pack_name); +} + int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1) { unsigned char sha1[20]; @@ -2538,7 +3014,7 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); if (returnsha1) hashcpy(returnsha1, sha1); - if (has_sha1_file(sha1)) + if (freshen_loose_object(sha1) || freshen_packed_object(sha1)) return 0; return write_loose_object(sha1, hdr, hdrlen, buf, len, 0); } @@ -2584,7 +3060,10 @@ int has_sha1_file(const unsigned char *sha1) if (find_pack_entry(sha1, &e)) return 1; - return has_loose_object(sha1); + if (has_loose_object(sha1)) + return 1; + reprepare_packed_git(); + return find_pack_entry(sha1, &e); } static void check_tree(const void *buf, size_t size) @@ -2654,6 +3133,29 @@ static int index_mem(unsigned char *sha1, void *buf, size_t size, return ret; } +static int index_stream_convert_blob(unsigned char *sha1, int fd, + const char *path, unsigned flags) +{ + int ret; + const int write_object = flags & HASH_WRITE_OBJECT; + struct strbuf sbuf = STRBUF_INIT; + + assert(path); + assert(would_convert_to_git_filter_fd(path)); + + convert_to_git_filter_fd(path, fd, &sbuf, + write_object ? safe_crlf : SAFE_CRLF_FALSE); + + if (write_object) + ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB), + sha1); + else + ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB), + sha1); + strbuf_release(&sbuf); + return ret; +} + static int index_pipe(unsigned char *sha1, int fd, enum object_type type, const char *path, unsigned flags) { @@ -2719,15 +3221,22 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags) { int ret; - size_t size = xsize_t(st->st_size); - if (!S_ISREG(st->st_mode)) + /* + * Call xsize_t() only when needed to avoid potentially unnecessary + * die() for large files. + */ + if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path)) + ret = index_stream_convert_blob(sha1, fd, path, flags); + else if (!S_ISREG(st->st_mode)) ret = index_pipe(sha1, fd, type, path, flags); - else if (size <= big_file_threshold || type != OBJ_BLOB || - (path && would_convert_to_git(path, NULL, 0, 0))) - ret = index_core(sha1, fd, size, type, path, flags); + else if (st->st_size <= big_file_threshold || type != OBJ_BLOB || + (path && would_convert_to_git(path))) + ret = index_core(sha1, fd, xsize_t(st->st_size), type, path, + flags); else - ret = index_stream(sha1, fd, size, type, path, flags); + ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path, + flags); close(fd); return ret; } @@ -2792,3 +3301,167 @@ void assert_sha1_type(const unsigned char *sha1, enum object_type expect) die("%s is not a valid '%s' object", sha1_to_hex(sha1), typename(expect)); } + +static int for_each_file_in_obj_subdir(int subdir_nr, + struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + size_t baselen = path->len; + DIR *dir = opendir(path->buf); + struct dirent *de; + int r = 0; + + if (!dir) { + if (errno == ENOENT) + return 0; + return error("unable to open %s: %s", path->buf, strerror(errno)); + } + + while ((de = readdir(dir))) { + if (is_dot_or_dotdot(de->d_name)) + continue; + + strbuf_setlen(path, baselen); + strbuf_addf(path, "/%s", de->d_name); + + if (strlen(de->d_name) == 38) { + char hex[41]; + unsigned char sha1[20]; + + snprintf(hex, sizeof(hex), "%02x%s", + subdir_nr, de->d_name); + if (!get_sha1_hex(hex, sha1)) { + if (obj_cb) { + r = obj_cb(sha1, path->buf, data); + if (r) + break; + } + continue; + } + } + + if (cruft_cb) { + r = cruft_cb(de->d_name, path->buf, data); + if (r) + break; + } + } + strbuf_setlen(path, baselen); + + if (!r && subdir_cb) + r = subdir_cb(subdir_nr, path->buf, data); + + closedir(dir); + return r; +} + +int for_each_loose_file_in_objdir_buf(struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + size_t baselen = path->len; + int r = 0; + int i; + + for (i = 0; i < 256; i++) { + strbuf_addf(path, "/%02x", i); + r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb, + subdir_cb, data); + strbuf_setlen(path, baselen); + if (r) + break; + } + + return r; +} + +int for_each_loose_file_in_objdir(const char *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + struct strbuf buf = STRBUF_INIT; + int r; + + strbuf_addstr(&buf, path); + r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb, + subdir_cb, data); + strbuf_release(&buf); + + return r; +} + +struct loose_alt_odb_data { + each_loose_object_fn *cb; + void *data; +}; + +static int loose_from_alt_odb(struct alternate_object_database *alt, + void *vdata) +{ + struct loose_alt_odb_data *data = vdata; + struct strbuf buf = STRBUF_INIT; + int r; + + /* copy base not including trailing '/' */ + strbuf_add(&buf, alt->base, alt->name - alt->base - 1); + r = for_each_loose_file_in_objdir_buf(&buf, + data->cb, NULL, NULL, + data->data); + strbuf_release(&buf); + return r; +} + +int for_each_loose_object(each_loose_object_fn cb, void *data) +{ + struct loose_alt_odb_data alt; + int r; + + r = for_each_loose_file_in_objdir(get_object_directory(), + cb, NULL, NULL, data); + if (r) + return r; + + alt.cb = cb; + alt.data = data; + return foreach_alt_odb(loose_from_alt_odb, &alt); +} + +static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data) +{ + uint32_t i; + int r = 0; + + for (i = 0; i < p->num_objects; i++) { + const unsigned char *sha1 = nth_packed_object_sha1(p, i); + + if (!sha1) + return error("unable to get sha1 of object %u in %s", + i, p->pack_name); + + r = cb(sha1, p, i, data); + if (r) + break; + } + return r; +} + +int for_each_packed_object(each_packed_object_fn cb, void *data) +{ + struct packed_git *p; + int r = 0; + + prepare_packed_git(); + for (p = packed_git; p; p = p->next) { + r = for_each_object_in_pack(p, cb, data); + if (r) + break; + } + return r; +} |