summaryrefslogtreecommitdiff
path: root/sha1_file.c
diff options
context:
space:
mode:
Diffstat (limited to 'sha1_file.c')
-rw-r--r--sha1_file.c1751
1 files changed, 1195 insertions, 556 deletions
diff --git a/sha1_file.c b/sha1_file.c
index daacc0cfb0..9c86d1924a 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -8,6 +8,7 @@
*/
#include "cache.h"
#include "string-list.h"
+#include "lockfile.h"
#include "delta.h"
#include "pack.h"
#include "blob.h"
@@ -22,6 +23,9 @@
#include "bulk-checkin.h"
#include "streaming.h"
#include "dir.h"
+#include "mru.h"
+#include "list.h"
+#include "mergesort.h"
#ifndef O_NOATIME
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
@@ -35,9 +39,13 @@
static inline uintmax_t sz_fmt(size_t s) { return s; }
const unsigned char null_sha1[20];
-
-static const char *no_log_pack_access = "no_log_pack_access";
-static const char *log_pack_access;
+const struct object_id null_oid;
+const struct object_id empty_tree_oid = {
+ EMPTY_TREE_SHA1_BIN_LITERAL
+};
+const struct object_id empty_blob_oid = {
+ EMPTY_BLOB_SHA1_BIN_LITERAL
+};
/*
* This is meant to hold a *small* number of objects that you would
@@ -60,8 +68,6 @@ static struct cached_object empty_tree = {
0
};
-static struct packed_git *last_found_pack;
-
static struct cached_object *find_cached_object(const unsigned char *sha1)
{
int i;
@@ -105,141 +111,163 @@ int mkdir_in_gitdir(const char *path)
return adjust_shared_perm(path);
}
-int safe_create_leading_directories(char *path)
+enum scld_error safe_create_leading_directories(char *path)
{
- char *pos = path + offset_1st_component(path);
- struct stat st;
+ char *next_component = path + offset_1st_component(path);
+ enum scld_error ret = SCLD_OK;
+
+ while (ret == SCLD_OK && next_component) {
+ struct stat st;
+ char *slash = next_component, slash_character;
+
+ while (*slash && !is_dir_sep(*slash))
+ slash++;
- while (pos) {
- pos = strchr(pos, '/');
- if (!pos)
+ if (!*slash)
break;
- while (*++pos == '/')
- ;
- if (!*pos)
+
+ next_component = slash + 1;
+ while (is_dir_sep(*next_component))
+ next_component++;
+ if (!*next_component)
break;
- *--pos = '\0';
+
+ slash_character = *slash;
+ *slash = '\0';
if (!stat(path, &st)) {
/* path exists */
- if (!S_ISDIR(st.st_mode)) {
- *pos = '/';
- return -3;
- }
- }
- else if (mkdir(path, 0777)) {
+ if (!S_ISDIR(st.st_mode))
+ ret = SCLD_EXISTS;
+ } else if (mkdir(path, 0777)) {
if (errno == EEXIST &&
- !stat(path, &st) && S_ISDIR(st.st_mode)) {
+ !stat(path, &st) && S_ISDIR(st.st_mode))
; /* somebody created it since we checked */
- } else {
- *pos = '/';
- return -1;
- }
- }
- else if (adjust_shared_perm(path)) {
- *pos = '/';
- return -2;
+ else if (errno == ENOENT)
+ /*
+ * Either mkdir() failed because
+ * somebody just pruned the containing
+ * directory, or stat() failed because
+ * the file that was in our way was
+ * just removed. Either way, inform
+ * the caller that it might be worth
+ * trying again:
+ */
+ ret = SCLD_VANISHED;
+ else
+ ret = SCLD_FAILED;
+ } else if (adjust_shared_perm(path)) {
+ ret = SCLD_PERMS;
}
- *pos++ = '/';
+ *slash = slash_character;
}
- return 0;
+ return ret;
}
-int safe_create_leading_directories_const(const char *path)
+enum scld_error safe_create_leading_directories_const(const char *path)
{
/* path points to cache entries, so xstrdup before messing with it */
char *buf = xstrdup(path);
- int result = safe_create_leading_directories(buf);
+ enum scld_error result = safe_create_leading_directories(buf);
free(buf);
return result;
}
-static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
+static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
{
int i;
for (i = 0; i < 20; i++) {
static char hex[] = "0123456789abcdef";
unsigned int val = sha1[i];
- char *pos = pathbuf + i*2 + (i > 0);
- *pos++ = hex[val >> 4];
- *pos = hex[val & 0xf];
+ strbuf_addch(buf, hex[val >> 4]);
+ strbuf_addch(buf, hex[val & 0xf]);
+ if (!i)
+ strbuf_addch(buf, '/');
}
}
-/*
- * NOTE! This returns a statically allocated buffer, so you have to be
- * careful about using it. Do an "xstrdup()" if you need to save the
- * filename.
- *
- * Also note that this returns the location for creating. Reading
- * SHA1 file can happen from any alternate directory listed in the
- * DB_ENVIRONMENT environment variable if it is not found in
- * the primary object database.
- */
-char *sha1_file_name(const unsigned char *sha1)
+const char *sha1_file_name(const unsigned char *sha1)
{
- static char buf[PATH_MAX];
- const char *objdir;
- int len;
+ static struct strbuf buf = STRBUF_INIT;
- objdir = get_object_directory();
- len = strlen(objdir);
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "%s/", get_object_directory());
- /* '/' + sha1(2) + '/' + sha1(38) + '\0' */
- if (len + 43 > PATH_MAX)
- die("insanely long object directory %s", objdir);
- memcpy(buf, objdir, len);
- buf[len] = '/';
- buf[len+3] = '/';
- buf[len+42] = '\0';
- fill_sha1_path(buf + len + 1, sha1);
- return buf;
+ fill_sha1_path(&buf, sha1);
+ return buf.buf;
}
-static char *sha1_get_pack_name(const unsigned char *sha1,
- char **name, char **base, const char *which)
+struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
{
- static const char hex[] = "0123456789abcdef";
- char *buf;
- int i;
-
- if (!*base) {
- const char *sha1_file_directory = get_object_directory();
- int len = strlen(sha1_file_directory);
- *base = xmalloc(len + 60);
- sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s",
- sha1_file_directory, which);
- *name = *base + len + 11;
- }
-
- buf = *name;
+ strbuf_setlen(&alt->scratch, alt->base_len);
+ return &alt->scratch;
+}
- for (i = 0; i < 20; i++) {
- unsigned int val = *sha1++;
- *buf++ = hex[val >> 4];
- *buf++ = hex[val & 0xf];
- }
+static const char *alt_sha1_path(struct alternate_object_database *alt,
+ const unsigned char *sha1)
+{
+ struct strbuf *buf = alt_scratch_buf(alt);
+ fill_sha1_path(buf, sha1);
+ return buf->buf;
+}
- return *base;
+/*
+ * Return the name of the pack or index file with the specified sha1
+ * in its filename. *base and *name are scratch space that must be
+ * provided by the caller. which should be "pack" or "idx".
+ */
+static char *sha1_get_pack_name(const unsigned char *sha1,
+ struct strbuf *buf,
+ const char *which)
+{
+ strbuf_reset(buf);
+ strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
+ sha1_to_hex(sha1), which);
+ return buf->buf;
}
char *sha1_pack_name(const unsigned char *sha1)
{
- static char *name, *base;
-
- return sha1_get_pack_name(sha1, &name, &base, "pack");
+ static struct strbuf buf = STRBUF_INIT;
+ return sha1_get_pack_name(sha1, &buf, "pack");
}
char *sha1_pack_index_name(const unsigned char *sha1)
{
- static char *name, *base;
-
- return sha1_get_pack_name(sha1, &name, &base, "idx");
+ static struct strbuf buf = STRBUF_INIT;
+ return sha1_get_pack_name(sha1, &buf, "idx");
}
struct alternate_object_database *alt_odb_list;
static struct alternate_object_database **alt_odb_tail;
-static int git_open_noatime(const char *name);
+/*
+ * Return non-zero iff the path is usable as an alternate object database.
+ */
+static int alt_odb_usable(struct strbuf *path, const char *normalized_objdir)
+{
+ struct alternate_object_database *alt;
+
+ /* Detect cases where alternate disappeared */
+ if (!is_directory(path->buf)) {
+ error("object directory %s does not exist; "
+ "check .git/objects/info/alternates.",
+ path->buf);
+ return 0;
+ }
+
+ /*
+ * Prevent the common mistake of listing the same
+ * thing twice, or object directory itself.
+ */
+ for (alt = alt_odb_list; alt; alt = alt->next) {
+ if (!fspathcmp(path->buf, alt->path))
+ return 0;
+ }
+ if (!fspathcmp(path->buf, normalized_objdir))
+ return 0;
+
+ return 1;
+}
/*
* Prepare alternate object database registry.
@@ -256,12 +284,10 @@ static int git_open_noatime(const char *name);
* SHA1, an extra slash for the first level indirection, and the
* terminating NUL.
*/
-static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth)
+static int link_alt_odb_entry(const char *entry, const char *relative_base,
+ int depth, const char *normalized_objdir)
{
- const char *objdir = get_object_directory();
struct alternate_object_database *ent;
- struct alternate_object_database *alt;
- int pfxlen, entlen;
struct strbuf pathbuf = STRBUF_INIT;
if (!is_absolute_path(entry) && relative_base) {
@@ -270,48 +296,26 @@ static int link_alt_odb_entry(const char *entry, const char *relative_base, int
}
strbuf_addstr(&pathbuf, entry);
- normalize_path_copy(pathbuf.buf, pathbuf.buf);
-
- pfxlen = strlen(pathbuf.buf);
+ if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
+ error("unable to normalize alternate object path: %s",
+ pathbuf.buf);
+ strbuf_release(&pathbuf);
+ return -1;
+ }
/*
* The trailing slash after the directory name is given by
* this function at the end. Remove duplicates.
*/
- while (pfxlen && pathbuf.buf[pfxlen-1] == '/')
- pfxlen -= 1;
-
- entlen = pfxlen + 43; /* '/' + 2 hex + '/' + 38 hex + NUL */
- ent = xmalloc(sizeof(*ent) + entlen);
- memcpy(ent->base, pathbuf.buf, pfxlen);
- strbuf_release(&pathbuf);
+ while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
+ strbuf_setlen(&pathbuf, pathbuf.len - 1);
- ent->name = ent->base + pfxlen + 1;
- ent->base[pfxlen + 3] = '/';
- ent->base[pfxlen] = ent->base[entlen-1] = 0;
-
- /* Detect cases where alternate disappeared */
- if (!is_directory(ent->base)) {
- error("object directory %s does not exist; "
- "check .git/objects/info/alternates.",
- ent->base);
- free(ent);
+ if (!alt_odb_usable(&pathbuf, normalized_objdir)) {
+ strbuf_release(&pathbuf);
return -1;
}
- /* Prevent the common mistake of listing the same
- * thing twice, or object directory itself.
- */
- for (alt = alt_odb_list; alt; alt = alt->next) {
- if (!memcmp(ent->base, alt->base, pfxlen)) {
- free(ent);
- return -1;
- }
- }
- if (!strcmp(ent->base, objdir)) {
- free(ent);
- return -1;
- }
+ ent = alloc_alt_odb(pathbuf.buf);
/* add the alternate entry */
*alt_odb_tail = ent;
@@ -319,10 +323,9 @@ static int link_alt_odb_entry(const char *entry, const char *relative_base, int
ent->next = NULL;
/* recursively add alternates */
- read_info_alternates(ent->base, depth + 1);
-
- ent->base[pfxlen] = '/';
+ read_info_alternates(pathbuf.buf, depth + 1);
+ strbuf_release(&pathbuf);
return 0;
}
@@ -332,6 +335,7 @@ static void link_alt_odb_entries(const char *alt, int len, int sep,
struct string_list entries = STRING_LIST_INIT_NODUP;
char *alt_copy;
int i;
+ struct strbuf objdirbuf = STRBUF_INIT;
if (depth > 5) {
error("%s: ignoring alternate object stores, nesting too deep.",
@@ -339,21 +343,22 @@ static void link_alt_odb_entries(const char *alt, int len, int sep,
return;
}
+ strbuf_add_absolute_path(&objdirbuf, get_object_directory());
+ if (strbuf_normalize_path(&objdirbuf) < 0)
+ die("unable to normalize object directory: %s",
+ objdirbuf.buf);
+
alt_copy = xmemdupz(alt, len);
string_list_split_in_place(&entries, alt_copy, sep, -1);
for (i = 0; i < entries.nr; i++) {
const char *entry = entries.items[i].string;
if (entry[0] == '\0' || entry[0] == '#')
continue;
- if (!is_absolute_path(entry) && depth) {
- error("%s: ignoring relative alternate object store %s",
- relative_base, entry);
- } else {
- link_alt_odb_entry(entry, relative_base, depth);
- }
+ link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
}
string_list_clear(&entries, 0);
free(alt_copy);
+ strbuf_release(&objdirbuf);
}
void read_info_alternates(const char * relative_base, int depth)
@@ -361,15 +366,12 @@ void read_info_alternates(const char * relative_base, int depth)
char *map;
size_t mapsz;
struct stat st;
- const char alt_file_name[] = "info/alternates";
- /* Given that relative_base is no longer than PATH_MAX,
- ensure that "path" has enough space to append "/", the
- file name, "info/alternates", and a trailing NUL. */
- char path[PATH_MAX + 1 + sizeof alt_file_name];
+ char *path;
int fd;
- sprintf(path, "%s/%s", relative_base, alt_file_name);
- fd = git_open_noatime(path);
+ path = xstrfmt("%s/info/alternates", relative_base);
+ fd = git_open(path);
+ free(path);
if (fd < 0)
return;
if (fstat(fd, &st) || (st.st_size == 0)) {
@@ -385,26 +387,162 @@ void read_info_alternates(const char * relative_base, int depth)
munmap(map, mapsz);
}
+struct alternate_object_database *alloc_alt_odb(const char *dir)
+{
+ struct alternate_object_database *ent;
+
+ FLEX_ALLOC_STR(ent, path, dir);
+ strbuf_init(&ent->scratch, 0);
+ strbuf_addf(&ent->scratch, "%s/", dir);
+ ent->base_len = ent->scratch.len;
+
+ return ent;
+}
+
void add_to_alternates_file(const char *reference)
{
struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
- int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR);
- char *alt = mkpath("%s\n", reference);
- write_or_die(fd, alt, strlen(alt));
- if (commit_lock_file(lock))
- die("could not close alternates file");
- if (alt_odb_tail)
- link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
+ char *alts = git_pathdup("objects/info/alternates");
+ FILE *in, *out;
+
+ hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
+ out = fdopen_lock_file(lock, "w");
+ if (!out)
+ die_errno("unable to fdopen alternates lockfile");
+
+ in = fopen(alts, "r");
+ if (in) {
+ struct strbuf line = STRBUF_INIT;
+ int found = 0;
+
+ while (strbuf_getline(&line, in) != EOF) {
+ if (!strcmp(reference, line.buf)) {
+ found = 1;
+ break;
+ }
+ fprintf_or_die(out, "%s\n", line.buf);
+ }
+
+ strbuf_release(&line);
+ fclose(in);
+
+ if (found) {
+ rollback_lock_file(lock);
+ lock = NULL;
+ }
+ }
+ else if (errno != ENOENT)
+ die_errno("unable to read alternates file");
+
+ if (lock) {
+ fprintf_or_die(out, "%s\n", reference);
+ if (commit_lock_file(lock))
+ die_errno("unable to move new alternates file into place");
+ if (alt_odb_tail)
+ link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
+ }
+ free(alts);
+}
+
+void add_to_alternates_memory(const char *reference)
+{
+ /*
+ * Make sure alternates are initialized, or else our entry may be
+ * overwritten when they are.
+ */
+ prepare_alt_odb();
+
+ link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
+}
+
+/*
+ * Compute the exact path an alternate is at and returns it. In case of
+ * error NULL is returned and the human readable error is added to `err`
+ * `path` may be relative and should point to $GITDIR.
+ * `err` must not be null.
+ */
+char *compute_alternate_path(const char *path, struct strbuf *err)
+{
+ char *ref_git = NULL;
+ const char *repo, *ref_git_s;
+ int seen_error = 0;
+
+ ref_git_s = real_path_if_valid(path);
+ if (!ref_git_s) {
+ seen_error = 1;
+ strbuf_addf(err, _("path '%s' does not exist"), path);
+ goto out;
+ } else
+ /*
+ * Beware: read_gitfile(), real_path() and mkpath()
+ * return static buffer
+ */
+ ref_git = xstrdup(ref_git_s);
+
+ repo = read_gitfile(ref_git);
+ if (!repo)
+ repo = read_gitfile(mkpath("%s/.git", ref_git));
+ if (repo) {
+ free(ref_git);
+ ref_git = xstrdup(repo);
+ }
+
+ if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
+ char *ref_git_git = mkpathdup("%s/.git", ref_git);
+ free(ref_git);
+ ref_git = ref_git_git;
+ } else if (!is_directory(mkpath("%s/objects", ref_git))) {
+ struct strbuf sb = STRBUF_INIT;
+ seen_error = 1;
+ if (get_common_dir(&sb, ref_git)) {
+ strbuf_addf(err,
+ _("reference repository '%s' as a linked "
+ "checkout is not supported yet."),
+ path);
+ goto out;
+ }
+
+ strbuf_addf(err, _("reference repository '%s' is not a "
+ "local repository."), path);
+ goto out;
+ }
+
+ if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
+ strbuf_addf(err, _("reference repository '%s' is shallow"),
+ path);
+ seen_error = 1;
+ goto out;
+ }
+
+ if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
+ strbuf_addf(err,
+ _("reference repository '%s' is grafted"),
+ path);
+ seen_error = 1;
+ goto out;
+ }
+
+out:
+ if (seen_error) {
+ free(ref_git);
+ ref_git = NULL;
+ }
+
+ return ref_git;
}
-void foreach_alt_odb(alt_odb_fn fn, void *cb)
+int foreach_alt_odb(alt_odb_fn fn, void *cb)
{
struct alternate_object_database *ent;
+ int r = 0;
prepare_alt_odb();
- for (ent = alt_odb_list; ent; ent = ent->next)
- if (fn(ent, cb))
- return;
+ for (ent = alt_odb_list; ent; ent = ent->next) {
+ r = fn(ent, cb);
+ if (r)
+ break;
+ }
+ return r;
}
void prepare_alt_odb(void)
@@ -423,28 +561,61 @@ void prepare_alt_odb(void)
read_info_alternates(get_object_directory(), 0);
}
-static int has_loose_object_local(const unsigned char *sha1)
+/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
+static int freshen_file(const char *fn)
{
- char *name = sha1_file_name(sha1);
- return !access(name, F_OK);
+ struct utimbuf t;
+ t.actime = t.modtime = time(NULL);
+ return !utime(fn, &t);
}
-int has_loose_object_nonlocal(const unsigned char *sha1)
+/*
+ * All of the check_and_freshen functions return 1 if the file exists and was
+ * freshened (if freshening was requested), 0 otherwise. If they return
+ * 0, you should not assume that it is safe to skip a write of the object (it
+ * either does not exist on disk, or has a stale mtime and may be subject to
+ * pruning).
+ */
+static int check_and_freshen_file(const char *fn, int freshen)
+{
+ if (access(fn, F_OK))
+ return 0;
+ if (freshen && !freshen_file(fn))
+ return 0;
+ return 1;
+}
+
+static int check_and_freshen_local(const unsigned char *sha1, int freshen)
+{
+ return check_and_freshen_file(sha1_file_name(sha1), freshen);
+}
+
+static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
{
struct alternate_object_database *alt;
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
- fill_sha1_path(alt->name, sha1);
- if (!access(alt->base, F_OK))
+ const char *path = alt_sha1_path(alt, sha1);
+ if (check_and_freshen_file(path, freshen))
return 1;
}
return 0;
}
+static int check_and_freshen(const unsigned char *sha1, int freshen)
+{
+ return check_and_freshen_local(sha1, freshen) ||
+ check_and_freshen_nonlocal(sha1, freshen);
+}
+
+int has_loose_object_nonlocal(const unsigned char *sha1)
+{
+ return check_and_freshen_nonlocal(sha1, 0);
+}
+
static int has_loose_object(const unsigned char *sha1)
{
- return has_loose_object_local(sha1) ||
- has_loose_object_nonlocal(sha1);
+ return check_and_freshen(sha1, 0);
}
static unsigned int pack_used_ctr;
@@ -457,6 +628,9 @@ static size_t peak_pack_mapped;
static size_t pack_mapped;
struct packed_git *packed_git;
+static struct mru packed_git_mru_storage;
+struct mru *packed_git_mru = &packed_git_mru_storage;
+
void pack_report(void)
{
fprintf(stderr,
@@ -478,13 +652,18 @@ void pack_report(void)
sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
}
-static int check_packed_git_idx(const char *path, struct packed_git *p)
+/*
+ * Open and mmap the index file at path, perform a couple of
+ * consistency checks, then record its information to p. Return 0 on
+ * success.
+ */
+static int check_packed_git_idx(const char *path, struct packed_git *p)
{
void *idx_map;
struct pack_idx_header *hdr;
size_t idx_size;
uint32_t version, nr, i, *index;
- int fd = git_open_noatime(path);
+ int fd = git_open(path);
struct stat st;
if (fd < 0)
@@ -583,13 +762,15 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
int open_pack_index(struct packed_git *p)
{
char *idx_name;
+ size_t len;
int ret;
if (p->index_data)
return 0;
- idx_name = xstrdup(p->pack_name);
- strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
+ if (!strip_suffix(p->pack_name, ".pack", &len))
+ die("BUG: pack_name does not end in .pack");
+ idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
ret = check_packed_git_idx(idx_name, p);
free(idx_name);
return ret;
@@ -644,21 +825,44 @@ void release_pack_memory(size_t need)
; /* nothing */
}
-void *xmmap(void *start, size_t length,
- int prot, int flags, int fd, off_t offset)
+static void mmap_limit_check(size_t length)
{
- void *ret = mmap(start, length, prot, flags, fd, offset);
+ static size_t limit = 0;
+ if (!limit) {
+ limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
+ if (!limit)
+ limit = SIZE_MAX;
+ }
+ if (length > limit)
+ die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
+ (uintmax_t)length, (uintmax_t)limit);
+}
+
+void *xmmap_gently(void *start, size_t length,
+ int prot, int flags, int fd, off_t offset)
+{
+ void *ret;
+
+ mmap_limit_check(length);
+ ret = mmap(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED) {
if (!length)
return NULL;
release_pack_memory(length);
ret = mmap(start, length, prot, flags, fd, offset);
- if (ret == MAP_FAILED)
- die_errno("Out of memory? mmap failed");
}
return ret;
}
+void *xmmap(void *start, size_t length,
+ int prot, int flags, int fd, off_t offset)
+{
+ void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
+ if (ret == MAP_FAILED)
+ die_errno("mmap failed");
+ return ret;
+}
+
void close_pack_windows(struct packed_git *p)
{
while (p->windows) {
@@ -675,6 +879,37 @@ void close_pack_windows(struct packed_git *p)
}
}
+static int close_pack_fd(struct packed_git *p)
+{
+ if (p->pack_fd < 0)
+ return 0;
+
+ close(p->pack_fd);
+ pack_open_fds--;
+ p->pack_fd = -1;
+
+ return 1;
+}
+
+static void close_pack(struct packed_git *p)
+{
+ close_pack_windows(p);
+ close_pack_fd(p);
+ close_pack_index(p);
+}
+
+void close_all_packs(void)
+{
+ struct packed_git *p;
+
+ for (p = packed_git; p; p = p->next)
+ if (p->do_not_close)
+ die("BUG: want to close pack marked 'do-not-close'");
+ else
+ close_pack(p);
+}
+
+
/*
* The LRU pack is the one with the oldest MRU window, preferring packs
* with no used windows, or the oldest mtime if it has no windows allocated.
@@ -742,12 +977,8 @@ static int close_one_pack(void)
find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
}
- if (lru_p) {
- close(lru_p->pack_fd);
- pack_open_fds--;
- lru_p->pack_fd = -1;
- return 1;
- }
+ if (lru_p)
+ return close_pack_fd(lru_p);
return 0;
}
@@ -769,53 +1000,41 @@ void close_pack_index(struct packed_git *p)
}
}
-/*
- * This is used by git-repack in case a newly created pack happens to
- * contain the same set of objects as an existing one. In that case
- * the resulting file might be different even if its name would be the
- * same. It is best to close any reference to the old pack before it is
- * replaced on disk. Of course no index pointers nor windows for given pack
- * must subsist at this point. If ever objects from this pack are requested
- * again, the new version of the pack will be reinitialized through
- * reprepare_packed_git().
- */
-void free_pack_by_name(const char *pack_name)
-{
- struct packed_git *p, **pp = &packed_git;
-
- while (*pp) {
- p = *pp;
- if (strcmp(pack_name, p->pack_name) == 0) {
- clear_delta_base_cache();
- close_pack_windows(p);
- if (p->pack_fd != -1) {
- close(p->pack_fd);
- pack_open_fds--;
- }
- close_pack_index(p);
- free(p->bad_object_sha1);
- *pp = p->next;
- if (last_found_pack == p)
- last_found_pack = NULL;
- free(p);
- return;
- }
- pp = &p->next;
- }
-}
-
static unsigned int get_max_fd_limit(void)
{
#ifdef RLIMIT_NOFILE
- struct rlimit lim;
+ {
+ struct rlimit lim;
+
+ if (!getrlimit(RLIMIT_NOFILE, &lim))
+ return lim.rlim_cur;
+ }
+#endif
- if (getrlimit(RLIMIT_NOFILE, &lim))
- die_errno("cannot get RLIMIT_NOFILE");
+#ifdef _SC_OPEN_MAX
+ {
+ long open_max = sysconf(_SC_OPEN_MAX);
+ if (0 < open_max)
+ return open_max;
+ /*
+ * Otherwise, we got -1 for one of the two
+ * reasons:
+ *
+ * (1) sysconf() did not understand _SC_OPEN_MAX
+ * and signaled an error with -1; or
+ * (2) sysconf() said there is no limit.
+ *
+ * We _could_ clear errno before calling sysconf() to
+ * tell these two cases apart and return a huge number
+ * in the latter case to let the caller cap it to a
+ * value that is not so selfish, but letting the
+ * fallback OPEN_MAX codepath take care of these cases
+ * is a lot simpler.
+ */
+ }
+#endif
- return lim.rlim_cur;
-#elif defined(_SC_OPEN_MAX)
- return sysconf(_SC_OPEN_MAX);
-#elif defined(OPEN_MAX)
+#ifdef OPEN_MAX
return OPEN_MAX;
#else
return 1; /* see the caller ;-) */
@@ -850,7 +1069,7 @@ static int open_packed_git_1(struct packed_git *p)
while (pack_max_fds <= pack_open_fds && close_one_pack())
; /* nothing */
- p->pack_fd = git_open_noatime(p->pack_name);
+ p->pack_fd = git_open(p->pack_name);
if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
return -1;
pack_open_fds++;
@@ -903,11 +1122,7 @@ static int open_packed_git(struct packed_git *p)
{
if (!open_packed_git_1(p))
return 0;
- if (p->pack_fd != -1) {
- close(p->pack_fd);
- pack_open_fds--;
- p->pack_fd = -1;
- }
+ close_pack_fd(p);
return -1;
}
@@ -940,6 +1155,8 @@ unsigned char *use_pack(struct packed_git *p,
die("packfile %s cannot be accessed", p->pack_name);
if (offset > (p->pack_size - 20))
die("offset beyond end of packfile (truncated pack?)");
+ if (offset < 0)
+ die(_("offset before end of packfile (broken .idx?)"));
if (!win || !in_window(win, offset)) {
if (win)
@@ -969,15 +1186,11 @@ unsigned char *use_pack(struct packed_git *p,
PROT_READ, MAP_PRIVATE,
p->pack_fd, win->offset);
if (win->base == MAP_FAILED)
- die("packfile %s cannot be mapped: %s",
- p->pack_name,
- strerror(errno));
+ die_errno("packfile %s cannot be mapped",
+ p->pack_name);
if (!win->offset && win->len == p->pack_size
- && !p->do_not_close) {
- close(p->pack_fd);
- pack_open_fds--;
- p->pack_fd = -1;
- }
+ && !p->do_not_close)
+ close_pack_fd(p);
pack_mmap_calls++;
pack_open_windows++;
if (pack_mapped > peak_pack_mapped)
@@ -1001,7 +1214,7 @@ unsigned char *use_pack(struct packed_git *p,
static struct packed_git *alloc_packed_git(int extra)
{
- struct packed_git *p = xmalloc(sizeof(*p) + extra);
+ struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
memset(p, 0, sizeof(*p));
p->pack_fd = -1;
return p;
@@ -1012,11 +1225,12 @@ static void try_to_free_pack_memory(size_t size)
release_pack_memory(size);
}
-struct packed_git *add_packed_git(const char *path, int path_len, int local)
+struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
{
static int have_set_try_to_free_routine;
struct stat st;
- struct packed_git *p = alloc_packed_git(path_len + 2);
+ size_t alloc;
+ struct packed_git *p;
if (!have_set_try_to_free_routine) {
have_set_try_to_free_routine = 1;
@@ -1027,18 +1241,22 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
* Make sure a corresponding .pack file exists and that
* the index looks sane.
*/
- path_len -= strlen(".idx");
- if (path_len < 1) {
- free(p);
+ if (!strip_suffix_mem(path, &path_len, ".idx"))
return NULL;
- }
+
+ /*
+ * ".pack" is long enough to hold any suffix we're adding (and
+ * the use xsnprintf double-checks that)
+ */
+ alloc = st_add3(path_len, strlen(".pack"), 1);
+ p = alloc_packed_git(alloc);
memcpy(p->pack_name, path, path_len);
- strcpy(p->pack_name + path_len, ".keep");
+ xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
if (!access(p->pack_name, F_OK))
p->pack_keep = 1;
- strcpy(p->pack_name + path_len, ".pack");
+ xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
free(p);
return NULL;
@@ -1058,9 +1276,10 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
{
const char *path = sha1_pack_name(sha1);
- struct packed_git *p = alloc_packed_git(strlen(path) + 1);
+ size_t alloc = st_add(strlen(path), 1);
+ struct packed_git *p = alloc_packed_git(alloc);
- strcpy(p->pack_name, path);
+ memcpy(p->pack_name, path, alloc); /* includes NUL */
hashcpy(p->sha1, sha1);
if (check_packed_git_idx(idx_path, p)) {
free(p);
@@ -1079,27 +1298,16 @@ void install_packed_git(struct packed_git *pack)
packed_git = pack;
}
-void (*report_garbage)(const char *desc, const char *path);
+void (*report_garbage)(unsigned seen_bits, const char *path);
static void report_helper(const struct string_list *list,
int seen_bits, int first, int last)
{
- const char *msg;
- switch (seen_bits) {
- case 0:
- msg = "no corresponding .idx nor .pack";
- break;
- case 1:
- msg = "no corresponding .idx";
- break;
- case 2:
- msg = "no corresponding .pack";
- break;
- default:
+ if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
return;
- }
+
for (; first < last; first++)
- report_garbage(msg, list->items[first].string);
+ report_garbage(seen_bits, list->items[first].string);
}
static void report_pack_garbage(struct string_list *list)
@@ -1109,7 +1317,7 @@ static void report_pack_garbage(struct string_list *list)
if (!report_garbage)
return;
- sort_string_list(list);
+ string_list_sort(list);
for (i = 0; i < list->nr; i++) {
const char *path = list->items[i].string;
@@ -1122,7 +1330,7 @@ static void report_pack_garbage(struct string_list *list)
if (baselen == -1) {
const char *dot = strrchr(path, '.');
if (!dot) {
- report_garbage("garbage found", path);
+ report_garbage(PACKDIR_FILE_GARBAGE, path);
continue;
}
baselen = dot - path + 1;
@@ -1138,48 +1346,42 @@ static void report_pack_garbage(struct string_list *list)
static void prepare_packed_git_one(char *objdir, int local)
{
- /* Ensure that this buffer is large enough so that we can
- append "/pack/" without clobbering the stack even if
- strlen(objdir) were PATH_MAX. */
- char path[PATH_MAX + 1 + 4 + 1 + 1];
- int len;
+ struct strbuf path = STRBUF_INIT;
+ size_t dirnamelen;
DIR *dir;
struct dirent *de;
struct string_list garbage = STRING_LIST_INIT_DUP;
- sprintf(path, "%s/pack", objdir);
- len = strlen(path);
- dir = opendir(path);
+ strbuf_addstr(&path, objdir);
+ strbuf_addstr(&path, "/pack");
+ dir = opendir(path.buf);
if (!dir) {
if (errno != ENOENT)
- error("unable to open object pack directory: %s: %s",
- path, strerror(errno));
+ error_errno("unable to open object pack directory: %s",
+ path.buf);
+ strbuf_release(&path);
return;
}
- path[len++] = '/';
+ strbuf_addch(&path, '/');
+ dirnamelen = path.len;
while ((de = readdir(dir)) != NULL) {
- int namelen = strlen(de->d_name);
struct packed_git *p;
-
- if (len + namelen + 1 > sizeof(path)) {
- if (report_garbage) {
- struct strbuf sb = STRBUF_INIT;
- strbuf_addf(&sb, "%.*s/%s", len - 1, path, de->d_name);
- report_garbage("path too long", sb.buf);
- strbuf_release(&sb);
- }
- continue;
- }
+ size_t base_len;
if (is_dot_or_dotdot(de->d_name))
continue;
- strcpy(path + len, de->d_name);
+ strbuf_setlen(&path, dirnamelen);
+ strbuf_addstr(&path, de->d_name);
- if (has_extension(de->d_name, ".idx")) {
+ base_len = path.len;
+ if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
/* Don't reopen a pack we already have. */
for (p = packed_git; p; p = p->next) {
- if (!memcmp(path, p->pack_name, len + namelen - 4))
+ size_t len;
+ if (strip_suffix(p->pack_name, ".pack", &len) &&
+ len == base_len &&
+ !memcmp(p->pack_name, path.buf, len))
break;
}
if (p == NULL &&
@@ -1187,29 +1389,67 @@ static void prepare_packed_git_one(char *objdir, int local)
* See if it really is a valid .idx file with
* corresponding .pack file that we can map.
*/
- (p = add_packed_git(path, len + namelen, local)) != NULL)
+ (p = add_packed_git(path.buf, path.len, local)) != NULL)
install_packed_git(p);
}
if (!report_garbage)
continue;
- if (has_extension(de->d_name, ".idx") ||
- has_extension(de->d_name, ".pack") ||
- has_extension(de->d_name, ".keep"))
- string_list_append(&garbage, path);
+ if (ends_with(de->d_name, ".idx") ||
+ ends_with(de->d_name, ".pack") ||
+ ends_with(de->d_name, ".bitmap") ||
+ ends_with(de->d_name, ".keep"))
+ string_list_append(&garbage, path.buf);
else
- report_garbage("garbage found", path);
+ report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
}
closedir(dir);
report_pack_garbage(&garbage);
string_list_clear(&garbage, 0);
+ strbuf_release(&path);
+}
+
+static int approximate_object_count_valid;
+
+/*
+ * Give a fast, rough count of the number of objects in the repository. This
+ * ignores loose objects completely. If you have a lot of them, then either
+ * you should repack because your performance will be awful, or they are
+ * all unreachable objects about to be pruned, in which case they're not really
+ * interesting as a measure of repo size in the first place.
+ */
+unsigned long approximate_object_count(void)
+{
+ static unsigned long count;
+ if (!approximate_object_count_valid) {
+ struct packed_git *p;
+
+ prepare_packed_git();
+ count = 0;
+ for (p = packed_git; p; p = p->next) {
+ if (open_pack_index(p))
+ continue;
+ count += p->num_objects;
+ }
+ }
+ return count;
+}
+
+static void *get_next_packed_git(const void *p)
+{
+ return ((const struct packed_git *)p)->next;
+}
+
+static void set_next_packed_git(void *p, void *next)
+{
+ ((struct packed_git *)p)->next = next;
}
static int sort_pack(const void *a_, const void *b_)
{
- struct packed_git *a = *((struct packed_git **)a_);
- struct packed_git *b = *((struct packed_git **)b_);
+ const struct packed_git *a = a_;
+ const struct packed_git *b = b_;
int st;
/*
@@ -1236,28 +1476,17 @@ static int sort_pack(const void *a_, const void *b_)
static void rearrange_packed_git(void)
{
- struct packed_git **ary, *p;
- int i, n;
-
- for (n = 0, p = packed_git; p; p = p->next)
- n++;
- if (n < 2)
- return;
-
- /* prepare an array of packed_git for easier sorting */
- ary = xcalloc(n, sizeof(struct packed_git *));
- for (n = 0, p = packed_git; p; p = p->next)
- ary[n++] = p;
-
- qsort(ary, n, sizeof(struct packed_git *), sort_pack);
+ packed_git = llist_mergesort(packed_git, get_next_packed_git,
+ set_next_packed_git, sort_pack);
+}
- /* link them back again */
- for (i = 0; i < n - 1; i++)
- ary[i]->next = ary[i + 1];
- ary[n - 1]->next = NULL;
- packed_git = ary[0];
+static void prepare_packed_git_mru(void)
+{
+ struct packed_git *p;
- free(ary);
+ mru_clear(packed_git_mru);
+ for (p = packed_git; p; p = p->next)
+ mru_append(packed_git_mru, p);
}
static int prepare_packed_git_run_once = 0;
@@ -1269,18 +1498,16 @@ void prepare_packed_git(void)
return;
prepare_packed_git_one(get_object_directory(), 1);
prepare_alt_odb();
- for (alt = alt_odb_list; alt; alt = alt->next) {
- alt->name[-1] = 0;
- prepare_packed_git_one(alt->base, 0);
- alt->name[-1] = '/';
- }
+ for (alt = alt_odb_list; alt; alt = alt->next)
+ prepare_packed_git_one(alt->path, 0);
rearrange_packed_git();
+ prepare_packed_git_mru();
prepare_packed_git_run_once = 1;
}
void reprepare_packed_git(void)
{
- discard_revindex();
+ approximate_object_count_valid = 0;
prepare_packed_git_run_once = 0;
prepare_packed_git();
}
@@ -1290,10 +1517,12 @@ static void mark_bad_packed_object(struct packed_git *p,
{
unsigned i;
for (i = 0; i < p->num_bad_objects; i++)
- if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
+ if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
return;
- p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1));
- hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1);
+ p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
+ st_mult(GIT_SHA1_RAWSZ,
+ st_add(p->num_bad_objects, 1)));
+ hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
p->num_bad_objects++;
}
@@ -1335,7 +1564,7 @@ int check_sha1_signature(const unsigned char *sha1, void *map,
return -1;
/* Generate the header */
- hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1;
+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
/* Sha1.. */
git_SHA1_Init(&c);
@@ -1357,39 +1586,45 @@ int check_sha1_signature(const unsigned char *sha1, void *map,
return hashcmp(sha1, real_sha1) ? -1 : 0;
}
-static int git_open_noatime(const char *name)
+int git_open(const char *name)
{
- static int sha1_file_open_flag = O_NOATIME;
+ static int sha1_file_open_flag = O_NOATIME | O_CLOEXEC;
for (;;) {
- int fd = open(name, O_RDONLY | sha1_file_open_flag);
+ int fd;
+
+ errno = 0;
+ fd = open(name, O_RDONLY | sha1_file_open_flag);
if (fd >= 0)
return fd;
- /* Might the failure be due to O_NOATIME? */
- if (errno != ENOENT && sha1_file_open_flag) {
- sha1_file_open_flag = 0;
+ /* Try again w/o O_CLOEXEC: the kernel might not support it */
+ if ((sha1_file_open_flag & O_CLOEXEC) && errno == EINVAL) {
+ sha1_file_open_flag &= ~O_CLOEXEC;
continue;
}
+ /* Might the failure be due to O_NOATIME? */
+ if (errno != ENOENT && (sha1_file_open_flag & O_NOATIME)) {
+ sha1_file_open_flag &= ~O_NOATIME;
+ continue;
+ }
return -1;
}
}
static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
{
- char *name = sha1_file_name(sha1);
struct alternate_object_database *alt;
- if (!lstat(name, st))
+ if (!lstat(sha1_file_name(sha1), st))
return 0;
prepare_alt_odb();
errno = ENOENT;
for (alt = alt_odb_list; alt; alt = alt->next) {
- name = alt->name;
- fill_sha1_path(name, sha1);
- if (!lstat(alt->base, st))
+ const char *path = alt_sha1_path(alt, sha1);
+ if (!lstat(path, st))
return 0;
}
@@ -1399,22 +1634,24 @@ static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
static int open_sha1_file(const unsigned char *sha1)
{
int fd;
- char *name = sha1_file_name(sha1);
struct alternate_object_database *alt;
+ int most_interesting_errno;
- fd = git_open_noatime(name);
+ fd = git_open(sha1_file_name(sha1));
if (fd >= 0)
return fd;
+ most_interesting_errno = errno;
prepare_alt_odb();
- errno = ENOENT;
for (alt = alt_odb_list; alt; alt = alt->next) {
- name = alt->name;
- fill_sha1_path(name, sha1);
- fd = git_open_noatime(alt->base);
+ const char *path = alt_sha1_path(alt, sha1);
+ fd = git_open(path);
if (fd >= 0)
return fd;
+ if (most_interesting_errno == ENOENT)
+ most_interesting_errno = errno;
}
+ errno = most_interesting_errno;
return -1;
}
@@ -1467,7 +1704,9 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf,
return used;
}
-int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
+static int unpack_sha1_short_header(git_zstream *stream,
+ unsigned char *map, unsigned long mapsize,
+ void *buffer, unsigned long bufsiz)
{
/* Get the data stream */
memset(stream, 0, sizeof(*stream));
@@ -1480,6 +1719,58 @@ int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long ma
return git_inflate(stream, 0);
}
+int unpack_sha1_header(git_zstream *stream,
+ unsigned char *map, unsigned long mapsize,
+ void *buffer, unsigned long bufsiz)
+{
+ int status = unpack_sha1_short_header(stream, map, mapsize,
+ buffer, bufsiz);
+
+ if (status < Z_OK)
+ return status;
+
+ /* Make sure we have the terminating NUL */
+ if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
+ return -1;
+ return 0;
+}
+
+static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
+ unsigned long mapsize, void *buffer,
+ unsigned long bufsiz, struct strbuf *header)
+{
+ int status;
+
+ status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
+ if (status < Z_OK)
+ return -1;
+
+ /*
+ * Check if entire header is unpacked in the first iteration.
+ */
+ if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
+ return 0;
+
+ /*
+ * buffer[0..bufsiz] was not large enough. Copy the partial
+ * result out to header, and then append the result of further
+ * reading the stream.
+ */
+ strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
+ stream->next_out = buffer;
+ stream->avail_out = bufsiz;
+
+ do {
+ status = git_inflate(stream, 0);
+ strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
+ if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
+ return 0;
+ stream->next_out = buffer;
+ stream->avail_out = bufsiz;
+ } while (status != Z_STREAM_END);
+ return -1;
+}
+
static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
{
int bytes = strlen(buffer) + 1;
@@ -1530,27 +1821,40 @@ static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long s
* too permissive for what we want to check. So do an anal
* object header parse by hand.
*/
-int parse_sha1_header(const char *hdr, unsigned long *sizep)
+static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
+ unsigned int flags)
{
- char type[10];
- int i;
+ const char *type_buf = hdr;
unsigned long size;
+ int type, type_len = 0;
/*
- * The type can be at most ten bytes (including the
- * terminating '\0' that we add), and is followed by
+ * The type can be of any size but is followed by
* a space.
*/
- i = 0;
for (;;) {
char c = *hdr++;
+ if (!c)
+ return -1;
if (c == ' ')
break;
- type[i++] = c;
- if (i >= sizeof(type))
- return -1;
+ type_len++;
}
- type[i] = 0;
+
+ type = type_from_string_gently(type_buf, type_len, 1);
+ if (oi->typename)
+ strbuf_add(oi->typename, type_buf, type_len);
+ /*
+ * Set type to 0 if its an unknown object and
+ * we're obtaining the type using '--allow-unknown-type'
+ * option.
+ */
+ if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
+ type = 0;
+ else if (type < 0)
+ die("invalid object type");
+ if (oi->typep)
+ *oi->typep = type;
/*
* The length must follow immediately, and be in canonical
@@ -1568,12 +1872,22 @@ int parse_sha1_header(const char *hdr, unsigned long *sizep)
size = size * 10 + c;
}
}
- *sizep = size;
+
+ if (oi->sizep)
+ *oi->sizep = size;
/*
* The length must be followed by a zero byte
*/
- return *hdr ? -1 : type_from_string(type);
+ return *hdr ? -1 : type;
+}
+
+int parse_sha1_header(const char *hdr, unsigned long *sizep)
+{
+ struct object_info oi = OBJECT_INFO_INIT;
+
+ oi.sizep = sizep;
+ return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
}
static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
@@ -1667,6 +1981,38 @@ static off_t get_delta_base(struct packed_git *p,
return base_offset;
}
+/*
+ * Like get_delta_base above, but we return the sha1 instead of the pack
+ * offset. This means it is cheaper for REF deltas (we do not have to do
+ * the final object lookup), but more expensive for OFS deltas (we
+ * have to load the revidx to convert the offset back into a sha1).
+ */
+static const unsigned char *get_delta_base_sha1(struct packed_git *p,
+ struct pack_window **w_curs,
+ off_t curpos,
+ enum object_type type,
+ off_t delta_obj_offset)
+{
+ if (type == OBJ_REF_DELTA) {
+ unsigned char *base = use_pack(p, w_curs, curpos, NULL);
+ return base;
+ } else if (type == OBJ_OFS_DELTA) {
+ struct revindex_entry *revidx;
+ off_t base_offset = get_delta_base(p, w_curs, &curpos,
+ type, delta_obj_offset);
+
+ if (!base_offset)
+ return NULL;
+
+ revidx = find_pack_revindex(p, base_offset);
+ if (!revidx)
+ return NULL;
+
+ return nth_packed_object_sha1(p, revidx->nr);
+ } else
+ return NULL;
+}
+
int unpack_object_header(struct packed_git *p,
struct pack_window **w_curs,
off_t *curpos,
@@ -1727,7 +2073,7 @@ static enum object_type packed_to_object_type(struct packed_git *p,
/* Push the object we're going to leave behind */
if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
poi_stack_alloc = alloc_nr(poi_stack_nr);
- poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc);
+ ALLOC_ARRAY(poi_stack, poi_stack_alloc);
memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
} else {
ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
@@ -1778,8 +2124,8 @@ unwind:
goto out;
}
-static int packed_object_info(struct packed_git *p, off_t obj_offset,
- struct object_info *oi)
+int packed_object_info(struct packed_git *p, off_t obj_offset,
+ struct object_info *oi)
{
struct pack_window *w_curs = NULL;
unsigned long size;
@@ -1824,6 +2170,22 @@ static int packed_object_info(struct packed_git *p, off_t obj_offset,
}
}
+ if (oi->delta_base_sha1) {
+ if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
+ const unsigned char *base;
+
+ base = get_delta_base_sha1(p, &w_curs, curpos,
+ type, obj_offset);
+ if (!base) {
+ type = OBJ_BAD;
+ goto out;
+ }
+
+ hashcpy(oi->delta_base_sha1, base);
+ } else
+ hashclr(oi->delta_base_sha1);
+ }
+
out:
unuse_pack(&w_curs);
return type;
@@ -1838,7 +2200,9 @@ static void *unpack_compressed_entry(struct packed_git *p,
git_zstream stream;
unsigned char *buffer, *in;
- buffer = xmallocz(size);
+ buffer = xmallocz_gently(size);
+ if (!buffer)
+ return NULL;
memset(&stream, 0, sizeof(stream));
stream.next_out = buffer;
stream.avail_out = size + 1;
@@ -1861,136 +2225,142 @@ static void *unpack_compressed_entry(struct packed_git *p,
return buffer;
}
-#define MAX_DELTA_CACHE (256)
-
+static struct hashmap delta_base_cache;
static size_t delta_base_cached;
-static struct delta_base_cache_lru_list {
- struct delta_base_cache_lru_list *prev;
- struct delta_base_cache_lru_list *next;
-} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
+static LIST_HEAD(delta_base_cache_lru);
-static struct delta_base_cache_entry {
- struct delta_base_cache_lru_list lru;
- void *data;
+struct delta_base_cache_key {
struct packed_git *p;
off_t base_offset;
+};
+
+struct delta_base_cache_entry {
+ struct hashmap hash;
+ struct delta_base_cache_key key;
+ struct list_head lru;
+ void *data;
unsigned long size;
enum object_type type;
-} delta_base_cache[MAX_DELTA_CACHE];
+};
-static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
+static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
{
- unsigned long hash;
+ unsigned int hash;
- hash = (unsigned long)p + (unsigned long)base_offset;
+ hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
hash += (hash >> 8) + (hash >> 16);
- return hash % MAX_DELTA_CACHE;
+ return hash;
}
static struct delta_base_cache_entry *
get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
{
- unsigned long hash = pack_entry_hash(p, base_offset);
- return delta_base_cache + hash;
+ struct hashmap_entry entry;
+ struct delta_base_cache_key key;
+
+ if (!delta_base_cache.cmpfn)
+ return NULL;
+
+ hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
+ key.p = p;
+ key.base_offset = base_offset;
+ return hashmap_get(&delta_base_cache, &entry, &key);
+}
+
+static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
+ const struct delta_base_cache_key *b)
+{
+ return a->p == b->p && a->base_offset == b->base_offset;
}
-static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent,
- struct packed_git *p, off_t base_offset)
+static int delta_base_cache_hash_cmp(const void *va, const void *vb,
+ const void *vkey)
{
- return (ent->data && ent->p == p && ent->base_offset == base_offset);
+ const struct delta_base_cache_entry *a = va, *b = vb;
+ const struct delta_base_cache_key *key = vkey;
+ if (key)
+ return !delta_base_cache_key_eq(&a->key, key);
+ else
+ return !delta_base_cache_key_eq(&a->key, &b->key);
}
static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
{
- struct delta_base_cache_entry *ent;
- ent = get_delta_base_cache_entry(p, base_offset);
- return eq_delta_base_cache_entry(ent, p, base_offset);
+ return !!get_delta_base_cache_entry(p, base_offset);
}
-static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent)
+/*
+ * Remove the entry from the cache, but do _not_ free the associated
+ * entry data. The caller takes ownership of the "data" buffer, and
+ * should copy out any fields it wants before detaching.
+ */
+static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
{
- ent->data = NULL;
- ent->lru.next->prev = ent->lru.prev;
- ent->lru.prev->next = ent->lru.next;
+ hashmap_remove(&delta_base_cache, ent, &ent->key);
+ list_del(&ent->lru);
delta_base_cached -= ent->size;
+ free(ent);
}
static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
- unsigned long *base_size, enum object_type *type, int keep_cache)
+ unsigned long *base_size, enum object_type *type)
{
struct delta_base_cache_entry *ent;
- void *ret;
ent = get_delta_base_cache_entry(p, base_offset);
-
- if (!eq_delta_base_cache_entry(ent, p, base_offset))
+ if (!ent)
return unpack_entry(p, base_offset, type, base_size);
- ret = ent->data;
-
- if (!keep_cache)
- clear_delta_base_cache_entry(ent);
- else
- ret = xmemdupz(ent->data, ent->size);
*type = ent->type;
*base_size = ent->size;
- return ret;
+ return xmemdupz(ent->data, ent->size);
}
static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
{
- if (ent->data) {
- free(ent->data);
- ent->data = NULL;
- ent->lru.next->prev = ent->lru.prev;
- ent->lru.prev->next = ent->lru.next;
- delta_base_cached -= ent->size;
- }
+ free(ent->data);
+ detach_delta_base_cache_entry(ent);
}
void clear_delta_base_cache(void)
{
- unsigned long p;
- for (p = 0; p < MAX_DELTA_CACHE; p++)
- release_delta_base_cache(&delta_base_cache[p]);
+ struct hashmap_iter iter;
+ struct delta_base_cache_entry *entry;
+ for (entry = hashmap_iter_first(&delta_base_cache, &iter);
+ entry;
+ entry = hashmap_iter_next(&iter)) {
+ release_delta_base_cache(entry);
+ }
}
static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
void *base, unsigned long base_size, enum object_type type)
{
- unsigned long hash = pack_entry_hash(p, base_offset);
- struct delta_base_cache_entry *ent = delta_base_cache + hash;
- struct delta_base_cache_lru_list *lru;
+ struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
+ struct list_head *lru, *tmp;
- release_delta_base_cache(ent);
delta_base_cached += base_size;
- for (lru = delta_base_cache_lru.next;
- delta_base_cached > delta_base_cache_limit
- && lru != &delta_base_cache_lru;
- lru = lru->next) {
- struct delta_base_cache_entry *f = (void *)lru;
- if (f->type == OBJ_BLOB)
- release_delta_base_cache(f);
- }
- for (lru = delta_base_cache_lru.next;
- delta_base_cached > delta_base_cache_limit
- && lru != &delta_base_cache_lru;
- lru = lru->next) {
- struct delta_base_cache_entry *f = (void *)lru;
+ list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
+ struct delta_base_cache_entry *f =
+ list_entry(lru, struct delta_base_cache_entry, lru);
+ if (delta_base_cached <= delta_base_cache_limit)
+ break;
release_delta_base_cache(f);
}
- ent->p = p;
- ent->base_offset = base_offset;
+ ent->key.p = p;
+ ent->key.base_offset = base_offset;
ent->type = type;
ent->data = base;
ent->size = base_size;
- ent->lru.next = &delta_base_cache_lru;
- ent->lru.prev = delta_base_cache_lru.prev;
- delta_base_cache_lru.prev->next = &ent->lru;
- delta_base_cache_lru.prev = &ent->lru;
+ list_add_tail(&ent->lru, &delta_base_cache_lru);
+
+ if (!delta_base_cache.cmpfn)
+ hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0);
+ hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
+ hashmap_add(&delta_base_cache, ent);
}
static void *read_object(const unsigned char *sha1, enum object_type *type,
@@ -1998,27 +2368,9 @@ static void *read_object(const unsigned char *sha1, enum object_type *type,
static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
{
- static FILE *log_file;
-
- if (!log_pack_access)
- log_pack_access = getenv("GIT_TRACE_PACK_ACCESS");
- if (!log_pack_access)
- log_pack_access = no_log_pack_access;
- if (log_pack_access == no_log_pack_access)
- return;
-
- if (!log_file) {
- log_file = fopen(log_pack_access, "w");
- if (!log_file) {
- error("cannot open pack access log '%s' for writing: %s",
- log_pack_access, strerror(errno));
- log_pack_access = no_log_pack_access;
- return;
- }
- }
- fprintf(log_file, "%s %"PRIuMAX"\n",
- p->pack_name, (uintmax_t)obj_offset);
- fflush(log_file);
+ static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
+ trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
+ p->pack_name, (uintmax_t)obj_offset);
}
int do_check_packed_object_crc;
@@ -2043,8 +2395,7 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
int base_from_cache = 0;
- if (log_pack_access != no_log_pack_access)
- write_pack_access_log(p, obj_offset);
+ write_pack_access_log(p, obj_offset);
/* PHASE 1: drill down to the innermost base object */
for (;;) {
@@ -2053,18 +2404,18 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
struct delta_base_cache_entry *ent;
ent = get_delta_base_cache_entry(p, curpos);
- if (eq_delta_base_cache_entry(ent, p, curpos)) {
+ if (ent) {
type = ent->type;
data = ent->data;
size = ent->size;
- clear_delta_base_cache_entry(ent);
+ detach_delta_base_cache_entry(ent);
base_from_cache = 1;
break;
}
if (do_check_packed_object_crc && p->index_version > 1) {
struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
- unsigned long len = revidx[1].offset - obj_offset;
+ off_t len = revidx[1].offset - obj_offset;
if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
const unsigned char *sha1 =
nth_packed_object_sha1(p, revidx->nr);
@@ -2094,7 +2445,7 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
if (delta_stack_nr >= delta_stack_alloc
&& delta_stack == small_delta_stack) {
delta_stack_alloc = alloc_nr(delta_stack_nr);
- delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc);
+ ALLOC_ARRAY(delta_stack, delta_stack_alloc);
memcpy(delta_stack, small_delta_stack,
sizeof(*delta_stack)*delta_stack_nr);
} else {
@@ -2113,7 +2464,7 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
case OBJ_OFS_DELTA:
case OBJ_REF_DELTA:
if (data)
- die("BUG in unpack_entry: left loop at a valid delta");
+ die("BUG: unpack_entry: left loop at a valid delta");
break;
case OBJ_COMMIT:
case OBJ_TREE:
@@ -2205,6 +2556,10 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
*final_size = size;
unuse_pack(&w_curs);
+
+ if (delta_stack != small_delta_stack)
+ free(delta_stack);
+
return data;
}
@@ -2228,6 +2583,20 @@ const unsigned char *nth_packed_object_sha1(struct packed_git *p,
}
}
+void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
+{
+ const unsigned char *ptr = vptr;
+ const unsigned char *start = p->index_data;
+ const unsigned char *end = start + p->index_size;
+ if (ptr < start)
+ die(_("offset before start of pack index for %s (corrupt index?)"),
+ p->pack_name);
+ /* No need to check for underflow; .idx files must be at least 8 bytes */
+ if (ptr >= end - 8)
+ die(_("offset beyond end of pack index for %s (truncated index?)"),
+ p->pack_name);
+}
+
off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
{
const unsigned char *index = p->index_data;
@@ -2241,6 +2610,7 @@ off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
if (!(off & 0x80000000))
return off;
index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
+ check_pack_index_ptr(p, index);
return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
ntohl(*((uint32_t *)(index + 4)));
}
@@ -2354,33 +2724,31 @@ static int fill_pack_entry(const unsigned char *sha1,
* answer, as it may have been deleted since the index was
* loaded!
*/
- if (!is_pack_valid(p)) {
- warning("packfile %s cannot be accessed", p->pack_name);
+ if (!is_pack_valid(p))
return 0;
- }
e->offset = offset;
e->p = p;
hashcpy(e->sha1, sha1);
return 1;
}
+/*
+ * Iff a pack file contains the object named by sha1, return true and
+ * store its location to e.
+ */
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
{
- struct packed_git *p;
+ struct mru_entry *p;
prepare_packed_git();
if (!packed_git)
return 0;
- if (last_found_pack && fill_pack_entry(sha1, e, last_found_pack))
- return 1;
-
- for (p = packed_git; p; p = p->next) {
- if (p == last_found_pack || !fill_pack_entry(sha1, e, p))
- continue;
-
- last_found_pack = p;
- return 1;
+ for (p = packed_git_mru->head; p; p = p->next) {
+ if (fill_pack_entry(sha1, e, p->item)) {
+ mru_mark(packed_git_mru, p);
+ return 1;
+ }
}
return 0;
}
@@ -2399,13 +2767,18 @@ struct packed_git *find_sha1_pack(const unsigned char *sha1,
}
static int sha1_loose_object_info(const unsigned char *sha1,
- struct object_info *oi)
+ struct object_info *oi,
+ int flags)
{
- int status;
- unsigned long mapsize, size;
+ int status = 0;
+ unsigned long mapsize;
void *map;
git_zstream stream;
char hdr[32];
+ struct strbuf hdrbuf = STRBUF_INIT;
+
+ if (oi->delta_base_sha1)
+ hashclr(oi->delta_base_sha1);
/*
* If we don't care about type or size, then we don't
@@ -2415,7 +2788,7 @@ static int sha1_loose_object_info(const unsigned char *sha1,
* return value implicitly indicates whether the
* object even exists.
*/
- if (!oi->typep && !oi->sizep) {
+ if (!oi->typep && !oi->typename && !oi->sizep) {
struct stat st;
if (stat_sha1_file(sha1, &st) < 0)
return -1;
@@ -2429,27 +2802,38 @@ static int sha1_loose_object_info(const unsigned char *sha1,
return -1;
if (oi->disk_sizep)
*oi->disk_sizep = mapsize;
- if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
+ if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
+ if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
+ status = error("unable to unpack %s header with --allow-unknown-type",
+ sha1_to_hex(sha1));
+ } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
status = error("unable to unpack %s header",
sha1_to_hex(sha1));
- else if ((status = parse_sha1_header(hdr, &size)) < 0)
+ if (status < 0)
+ ; /* Do nothing */
+ else if (hdrbuf.len) {
+ if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
+ status = error("unable to parse %s header with --allow-unknown-type",
+ sha1_to_hex(sha1));
+ } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
status = error("unable to parse %s header", sha1_to_hex(sha1));
- else if (oi->sizep)
- *oi->sizep = size;
git_inflate_end(&stream);
munmap(map, mapsize);
- if (oi->typep)
+ if (status && oi->typep)
*oi->typep = status;
+ strbuf_release(&hdrbuf);
return 0;
}
-int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
+int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
{
struct cached_object *co;
struct pack_entry e;
int rtype;
+ enum object_type real_type;
+ const unsigned char *real = lookup_replace_object_extended(sha1, flags);
- co = find_cached_object(sha1);
+ co = find_cached_object(real);
if (co) {
if (oi->typep)
*(oi->typep) = co->type;
@@ -2457,27 +2841,40 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
*(oi->sizep) = co->size;
if (oi->disk_sizep)
*(oi->disk_sizep) = 0;
+ if (oi->delta_base_sha1)
+ hashclr(oi->delta_base_sha1);
+ if (oi->typename)
+ strbuf_addstr(oi->typename, typename(co->type));
oi->whence = OI_CACHED;
return 0;
}
- if (!find_pack_entry(sha1, &e)) {
+ if (!find_pack_entry(real, &e)) {
/* Most likely it's a loose object. */
- if (!sha1_loose_object_info(sha1, oi)) {
+ if (!sha1_loose_object_info(real, oi, flags)) {
oi->whence = OI_LOOSE;
return 0;
}
/* Not a loose object; someone else may have just packed it. */
reprepare_packed_git();
- if (!find_pack_entry(sha1, &e))
+ if (!find_pack_entry(real, &e))
return -1;
}
+ /*
+ * packed_object_info() does not follow the delta chain to
+ * find out the real type, unless it is given oi->typep.
+ */
+ if (oi->typename && !oi->typep)
+ oi->typep = &real_type;
+
rtype = packed_object_info(e.p, e.offset, oi);
if (rtype < 0) {
- mark_bad_packed_object(e.p, sha1);
- return sha1_object_info_extended(sha1, oi);
+ mark_bad_packed_object(e.p, real);
+ if (oi->typep == &real_type)
+ oi->typep = NULL;
+ return sha1_object_info_extended(real, oi, 0);
} else if (in_delta_base_cache(e.p, e.offset)) {
oi->whence = OI_DBCACHED;
} else {
@@ -2487,6 +2884,10 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
rtype == OBJ_OFS_DELTA);
}
+ if (oi->typename)
+ strbuf_addstr(oi->typename, typename(*oi->typep));
+ if (oi->typep == &real_type)
+ oi->typep = NULL;
return 0;
}
@@ -2495,11 +2896,11 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
{
enum object_type type;
- struct object_info oi = {NULL};
+ struct object_info oi = OBJECT_INFO_INIT;
oi.typep = &type;
oi.sizep = sizep;
- if (sha1_object_info_extended(sha1, &oi) < 0)
+ if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
return -1;
return type;
}
@@ -2512,7 +2913,7 @@ static void *read_packed_sha1(const unsigned char *sha1,
if (!find_pack_entry(sha1, &e))
return NULL;
- data = cache_or_unpack_entry(e.p, e.offset, size, type, 1);
+ data = cache_or_unpack_entry(e.p, e.offset, size, type);
if (!data) {
/*
* We're probably in deep shit, but let's try to fetch
@@ -2536,12 +2937,7 @@ int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
hash_sha1_file(buf, len, typename(type), sha1);
if (has_sha1_file(sha1) || find_cached_object(sha1))
return 0;
- if (cached_object_alloc <= cached_object_nr) {
- cached_object_alloc = alloc_nr(cached_object_alloc);
- cached_objects = xrealloc(cached_objects,
- sizeof(*cached_objects) *
- cached_object_alloc);
- }
+ ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
co = &cached_objects[cached_object_nr++];
co->size = len;
co->type = type;
@@ -2589,10 +2985,8 @@ void *read_sha1_file_extended(const unsigned char *sha1,
unsigned flag)
{
void *data;
- char *path;
const struct packed_git *p;
- const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
- ? lookup_replace_object(sha1) : sha1;
+ const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
errno = 0;
data = read_object(repl, type, size);
@@ -2608,7 +3002,8 @@ void *read_sha1_file_extended(const unsigned char *sha1,
sha1_to_hex(repl), sha1_to_hex(sha1));
if (has_loose_object(repl)) {
- path = sha1_file_name(sha1);
+ const char *path = sha1_file_name(sha1);
+
die("loose object %s (stored in %s) is corrupt",
sha1_to_hex(repl), path);
}
@@ -2675,7 +3070,7 @@ static void write_sha1_file_prepare(const void *buf, unsigned long len,
git_SHA_CTX c;
/* Generate the header */
- *hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
+ *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
/* Sha1.. */
git_SHA1_Init(&c);
@@ -2686,11 +3081,8 @@ static void write_sha1_file_prepare(const void *buf, unsigned long len,
/*
* Move the just written object into its final resting place.
- * NEEDSWORK: this should be renamed to finalize_temp_file() as
- * "moving" is only a part of what it does, when no patch between
- * master to pu changes the call sites of this function.
*/
-int move_temp_to_file(const char *tmpfile, const char *filename)
+int finalize_object_file(const char *tmpfile, const char *filename)
{
int ret = 0;
@@ -2719,7 +3111,7 @@ int move_temp_to_file(const char *tmpfile, const char *filename)
unlink_or_warn(tmpfile);
if (ret) {
if (ret != EEXIST) {
- return error("unable to write sha1 filename %s: %s", filename, strerror(ret));
+ return error_errno("unable to write sha1 filename %s", filename);
}
/* FIXME!!! Collision check here ? */
}
@@ -2733,7 +3125,7 @@ out:
static int write_buffer(int fd, const void *buf, size_t len)
{
if (write_in_full(fd, buf, len) < 0)
- return error("file write error (%s)", strerror(errno));
+ return error_errno("file write error");
return 0;
}
@@ -2741,7 +3133,7 @@ int hash_sha1_file(const void *buf, unsigned long len, const char *type,
unsigned char *sha1)
{
char hdr[32];
- int hdrlen;
+ int hdrlen = sizeof(hdr);
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
return 0;
}
@@ -2771,29 +3163,31 @@ static inline int directory_size(const char *filename)
* We want to avoid cross-directory filename renames, because those
* can have problems on various filesystems (FAT, NFS, Coda).
*/
-static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename)
+static int create_tmpfile(struct strbuf *tmp, const char *filename)
{
int fd, dirlen = directory_size(filename);
- if (dirlen + 20 > bufsiz) {
- errno = ENAMETOOLONG;
- return -1;
- }
- memcpy(buffer, filename, dirlen);
- strcpy(buffer + dirlen, "tmp_obj_XXXXXX");
- fd = git_mkstemp_mode(buffer, 0444);
+ strbuf_reset(tmp);
+ strbuf_add(tmp, filename, dirlen);
+ strbuf_addstr(tmp, "tmp_obj_XXXXXX");
+ fd = git_mkstemp_mode(tmp->buf, 0444);
if (fd < 0 && dirlen && errno == ENOENT) {
- /* Make sure the directory exists */
- memcpy(buffer, filename, dirlen);
- buffer[dirlen-1] = 0;
- if (mkdir(buffer, 0777) && errno != EEXIST)
+ /*
+ * Make sure the directory exists; note that the contents
+ * of the buffer are undefined after mkstemp returns an
+ * error, so we have to rewrite the whole buffer from
+ * scratch.
+ */
+ strbuf_reset(tmp);
+ strbuf_add(tmp, filename, dirlen - 1);
+ if (mkdir(tmp->buf, 0777) && errno != EEXIST)
return -1;
- if (adjust_shared_perm(buffer))
+ if (adjust_shared_perm(tmp->buf))
return -1;
/* Try again */
- strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX");
- fd = git_mkstemp_mode(buffer, 0444);
+ strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
+ fd = git_mkstemp_mode(tmp->buf, 0444);
}
return fd;
}
@@ -2806,20 +3200,18 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
git_zstream stream;
git_SHA_CTX c;
unsigned char parano_sha1[20];
- char *filename;
- static char tmp_file[PATH_MAX];
+ static struct strbuf tmp_file = STRBUF_INIT;
+ const char *filename = sha1_file_name(sha1);
- filename = sha1_file_name(sha1);
- fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename);
+ fd = create_tmpfile(&tmp_file, filename);
if (fd < 0) {
if (errno == EACCES)
return error("insufficient permission for adding an object to repository database %s", get_object_directory());
else
- return error("unable to create temporary file: %s", strerror(errno));
+ return error_errno("unable to create temporary file");
}
/* Set it up */
- memset(&stream, 0, sizeof(stream));
git_deflate_init(&stream, zlib_compression_level);
stream.next_out = compressed;
stream.avail_out = sizeof(compressed);
@@ -2860,31 +3252,67 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
struct utimbuf utb;
utb.actime = mtime;
utb.modtime = mtime;
- if (utime(tmp_file, &utb) < 0)
- warning("failed utime() on %s: %s",
- tmp_file, strerror(errno));
+ if (utime(tmp_file.buf, &utb) < 0)
+ warning_errno("failed utime() on %s", tmp_file.buf);
}
- return move_temp_to_file(tmp_file, filename);
+ return finalize_object_file(tmp_file.buf, filename);
}
-int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
+static int freshen_loose_object(const unsigned char *sha1)
+{
+ return check_and_freshen(sha1, 1);
+}
+
+static int freshen_packed_object(const unsigned char *sha1)
+{
+ struct pack_entry e;
+ if (!find_pack_entry(sha1, &e))
+ return 0;
+ if (e.p->freshened)
+ return 1;
+ if (!freshen_file(e.p->pack_name))
+ return 0;
+ e.p->freshened = 1;
+ return 1;
+}
+
+int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
{
- unsigned char sha1[20];
char hdr[32];
- int hdrlen;
+ int hdrlen = sizeof(hdr);
/* Normally if we have it in the pack then we do not bother writing
* it out into .git/objects/??/?{38} file.
*/
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
- if (returnsha1)
- hashcpy(returnsha1, sha1);
- if (has_sha1_file(sha1))
+ if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
}
+int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
+ unsigned char *sha1, unsigned flags)
+{
+ char *header;
+ int hdrlen, status = 0;
+
+ /* type string, SP, %lu of the length plus NUL must fit this */
+ hdrlen = strlen(type) + 32;
+ header = xmalloc(hdrlen);
+ write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
+
+ if (!(flags & HASH_WRITE_OBJECT))
+ goto cleanup;
+ if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
+ goto cleanup;
+ status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
+
+cleanup:
+ free(header);
+ return status;
+}
+
int force_object_loose(const unsigned char *sha1, time_t mtime)
{
void *buf;
@@ -2899,7 +3327,7 @@ int force_object_loose(const unsigned char *sha1, time_t mtime)
buf = read_packed_sha1(sha1, &type, &len);
if (!buf)
return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
- hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
free(buf);
@@ -2920,7 +3348,7 @@ int has_sha1_pack(const unsigned char *sha1)
return find_pack_entry(sha1, &e);
}
-int has_sha1_file(const unsigned char *sha1)
+int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
{
struct pack_entry e;
@@ -2928,10 +3356,22 @@ int has_sha1_file(const unsigned char *sha1)
return 1;
if (has_loose_object(sha1))
return 1;
+ if (flags & HAS_SHA1_QUICK)
+ return 0;
reprepare_packed_git();
return find_pack_entry(sha1, &e);
}
+int has_object_file(const struct object_id *oid)
+{
+ return has_sha1_file(oid->hash);
+}
+
+int has_object_file_with_flags(const struct object_id *oid, int flags)
+{
+ return has_sha1_file_with_flags(oid->hash, flags);
+}
+
static void check_tree(const void *buf, size_t size)
{
struct tree_desc desc;
@@ -2999,6 +3439,29 @@ static int index_mem(unsigned char *sha1, void *buf, size_t size,
return ret;
}
+static int index_stream_convert_blob(unsigned char *sha1, int fd,
+ const char *path, unsigned flags)
+{
+ int ret;
+ const int write_object = flags & HASH_WRITE_OBJECT;
+ struct strbuf sbuf = STRBUF_INIT;
+
+ assert(path);
+ assert(would_convert_to_git_filter_fd(path));
+
+ convert_to_git_filter_fd(path, fd, &sbuf,
+ write_object ? safe_crlf : SAFE_CRLF_FALSE);
+
+ if (write_object)
+ ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ sha1);
+ else
+ ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ sha1);
+ strbuf_release(&sbuf);
+ return ret;
+}
+
static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
const char *path, unsigned flags)
{
@@ -3022,13 +3485,13 @@ static int index_core(unsigned char *sha1, int fd, size_t size,
int ret;
if (!size) {
- ret = index_mem(sha1, NULL, size, type, path, flags);
+ ret = index_mem(sha1, "", size, type, path, flags);
} else if (size <= SMALL_FILE_SIZE) {
char *buf = xmalloc(size);
if (size == read_in_full(fd, buf, size))
ret = index_mem(sha1, buf, size, type, path, flags);
else
- ret = error("short read %s", strerror(errno));
+ ret = error_errno("short read");
free(buf);
} else {
void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
@@ -3064,15 +3527,22 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st,
enum object_type type, const char *path, unsigned flags)
{
int ret;
- size_t size = xsize_t(st->st_size);
- if (!S_ISREG(st->st_mode))
+ /*
+ * Call xsize_t() only when needed to avoid potentially unnecessary
+ * die() for large files.
+ */
+ if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
+ ret = index_stream_convert_blob(sha1, fd, path, flags);
+ else if (!S_ISREG(st->st_mode))
ret = index_pipe(sha1, fd, type, path, flags);
- else if (size <= big_file_threshold || type != OBJ_BLOB ||
- (path && would_convert_to_git(path, NULL, 0, 0)))
- ret = index_core(sha1, fd, size, type, path, flags);
+ else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
+ (path && would_convert_to_git(path)))
+ ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
+ flags);
else
- ret = index_stream(sha1, fd, size, type, path, flags);
+ ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
+ flags);
close(fd);
return ret;
}
@@ -3086,18 +3556,14 @@ int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned
case S_IFREG:
fd = open(path, O_RDONLY);
if (fd < 0)
- return error("open(\"%s\"): %s", path,
- strerror(errno));
+ return error_errno("open(\"%s\")", path);
if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
return error("%s: failed to insert into database",
path);
break;
case S_IFLNK:
- if (strbuf_readlink(&sb, path, st->st_size)) {
- char *errstr = strerror(errno);
- return error("readlink(\"%s\"): %s", path,
- errstr);
- }
+ if (strbuf_readlink(&sb, path, st->st_size))
+ return error_errno("readlink(\"%s\")", path);
if (!(flags & HASH_WRITE_OBJECT))
hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
@@ -3137,3 +3603,176 @@ void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
die("%s is not a valid '%s' object", sha1_to_hex(sha1),
typename(expect));
}
+
+static int for_each_file_in_obj_subdir(int subdir_nr,
+ struct strbuf *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ size_t baselen = path->len;
+ DIR *dir = opendir(path->buf);
+ struct dirent *de;
+ int r = 0;
+
+ if (!dir) {
+ if (errno == ENOENT)
+ return 0;
+ return error_errno("unable to open %s", path->buf);
+ }
+
+ while ((de = readdir(dir))) {
+ if (is_dot_or_dotdot(de->d_name))
+ continue;
+
+ strbuf_setlen(path, baselen);
+ strbuf_addf(path, "/%s", de->d_name);
+
+ if (strlen(de->d_name) == 38) {
+ char hex[41];
+ unsigned char sha1[20];
+
+ snprintf(hex, sizeof(hex), "%02x%s",
+ subdir_nr, de->d_name);
+ if (!get_sha1_hex(hex, sha1)) {
+ if (obj_cb) {
+ r = obj_cb(sha1, path->buf, data);
+ if (r)
+ break;
+ }
+ continue;
+ }
+ }
+
+ if (cruft_cb) {
+ r = cruft_cb(de->d_name, path->buf, data);
+ if (r)
+ break;
+ }
+ }
+ closedir(dir);
+
+ strbuf_setlen(path, baselen);
+ if (!r && subdir_cb)
+ r = subdir_cb(subdir_nr, path->buf, data);
+
+ return r;
+}
+
+int for_each_loose_file_in_objdir_buf(struct strbuf *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ size_t baselen = path->len;
+ int r = 0;
+ int i;
+
+ for (i = 0; i < 256; i++) {
+ strbuf_addf(path, "/%02x", i);
+ r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
+ subdir_cb, data);
+ strbuf_setlen(path, baselen);
+ if (r)
+ break;
+ }
+
+ return r;
+}
+
+int for_each_loose_file_in_objdir(const char *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ struct strbuf buf = STRBUF_INIT;
+ int r;
+
+ strbuf_addstr(&buf, path);
+ r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
+ subdir_cb, data);
+ strbuf_release(&buf);
+
+ return r;
+}
+
+struct loose_alt_odb_data {
+ each_loose_object_fn *cb;
+ void *data;
+};
+
+static int loose_from_alt_odb(struct alternate_object_database *alt,
+ void *vdata)
+{
+ struct loose_alt_odb_data *data = vdata;
+ struct strbuf buf = STRBUF_INIT;
+ int r;
+
+ strbuf_addstr(&buf, alt->path);
+ r = for_each_loose_file_in_objdir_buf(&buf,
+ data->cb, NULL, NULL,
+ data->data);
+ strbuf_release(&buf);
+ return r;
+}
+
+int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
+{
+ struct loose_alt_odb_data alt;
+ int r;
+
+ r = for_each_loose_file_in_objdir(get_object_directory(),
+ cb, NULL, NULL, data);
+ if (r)
+ return r;
+
+ if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
+ return 0;
+
+ alt.cb = cb;
+ alt.data = data;
+ return foreach_alt_odb(loose_from_alt_odb, &alt);
+}
+
+static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
+{
+ uint32_t i;
+ int r = 0;
+
+ for (i = 0; i < p->num_objects; i++) {
+ const unsigned char *sha1 = nth_packed_object_sha1(p, i);
+
+ if (!sha1)
+ return error("unable to get sha1 of object %u in %s",
+ i, p->pack_name);
+
+ r = cb(sha1, p, i, data);
+ if (r)
+ break;
+ }
+ return r;
+}
+
+int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
+{
+ struct packed_git *p;
+ int r = 0;
+ int pack_errors = 0;
+
+ prepare_packed_git();
+ for (p = packed_git; p; p = p->next) {
+ if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
+ continue;
+ if (open_pack_index(p)) {
+ pack_errors = 1;
+ continue;
+ }
+ r = for_each_object_in_pack(p, cb, data);
+ if (r)
+ break;
+ }
+ return r ? r : pack_errors;
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-29 02:59:04 +0000