diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Documentation/git-status.txt | 5 | ||||
| -rw-r--r-- | Documentation/git-update-index.txt | 14 | ||||
| -rw-r--r-- | Documentation/technical/index-format.txt | 62 | ||||
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | builtin/commit.c | 5 | ||||
| -rw-r--r-- | builtin/update-index.c | 188 | ||||
| -rw-r--r-- | cache.h | 6 | ||||
| -rw-r--r-- | compat/mingw.c | 11 | ||||
| -rw-r--r-- | compat/mingw.h | 9 | ||||
| -rw-r--r-- | dir.c | 990 | ||||
| -rw-r--r-- | dir.h | 82 | ||||
| -rw-r--r-- | ewah/ewah_io.c | 13 | ||||
| -rw-r--r-- | ewah/ewok.h | 2 | ||||
| -rw-r--r-- | git-compat-util.h | 1 | ||||
| -rw-r--r-- | read-cache.c | 55 | ||||
| -rw-r--r-- | split-index.c | 11 | ||||
| -rwxr-xr-x | t/t7063-status-untracked-cache.sh | 353 | ||||
| -rw-r--r-- | test-dump-untracked-cache.c | 62 | ||||
| -rw-r--r-- | unpack-trees.c | 7 | ||||
| -rw-r--r-- | wt-status.c | 2 |
21 files changed, 1822 insertions, 58 deletions
diff --git a/.gitignore b/.gitignore index a05241916c..422c5382c1 100644 --- a/.gitignore +++ b/.gitignore @@ -184,6 +184,7 @@ /test-delta /test-dump-cache-tree /test-dump-split-index +/test-dump-untracked-cache /test-scrap-cache-tree /test-genrandom /test-hashmap diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 5221f950ce..335f312335 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -66,7 +66,10 @@ When `-u` option is not used, untracked files and directories are shown (i.e. the same as specifying `normal`), to help you avoid forgetting to add newly created files. Because it takes extra work to find untracked files in the filesystem, this mode may take some -time in a large working tree. You can use `no` to have `git status` +time in a large working tree. +Consider enabling untracked cache and split index if supported (see +`git update-index --untracked-cache` and `git update-index +--split-index`), Otherwise you can use `no` to have `git status` return more quickly without showing untracked files. + The default can be changed using the status.showUntrackedFiles diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index aff01798cd..1a296bc29a 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -170,6 +170,20 @@ may not support it yet. the shared index file. This mode is designed for very large indexes that take a significant amount of time to read or write. +--untracked-cache:: +--no-untracked-cache:: + Enable or disable untracked cache extension. This could speed + up for commands that involve determining untracked files such + as `git status`. The underlying operating system and file + system must change `st_mtime` field of a directory if files + are added or deleted in that directory. + +--force-untracked-cache:: + For safety, `--untracked-cache` performs tests on the working + directory to make sure untracked cache can be used. These + tests can take a few seconds. `--force-untracked-cache` can be + used to skip the tests. + \--:: Do not interpret any more arguments as options. diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt index 35112e4966..b7093af8b2 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/technical/index-format.txt @@ -233,3 +233,65 @@ Git index format The remaining index entries after replaced ones will be added to the final index. These added entries are also sorted by entry name then stage. + +== Untracked cache + + Untracked cache saves the untracked file list and necessary data to + verify the cache. The signature for this extension is { 'U', 'N', + 'T', 'R' }. + + The extension starts with + + - A sequence of NUL-terminated strings, preceded by the size of the + sequence in variable width encoding. Each string describes the + environment where the cache can be used. + + - Stat data of $GIT_DIR/info/exclude. See "Index entry" section from + ctime field until "file size". + + - Stat data of core.excludesfile + + - 32-bit dir_flags (see struct dir_struct) + + - 160-bit SHA-1 of $GIT_DIR/info/exclude. Null SHA-1 means the file + does not exist. + + - 160-bit SHA-1 of core.excludesfile. Null SHA-1 means the file does + not exist. + + - NUL-terminated string of per-dir exclude file name. This usually + is ".gitignore". + + - The number of following directory blocks, variable width + encoding. If this number is zero, the extension ends here with a + following NUL. + + - A number of directory blocks in depth-first-search order, each + consists of + + - The number of untracked entries, variable width encoding. + + - The number of sub-directory blocks, variable width encoding. + + - The directory name terminated by NUL. + + - A number of untrached file/dir names terminated by NUL. + +The remaining data of each directory block is grouped by type: + + - An ewah bitmap, the n-th bit marks whether the n-th directory has + valid untracked cache entries. + + - An ewah bitmap, the n-th bit records "check-only" bit of + read_directory_recursive() for the n-th directory. + + - An ewah bitmap, the n-th bit indicates whether SHA-1 and stat data + is valid for the n-th directory and exists in the next data. + + - An array of stat data. The n-th data corresponds with the n-th + "one" bit in the previous ewah bitmap. + + - An array of SHA-1. The n-th SHA-1 corresponds with the n-th "one" bit + in the previous ewah bitmap. + + - One NUL. @@ -574,6 +574,7 @@ TEST_PROGRAMS_NEED_X += test-date TEST_PROGRAMS_NEED_X += test-delta TEST_PROGRAMS_NEED_X += test-dump-cache-tree TEST_PROGRAMS_NEED_X += test-dump-split-index +TEST_PROGRAMS_NEED_X += test-dump-untracked-cache TEST_PROGRAMS_NEED_X += test-genrandom TEST_PROGRAMS_NEED_X += test-hashmap TEST_PROGRAMS_NEED_X += test-index-version diff --git a/builtin/commit.c b/builtin/commit.c index d6515a2a50..254477fd1d 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1366,13 +1366,14 @@ int cmd_status(int argc, const char **argv, const char *prefix) refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); fd = hold_locked_index(&index_lock, 0); - if (0 <= fd) - update_index_if_able(&the_index, &index_lock); s.is_initial = get_sha1(s.reference, sha1) ? 1 : 0; s.ignore_submodule_arg = ignore_submodule_arg; wt_status_collect(&s); + if (0 <= fd) + update_index_if_able(&the_index, &index_lock); + if (s.relative_paths) s.prefix = prefix; diff --git a/builtin/update-index.c b/builtin/update-index.c index 0665b31ea1..7431938fa6 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -33,6 +33,7 @@ static int mark_valid_only; static int mark_skip_worktree_only; #define MARK_FLAG 1 #define UNMARK_FLAG 2 +static struct strbuf mtime_dir = STRBUF_INIT; __attribute__((format (printf, 1, 2))) static void report(const char *fmt, ...) @@ -48,6 +49,166 @@ static void report(const char *fmt, ...) va_end(vp); } +static void remove_test_directory(void) +{ + if (mtime_dir.len) + remove_dir_recursively(&mtime_dir, 0); +} + +static const char *get_mtime_path(const char *path) +{ + static struct strbuf sb = STRBUF_INIT; + strbuf_reset(&sb); + strbuf_addf(&sb, "%s/%s", mtime_dir.buf, path); + return sb.buf; +} + +static void xmkdir(const char *path) +{ + path = get_mtime_path(path); + if (mkdir(path, 0700)) + die_errno(_("failed to create directory %s"), path); +} + +static int xstat_mtime_dir(struct stat *st) +{ + if (stat(mtime_dir.buf, st)) + die_errno(_("failed to stat %s"), mtime_dir.buf); + return 0; +} + +static int create_file(const char *path) +{ + int fd; + path = get_mtime_path(path); + fd = open(path, O_CREAT | O_RDWR, 0644); + if (fd < 0) + die_errno(_("failed to create file %s"), path); + return fd; +} + +static void xunlink(const char *path) +{ + path = get_mtime_path(path); + if (unlink(path)) + die_errno(_("failed to delete file %s"), path); +} + +static void xrmdir(const char *path) +{ + path = get_mtime_path(path); + if (rmdir(path)) + die_errno(_("failed to delete directory %s"), path); +} + +static void avoid_racy(void) +{ + /* + * not use if we could usleep(10) if USE_NSEC is defined. The + * field nsec could be there, but the OS could choose to + * ignore it? + */ + sleep(1); +} + +static int test_if_untracked_cache_is_supported(void) +{ + struct stat st; + struct stat_data base; + int fd, ret = 0; + + strbuf_addstr(&mtime_dir, "mtime-test-XXXXXX"); + if (!mkdtemp(mtime_dir.buf)) + die_errno("Could not make temporary directory"); + + fprintf(stderr, _("Testing ")); + atexit(remove_test_directory); + xstat_mtime_dir(&st); + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + fd = create_file("newfile"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + close(fd); + fputc('\n', stderr); + fprintf_ln(stderr,_("directory stat info does not " + "change after adding a new file")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + xmkdir("new-dir"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + close(fd); + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not change " + "after adding a new directory")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + write_or_die(fd, "data", 4); + close(fd); + xstat_mtime_dir(&st); + if (match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info changes " + "after updating a file")); + goto done; + } + fputc('.', stderr); + + avoid_racy(); + close(create_file("new-dir/new")); + xstat_mtime_dir(&st); + if (match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info changes after " + "adding a file inside subdirectory")); + goto done; + } + fputc('.', stderr); + + avoid_racy(); + xunlink("newfile"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not " + "change after deleting a file")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + xunlink("new-dir/new"); + xrmdir("new-dir"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not " + "change after deleting a directory")); + goto done; + } + + if (rmdir(mtime_dir.buf)) + die_errno(_("failed to delete directory %s"), mtime_dir.buf); + fprintf_ln(stderr, _(" OK")); + ret = 1; + +done: + strbuf_release(&mtime_dir); + return ret; +} + static int mark_ce_flags(const char *path, int flag, int mark) { int namelen = strlen(path); @@ -741,6 +902,7 @@ static int reupdate_callback(struct parse_opt_ctx_t *ctx, int cmd_update_index(int argc, const char **argv, const char *prefix) { int newfd, entries, has_errors = 0, line_termination = '\n'; + int untracked_cache = -1; int read_from_stdin = 0; int prefix_length = prefix ? strlen(prefix) : 0; int preferred_index_format = 0; @@ -832,6 +994,10 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) N_("write index in this format")), OPT_BOOL(0, "split-index", &split_index, N_("enable or disable split index")), + OPT_BOOL(0, "untracked-cache", &untracked_cache, + N_("enable/disable untracked cache")), + OPT_SET_INT(0, "force-untracked-cache", &untracked_cache, + N_("enable untracked cache without testing the filesystem"), 2), OPT_END() }; @@ -938,6 +1104,28 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) the_index.split_index = NULL; the_index.cache_changed |= SOMETHING_CHANGED; } + if (untracked_cache > 0) { + struct untracked_cache *uc; + + if (untracked_cache < 2) { + setup_work_tree(); + if (!test_if_untracked_cache_is_supported()) + return 1; + } + if (!the_index.untracked) { + uc = xcalloc(1, sizeof(*uc)); + strbuf_init(&uc->ident, 100); + uc->exclude_per_dir = ".gitignore"; + /* should be the same flags used by git-status */ + uc->dir_flags = DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + the_index.untracked = uc; + } + add_untracked_ident(the_index.untracked); + the_index.cache_changed |= UNTRACKED_CHANGED; + } else if (!untracked_cache && the_index.untracked) { + the_index.untracked = NULL; + the_index.cache_changed |= UNTRACKED_CHANGED; + } if (active_cache_changed) { if (newfd < 0) { @@ -297,8 +297,11 @@ static inline unsigned int canon_mode(unsigned int mode) #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; +struct untracked_cache; + struct index_state { struct cache_entry **cache; unsigned int version; @@ -312,6 +315,7 @@ struct index_state { struct hashmap name_hash; struct hashmap dir_hash; unsigned char sha1[20]; + struct untracked_cache *untracked; }; extern struct index_state the_index; @@ -563,6 +567,8 @@ extern void fill_stat_data(struct stat_data *sd, struct stat *st); * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); diff --git a/compat/mingw.c b/compat/mingw.c index 70f3191a4f..496e6f8bb0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2128,3 +2128,14 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); } + +int uname(struct utsname *buf) +{ + DWORD v = GetVersion(); + memset(buf, 0, sizeof(*buf)); + strcpy(buf->sysname, "Windows"); + sprintf(buf->release, "%u.%u", v & 0xff, (v >> 8) & 0xff); + /* assuming NT variants only.. */ + sprintf(buf->version, "%u", (v >> 16) & 0x7fff); + return 0; +} diff --git a/compat/mingw.h b/compat/mingw.h index 98c5e44294..738865c6c0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -76,6 +76,14 @@ struct itimerval { }; #define ITIMER_REAL 0 +struct utsname { + char sysname[16]; + char nodename[1]; + char release[16]; + char version[16]; + char machine[1]; +}; + /* * sanitize preprocessor namespace polluted by Windows headers defining * macros which collide with git local versions @@ -175,6 +183,7 @@ struct passwd *getpwuid(uid_t uid); int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); +int uname(struct utsname *buf); /* * replacements of existing functions @@ -13,6 +13,8 @@ #include "wildmatch.h" #include "pathspec.h" #include "utf8.h" +#include "varint.h" +#include "ewah/ewok.h" struct path_simplify { int len; @@ -32,8 +34,22 @@ enum path_treatment { path_untracked }; +/* + * Support data structure for our opendir/readdir/closedir wrappers + */ +struct cached_dir { + DIR *fdir; + struct untracked_cache_dir *untracked; + int nr_files; + int nr_dirs; + + struct dirent *de; + const char *file; + struct untracked_cache_dir *ucd; +}; + static enum path_treatment read_directory_recursive(struct dir_struct *dir, - const char *path, int len, + const char *path, int len, struct untracked_cache_dir *untracked, int check_only, const struct path_simplify *simplify); static int get_dtype(struct dirent *de, const char *path, int len); @@ -510,7 +526,8 @@ void add_exclude(const char *string, const char *base, x->el = el; } -static void *read_skip_worktree_file_from_index(const char *path, size_t *size) +static void *read_skip_worktree_file_from_index(const char *path, size_t *size, + struct sha1_stat *sha1_stat) { int pos, len; unsigned long sz; @@ -529,6 +546,10 @@ static void *read_skip_worktree_file_from_index(const char *path, size_t *size) return NULL; } *size = xsize_t(sz); + if (sha1_stat) { + memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + } return data; } @@ -573,11 +594,93 @@ static void trim_trailing_spaces(char *buf) *last_space = '\0'; } -int add_excludes_from_file_to_list(const char *fname, - const char *base, - int baselen, - struct exclude_list *el, - int check_index) +/* + * Given a subdirectory name and "dir" of the current directory, + * search the subdir in "dir" and return it, or create a new one if it + * does not exist in "dir". + * + * If "name" has the trailing slash, it'll be excluded in the search. + */ +static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc, + struct untracked_cache_dir *dir, + const char *name, int len) +{ + int first, last; + struct untracked_cache_dir *d; + if (!dir) + return NULL; + if (len && name[len - 1] == '/') + len--; + first = 0; + last = dir->dirs_nr; + while (last > first) { + int cmp, next = (last + first) >> 1; + d = dir->dirs[next]; + cmp = strncmp(name, d->name, len); + if (!cmp && strlen(d->name) > len) + cmp = -1; + if (!cmp) + return d; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + + uc->dir_created++; + d = xmalloc(sizeof(*d) + len + 1); + memset(d, 0, sizeof(*d)); + memcpy(d->name, name, len); + d->name[len] = '\0'; + + ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc); + memmove(dir->dirs + first + 1, dir->dirs + first, + (dir->dirs_nr - first) * sizeof(*dir->dirs)); + dir->dirs_nr++; + dir->dirs[first] = d; + return d; +} + +static void do_invalidate_gitignore(struct untracked_cache_dir *dir) +{ + int i; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + do_invalidate_gitignore(dir->dirs[i]); +} + +static void invalidate_gitignore(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + uc->gitignore_invalidated++; + do_invalidate_gitignore(dir); +} + +static void invalidate_directory(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + int i; + uc->dir_invalidated++; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + dir->dirs[i]->recurse = 0; +} + +/* + * Given a file with name "fname", read it (either from disk, or from + * the index if "check_index" is non-zero), parse it and store the + * exclude rules in "el". + * + * If "ss" is not NULL, compute SHA-1 of the exclude file and fill + * stat data from disk (only valid if add_excludes returns zero). If + * ss_valid is non-zero, "ss" must contain good value as input. + */ +static int add_excludes(const char *fname, const char *base, int baselen, + struct exclude_list *el, int check_index, + struct sha1_stat *sha1_stat) { struct stat st; int fd, i, lineno = 1; @@ -591,7 +694,7 @@ int add_excludes_from_file_to_list(const char *fname, if (0 <= fd) close(fd); if (!check_index || - (buf = read_skip_worktree_file_from_index(fname, &size)) == NULL) + (buf = read_skip_worktree_file_from_index(fname, &size, sha1_stat)) == NULL) return -1; if (size == 0) { free(buf); @@ -604,6 +707,11 @@ int add_excludes_from_file_to_list(const char *fname, } else { size = xsize_t(st.st_size); if (size == 0) { + if (sha1_stat) { + fill_stat_data(&sha1_stat->stat, &st); + hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN); + sha1_stat->valid = 1; + } close(fd); return 0; } @@ -615,6 +723,22 @@ int add_excludes_from_file_to_list(const char *fname, } buf[size++] = '\n'; close(fd); + if (sha1_stat) { + int pos; + if (sha1_stat->valid && + !match_stat_data_racy(&the_index, &sha1_stat->stat, &st)) + ; /* no content change, ss->sha1 still good */ + else if (check_index && + (pos = cache_name_pos(fname, strlen(fname))) >= 0 && + !ce_stage(active_cache[pos]) && + ce_uptodate(active_cache[pos]) && + !would_convert_to_git(fname)) + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + else + hash_sha1_file(buf, size, "blob", sha1_stat->sha1); + fill_stat_data(&sha1_stat->stat, &st); + sha1_stat->valid = 1; + } } el->filebuf = buf; @@ -638,6 +762,13 @@ int add_excludes_from_file_to_list(const char *fname, return 0; } +int add_excludes_from_file_to_list(const char *fname, const char *base, + int baselen, struct exclude_list *el, + int check_index) +{ + return add_excludes(fname, base, baselen, el, check_index, NULL); +} + struct exclude_list *add_exclude_list(struct dir_struct *dir, int group_type, const char *src) { @@ -655,14 +786,28 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir, /* * Used to set up core.excludesfile and .git/info/exclude lists. */ -void add_excludes_from_file(struct dir_struct *dir, const char *fname) +static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, + struct sha1_stat *sha1_stat) { struct exclude_list *el; + /* + * catch setup_standard_excludes() that's called before + * dir->untracked is assigned. That function behaves + * differently when dir->untracked is non-NULL. + */ + if (!dir->untracked) + dir->unmanaged_exclude_files++; el = add_exclude_list(dir, EXC_FILE, fname); - if (add_excludes_from_file_to_list(fname, "", 0, el, 0) < 0) + if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0) die("cannot use %s as an exclude file", fname); } +void add_excludes_from_file(struct dir_struct *dir, const char *fname) +{ + dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */ + add_excludes_from_file_1(dir, fname, NULL); +} + int match_basename(const char *basename, int basenamelen, const char *pattern, int prefix, int patternlen, int flags) @@ -837,6 +982,7 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) struct exclude_list_group *group; struct exclude_list *el; struct exclude_stack *stk = NULL; + struct untracked_cache_dir *untracked; int current; group = &dir->exclude_list_group[EXC_DIRS]; @@ -874,8 +1020,14 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) /* Read from the parent directories and push them down. */ current = stk ? stk->baselen : -1; strbuf_setlen(&dir->basebuf, current < 0 ? 0 : current); + if (dir->untracked) + untracked = stk ? stk->ucd : dir->untracked->root; + else + untracked = NULL; + while (current < baselen) { const char *cp; + struct sha1_stat sha1_stat; stk = xcalloc(1, sizeof(*stk)); if (current < 0) { @@ -886,10 +1038,15 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) if (!cp) die("oops in prep_exclude"); cp++; + untracked = + lookup_untracked(dir->untracked, untracked, + base + current, + cp - base - current); } stk->prev = dir->exclude_stack; stk->baselen = cp - base; stk->exclude_ix = group->nr; + stk->ucd = untracked; el = add_exclude_list(dir, EXC_DIRS, NULL); strbuf_add(&dir->basebuf, base + current, stk->baselen - current); assert(stk->baselen == dir->basebuf.len); @@ -912,7 +1069,23 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) } /* Try to read per-directory file */ - if (dir->exclude_per_dir) { + hashclr(sha1_stat.sha1); + sha1_stat.valid = 0; + if (dir->exclude_per_dir && + /* + * If we know that no files have been added in + * this directory (i.e. valid_cached_dir() has + * been executed and set untracked->valid) .. + */ + (!untracked || !untracked->valid || + /* + * .. and .gitignore does not exist before + * (i.e. null exclude_sha1 and skip_worktree is + * not set). Then we can skip loading .gitignore, + * which would result in ENOENT anyway. + * skip_worktree is taken care in read_directory() + */ + !is_null_sha1(untracked->exclude_sha1))) { /* * dir->basebuf gets reused by the traversal, but we * need fname to remain unchanged to ensure the src @@ -925,8 +1098,27 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) strbuf_addbuf(&sb, &dir->basebuf); strbuf_addstr(&sb, dir->exclude_per_dir); el->src = strbuf_detach(&sb, NULL); - add_excludes_from_file_to_list(el->src, el->src, - stk->baselen, el, 1); + add_excludes(el->src, el->src, stk->baselen, el, 1, + untracked ? &sha1_stat : NULL); + } + /* + * NEEDSWORK: when untracked cache is enabled, prep_exclude() + * will first be called in valid_cached_dir() then maybe many + * times more in last_exclude_matching(). When the cache is + * used, last_exclude_matching() will not be called and + * reading .gitignore content will be a waste. + * + * So when it's called by valid_cached_dir() and we can get + * .gitignore SHA-1 from the index (i.e. .gitignore is not + * modified on work tree), we could delay reading the + * .gitignore content until we absolutely need it in + * last_exclude_matching(). Be careful about ignore rule + * order, though, if you do that. + */ + if (untracked && + hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) { + invalidate_gitignore(dir->untracked, untracked); + hashcpy(untracked->exclude_sha1, sha1_stat.sha1); } dir->exclude_stack = stk; current = stk->baselen; @@ -1107,6 +1299,7 @@ static enum exist_status directory_exists_in_index(const char *dirname, int len) * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, + struct untracked_cache_dir *untracked, const char *dirname, int len, int exclude, const struct path_simplify *simplify) { @@ -1134,7 +1327,9 @@ static enum path_treatment treat_directory(struct dir_struct *dir, if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES)) return exclude ? path_excluded : path_untracked; - return read_directory_recursive(dir, dirname, len, 1, simplify); + untracked = lookup_untracked(dir->untracked, untracked, dirname, len); + return read_directory_recursive(dir, dirname, len, + untracked, 1, simplify); } /* @@ -1250,6 +1445,7 @@ static int get_dtype(struct dirent *de, const char *path, int len) } static enum path_treatment treat_one_path(struct dir_struct *dir, + struct untracked_cache_dir *untracked, struct strbuf *path, const struct path_simplify *simplify, int dtype, struct dirent *de) @@ -1302,7 +1498,7 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, return path_none; case DT_DIR: strbuf_addch(path, '/'); - return treat_directory(dir, path->buf, path->len, exclude, + return treat_directory(dir, untracked, path->buf, path->len, exclude, simplify); case DT_REG: case DT_LNK: @@ -1310,14 +1506,52 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, } } +static enum path_treatment treat_path_fast(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, + struct strbuf *path, + int baselen, + const struct path_simplify *simplify) +{ + strbuf_setlen(path, baselen); + if (!cdir->ucd) { + strbuf_addstr(path, cdir->file); + return path_untracked; + } + strbuf_addstr(path, cdir->ucd->name); + /* treat_one_path() does this before it calls treat_directory() */ + if (path->buf[path->len - 1] != '/') + strbuf_addch(path, '/'); + if (cdir->ucd->check_only) + /* + * check_only is set as a result of treat_directory() getting + * to its bottom. Verify again the same set of directories + * with check_only set. + */ + return read_directory_recursive(dir, path->buf, path->len, + cdir->ucd, 1, simplify); + /* + * We get path_recurse in the first run when + * directory_exists_in_index() returns index_nonexistent. We + * are sure that new changes in the index does not impact the + * outcome. Return now. + */ + return path_recurse; +} + static enum path_treatment treat_path(struct dir_struct *dir, - struct dirent *de, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, struct strbuf *path, int baselen, const struct path_simplify *simplify) { int dtype; + struct dirent *de = cdir->de; + if (!de) + return treat_path_fast(dir, untracked, cdir, path, + baselen, simplify); if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git")) return path_none; strbuf_setlen(path, baselen); @@ -1326,7 +1560,121 @@ static enum path_treatment treat_path(struct dir_struct *dir, return path_none; dtype = DTYPE(de); - return treat_one_path(dir, path, simplify, dtype, de); + return treat_one_path(dir, untracked, path, simplify, dtype, de); +} + +static void add_untracked(struct untracked_cache_dir *dir, const char *name) +{ + if (!dir) + return; + ALLOC_GROW(dir->untracked, dir->untracked_nr + 1, + dir->untracked_alloc); + dir->untracked[dir->untracked_nr++] = xstrdup(name); +} + +static int valid_cached_dir(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct strbuf *path, + int check_only) +{ + struct stat st; + + if (!untracked) + return 0; + + if (stat(path->len ? path->buf : ".", &st)) { + invalidate_directory(dir->untracked, untracked); + memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_stat_data_racy(&the_index, &untracked->stat_data, &st)) { + if (untracked->valid) + invalidate_directory(dir->untracked, untracked); + fill_stat_data(&untracked->stat_data, &st); + return 0; + } + + if (untracked->check_only != !!check_only) { + invalidate_directory(dir->untracked, untracked); + return 0; + } + + /* + * prep_exclude will be called eventually on this directory, + * but it's called much later in last_exclude_matching(). We + * need it now to determine the validity of the cache for this + * path. The next calls will be nearly no-op, the way + * prep_exclude() is designed. + */ + if (path->len && path->buf[path->len - 1] != '/') { + strbuf_addch(path, '/'); + prep_exclude(dir, path->buf, path->len); + strbuf_setlen(path, path->len - 1); + } else + prep_exclude(dir, path->buf, path->len); + + /* hopefully prep_exclude() haven't invalidated this entry... */ + return untracked->valid; +} + +static int open_cached_dir(struct cached_dir *cdir, + struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct strbuf *path, + int check_only) +{ + memset(cdir, 0, sizeof(*cdir)); + cdir->untracked = untracked; + if (valid_cached_dir(dir, untracked, path, check_only)) + return 0; + cdir->fdir = opendir(path->len ? path->buf : "."); + if (dir->untracked) + dir->untracked->dir_opened++; + if (!cdir->fdir) + return -1; + return 0; +} + +static int read_cached_dir(struct cached_dir *cdir) +{ + if (cdir->fdir) { + cdir->de = readdir(cdir->fdir); + if (!cdir->de) + return -1; + return 0; + } + while (cdir->nr_dirs < cdir->untracked->dirs_nr) { + struct untracked_cache_dir *d = cdir->untracked->dirs[cdir->nr_dirs]; + if (!d->recurse) { + cdir->nr_dirs++; + continue; + } + cdir->ucd = d; + cdir->nr_dirs++; + return 0; + } + cdir->ucd = NULL; + if (cdir->nr_files < cdir->untracked->untracked_nr) { + struct untracked_cache_dir *d = cdir->untracked; + cdir->file = d->untracked[cdir->nr_files++]; + return 0; + } + return -1; +} + +static void close_cached_dir(struct cached_dir *cdir) +{ + if (cdir->fdir) + closedir(cdir->fdir); + /* + * We have gone through this directory and found no untracked + * entries. Mark it valid. + */ + if (cdir->untracked) { + cdir->untracked->valid = 1; + cdir->untracked->recurse = 1; + } } /* @@ -1342,38 +1690,48 @@ static enum path_treatment treat_path(struct dir_struct *dir, */ static enum path_treatment read_directory_recursive(struct dir_struct *dir, const char *base, int baselen, - int check_only, + struct untracked_cache_dir *untracked, int check_only, const struct path_simplify *simplify) { - DIR *fdir; + struct cached_dir cdir; enum path_treatment state, subdir_state, dir_state = path_none; - struct dirent *de; struct strbuf path = STRBUF_INIT; strbuf_add(&path, base, baselen); - fdir = opendir(path.len ? path.buf : "."); - if (!fdir) + if (open_cached_dir(&cdir, dir, untracked, &path, check_only)) goto out; - while ((de = readdir(fdir)) != NULL) { + if (untracked) + untracked->check_only = !!check_only; + + while (!read_cached_dir(&cdir)) { /* check how the file or directory should be treated */ - state = treat_path(dir, de, &path, baselen, simplify); + state = treat_path(dir, untracked, &cdir, &path, baselen, simplify); + if (state > dir_state) dir_state = state; /* recurse into subdir if instructed by treat_path */ if (state == path_recurse) { - subdir_state = read_directory_recursive(dir, path.buf, - path.len, check_only, simplify); + struct untracked_cache_dir *ud; + ud = lookup_untracked(dir->untracked, untracked, + path.buf + baselen, + path.len - baselen); + subdir_state = + read_directory_recursive(dir, path.buf, path.len, + ud, check_only, simplify); if (subdir_state > dir_state) dir_state = subdir_state; } if (check_only) { /* abort early if maximum state has been reached */ - if (dir_state == path_untracked) + if (dir_state == path_untracked) { + if (cdir.fdir) + add_untracked(untracked, path.buf + baselen); break; + } /* skip the dir_add_* part */ continue; } @@ -1391,15 +1749,18 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, break; case path_untracked: - if (!(dir->flags & DIR_SHOW_IGNORED)) - dir_add_name(dir, path.buf, path.len); + if (dir->flags & DIR_SHOW_IGNORED) + break; + dir_add_name(dir, path.buf, path.len); + if (cdir.fdir) + add_untracked(untracked, path.buf + baselen); break; default: break; } } - closedir(fdir); + close_cached_dir(&cdir); out: strbuf_release(&path); @@ -1469,7 +1830,7 @@ static int treat_leading_path(struct dir_struct *dir, break; if (simplify_away(sb.buf, sb.len, simplify)) break; - if (treat_one_path(dir, &sb, simplify, + if (treat_one_path(dir, NULL, &sb, simplify, DT_DIR, NULL) == path_none) break; /* do not recurse into it */ if (len <= baselen) { @@ -1482,9 +1843,139 @@ static int treat_leading_path(struct dir_struct *dir, return rc; } +static const char *get_ident_string(void) +{ + static struct strbuf sb = STRBUF_INIT; + struct utsname uts; + + if (sb.len) + return sb.buf; + if (uname(&uts)) + die_errno(_("failed to get kernel name and information")); + strbuf_addf(&sb, "Location %s, system %s %s %s", get_git_work_tree(), + uts.sysname, uts.release, uts.version); + return sb.buf; +} + +static int ident_in_untracked(const struct untracked_cache *uc) +{ + const char *end = uc->ident.buf + uc->ident.len; + const char *p = uc->ident.buf; + + for (p = uc->ident.buf; p < end; p += strlen(p) + 1) + if (!strcmp(p, get_ident_string())) + return 1; + return 0; +} + +void add_untracked_ident(struct untracked_cache *uc) +{ + if (ident_in_untracked(uc)) + return; + strbuf_addstr(&uc->ident, get_ident_string()); + /* this strbuf contains a list of strings, save NUL too */ + strbuf_addch(&uc->ident, 0); +} + +static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *dir, + int base_len, + const struct pathspec *pathspec) +{ + struct untracked_cache_dir *root; + int i; + + if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE")) + return NULL; + + /* + * We only support $GIT_DIR/info/exclude and core.excludesfile + * as the global ignore rule files. Any other additions + * (e.g. from command line) invalidate the cache. This + * condition also catches running setup_standard_excludes() + * before setting dir->untracked! + */ + if (dir->unmanaged_exclude_files) + return NULL; + + /* + * Optimize for the main use case only: whole-tree git + * status. More work involved in treat_leading_path() if we + * use cache on just a subset of the worktree. pathspec + * support could make the matter even worse. + */ + if (base_len || (pathspec && pathspec->nr)) + return NULL; + + /* Different set of flags may produce different results */ + if (dir->flags != dir->untracked->dir_flags || + /* + * See treat_directory(), case index_nonexistent. Without + * this flag, we may need to also cache .git file content + * for the resolve_gitlink_ref() call, which we don't. + */ + !(dir->flags & DIR_SHOW_OTHER_DIRECTORIES) || + /* We don't support collecting ignore files */ + (dir->flags & (DIR_SHOW_IGNORED | DIR_SHOW_IGNORED_TOO | + DIR_COLLECT_IGNORED))) + return NULL; + + /* + * If we use .gitignore in the cache and now you change it to + * .gitexclude, everything will go wrong. + */ + if (dir->exclude_per_dir != dir->untracked->exclude_per_dir && + strcmp(dir->exclude_per_dir, dir->untracked->exclude_per_dir)) + return NULL; + + /* + * EXC_CMDL is not considered in the cache. If people set it, + * skip the cache. + */ + if (dir->exclude_list_group[EXC_CMDL].nr) + return NULL; + + /* + * An optimization in prep_exclude() does not play well with + * CE_SKIP_WORKTREE. It's a rare case anyway, if a single + * entry has that bit set, disable the whole untracked cache. + */ + for (i = 0; i < active_nr; i++) + if (ce_skip_worktree(active_cache[i])) + return NULL; + + if (!ident_in_untracked(dir->untracked)) { + warning(_("Untracked cache is disabled on this system.")); + return NULL; + } + + if (!dir->untracked->root) { + const int len = sizeof(*dir->untracked->root); + dir->untracked->root = xmalloc(len); + memset(dir->untracked->root, 0, len); + } + + /* Validate $GIT_DIR/info/exclude and core.excludesfile */ + root = dir->untracked->root; + if (hashcmp(dir->ss_info_exclude.sha1, + dir->untracked->ss_info_exclude.sha1)) { + invalidate_gitignore(dir->untracked, root); + dir->untracked->ss_info_exclude = dir->ss_info_exclude; + } + if (hashcmp(dir->ss_excludes_file.sha1, + dir->untracked->ss_excludes_file.sha1)) { + invalidate_gitignore(dir->untracked, root); + dir->untracked->ss_excludes_file = dir->ss_excludes_file; + } + + /* Make sure this directory is not dropped out at saving phase */ + root->recurse = 1; + return root; +} + int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec) { struct path_simplify *simplify; + struct untracked_cache_dir *untracked; /* * Check out create_simplify() @@ -1508,11 +1999,39 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru * create_simplify(). */ simplify = create_simplify(pathspec ? pathspec->_raw : NULL); + untracked = validate_untracked_cache(dir, len, pathspec); + if (!untracked) + /* + * make sure untracked cache code path is disabled, + * e.g. prep_exclude() + */ + dir->untracked = NULL; if (!len || treat_leading_path(dir, path, len, simplify)) - read_directory_recursive(dir, path, len, 0, simplify); + read_directory_recursive(dir, path, len, untracked, 0, simplify); free_simplify(simplify); qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name); qsort(dir->ignored, dir->ignored_nr, sizeof(struct dir_entry *), cmp_name); + if (dir->untracked) { + static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS); + trace_printf_key(&trace_untracked_stats, + "node creation: %u\n" + "gitignore invalidation: %u\n" + "directory invalidation: %u\n" + "opendir: %u\n", + dir->untracked->dir_created, + dir->untracked->gitignore_invalidated, + dir->untracked->dir_invalidated, + dir->untracked->dir_opened); + if (dir->untracked == the_index.untracked && + (dir->untracked->dir_opened || + dir->untracked->gitignore_invalidated || + dir->untracked->dir_invalidated)) + the_index.cache_changed |= UNTRACKED_CHANGED; + if (dir->untracked != the_index.untracked) { + free(dir->untracked); + dir->untracked = NULL; + } + } return dir->nr; } @@ -1678,12 +2197,14 @@ void setup_standard_excludes(struct dir_struct *dir) if (!excludes_file) excludes_file = xdg_config_home("ignore"); if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) - add_excludes_from_file(dir, excludes_file); + add_excludes_from_file_1(dir, excludes_file, + dir->untracked ? &dir->ss_excludes_file : NULL); /* per repository user preference */ path = git_path("info/exclude"); if (!access_or_warn(path, R_OK, 0)) - add_excludes_from_file(dir, path); + add_excludes_from_file_1(dir, path, + dir->untracked ? &dir->ss_info_exclude : NULL); } int remove_path(const char *name) @@ -1735,3 +2256,404 @@ void clear_directory(struct dir_struct *dir) } strbuf_release(&dir->basebuf); } + +struct ondisk_untracked_cache { + struct stat_data info_exclude_stat; + struct stat_data excludes_file_stat; + uint32_t dir_flags; + unsigned char info_exclude_sha1[20]; + unsigned char excludes_file_sha1[20]; + char exclude_per_dir[FLEX_ARRAY]; +}; + +#define ouc_size(len) (offsetof(struct ondisk_untracked_cache, exclude_per_dir) + len + 1) + +struct write_data { + int index; /* number of written untracked_cache_dir */ + struct ewah_bitmap *check_only; /* from untracked_cache_dir */ + struct ewah_bitmap *valid; /* from untracked_cache_dir */ + struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */ + struct strbuf out; + struct strbuf sb_stat; + struct strbuf sb_sha1; +}; + +static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from) +{ + to->sd_ctime.sec = htonl(from->sd_ctime.sec); + to->sd_ctime.nsec = htonl(from->sd_ctime.nsec); + to->sd_mtime.sec = htonl(from->sd_mtime.sec); + to->sd_mtime.nsec = htonl(from->sd_mtime.nsec); + to->sd_dev = htonl(from->sd_dev); + to->sd_ino = htonl(from->sd_ino); + to->sd_uid = htonl(from->sd_uid); + to->sd_gid = htonl(from->sd_gid); + to->sd_size = htonl(from->sd_size); +} + +static void write_one_dir(struct untracked_cache_dir *untracked, + struct write_data *wd) +{ + struct stat_data stat_data; + struct strbuf *out = &wd->out; + unsigned char intbuf[16]; + unsigned int intlen, value; + int i = wd->index++; + + /* + * untracked_nr should be reset whenever valid is clear, but + * for safety.. + */ + if (!untracked->valid) { + untracked->untracked_nr = 0; + untracked->check_only = 0; + } + + if (untracked->check_only) + ewah_set(wd->check_only, i); + if (untracked->valid) { + ewah_set(wd->valid, i); + stat_data_to_disk(&stat_data, &untracked->stat_data); + strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data)); + } + if (!is_null_sha1(untracked->exclude_sha1)) { + ewah_set(wd->sha1_valid, i); + strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20); + } + + intlen = encode_varint(untracked->untracked_nr, intbuf); + strbuf_add(out, intbuf, intlen); + + /* skip non-recurse directories */ + for (i = 0, value = 0; i < untracked->dirs_nr; i++) + if (untracked->dirs[i]->recurse) + value++; + intlen = encode_varint(value, intbuf); + strbuf_add(out, intbuf, intlen); + + strbuf_add(out, untracked->name, strlen(untracked->name) + 1); + + for (i = 0; i < untracked->untracked_nr; i++) + strbuf_add(out, untracked->untracked[i], + strlen(untracked->untracked[i]) + 1); + + for (i = 0; i < untracked->dirs_nr; i++) + if (untracked->dirs[i]->recurse) + write_one_dir(untracked->dirs[i], wd); +} + +void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked) +{ + struct ondisk_untracked_cache *ouc; + struct write_data wd; + unsigned char varbuf[16]; + int len = 0, varint_len; + if (untracked->exclude_per_dir) + len = strlen(untracked->exclude_per_dir); + ouc = xmalloc(sizeof(*ouc) + len + 1); + stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat); + stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat); + hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1); + hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1); + ouc->dir_flags = htonl(untracked->dir_flags); + memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1); + + varint_len = encode_varint(untracked->ident.len, varbuf); + strbuf_add(out, varbuf, varint_len); + strbuf_add(out, untracked->ident.buf, untracked->ident.len); + + strbuf_add(out, ouc, ouc_size(len)); + free(ouc); + ouc = NULL; + + if (!untracked->root) { + varint_len = encode_varint(0, varbuf); + strbuf_add(out, varbuf, varint_len); + return; + } + + wd.index = 0; + wd.check_only = ewah_new(); + wd.valid = ewah_new(); + wd.sha1_valid = ewah_new(); + strbuf_init(&wd.out, 1024); + strbuf_init(&wd.sb_stat, 1024); + strbuf_init(&wd.sb_sha1, 1024); + write_one_dir(untracked->root, &wd); + + varint_len = encode_varint(wd.index, varbuf); + strbuf_add(out, varbuf, varint_len); + strbuf_addbuf(out, &wd.out); + ewah_serialize_strbuf(wd.valid, out); + ewah_serialize_strbuf(wd.check_only, out); + ewah_serialize_strbuf(wd.sha1_valid, out); + strbuf_addbuf(out, &wd.sb_stat); + strbuf_addbuf(out, &wd.sb_sha1); + strbuf_addch(out, '\0'); /* safe guard for string lists */ + + ewah_free(wd.valid); + ewah_free(wd.check_only); + ewah_free(wd.sha1_valid); + strbuf_release(&wd.out); + strbuf_release(&wd.sb_stat); + strbuf_release(&wd.sb_sha1); +} + +static void free_untracked(struct untracked_cache_dir *ucd) +{ + int i; + if (!ucd) + return; + for (i = 0; i < ucd->dirs_nr; i++) + free_untracked(ucd->dirs[i]); + for (i = 0; i < ucd->untracked_nr; i++) + free(ucd->untracked[i]); + free(ucd->untracked); + free(ucd->dirs); + free(ucd); +} + +void free_untracked_cache(struct untracked_cache *uc) +{ + if (uc) + free_untracked(uc->root); + free(uc); +} + +struct read_data { + int index; + struct untracked_cache_dir **ucd; + struct ewah_bitmap *check_only; + struct ewah_bitmap *valid; + struct ewah_bitmap *sha1_valid; + const unsigned char *data; + const unsigned char *end; +}; + +static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from) +{ + to->sd_ctime.sec = get_be32(&from->sd_ctime.sec); + to->sd_ctime.nsec = get_be32(&from->sd_ctime.nsec); + to->sd_mtime.sec = get_be32(&from->sd_mtime.sec); + to->sd_mtime.nsec = get_be32(&from->sd_mtime.nsec); + to->sd_dev = get_be32(&from->sd_dev); + to->sd_ino = get_be32(&from->sd_ino); + to->sd_uid = get_be32(&from->sd_uid); + to->sd_gid = get_be32(&from->sd_gid); + to->sd_size = get_be32(&from->sd_size); +} + +static int read_one_dir(struct untracked_cache_dir **untracked_, + struct read_data *rd) +{ + struct untracked_cache_dir ud, *untracked; + const unsigned char *next, *data = rd->data, *end = rd->end; + unsigned int value; + int i, len; + + memset(&ud, 0, sizeof(ud)); + + next = data; + value = decode_varint(&next); + if (next > end) + return -1; + ud.recurse = 1; + ud.untracked_alloc = value; + ud.untracked_nr = value; + if (ud.untracked_nr) + ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr); + data = next; + + next = data; + ud.dirs_alloc = ud.dirs_nr = decode_varint(&next); + if (next > end) + return -1; + ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr); + data = next; + + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + *untracked_ = untracked = xmalloc(sizeof(*untracked) + len); + memcpy(untracked, &ud, sizeof(ud)); + memcpy(untracked->name, data, len + 1); + data = next; + + for (i = 0; i < untracked->untracked_nr; i++) { + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + untracked->untracked[i] = xstrdup((const char*)data); + data = next; + } + + rd->ucd[rd->index++] = untracked; + rd->data = data; + + for (i = 0; i < untracked->dirs_nr; i++) { + len = read_one_dir(untracked->dirs + i, rd); + if (len < 0) + return -1; + } + return 0; +} + +static void set_check_only(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + ud->check_only = 1; +} + +static void read_stat(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + sizeof(struct stat_data) > rd->end) { + rd->data = rd->end + 1; + return; + } + stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data); + rd->data += sizeof(struct stat_data); + ud->valid = 1; +} + +static void read_sha1(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + 20 > rd->end) { + rd->data = rd->end + 1; + return; + } + hashcpy(ud->exclude_sha1, rd->data); + rd->data += 20; +} + +static void load_sha1_stat(struct sha1_stat *sha1_stat, + const struct stat_data *stat, + const unsigned char *sha1) +{ + stat_data_from_disk(&sha1_stat->stat, stat); + hashcpy(sha1_stat->sha1, sha1); + sha1_stat->valid = 1; +} + +struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz) +{ + const struct ondisk_untracked_cache *ouc; + struct untracked_cache *uc; + struct read_data rd; + const unsigned char *next = data, *end = (const unsigned char *)data + sz; + const char *ident; + int ident_len, len; + + if (sz <= 1 || end[-1] != '\0') + return NULL; + end--; + + ident_len = decode_varint(&next); + if (next + ident_len > end) + return NULL; + ident = (const char *)next; + next += ident_len; + + ouc = (const struct ondisk_untracked_cache *)next; + if (next + ouc_size(0) > end) + return NULL; + + uc = xcalloc(1, sizeof(*uc)); + strbuf_init(&uc->ident, ident_len); + strbuf_add(&uc->ident, ident, ident_len); + load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat, + ouc->info_exclude_sha1); + load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat, + ouc->excludes_file_sha1); + uc->dir_flags = get_be32(&ouc->dir_flags); + uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir); + /* NUL after exclude_per_dir is covered by sizeof(*ouc) */ + next += ouc_size(strlen(ouc->exclude_per_dir)); + if (next >= end) + goto done2; + + len = decode_varint(&next); + if (next > end || len == 0) + goto done2; + + rd.valid = ewah_new(); + rd.check_only = ewah_new(); + rd.sha1_valid = ewah_new(); + rd.data = next; + rd.end = end; + rd.index = 0; + rd.ucd = xmalloc(sizeof(*rd.ucd) * len); + + if (read_one_dir(&uc->root, &rd) || rd.index != len) + goto done; + + next = rd.data; + len = ewah_read_mmap(rd.valid, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.check_only, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.sha1_valid, next, end - next); + if (len < 0) + goto done; + + ewah_each_bit(rd.check_only, set_check_only, &rd); + rd.data = next + len; + ewah_each_bit(rd.valid, read_stat, &rd); + ewah_each_bit(rd.sha1_valid, read_sha1, &rd); + next = rd.data; + +done: + free(rd.ucd); + ewah_free(rd.valid); + ewah_free(rd.check_only); + ewah_free(rd.sha1_valid); +done2: + if (next != end) { + free_untracked_cache(uc); + uc = NULL; + } + return uc; +} + +void untracked_cache_invalidate_path(struct index_state *istate, + const char *path) +{ + const char *sep; + struct untracked_cache_dir *d; + if (!istate->untracked || !istate->untracked->root) + return; + sep = strrchr(path, '/'); + if (sep) + d = lookup_untracked(istate->untracked, + istate->untracked->root, + path, sep - path); + else + d = istate->untracked->root; + istate->untracked->dir_invalidated++; + d->valid = 0; + d->untracked_nr = 0; +} + +void untracked_cache_remove_from_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); +} + +void untracked_cache_add_to_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); +} @@ -66,6 +66,7 @@ struct exclude_stack { struct exclude_stack *prev; /* the struct exclude_stack for the parent directory */ int baselen; int exclude_ix; /* index of exclude_list within EXC_DIRS exclude_list_group */ + struct untracked_cache_dir *ucd; }; struct exclude_list_group { @@ -73,6 +74,73 @@ struct exclude_list_group { struct exclude_list *el; }; +struct sha1_stat { + struct stat_data stat; + unsigned char sha1[20]; + int valid; +}; + +/* + * Untracked cache + * + * The following inputs are sufficient to determine what files in a + * directory are excluded: + * + * - The list of files and directories of the directory in question + * - The $GIT_DIR/index + * - dir_struct flags + * - The content of $GIT_DIR/info/exclude + * - The content of core.excludesfile + * - The content (or the lack) of .gitignore of all parent directories + * from $GIT_WORK_TREE + * - The check_only flag in read_directory_recursive (for + * DIR_HIDE_EMPTY_DIRECTORIES) + * + * The first input can be checked using directory mtime. In many + * filesystems, directory mtime (stat_data field) is updated when its + * files or direct subdirs are added or removed. + * + * The second one can be hooked from cache_tree_invalidate_path(). + * Whenever a file (or a submodule) is added or removed from a + * directory, we invalidate that directory. + * + * The remaining inputs are easy, their SHA-1 could be used to verify + * their contents (exclude_sha1[], info_exclude_sha1[] and + * excludes_file_sha1[]) + */ +struct untracked_cache_dir { + struct untracked_cache_dir **dirs; + char **untracked; + struct stat_data stat_data; + unsigned int untracked_alloc, dirs_nr, dirs_alloc; + unsigned int untracked_nr; + unsigned int check_only : 1; + /* all data except 'dirs' in this struct are good */ + unsigned int valid : 1; + unsigned int recurse : 1; + /* null SHA-1 means this directory does not have .gitignore */ + unsigned char exclude_sha1[20]; + char name[FLEX_ARRAY]; +}; + +struct untracked_cache { + struct sha1_stat ss_info_exclude; + struct sha1_stat ss_excludes_file; + const char *exclude_per_dir; + struct strbuf ident; + /* + * dir_struct#flags must match dir_flags or the untracked + * cache is ignored. + */ + unsigned dir_flags; + struct untracked_cache_dir *root; + /* Statistics */ + int dir_created; + int gitignore_invalidated; + int dir_invalidated; + int dir_opened; +}; + struct dir_struct { int nr, alloc; int ignored_nr, ignored_alloc; @@ -120,6 +188,12 @@ struct dir_struct { struct exclude_stack *exclude_stack; struct exclude *exclude; struct strbuf basebuf; + + /* Enable untracked file cache if set */ + struct untracked_cache *untracked; + struct sha1_stat ss_info_exclude; + struct sha1_stat ss_excludes_file; + unsigned unmanaged_exclude_files; }; /* @@ -226,4 +300,12 @@ static inline int dir_path_match(const struct dir_entry *ent, has_trailing_dir); } +void untracked_cache_invalidate_path(struct index_state *, const char *); +void untracked_cache_remove_from_index(struct index_state *, const char *); +void untracked_cache_add_to_index(struct index_state *, const char *); + +void free_untracked_cache(struct untracked_cache *); +struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz); +void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked); +void add_untracked_ident(struct untracked_cache *); #endif diff --git a/ewah/ewah_io.c b/ewah/ewah_io.c index 1c2d7afd4c..43481b9c60 100644 --- a/ewah/ewah_io.c +++ b/ewah/ewah_io.c @@ -19,6 +19,7 @@ */ #include "git-compat-util.h" #include "ewok.h" +#include "strbuf.h" int ewah_serialize_native(struct ewah_bitmap *self, int fd) { @@ -110,6 +111,18 @@ int ewah_serialize(struct ewah_bitmap *self, int fd) return ewah_serialize_to(self, write_helper, (void *)(intptr_t)fd); } +static int write_strbuf(void *user_data, const void *data, size_t len) +{ + struct strbuf *sb = user_data; + strbuf_add(sb, data, len); + return len; +} + +int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *sb) +{ + return ewah_serialize_to(self, write_strbuf, sb); +} + int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len) { const uint8_t *ptr = map; diff --git a/ewah/ewok.h b/ewah/ewok.h index 13c6e20412..e732525367 100644 --- a/ewah/ewok.h +++ b/ewah/ewok.h @@ -30,6 +30,7 @@ # define ewah_calloc xcalloc #endif +struct strbuf; typedef uint64_t eword_t; #define BITS_IN_WORD (sizeof(eword_t) * 8) @@ -98,6 +99,7 @@ int ewah_serialize_to(struct ewah_bitmap *self, void *out); int ewah_serialize(struct ewah_bitmap *self, int fd); int ewah_serialize_native(struct ewah_bitmap *self, int fd); +int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *); int ewah_deserialize(struct ewah_bitmap *self, int fd); int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len); diff --git a/git-compat-util.h b/git-compat-util.h index b7a97fbe83..17584adbd0 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -189,6 +189,7 @@ #elif defined(_MSC_VER) #include "compat/msvc.h" #else +#include <sys/utsname.h> #include <sys/wait.h> #include <sys/resource.h> #include <sys/socket.h> diff --git a/read-cache.c b/read-cache.c index 36ff89f29e..723d48dddf 100644 --- a/read-cache.c +++ b/read-cache.c @@ -39,11 +39,12 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, #define CACHE_EXT_TREE 0x54524545 /* "TREE" */ #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ +#define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \ - SPLIT_INDEX_ORDERED) + SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED) struct index_state the_index; static const char *alternate_index_output; @@ -79,6 +80,7 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n memcpy(new->name, new_name, namelen + 1); cache_tree_invalidate_path(istate, old->name); + untracked_cache_remove_from_index(istate, old->name); remove_index_entry_at(istate, nr); add_index_entry(istate, new, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE); } @@ -270,20 +272,34 @@ static int ce_match_stat_basic(const struct cache_entry *ce, struct stat *st) return changed; } -static int is_racy_timestamp(const struct index_state *istate, - const struct cache_entry *ce) +static int is_racy_stat(const struct index_state *istate, + const struct stat_data *sd) { - return (!S_ISGITLINK(ce->ce_mode) && - istate->timestamp.sec && + return (istate->timestamp.sec && #ifdef USE_NSEC /* nanosecond timestamped files can also be racy! */ - (istate->timestamp.sec < ce->ce_stat_data.sd_mtime.sec || - (istate->timestamp.sec == ce->ce_stat_data.sd_mtime.sec && - istate->timestamp.nsec <= ce->ce_stat_data.sd_mtime.nsec)) + (istate->timestamp.sec < sd->sd_mtime.sec || + (istate->timestamp.sec == sd->sd_mtime.sec && + istate->timestamp.nsec <= sd->sd_mtime.nsec)) #else - istate->timestamp.sec <= ce->ce_stat_data.sd_mtime.sec + istate->timestamp.sec <= sd->sd_mtime.sec #endif - ); + ); +} + +static int is_racy_timestamp(const struct index_state *istate, + const struct cache_entry *ce) +{ + return (!S_ISGITLINK(ce->ce_mode) && + is_racy_stat(istate, &ce->ce_stat_data)); +} + +int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st) +{ + if (is_racy_stat(istate, sd)) + return MTIME_CHANGED; + return match_stat_data(sd, st); } int ie_match_stat(const struct index_state *istate, @@ -538,6 +554,7 @@ int remove_file_from_index(struct index_state *istate, const char *path) if (pos < 0) pos = -pos-1; cache_tree_invalidate_path(istate, path); + untracked_cache_remove_from_index(istate, path); while (pos < istate->cache_nr && !strcmp(istate->cache[pos]->name, path)) remove_index_entry_at(istate, pos); return 0; @@ -982,6 +999,8 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e } pos = -pos-1; + untracked_cache_add_to_index(istate, ce->name); + /* * Inserting a merged entry ("stage 0") into the index * will always replace all non-merged entries.. @@ -1372,6 +1391,9 @@ static int read_index_extension(struct index_state *istate, if (read_link_extension(istate, data, sz)) return -1; break; + case CACHE_EXT_UNTRACKED: + istate->untracked = read_untracked_extension(data, sz); + break; default: if (*ext < 'A' || 'Z' < *ext) return error("index uses %.4s extension, which we do not understand", @@ -1667,6 +1689,8 @@ int discard_index(struct index_state *istate) istate->cache = NULL; istate->cache_alloc = 0; discard_split_index(istate); + free_untracked_cache(istate->untracked); + istate->untracked = NULL; return 0; } @@ -2053,6 +2077,17 @@ static int do_write_index(struct index_state *istate, int newfd, if (err) return -1; } + if (!strip_extensions && istate->untracked) { + struct strbuf sb = STRBUF_INIT; + + write_untracked_extension(&sb, istate->untracked); + err = write_index_ext_header(&c, newfd, CACHE_EXT_UNTRACKED, + sb.len) < 0 || + ce_write(&c, newfd, sb.buf, sb.len) < 0; + strbuf_release(&sb); + if (err) + return -1; + } if (ce_flush(&c, newfd, istate->sha1) || fstat(newfd, &st)) return -1; diff --git a/split-index.c b/split-index.c index 21485e2066..968b780a06 100644 --- a/split-index.c +++ b/split-index.c @@ -41,13 +41,6 @@ int read_link_extension(struct index_state *istate, return 0; } -static int write_strbuf(void *user_data, const void *data, size_t len) -{ - struct strbuf *sb = user_data; - strbuf_add(sb, data, len); - return len; -} - int write_link_extension(struct strbuf *sb, struct index_state *istate) { @@ -55,8 +48,8 @@ int write_link_extension(struct strbuf *sb, strbuf_add(sb, si->base_sha1, 20); if (!si->delete_bitmap && !si->replace_bitmap) return 0; - ewah_serialize_to(si->delete_bitmap, write_strbuf, sb); - ewah_serialize_to(si->replace_bitmap, write_strbuf, sb); + ewah_serialize_strbuf(si->delete_bitmap, sb); + ewah_serialize_strbuf(si->replace_bitmap, sb); return 0; } diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh new file mode 100755 index 0000000000..2b2ffd7d6b --- /dev/null +++ b/t/t7063-status-untracked-cache.sh @@ -0,0 +1,353 @@ +#!/bin/sh + +test_description='test untracked cache' + +. ./test-lib.sh + +avoid_racy() { + sleep 1 +} + +git update-index --untracked-cache +# It's fine if git update-index returns an error code other than one, +# it'll be caught in the first test. +if test $? -eq 1; then + skip_all='This system does not support untracked cache' + test_done +fi + +test_expect_success 'setup' ' + git init worktree && + cd worktree && + mkdir done dtwo dthree && + touch one two three done/one dtwo/two dthree/three && + git add one two done/one && + : >.git/info/exclude && + git update-index --untracked-cache +' + +test_expect_success 'untracked cache is empty' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 0000000000000000000000000000000000000000 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +EOF + test_cmp ../expect ../actual +' + +cat >../status.expect <<EOF && +A done/one +A one +A two +?? dthree/ +?? dtwo/ +?? three +EOF + +cat >../dump.expect <<EOF && +info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ 0000000000000000000000000000000000000000 recurse valid +dthree/ +dtwo/ +three +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +three +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + +test_expect_success 'status first time (empty cache)' ' + avoid_racy && + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 3 +gitignore invalidation: 1 +directory invalidation: 0 +opendir: 4 +EOF + test_cmp ../trace.expect ../trace +' + +test_expect_success 'untracked cache after first status' ' + test-dump-untracked-cache >../actual && + test_cmp ../dump.expect ../actual +' + +test_expect_success 'status second time (fully populated cache)' ' + avoid_racy && + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 0 +directory invalidation: 0 +opendir: 0 +EOF + test_cmp ../trace.expect ../trace +' + +test_expect_success 'untracked cache after second status' ' + test-dump-untracked-cache >../actual && + test_cmp ../dump.expect ../actual +' + +test_expect_success 'modify in root directory, one dir invalidation' ' + avoid_racy && + : >four && + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + cat >../status.expect <<EOF && +A done/one +A one +A two +?? dthree/ +?? dtwo/ +?? four +?? three +EOF + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 0 +directory invalidation: 1 +opendir: 1 +EOF + test_cmp ../trace.expect ../trace + +' + +test_expect_success 'verify untracked cache dump' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ 0000000000000000000000000000000000000000 recurse valid +dthree/ +dtwo/ +four +three +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +three +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'new .gitignore invalidates recursively' ' + avoid_racy && + echo four >.gitignore && + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + cat >../status.expect <<EOF && +A done/one +A one +A two +?? .gitignore +?? dthree/ +?? dtwo/ +?? three +EOF + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 1 +directory invalidation: 1 +opendir: 4 +EOF + test_cmp ../trace.expect ../trace + +' + +test_expect_success 'verify untracked cache dump' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid +.gitignore +dthree/ +dtwo/ +three +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +three +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'new info/exclude invalidates everything' ' + avoid_racy && + echo three >>.git/info/exclude && + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + cat >../status.expect <<EOF && +A done/one +A one +A two +?? .gitignore +?? dtwo/ +EOF + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 1 +directory invalidation: 0 +opendir: 4 +EOF + test_cmp ../trace.expect ../trace +' + +test_expect_success 'verify untracked cache dump' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid +.gitignore +dtwo/ +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'move two from tracked to untracked' ' + git rm --cached two && + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'status after the move' ' + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + cat >../status.expect <<EOF && +A done/one +A one +?? .gitignore +?? dtwo/ +?? two +EOF + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 0 +directory invalidation: 0 +opendir: 1 +EOF + test_cmp ../trace.expect ../trace +' + +test_expect_success 'verify untracked cache dump' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid +.gitignore +dtwo/ +two +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'move two from untracked to tracked' ' + git add two && + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_expect_success 'status after the move' ' + : >../trace && + GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + git status --porcelain >../actual && + cat >../status.expect <<EOF && +A done/one +A one +A two +?? .gitignore +?? dtwo/ +EOF + test_cmp ../status.expect ../actual && + cat >../trace.expect <<EOF && +node creation: 0 +gitignore invalidation: 0 +directory invalidation: 0 +opendir: 1 +EOF + test_cmp ../trace.expect ../trace +' + +test_expect_success 'verify untracked cache dump' ' + test-dump-untracked-cache >../actual && + cat >../expect <<EOF && +info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0 +core.excludesfile 0000000000000000000000000000000000000000 +exclude_per_dir .gitignore +flags 00000006 +/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid +.gitignore +dtwo/ +/done/ 0000000000000000000000000000000000000000 recurse valid +/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid +/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid +two +EOF + test_cmp ../expect ../actual +' + +test_done diff --git a/test-dump-untracked-cache.c b/test-dump-untracked-cache.c new file mode 100644 index 0000000000..25d855d98b --- /dev/null +++ b/test-dump-untracked-cache.c @@ -0,0 +1,62 @@ +#include "cache.h" +#include "dir.h" + +static int compare_untracked(const void *a_, const void *b_) +{ + const char *const *a = a_; + const char *const *b = b_; + return strcmp(*a, *b); +} + +static int compare_dir(const void *a_, const void *b_) +{ + const struct untracked_cache_dir *const *a = a_; + const struct untracked_cache_dir *const *b = b_; + return strcmp((*a)->name, (*b)->name); +} + +static void dump(struct untracked_cache_dir *ucd, struct strbuf *base) +{ + int i, len; + qsort(ucd->untracked, ucd->untracked_nr, sizeof(*ucd->untracked), + compare_untracked); + qsort(ucd->dirs, ucd->dirs_nr, sizeof(*ucd->dirs), + compare_dir); + len = base->len; + strbuf_addf(base, "%s/", ucd->name); + printf("%s %s", base->buf, + sha1_to_hex(ucd->exclude_sha1)); + if (ucd->recurse) + fputs(" recurse", stdout); + if (ucd->check_only) + fputs(" check_only", stdout); + if (ucd->valid) + fputs(" valid", stdout); + printf("\n"); + for (i = 0; i < ucd->untracked_nr; i++) + printf("%s\n", ucd->untracked[i]); + for (i = 0; i < ucd->dirs_nr; i++) + dump(ucd->dirs[i], base); + strbuf_setlen(base, len); +} + +int main(int ac, char **av) +{ + struct untracked_cache *uc; + struct strbuf base = STRBUF_INIT; + setup_git_directory(); + if (read_cache() < 0) + die("unable to read index file"); + uc = the_index.untracked; + if (!uc) { + printf("no untracked cache\n"); + return 0; + } + printf("info/exclude %s\n", sha1_to_hex(uc->ss_info_exclude.sha1)); + printf("core.excludesfile %s\n", sha1_to_hex(uc->ss_excludes_file.sha1)); + printf("exclude_per_dir %s\n", uc->exclude_per_dir); + printf("flags %08x\n", uc->dir_flags); + if (uc->root) + dump(uc->root, &base); + return 0; +} diff --git a/unpack-trees.c b/unpack-trees.c index be84ba2607..2927660d92 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -9,6 +9,7 @@ #include "refs.h" #include "attr.h" #include "split-index.h" +#include "dir.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -1259,8 +1260,10 @@ static int verify_uptodate_sparse(const struct cache_entry *ce, static void invalidate_ce_path(const struct cache_entry *ce, struct unpack_trees_options *o) { - if (ce) - cache_tree_invalidate_path(o->src_index, ce->name); + if (!ce) + return; + cache_tree_invalidate_path(o->src_index, ce->name); + untracked_cache_invalidate_path(o->src_index, ce->name); } /* diff --git a/wt-status.c b/wt-status.c index 38cb165f12..33452f169d 100644 --- a/wt-status.c +++ b/wt-status.c @@ -585,6 +585,8 @@ static void wt_status_collect_untracked(struct wt_status *s) DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; if (s->show_ignored_files) dir.flags |= DIR_SHOW_IGNORED_TOO; + else + dir.untracked = the_index.untracked; setup_standard_excludes(&dir); fill_directory(&dir, &s->pathspec); |