diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Documentation/git-status.txt | 5 | ||||
| -rw-r--r-- | Documentation/git-update-index.txt | 14 | ||||
| -rw-r--r-- | Documentation/technical/index-format.txt | 62 | ||||
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | builtin/commit.c | 5 | ||||
| -rw-r--r-- | builtin/update-index.c | 188 | ||||
| -rw-r--r-- | cache.h | 6 | ||||
| -rw-r--r-- | compat/mingw.c | 11 | ||||
| -rw-r--r-- | compat/mingw.h | 9 | ||||
| -rw-r--r-- | dir.c | 990 | ||||
| -rw-r--r-- | dir.h | 82 | ||||
| -rw-r--r-- | ewah/ewah_io.c | 13 | ||||
| -rw-r--r-- | ewah/ewok.h | 2 | ||||
| -rw-r--r-- | git-compat-util.h | 1 | ||||
| -rw-r--r-- | read-cache.c | 55 | ||||
| -rw-r--r-- | split-index.c | 11 | ||||
| -rwxr-xr-x | t/t7063-status-untracked-cache.sh | 353 | ||||
| -rw-r--r-- | test-dump-untracked-cache.c | 62 | ||||
| -rw-r--r-- | unpack-trees.c | 7 | ||||
| -rw-r--r-- | wt-status.c | 2 |
21 files changed, 1822 insertions, 58 deletions
diff --git a/.gitignore b/.gitignore index a05241916c..422c5382c1 100644 --- a/.gitignore +++ b/.gitignore @@ -184,6 +184,7 @@ /test-delta /test-dump-cache-tree /test-dump-split-index +/test-dump-untracked-cache /test-scrap-cache-tree /test-genrandom /test-hashmap diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 5221f950ce..335f312335 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -66,7 +66,10 @@ When `-u` option is not used, untracked files and directories are shown (i.e. the same as specifying `normal`), to help you avoid forgetting to add newly created files. Because it takes extra work to find untracked files in the filesystem, this mode may take some -time in a large working tree. You can use `no` to have `git status` +time in a large working tree. +Consider enabling untracked cache and split index if supported (see +`git update-index --untracked-cache` and `git update-index +--split-index`), Otherwise you can use `no` to have `git status` return more quickly without showing untracked files. + The default can be changed using the status.showUntrackedFiles diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index aff01798cd..1a296bc29a 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -170,6 +170,20 @@ may not support it yet. the shared index file. This mode is designed for very large indexes that take a significant amount of time to read or write. +--untracked-cache:: +--no-untracked-cache:: + Enable or disable untracked cache extension. This could speed + up for commands that involve determining untracked files such + as `git status`. The underlying operating system and file + system must change `st_mtime` field of a directory if files + are added or deleted in that directory. + +--force-untracked-cache:: + For safety, `--untracked-cache` performs tests on the working + directory to make sure untracked cache can be used. These + tests can take a few seconds. `--force-untracked-cache` can be + used to skip the tests. + \--:: Do not interpret any more arguments as options. diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt index 35112e4966..b7093af8b2 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/technical/index-format.txt @@ -233,3 +233,65 @@ Git index format The remaining index entries after replaced ones will be added to the final index. These added entries are also sorted by entry name then stage. + +== Untracked cache + + Untracked cache saves the untracked file list and necessary data to + verify the cache. The signature for this extension is { 'U', 'N', + 'T', 'R' }. + + The extension starts with + + - A sequence of NUL-terminated strings, preceded by the size of the + sequence in variable width encoding. Each string describes the + environment where the cache can be used. + + - Stat data of $GIT_DIR/info/exclude. See "Index entry" section from + ctime field until "file size". + + - Stat data of core.excludesfile + + - 32-bit dir_flags (see struct dir_struct) + + - 160-bit SHA-1 of $GIT_DIR/info/exclude. Null SHA-1 means the file + does not exist. + + - 160-bit SHA-1 of core.excludesfile. Null SHA-1 means the file does + not exist. + + - NUL-terminated string of per-dir exclude file name. This usually + is ".gitignore". + + - The number of following directory blocks, variable width + encoding. If this number is zero, the extension ends here with a + following NUL. + + - A number of directory blocks in depth-first-search order, each + consists of + + - The number of untracked entries, variable width encoding. + + - The number of sub-directory blocks, variable width encoding. + + - The directory name terminated by NUL. + + - A number of untrached file/dir names terminated by NUL. + +The remaining data of each directory block is grouped by type: + + - An ewah bitmap, the n-th bit marks whether the n-th directory has + valid untracked cache entries. + + - An ewah bitmap, the n-th bit records "check-only" bit of + read_directory_recursive() for the n-th directory. + + - An ewah bitmap, the n-th bit indicates whether SHA-1 and stat data + is valid for the n-th directory and exists in the next data. + + - An array of stat data. The n-th data corresponds with the n-th + "one" bit in the previous ewah bitmap. + + - An array of SHA-1. The n-th SHA-1 corresponds with the n-th "one" bit + in the previous ewah bitmap. + + - One NUL. @@ -574,6 +574,7 @@ TEST_PROGRAMS_NEED_X += test-date TEST_PROGRAMS_NEED_X += test-delta TEST_PROGRAMS_NEED_X += test-dump-cache-tree TEST_PROGRAMS_NEED_X += test-dump-split-index +TEST_PROGRAMS_NEED_X += test-dump-untracked-cache TEST_PROGRAMS_NEED_X += test-genrandom TEST_PROGRAMS_NEED_X += test-hashmap TEST_PROGRAMS_NEED_X += test-index-version diff --git a/builtin/commit.c b/builtin/commit.c index d6515a2a50..254477fd1d 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1366,13 +1366,14 @@ int cmd_status(int argc, const char **argv, const char *prefix) refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); fd = hold_locked_index(&index_lock, 0); - if (0 <= fd) - update_index_if_able(&the_index, &index_lock); s.is_initial = get_sha1(s.reference, sha1) ? 1 : 0; s.ignore_submodule_arg = ignore_submodule_arg; wt_status_collect(&s); + if (0 <= fd) + update_index_if_able(&the_index, &index_lock); + if (s.relative_paths) s.prefix = prefix; diff --git a/builtin/update-index.c b/builtin/update-index.c index 0665b31ea1..7431938fa6 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -33,6 +33,7 @@ static int mark_valid_only; static int mark_skip_worktree_only; #define MARK_FLAG 1 #define UNMARK_FLAG 2 +static struct strbuf mtime_dir = STRBUF_INIT; __attribute__((format (printf, 1, 2))) static void report(const char *fmt, ...) @@ -48,6 +49,166 @@ static void report(const char *fmt, ...) va_end(vp); } +static void remove_test_directory(void) +{ + if (mtime_dir.len) + remove_dir_recursively(&mtime_dir, 0); +} + +static const char *get_mtime_path(const char *path) +{ + static struct strbuf sb = STRBUF_INIT; + strbuf_reset(&sb); + strbuf_addf(&sb, "%s/%s", mtime_dir.buf, path); + return sb.buf; +} + +static void xmkdir(const char *path) +{ + path = get_mtime_path(path); + if (mkdir(path, 0700)) + die_errno(_("failed to create directory %s"), path); +} + +static int xstat_mtime_dir(struct stat *st) +{ + if (stat(mtime_dir.buf, st)) + die_errno(_("failed to stat %s"), mtime_dir.buf); + return 0; +} + +static int create_file(const char *path) +{ + int fd; + path = get_mtime_path(path); + fd = open(path, O_CREAT | O_RDWR, 0644); + if (fd < 0) + die_errno(_("failed to create file %s"), path); + return fd; +} + +static void xunlink(const char *path) +{ + path = get_mtime_path(path); + if (unlink(path)) + die_errno(_("failed to delete file %s"), path); +} + +static void xrmdir(const char *path) +{ + path = get_mtime_path(path); + if (rmdir(path)) + die_errno(_("failed to delete directory %s"), path); +} + +static void avoid_racy(void) +{ + /* + * not use if we could usleep(10) if USE_NSEC is defined. The + * field nsec could be there, but the OS could choose to + * ignore it? + */ + sleep(1); +} + +static int test_if_untracked_cache_is_supported(void) +{ + struct stat st; + struct stat_data base; + int fd, ret = 0; + + strbuf_addstr(&mtime_dir, "mtime-test-XXXXXX"); + if (!mkdtemp(mtime_dir.buf)) + die_errno("Could not make temporary directory"); + + fprintf(stderr, _("Testing ")); + atexit(remove_test_directory); + xstat_mtime_dir(&st); + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + fd = create_file("newfile"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + close(fd); + fputc('\n', stderr); + fprintf_ln(stderr,_("directory stat info does not " + "change after adding a new file")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + xmkdir("new-dir"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + close(fd); + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not change " + "after adding a new directory")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + write_or_die(fd, "data", 4); + close(fd); + xstat_mtime_dir(&st); + if (match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info changes " + "after updating a file")); + goto done; + } + fputc('.', stderr); + + avoid_racy(); + close(create_file("new-dir/new")); + xstat_mtime_dir(&st); + if (match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info changes after " + "adding a file inside subdirectory")); + goto done; + } + fputc('.', stderr); + + avoid_racy(); + xunlink("newfile"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not " + "change after deleting a file")); + goto done; + } + fill_stat_data(&base, &st); + fputc('.', stderr); + + avoid_racy(); + xunlink("new-dir/new"); + xrmdir("new-dir"); + xstat_mtime_dir(&st); + if (!match_stat_data(&base, &st)) { + fputc('\n', stderr); + fprintf_ln(stderr, _("directory stat info does not " + "change after deleting a directory")); + goto done; + } + + if (rmdir(mtime_dir.buf)) + die_errno(_("failed to delete directory %s"), mtime_dir.buf); + fprintf_ln(stderr, _(" OK")); + ret = 1; + +done: + strbuf_release(&mtime_dir); + return ret; +} + static int mark_ce_flags(const char *path, int flag, int mark) { int namelen = strlen(path); @@ -741,6 +902,7 @@ static int reupdate_callback(struct parse_opt_ctx_t *ctx, int cmd_update_index(int argc, const char **argv, const char *prefix) { int newfd, entries, has_errors = 0, line_termination = '\n'; + int untracked_cache = -1; int read_from_stdin = 0; int prefix_length = prefix ? strlen(prefix) : 0; int preferred_index_format = 0; @@ -832,6 +994,10 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) N_("write index in this format")), OPT_BOOL(0, "split-index", &split_index, N_("enable or disable split index")), + OPT_BOOL(0, "untracked-cache", &untracked_cache, + N_("enable/disable untracked cache")), + OPT_SET_INT(0, "force-untracked-cache", &untracked_cache, + N_("enable untracked cache without testing the filesystem"), 2), OPT_END() }; @@ -938,6 +1104,28 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) the_index.split_index = NULL; the_index.cache_changed |= SOMETHING_CHANGED; } + if (untracked_cache > 0) { + struct untracked_cache *uc; + + if (untracked_cache < 2) { + setup_work_tree(); + if (!test_if_untracked_cache_is_supported()) + return 1; + } + if (!the_index.untracked) { + uc = xcalloc(1, sizeof(*uc)); + strbuf_init(&uc->ident, 100); + uc->exclude_per_dir = ".gitignore"; + /* should be the same flags used by git-status */ + uc->dir_flags = DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + the_index.untracked = uc; + } + add_untracked_ident(the_index.untracked); + the_index.cache_changed |= UNTRACKED_CHANGED; + } else if (!untracked_cache && the_index.untracked) { + the_index.untracked = NULL; + the_index.cache_changed |= UNTRACKED_CHANGED; + } if (active_cache_changed) { if (newfd < 0) { @@ -297,8 +297,11 @@ static inline unsigned int canon_mode(unsigned int mode) #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; +struct untracked_cache; + struct index_state { struct cache_entry **cache; unsigned int version; @@ -312,6 +315,7 @@ struct index_state { struct hashmap name_hash; struct hashmap dir_hash; unsigned char sha1[20]; + struct untracked_cache *untracked; }; extern struct index_state the_index; @@ -563,6 +567,8 @@ extern void fill_stat_data(struct stat_data *sd, struct stat *st); * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); diff --git a/compat/mingw.c b/compat/mingw.c index 70f3191a4f..496e6f8bb0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2128,3 +2128,14 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); } + +int uname(struct utsname *buf) +{ + DWORD v = GetVersion(); + memset(buf, 0, sizeof(*buf)); + strcpy(buf->sysname, "Windows"); + sprintf(buf->release, "%u.%u", v & 0xff, (v >> 8) & 0xff); + /* assuming NT variants only.. */ + sprintf(buf->version, "%u", (v >> 16) & 0x7fff); + return 0; +} diff --git a/compat/mingw.h b/compat/mingw.h index 98c5e44294..738865c6c0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -76,6 +76,14 @@ struct itimerval { }; #define ITIMER_REAL 0 +struct utsname { + char sysname[16]; + char nodename[1]; + char release[16]; + char version[16]; + char machine[1]; +}; + /* * sanitize preprocessor namespace polluted by Windows headers defining * macros which collide with git local versions @@ -175,6 +183,7 @@ struct passwd *getpwuid(uid_t uid); int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); +int uname(struct utsname *buf); /* * replacements of existing functions @@ -13,6 +13,8 @@ #include "wildmatch.h" #include "pathspec.h" #include "utf8.h" +#include "varint.h" +#include "ewah/ewok.h" struct path_simplify { int len; @@ -32,8 +34,22 @@ enum path_treatment { path_untracked }; +/* + * Support data structure for our opendir/readdir/closedir wrappers + */ +struct cached_dir { + DIR *fdir; + struct untracked_cache_dir *untracked; + int nr_files; + int nr_dirs; + + struct dirent *de; + const char *file; + struct untracked_cache_dir *ucd; +}; + static enum path_treatment read_directory_recursive(struct dir_struct *dir, - const char *path, int len, + const char *path, int len, struct untracked_cache_dir *untracked, int check_only, const struct path_simplify *simplify); static int get_dtype(struct dirent *de, const char *path, int len); @@ -510,7 +526,8 @@ void add_exclude(const char *string, const char *base, x->el = el; } -static void *read_skip_worktree_file_from_index(const char *path, size_t *size) +static void *read_skip_worktree_file_from_index(const char *path, size_t *size, + struct sha1_stat *sha1_stat) { int pos, len; unsigned long sz; @@ -529,6 +546,10 @@ static void *read_skip_worktree_file_from_index(const char *path, size_t *size) return NULL; } *size = xsize_t(sz); + if (sha1_stat) { + memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + } return data; } @@ -573,11 +594,93 @@ static void trim_trailing_spaces(char *buf) *last_space = '\0'; } -int add_excludes_from_file_to_list(const char *fname, - const char *base, - int baselen, - struct exclude_list *el, - int check_index) +/* + * Given a subdirectory name and "dir" of the current directory, + * search the subdir in "dir" and return it, or create a new one if it + * does not exist in "dir". + * + * If "name" has the trailing slash, it'll be excluded in the search. + */ +static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc, + struct untracked_cache_dir *dir, + const char *name, int len) +{ + int first, last; + struct untracked_cache_dir *d; + if (!dir) + return NULL; + if (len && name[len - 1] == '/') + len--; + first = 0; + last = dir->dirs_nr; + while (last > first) { + int cmp, next = (last + first) >> 1; + d = dir->dirs[next]; + cmp = strncmp(name, d->name, len); + if (!cmp && strlen(d->name) > len) + cmp = -1; + if (!cmp) + return d; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + + uc->dir_created++; + d = xmalloc(sizeof(*d) + len + 1); + memset(d, 0, sizeof(*d)); + memcpy(d->name, name, len); + d->name[len] = '\0'; + + ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc); + memmove(dir->dirs + first + 1, dir->dirs + first, + (dir->dirs_nr - first) * sizeof(*dir->dirs)); + dir->dirs_nr++; + dir->dirs[first] = d; + return d; +} + +static void do_invalidate_gitignore(struct untracked_cache_dir *dir) +{ + int i; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + do_invalidate_gitignore(dir->dirs[i]); +} + +static void invalidate_gitignore(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + uc->gitignore_invalidated++; + do_invalidate_gitignore(dir); +} + +static void invalidate_directory(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + int i; + uc->dir_invalidated++; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + dir->dirs[i]->recurse = 0; +} + +/* + * Given a file with name "fname", read it (either from disk, or from + * the index if "check_index" is non-zero), parse it and store the + * exclude rules in "el". + * + * If "ss" is not NULL, compute SHA-1 of the exclude file and fill + * stat data from disk (only valid if add_excludes returns zero). If + * ss_valid is non-zero, "ss" must contain good value as input. + */ +static int add_excludes(const char *fname, const char *base, int baselen, + struct exclude_list *el, int check_index, + struct sha1_stat *sha1_stat) { struct stat st; int fd, i, lineno = 1; @@ -591,7 +694,7 @@ int add_excludes_from_file_to_list(const char *fname, if (0 <= fd) close(fd); if (!check_index || - (buf = read_skip_worktree_file_from_index(fname, &size)) == NULL) + (buf = read_skip_worktree_file_from_index(fname, &size, sha1_stat)) == NULL) return -1; if (size == 0) { free(buf); @@ -604,6 +707,11 @@ int add_excludes_from_file_to_list(const char *fname, } else { size = xsize_t(st.st_size); if (size == 0) { + if (sha1_stat) { + fill_stat_data(&sha1_stat->stat, &st); + hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN); + sha1_stat->valid = 1; + } close(fd); return 0; } @@ -615,6 +723,22 @@ int add_excludes_from_file_to_list(const char *fname, } buf[size++] = '\n'; close(fd); + if (sha1_stat) { + int pos; + if (sha1_stat->valid && + !match_stat_data_racy(&the_index, &sha1_stat->stat, &st)) + ; /* no content change, ss->sha1 still good */ + else if (check_index && + (pos = cache_name_pos(fname, strlen(fname))) >= 0 && + !ce_stage(active_cache[pos]) && + ce_uptodate(active_cache[pos]) && + !would_convert_to_git(fname)) + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + else + hash_sha1_file(buf, size, "blob", sha1_stat->sha1); + fill_stat_data(&sha1_stat->stat, &st); + sha1_stat->valid = 1; + } } el->filebuf = buf; @@ -638,6 +762,13 @@ int add_excludes_from_file_to_list(const char *fname, return 0; } +int add_excludes_from_file_to_list(const char *fname, const char *base, + int baselen, struct exclude_list *el, + int check_index) +{ + return add_excludes(fname, base, baselen, el, check_index, NULL); +} + struct exclude_list *add_exclude_list(struct dir_struct *dir, int group_type, const char *src) { @@ -655,14 +786,28 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir, /* * Used to set up core.excludesfile and .git/info/exclude lists. */ -void add_excludes_from_file(struct dir_struct *dir, const char *fname) +static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, + struct sha1_stat *sha1_stat) { struct exclude_list *el; + /* + * catch setup_standard_excludes() that's called before + * dir->untracked is assigned. That function behaves + * differently when dir->untracked is non-NULL. + */ + if (!dir->untracked) + dir->unmanaged_exclude_files++; el = add_exclude_list(dir, EXC_FILE, fname); - if (add_excludes_from_file_to_list(fname, "", 0, el, 0) < 0) + if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0) die("cannot use %s as an exclude file", fname); } +void add_excludes_from_file(struct dir_struct *dir, const char *fname) +{ + dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */ + add_excludes_from_file_1(dir, fname, NULL); +} + int match_basename(const char *basename, int basenamelen, const char *pattern, int prefix, int patternlen, int flags) @@ -837,6 +982,7 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) struct exclude_list_group *group; struct exclude_list *el; struct exclude_stack *stk = NULL; + struct untracked_cache_dir *untracked; int current; group = &dir->exclude_list_group[EXC_DIRS]; @@ -874,8 +1020,14 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) /* Read from the parent directories and push them down. */ current = stk ? stk->baselen : -1; strbuf_setlen(&dir->basebuf, current < 0 ? 0 : current); + if (dir->untracked) + untracked = stk ? stk->ucd : dir->untracked->root; + else + untracked = NULL; + while (current < baselen) { const char *cp; + struct sha1_stat sha1_stat; stk = xcalloc(1, sizeof(*stk)); if (current < 0) { @@ -886,10 +1038,15 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) if (!cp) die("oops in prep_exclude"); cp++; + untracked = + lookup_untracked(dir->untracked, untracked, + base + current, + cp - base - current); } stk->prev = dir->exclude_stack; stk->baselen = cp - base; stk->exclude_ix = group->nr; + stk->ucd = untracked; el = add_exclude_list(dir, EXC_DIRS, NULL); strbuf_add(&dir->basebuf, base + current, stk->baselen - current); assert(stk->baselen == dir->basebuf.len); @@ -912,7 +1069,23 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) } /* Try to read per-directory file */ - if (dir->exclude_per_dir) { + hashclr(sha1_stat.sha1); + sha1_stat.valid = 0; + if (dir->exclude_per_dir && + /* + * If we know that no files have been added in + * this directory (i.e. valid_cached_dir() has + * been executed and set untracked->valid) .. + */ + (!untracked || !untracked->valid || + /* + * .. and .gitignore does not exist before + * (i.e. null exclude_sha1 and skip_worktree is + * not set). Then we can skip loading .gitignore, + * which would result in ENOENT anyway. + * skip_worktree is taken care in read_directory() + */ + !is_null_sha1(untracked->exclude_sha1))) { /* * dir->basebuf gets reused by the traversal, but we * need fname to remain unchanged to ensure the src @@ -925,8 +1098,27 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) strbuf_addbuf(&sb, &dir->basebuf); strbuf_addstr(&sb, dir->exclude_per_dir); el->src = strbuf_detach(&sb, NULL); - add_excludes_from_file_to_list(el->src, el->src, - stk->baselen, el, 1); + add_excludes(el->src, el->src, stk->baselen, el, 1, + untracked ? &sha1_stat : NULL); + } + /* + * NEEDSWORK: when untracked cache is enabled, prep_exclude() + * will first be called in valid_cached_dir() then maybe many + * times more in last_exclude_matching(). When the cache is + * used, last_exclude_matching() will not be called and + * reading .gitignore content will be a waste. + * + * So when it's called by valid_cached_dir() and we can get + * .gitignore SHA-1 from the index (i.e. .gitignore is not + * modified on work tree), we could delay reading the + * .gitignore content until we absolutely need it in + * last_exclude_matching(). Be careful about ignore rule + * order, though, if you do that. + */ + if (untracked && + hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) { + invalidate_gitignore(dir->untracked, untracked); + hashcpy(untracked->exclude_sha1, sha1_stat.sha1); } dir->exclude_stack = stk; current = stk->baselen; @@ -1107,6 +1299,7 @@ static enum exist_status directory_exists_in_index(const char *dirname, int len) * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, + struct untracked_cache_dir *untracked, const char *dirname, int len, int exclude, const struct path_simplify *simplify) { @@ -1134,7 +1327,9 @@ static enum path_treatment treat_directory(struct dir_struct *dir, if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES)) return exclude ? path_excluded : path_untracked; - return read_directory_recursive(dir, dirname, len, 1, simplify); + untracked = lookup_untracked(dir->untracked, untracked, dirname, len); + return read_directory_recursive(dir, dirname, len, + untracked, 1, simplify); } /* @@ -1250,6 +1445,7 @@ static int get_dtype(struct dirent *de, const char *path, int len) } static enum path_treatment treat_one_path(struct dir_struct *dir, + struct untracked_cache_dir *untracked, struct strbuf *path, const struct path_simplify *simplify, int dtype, struct dirent *de) @@ -1302,7 +1498,7 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, return path_none; case DT_DIR: strbuf_addch(path, '/'); - return treat_directory(dir, path->buf, path->len, exclude, + return treat_directory(dir, untracked, path->buf, path->len, exclude, simplify); case DT_REG: case DT_LNK: @@ -1310,14 +1506,52 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, } } +static enum path_treatment treat_path_fast(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, + struct strbuf *path, + int baselen, + const struct path_simplify *simplify) +{ + strbuf_setlen(path, baselen); + if (!cdir->ucd) { + strbuf_addstr(path, cdir->file); + return path_untracked; + } + strbuf_addstr(path, cdir->ucd->name); + /* treat_one_path() does this before it calls treat_directory() */ + if (path->buf[path->len - 1] != '/') + strbuf_addch(path, '/'); + if (cdir->ucd->check_only) + /* + * check_only is set as a result of treat_directory() getting + * to its bottom. Verify again the same set of directories + * with check_only set. + */ + return read_directory_recursive(dir, path->buf, path->len, + cdir->ucd, 1, simplify); + /* + * We get path_recurse in the first run when + * directory_exists_in_index() returns index_nonexistent. We + * are sure that new changes in the index does not impact the + * outcome. Return now. + */ + return path_recurse; +} + static enum path_treatment treat_path(struct dir_struct *dir, - struct dirent *de, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, struct strbuf *path, int baselen, const struct path_simplify *simplify) { int dtype; + struct dirent *de = cdir->de; + if (!de) + return treat_path_fast(dir, untracked, cdir, path, + baselen, simplify); if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git")) return path_none; strbuf_setlen(path, baselen); @@ -1326,7 +1560,121 @@ static enum path_treatment treat_path(struct dir_struct *dir, return path_none; dtype = DTYPE(de); - return treat_one_path(dir, path, simplify, dtype, de); + return treat_one_path(dir, untracked, path, simplify, dtype, de); +} + +static void add_untracked(struct untracked_cache_dir *dir, const char *name) +{ + if (!dir) + return; + ALLOC_GROW(dir->untracked, dir->untracked_nr + 1, + dir->untracked_alloc); + dir->untracked[dir->untracked_nr++] = xstrdup(name); +} + +static int valid_cached_dir(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct strbuf *path, + int check_only) +{ + struct stat st; + + if (!untracked) + return 0; + + if (stat(path->len ? path->buf : ".", &st)) { + invalidate_directory(dir->untracked, untracked); + memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_stat_data_racy(&the_index, &untracked->stat_data, &st)) { + if (untracked->valid) + invalidate_directory(dir->untracked, untracked); + fill_stat_data(&untracked->stat_data, &st); + return 0; + } + + if (untracked->check_only != !!check_only) { + invalidate_directory(dir->untracked, untracked); + return 0; + } + + /* + * prep_exclude will be called eventually on this directory, + * but it's called much later in last_exclude_matching(). We + * need it now to determine the validity of the cache for this |