diff options
Diffstat (limited to 'dir.c')
| -rw-r--r-- | dir.c | 1605 |
1 files changed, 1299 insertions, 306 deletions
@@ -11,6 +11,10 @@ #include "dir.h" #include "refs.h" #include "wildmatch.h" +#include "pathspec.h" +#include "utf8.h" +#include "varint.h" +#include "ewah/ewok.h" struct path_simplify { int len; @@ -30,8 +34,22 @@ enum path_treatment { path_untracked }; +/* + * Support data structure for our opendir/readdir/closedir wrappers + */ +struct cached_dir { + DIR *fdir; + struct untracked_cache_dir *untracked; + int nr_files; + int nr_dirs; + + struct dirent *de; + const char *file; + struct untracked_cache_dir *ucd; +}; + static enum path_treatment read_directory_recursive(struct dir_struct *dir, - const char *path, int len, + const char *path, int len, struct untracked_cache_dir *untracked, int check_only, const struct path_simplify *simplify); static int get_dtype(struct dirent *de, const char *path, int len); @@ -48,29 +66,38 @@ int strncmp_icase(const char *a, const char *b, size_t count) int fnmatch_icase(const char *pattern, const char *string, int flags) { - return fnmatch(pattern, string, flags | (ignore_case ? FNM_CASEFOLD : 0)); + return wildmatch(pattern, string, + flags | (ignore_case ? WM_CASEFOLD : 0), + NULL); } -inline int git_fnmatch(const char *pattern, const char *string, - int flags, int prefix) +int git_fnmatch(const struct pathspec_item *item, + const char *pattern, const char *string, + int prefix) { - int fnm_flags = 0; - if (flags & GFNM_PATHNAME) - fnm_flags |= FNM_PATHNAME; if (prefix > 0) { - if (strncmp(pattern, string, prefix)) - return FNM_NOMATCH; + if (ps_strncmp(item, pattern, string, prefix)) + return WM_NOMATCH; pattern += prefix; string += prefix; } - if (flags & GFNM_ONESTAR) { + if (item->flags & PATHSPEC_ONESTAR) { int pattern_len = strlen(++pattern); int string_len = strlen(string); return string_len < pattern_len || - strcmp(pattern, - string + string_len - pattern_len); + ps_strcmp(item, pattern, + string + string_len - pattern_len); } - return fnmatch(pattern, string, fnm_flags); + if (item->magic & PATHSPEC_GLOB) + return wildmatch(pattern, string, + WM_PATHNAME | + (item->magic & PATHSPEC_ICASE ? WM_CASEFOLD : 0), + NULL); + else + /* wildmatch has not learned no FNM_PATHNAME mode yet */ + return wildmatch(pattern, string, + item->magic & PATHSPEC_ICASE ? WM_CASEFOLD : 0, + NULL); } static int fnmatch_icase_mem(const char *pattern, int patternlen, @@ -102,26 +129,43 @@ static int fnmatch_icase_mem(const char *pattern, int patternlen, return match_status; } -static size_t common_prefix_len(const char **pathspec) +static size_t common_prefix_len(const struct pathspec *pathspec) { - const char *n, *first; + int n; size_t max = 0; - int literal = limit_pathspec_to_literal(); - if (!pathspec) - return max; - - first = *pathspec; - while ((n = *pathspec++)) { - size_t i, len = 0; - for (i = 0; first == n || i < max; i++) { - char c = n[i]; - if (!c || c != first[i] || (!literal && is_glob_special(c))) + /* + * ":(icase)path" is treated as a pathspec full of + * wildcard. In other words, only prefix is considered common + * prefix. If the pathspec is abc/foo abc/bar, running in + * subdir xyz, the common prefix is still xyz, not xuz/abc as + * in non-:(icase). + */ + GUARD_PATHSPEC(pathspec, + PATHSPEC_FROMTOP | + PATHSPEC_MAXDEPTH | + PATHSPEC_LITERAL | + PATHSPEC_GLOB | + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); + + for (n = 0; n < pathspec->nr; n++) { + size_t i = 0, len = 0, item_len; + if (pathspec->items[n].magic & PATHSPEC_EXCLUDE) + continue; + if (pathspec->items[n].magic & PATHSPEC_ICASE) + item_len = pathspec->items[n].prefix; + else + item_len = pathspec->items[n].nowildcard_len; + while (i < item_len && (n == 0 || i < max)) { + char c = pathspec->items[n].match[i]; + if (c != pathspec->items[0].match[i]) break; if (c == '/') len = i + 1; + i++; } - if (first == n || len < max) { + if (n == 0 || len < max) { max = len; if (!max) break; @@ -134,14 +178,14 @@ static size_t common_prefix_len(const char **pathspec) * Returns a copy of the longest leading path common among all * pathspecs. */ -char *common_prefix(const char **pathspec) +char *common_prefix(const struct pathspec *pathspec) { unsigned long len = common_prefix_len(pathspec); - return len ? xmemdupz(*pathspec, len) : NULL; + return len ? xmemdupz(pathspec->items[0].match, len) : NULL; } -int fill_directory(struct dir_struct *dir, const char **pathspec) +int fill_directory(struct dir_struct *dir, const struct pathspec *pathspec) { size_t len; @@ -152,7 +196,7 @@ int fill_directory(struct dir_struct *dir, const char **pathspec) len = common_prefix_len(pathspec); /* Read the directory and prune it */ - read_directory(dir, pathspec ? *pathspec : "", len, pathspec); + read_directory(dir, pathspec->nr ? pathspec->_raw[0] : "", len, pathspec); return len; } @@ -171,112 +215,8 @@ int within_depth(const char *name, int namelen, return 1; } -/* - * Does 'match' match the given name? - * A match is found if - * - * (1) the 'match' string is leading directory of 'name', or - * (2) the 'match' string is a wildcard and matches 'name', or - * (3) the 'match' string is exactly the same as 'name'. - * - * and the return value tells which case it was. - * - * It returns 0 when there is no match. - */ -static int match_one(const char *match, const char *name, int namelen) -{ - int matchlen; - int literal = limit_pathspec_to_literal(); - - /* If the match was just the prefix, we matched */ - if (!*match) - return MATCHED_RECURSIVELY; - - if (ignore_case) { - for (;;) { - unsigned char c1 = tolower(*match); - unsigned char c2 = tolower(*name); - if (c1 == '\0' || (!literal && is_glob_special(c1))) - break; - if (c1 != c2) - return 0; - match++; - name++; - namelen--; - } - } else { - for (;;) { - unsigned char c1 = *match; - unsigned char c2 = *name; - if (c1 == '\0' || (!literal && is_glob_special(c1))) - break; - if (c1 != c2) - return 0; - match++; - name++; - namelen--; - } - } - - /* - * If we don't match the matchstring exactly, - * we need to match by fnmatch - */ - matchlen = strlen(match); - if (strncmp_icase(match, name, matchlen)) { - if (literal) - return 0; - return !fnmatch_icase(match, name, 0) ? MATCHED_FNMATCH : 0; - } - - if (namelen == matchlen) - return MATCHED_EXACTLY; - if (match[matchlen-1] == '/' || name[matchlen] == '/') - return MATCHED_RECURSIVELY; - return 0; -} - -/* - * Given a name and a list of pathspecs, returns the nature of the - * closest (i.e. most specific) match of the name to any of the - * pathspecs. - * - * The caller typically calls this multiple times with the same - * pathspec and seen[] array but with different name/namelen - * (e.g. entries from the index) and is interested in seeing if and - * how each pathspec matches all the names it calls this function - * with. A mark is left in the seen[] array for each pathspec element - * indicating the closest type of match that element achieved, so if - * seen[n] remains zero after multiple invocations, that means the nth - * pathspec did not match any names, which could indicate that the - * user mistyped the nth pathspec. - */ -int match_pathspec(const char **pathspec, const char *name, int namelen, - int prefix, char *seen) -{ - int i, retval = 0; - - if (!pathspec) - return 1; - - name += prefix; - namelen -= prefix; - - for (i = 0; pathspec[i] != NULL; i++) { - int how; - const char *match = pathspec[i] + prefix; - if (seen && seen[i] == MATCHED_EXACTLY) - continue; - how = match_one(match, name, namelen); - if (how) { - if (retval < how) - retval = how; - if (seen && seen[i] < how) - seen[i] = how; - } - } - return retval; -} +#define DO_MATCH_EXCLUDE 1 +#define DO_MATCH_DIRECTORY 2 /* * Does 'match' match the given name? @@ -291,27 +231,63 @@ int match_pathspec(const char **pathspec, const char *name, int namelen, * It returns 0 when there is no match. */ static int match_pathspec_item(const struct pathspec_item *item, int prefix, - const char *name, int namelen) + const char *name, int namelen, unsigned flags) { /* name/namelen has prefix cut off by caller */ const char *match = item->match + prefix; int matchlen = item->len - prefix; + /* + * The normal call pattern is: + * 1. prefix = common_prefix_len(ps); + * 2. prune something, or fill_directory + * 3. match_pathspec() + * + * 'prefix' at #1 may be shorter than the command's prefix and + * it's ok for #2 to match extra files. Those extras will be + * trimmed at #3. + * + * Suppose the pathspec is 'foo' and '../bar' running from + * subdir 'xyz'. The common prefix at #1 will be empty, thanks + * to "../". We may have xyz/foo _and_ XYZ/foo after #2. The + * user does not want XYZ/foo, only the "foo" part should be + * case-insensitive. We need to filter out XYZ/foo here. In + * other words, we do not trust the caller on comparing the + * prefix part when :(icase) is involved. We do exact + * comparison ourselves. + * + * Normally the caller (common_prefix_len() in fact) does + * _exact_ matching on name[-prefix+1..-1] and we do not need + * to check that part. Be defensive and check it anyway, in + * case common_prefix_len is changed, or a new caller is + * introduced that does not use common_prefix_len. + * + * If the penalty turns out too high when prefix is really + * long, maybe change it to + * strncmp(match, name, item->prefix - prefix) + */ + if (item->prefix && (item->magic & PATHSPEC_ICASE) && + strncmp(item->match, name - prefix, item->prefix)) + return 0; + /* If the match was just the prefix, we matched */ if (!*match) return MATCHED_RECURSIVELY; - if (matchlen <= namelen && !strncmp(match, name, matchlen)) { + if (matchlen <= namelen && !ps_strncmp(item, match, name, matchlen)) { if (matchlen == namelen) return MATCHED_EXACTLY; if (match[matchlen-1] == '/' || name[matchlen] == '/') return MATCHED_RECURSIVELY; - } + } else if ((flags & DO_MATCH_DIRECTORY) && + match[matchlen - 1] == '/' && + namelen == matchlen - 1 && + !ps_strncmp(item, match, name, namelen)) + return MATCHED_EXACTLY; if (item->nowildcard_len < item->len && - !git_fnmatch(match, name, - item->flags & PATHSPEC_ONESTAR ? GFNM_ONESTAR : 0, + !git_fnmatch(item, match, name, item->nowildcard_len - prefix)) return MATCHED_FNMATCH; @@ -333,14 +309,25 @@ static int match_pathspec_item(const struct pathspec_item *item, int prefix, * pathspec did not match any names, which could indicate that the * user mistyped the nth pathspec. */ -int match_pathspec_depth(const struct pathspec *ps, - const char *name, int namelen, - int prefix, char *seen) +static int do_match_pathspec(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen, + unsigned flags) { - int i, retval = 0; + int i, retval = 0, exclude = flags & DO_MATCH_EXCLUDE; + + GUARD_PATHSPEC(ps, + PATHSPEC_FROMTOP | + PATHSPEC_MAXDEPTH | + PATHSPEC_LITERAL | + PATHSPEC_GLOB | + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); if (!ps->nr) { - if (!ps->recursive || ps->max_depth == -1) + if (!ps->recursive || + !(ps->magic & PATHSPEC_MAXDEPTH) || + ps->max_depth == -1) return MATCHED_RECURSIVELY; if (within_depth(name, namelen, 0, ps->max_depth)) @@ -354,10 +341,24 @@ int match_pathspec_depth(const struct pathspec *ps, for (i = ps->nr - 1; i >= 0; i--) { int how; + + if ((!exclude && ps->items[i].magic & PATHSPEC_EXCLUDE) || + ( exclude && !(ps->items[i].magic & PATHSPEC_EXCLUDE))) + continue; + if (seen && seen[i] == MATCHED_EXACTLY) continue; - how = match_pathspec_item(ps->items+i, prefix, name, namelen); - if (ps->recursive && ps->max_depth != -1 && + /* + * Make exclude patterns optional and never report + * "pathspec ':(exclude)foo' matches no files" + */ + if (seen && ps->items[i].magic & PATHSPEC_EXCLUDE) + seen[i] = MATCHED_FNMATCH; + how = match_pathspec_item(ps->items+i, prefix, name, + namelen, flags); + if (ps->recursive && + (ps->magic & PATHSPEC_MAXDEPTH) && + ps->max_depth != -1 && how && how != MATCHED_FNMATCH) { int len = ps->items[i].len; if (name[len] == '/') @@ -377,10 +378,67 @@ int match_pathspec_depth(const struct pathspec *ps, return retval; } +int match_pathspec(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen, int is_dir) +{ + int positive, negative; + unsigned flags = is_dir ? DO_MATCH_DIRECTORY : 0; + positive = do_match_pathspec(ps, name, namelen, + prefix, seen, flags); + if (!(ps->magic & PATHSPEC_EXCLUDE) || !positive) + return positive; + negative = do_match_pathspec(ps, name, namelen, + prefix, seen, + flags | DO_MATCH_EXCLUDE); + return negative ? 0 : positive; +} + +int report_path_error(const char *ps_matched, + const struct pathspec *pathspec, + const char *prefix) +{ + /* + * Make sure all pathspec matched; otherwise it is an error. + */ + int num, errors = 0; + for (num = 0; num < pathspec->nr; num++) { + int other, found_dup; + + if (ps_matched[num]) + continue; + /* + * The caller might have fed identical pathspec + * twice. Do not barf on such a mistake. + * FIXME: parse_pathspec should have eliminated + * duplicate pathspec. + */ + for (found_dup = other = 0; + !found_dup && other < pathspec->nr; + other++) { + if (other == num || !ps_matched[other]) + continue; + if (!strcmp(pathspec->items[other].original, + pathspec->items[num].original)) + /* + * Ok, we have a match already. + */ + found_dup = 1; + } + if (found_dup) + continue; + + error("pathspec '%s' did not match any file(s) known to git.", + pathspec->items[num].original); + errors++; + } + return errors; +} + /* * Return the length of the "simple" part of a path match limiter. */ -static int simple_length(const char *match) +int simple_length(const char *match) { int len = -1; @@ -392,7 +450,7 @@ static int simple_length(const char *match) } } -static int no_wildcard(const char *string) +int no_wildcard(const char *string) { return string[simple_length(string)] == '\0'; } @@ -466,26 +524,30 @@ void add_exclude(const char *string, const char *base, x->el = el; } -static void *read_skip_worktree_file_from_index(const char *path, size_t *size) +static void *read_skip_worktree_file_from_index(const char *path, size_t *size, + struct sha1_stat *sha1_stat) { int pos, len; unsigned long sz; enum object_type type; void *data; - struct index_state *istate = &the_index; len = strlen(path); - pos = index_name_pos(istate, path, len); + pos = cache_name_pos(path, len); if (pos < 0) return NULL; - if (!ce_skip_worktree(istate->cache[pos])) + if (!ce_skip_worktree(active_cache[pos])) return NULL; - data = read_sha1_file(istate->cache[pos]->sha1, &type, &sz); + data = read_sha1_file(active_cache[pos]->sha1, &type, &sz); if (!data || type != OBJ_BLOB) { free(data); return NULL; } *size = xsize_t(sz); + if (sha1_stat) { + memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + } return data; } @@ -507,11 +569,116 @@ void clear_exclude_list(struct exclude_list *el) el->filebuf = NULL; } -int add_excludes_from_file_to_list(const char *fname, - const char *base, - int baselen, - struct exclude_list *el, - int check_index) +static void trim_trailing_spaces(char *buf) +{ + char *p, *last_space = NULL; + + for (p = buf; *p; p++) + switch (*p) { + case ' ': + if (!last_space) + last_space = p; + break; + case '\\': + p++; + if (!*p) + return; + /* fallthrough */ + default: + last_space = NULL; + } + + if (last_space) + *last_space = '\0'; +} + +/* + * Given a subdirectory name and "dir" of the current directory, + * search the subdir in "dir" and return it, or create a new one if it + * does not exist in "dir". + * + * If "name" has the trailing slash, it'll be excluded in the search. + */ +static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc, + struct untracked_cache_dir *dir, + const char *name, int len) +{ + int first, last; + struct untracked_cache_dir *d; + if (!dir) + return NULL; + if (len && name[len - 1] == '/') + len--; + first = 0; + last = dir->dirs_nr; + while (last > first) { + int cmp, next = (last + first) >> 1; + d = dir->dirs[next]; + cmp = strncmp(name, d->name, len); + if (!cmp && strlen(d->name) > len) + cmp = -1; + if (!cmp) + return d; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + + uc->dir_created++; + d = xmalloc(sizeof(*d) + len + 1); + memset(d, 0, sizeof(*d)); + memcpy(d->name, name, len); + d->name[len] = '\0'; + + ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc); + memmove(dir->dirs + first + 1, dir->dirs + first, + (dir->dirs_nr - first) * sizeof(*dir->dirs)); + dir->dirs_nr++; + dir->dirs[first] = d; + return d; +} + +static void do_invalidate_gitignore(struct untracked_cache_dir *dir) +{ + int i; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + do_invalidate_gitignore(dir->dirs[i]); +} + +static void invalidate_gitignore(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + uc->gitignore_invalidated++; + do_invalidate_gitignore(dir); +} + +static void invalidate_directory(struct untracked_cache *uc, + struct untracked_cache_dir *dir) +{ + int i; + uc->dir_invalidated++; + dir->valid = 0; + dir->untracked_nr = 0; + for (i = 0; i < dir->dirs_nr; i++) + dir->dirs[i]->recurse = 0; +} + +/* + * Given a file with name "fname", read it (either from disk, or from + * the index if "check_index" is non-zero), parse it and store the + * exclude rules in "el". + * + * If "ss" is not NULL, compute SHA-1 of the exclude file and fill + * stat data from disk (only valid if add_excludes returns zero). If + * ss_valid is non-zero, "ss" must contain good value as input. + */ +static int add_excludes(const char *fname, const char *base, int baselen, + struct exclude_list *el, int check_index, + struct sha1_stat *sha1_stat) { struct stat st; int fd, i, lineno = 1; @@ -525,7 +692,7 @@ int add_excludes_from_file_to_list(const char *fname, if (0 <= fd) close(fd); if (!check_index || - (buf = read_skip_worktree_file_from_index(fname, &size)) == NULL) + (buf = read_skip_worktree_file_from_index(fname, &size, sha1_stat)) == NULL) return -1; if (size == 0) { free(buf); @@ -535,10 +702,14 @@ int add_excludes_from_file_to_list(const char *fname, buf = xrealloc(buf, size+1); buf[size++] = '\n'; } - } - else { + } else { size = xsize_t(st.st_size); if (size == 0) { + if (sha1_stat) { + fill_stat_data(&sha1_stat->stat, &st); + hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN); + sha1_stat->valid = 1; + } close(fd); return 0; } @@ -550,14 +721,36 @@ int add_excludes_from_file_to_list(const char *fname, } buf[size++] = '\n'; close(fd); + if (sha1_stat) { + int pos; + if (sha1_stat->valid && + !match_stat_data_racy(&the_index, &sha1_stat->stat, &st)) + ; /* no content change, ss->sha1 still good */ + else if (check_index && + (pos = cache_name_pos(fname, strlen(fname))) >= 0 && + !ce_stage(active_cache[pos]) && + ce_uptodate(active_cache[pos]) && + !would_convert_to_git(fname)) + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); + else + hash_sha1_file(buf, size, "blob", sha1_stat->sha1); + fill_stat_data(&sha1_stat->stat, &st); + sha1_stat->valid = 1; + } } el->filebuf = buf; + + if (skip_utf8_bom(&buf, size)) + size -= buf - el->filebuf; + entry = buf; + for (i = 0; i < size; i++) { if (buf[i] == '\n') { if (entry != buf + i && entry[0] != '#') { buf[i - (i && buf[i-1] == '\r')] = 0; + trim_trailing_spaces(entry); add_exclude(entry, base, baselen, el, lineno); } lineno++; @@ -567,6 +760,13 @@ int add_excludes_from_file_to_list(const char *fname, return 0; } +int add_excludes_from_file_to_list(const char *fname, const char *base, + int baselen, struct exclude_list *el, + int check_index) +{ + return add_excludes(fname, base, baselen, el, check_index, NULL); +} + struct exclude_list *add_exclude_list(struct dir_struct *dir, int group_type, const char *src) { @@ -584,14 +784,28 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir, /* * Used to set up core.excludesfile and .git/info/exclude lists. */ -void add_excludes_from_file(struct dir_struct *dir, const char *fname) +static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, + struct sha1_stat *sha1_stat) { struct exclude_list *el; + /* + * catch setup_standard_excludes() that's called before + * dir->untracked is assigned. That function behaves + * differently when dir->untracked is non-NULL. + */ + if (!dir->untracked) + dir->unmanaged_exclude_files++; el = add_exclude_list(dir, EXC_FILE, fname); - if (add_excludes_from_file_to_list(fname, "", 0, el, 0) < 0) + if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0) die("cannot use %s as an exclude file", fname); } +void add_excludes_from_file(struct dir_struct *dir, const char *fname) +{ + dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */ + add_excludes_from_file_1(dir, fname, NULL); +} + int match_basename(const char *basename, int basenamelen, const char *pattern, int prefix, int patternlen, int flags) @@ -766,21 +980,24 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) struct exclude_list_group *group; struct exclude_list *el; struct exclude_stack *stk = NULL; + struct untracked_cache_dir *untracked; int current; group = &dir->exclude_list_group[EXC_DIRS]; - /* Pop the exclude lists from the EXCL_DIRS exclude_list_group + /* + * Pop the exclude lists from the EXCL_DIRS exclude_list_group * which originate from directories not in the prefix of the - * path being checked. */ + * path being checked. + */ while ((stk = dir->exclude_stack) != NULL) { if (stk->baselen <= baselen && - !strncmp(dir->basebuf, base, stk->baselen)) + !strncmp(dir->basebuf.buf, base, stk->baselen)) break; el = &group->el[dir->exclude_stack->exclude_ix]; dir->exclude_stack = stk->prev; dir->exclude = NULL; - free((char *)el->src); /* see strdup() below */ + free((char *)el->src); /* see strbuf_detach() below */ clear_exclude_list(el); free(stk); group->nr--; @@ -790,68 +1007,121 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) if (dir->exclude) return; + /* + * Lazy initialization. All call sites currently just + * memset(dir, 0, sizeof(*dir)) before use. Changing all of + * them seems lots of work for little benefit. + */ + if (!dir->basebuf.buf) + strbuf_init(&dir->basebuf, PATH_MAX); + /* Read from the parent directories and push them down. */ current = stk ? stk->baselen : -1; + strbuf_setlen(&dir->basebuf, current < 0 ? 0 : current); + if (dir->untracked) + untracked = stk ? stk->ucd : dir->untracked->root; + else + untracked = NULL; + while (current < baselen) { - struct exclude_stack *stk = xcalloc(1, sizeof(*stk)); const char *cp; + struct sha1_stat sha1_stat; + stk = xcalloc(1, sizeof(*stk)); if (current < 0) { cp = base; current = 0; - } - else { + } else { cp = strchr(base + current + 1, '/'); if (!cp) die("oops in prep_exclude"); cp++; + untracked = + lookup_untracked(dir->untracked, untracked, + base + current, + cp - base - current); } stk->prev = dir->exclude_stack; stk->baselen = cp - base; stk->exclude_ix = group->nr; + stk->ucd = untracked; el = add_exclude_list(dir, EXC_DIRS, NULL); - memcpy(dir->basebuf + current, base + current, - stk->baselen - current); + strbuf_add(&dir->basebuf, base + current, stk->baselen - current); + assert(stk->baselen == dir->basebuf.len); /* Abort if the directory is excluded */ if (stk->baselen) { int dt = DT_DIR; - dir->basebuf[stk->baselen - 1] = 0; + dir->basebuf.buf[stk->baselen - 1] = 0; dir->exclude = last_exclude_matching_from_lists(dir, - dir->basebuf, stk->baselen - 1, - dir->basebuf + current, &dt); - dir->basebuf[stk->baselen - 1] = '/'; + dir->basebuf.buf, stk->baselen - 1, + dir->basebuf.buf + current, &dt); + dir->basebuf.buf[stk->baselen - 1] = '/'; if (dir->exclude && dir->exclude->flags & EXC_FLAG_NEGATIVE) dir->exclude = NULL; if (dir->exclude) { - dir->basebuf[stk->baselen] = 0; dir->exclude_stack = stk; return; } } - /* Try to read per-directory file unless path is too long */ + /* Try to read per-directory file */ + hashclr(sha1_stat.sha1); + sha1_stat.valid = 0; if (dir->exclude_per_dir && - stk->baselen + strlen(dir->exclude_per_dir) < PATH_MAX) { - strcpy(dir->basebuf + stk->baselen, - dir->exclude_per_dir); + /* + * If we know that no files have been added in + * this directory (i.e. valid_cached_dir() has + * been executed and set untracked->valid) .. + */ + (!untracked || !untracked->valid || + /* + * .. and .gitignore does not exist before + * (i.e. null exclude_sha1 and skip_worktree is + * not set). Then we can skip loading .gitignore, + * which would result in ENOENT anyway. + * skip_worktree is taken care in read_directory() + */ + !is_null_sha1(untracked->exclude_sha1))) { /* * dir->basebuf gets reused by the traversal, but we * need fname to remain unchanged to ensure the src * member of each struct exclude correctly * back-references its source file. Other invocations * of add_exclude_list provide stable strings, so we - * strdup() and free() here in the caller. + * strbuf_detach() and free() here in the caller. */ - el->src = strdup(dir->basebuf); - add_excludes_from_file_to_list(dir->basebuf, - dir->basebuf, stk->baselen, el, 1); + struct strbuf sb = STRBUF_INIT; + strbuf_addbuf(&sb, &dir->basebuf); + strbuf_addstr(&sb, dir->exclude_per_dir); + el->src = strbuf_detach(&sb, NULL); + add_excludes(el->src, el->src, stk->baselen, el, 1, + untracked ? &sha1_stat : NULL); + } + /* + * NEEDSWORK: when untracked cache is enabled, prep_exclude() + * will first be called in valid_cached_dir() then maybe many + * times more in last_exclude_matching(). When the cache is + * used, last_exclude_matching() will not be called and + * reading .gitignore content will be a waste. + * + * So when it's called by valid_cached_dir() and we can get + * .gitignore SHA-1 from the index (i.e. .gitignore is not + * modified on work tree), we could delay reading the + * .gitignore content until we absolutely need it in + * last_exclude_matching(). Be careful about ignore rule + * order, though, if you do that. + */ + if (untracked && + hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) { + invalidate_gitignore(dir->untracked, untracked); + hashcpy(untracked->exclude_sha1, sha1_stat.sha1); } dir->exclude_stack = stk; current = stk->baselen; } - dir->basebuf[baselen] = '\0'; + strbuf_setlen(&dir->basebuf, baselen); } /* @@ -904,7 +1174,7 @@ static struct dir_entry *dir_entry_new(const char *pathname, int len) static struct dir_entry *dir_add_name(struct dir_struct *dir, const char *pathname, int len) { - if (cache_name_exists(pathname, len, ignore_case)) + if (cache_file_exists(pathname, len, ignore_case)) return NULL; ALLOC_GROW(dir->entries, dir->nr+1, dir->alloc); @@ -927,13 +1197,13 @@ enum exist_status { }; /* - * Do not use the alphabetically stored index to look up + * Do not use the alphabetically sorted index to look up * the directory name; instead, use the case insensitive - * name hash. + * directory hash. */ static enum exist_status directory_exists_in_index_icase(const char *dirname, int len) { - const struct cache_entry *ce = index_name_exists(&the_index, dirname, len + 1, ignore_case); + const struct cache_entry *ce = cache_dir_exists(dirname, len); unsigned char endchar; if (!ce) @@ -1027,6 +1297,7 @@ static enum exist_status directory_exists_in_index(const char *dirname, int len) * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, + struct untracked_cache_dir *untracked, const char *dirname, int len, int exclude, const struct path_simplify *simplify) { @@ -1054,7 +1325,9 @@ static enum path_treatment treat_directory(struct dir_struct *dir, if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES)) return exclude ? path_excluded : path_untracked; - return read_directory_recursive(dir, dirname, len, 1, simplify); + untracked = lookup_untracked(dir->untracked, untracked, dirname, len); + return read_directory_recursive(dir, dirname, len, + untracked, 1, simplify); } /* @@ -1115,7 +1388,7 @@ static int get_index_dtype(const char *path, int len) int pos; const struct cache_entry *ce; - ce = cache_name_exists(path, len, 0); + ce = cache_file_exists(path, len, 0); if (ce) { if (!ce_uptodate(ce)) return DT_UNKNOWN; @@ -1170,17 +1443,43 @@ static int get_dtype(struct dirent *de, const char *path, int len) } static enum path_treatment treat_one_path(struct dir_struct *dir, + struct untracked_cache_dir *untracked, struct strbuf *path, const struct path_simplify *simplify, int dtype, struct dirent *de) { int exclude; + int has_path_in_index = !!cache_file_exists(path->buf, path->len, ignore_case); + if (dtype == DT_UNKNOWN) dtype = get_dtype(de, path->buf, path->len); /* Always exclude indexed files */ - if (dtype != DT_DIR && - cache_name_exists(path->buf, path->len, ignore_case)) + if (dtype != DT_DIR && has_path_in_index) + return path_none; + + /* + * When we are looking at a directory P in the working tree, + * there are three cases: + * + * (1) P exists in the index. Everything inside the directory P in + * the working tree needs to go when P is checked out from the + * index. + * + * (2) P does not exist in the index, but there is P/Q in the index. + * We know P will stay a directory when we check out the contents + * of the index, but we do not know yet if there is a directory + * P/Q in the working tree to be killed, so we need to recurse. + * + * (3) P does not exist in the index, and there is no P/Q in the index + * to require P to be a directory, either. Only in this case, we + * know that everything inside P will not be killed without + * recursing. + */ + if ((dir->flags & DIR_COLLECT_KILLED_ONLY) && + (dtype == DT_DIR) && + !has_path_in_index && + (directory_exists_in_index(path->buf, path->len) == index_nonexistent)) return path_none; exclude = is_excluded(dir, path->buf, &dtype); @@ -1197,7 +1496,7 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, return path_none; case DT_DIR: strbuf_addch(path, '/'); - return treat_directory(dir, path->buf, path->len, exclude, + return treat_directory(dir, untracked, path->buf, path->len, exclude, simplify); case DT_REG: case DT_LNK: @@ -1205,14 +1504,52 @@ static enum path_treatment treat_one_path(struct dir_struct *dir, } } +static enum path_treatment treat_path_fast(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, + struct strbuf *path, + int baselen, + const struct path_simplify *simplify) +{ + strbuf_setlen(path, baselen); + if (!cdir->ucd) { + strbuf_addstr(path, cdir->file); + return path_untracked; + } + strbuf_addstr(path, cdir->ucd->name); + /* treat_one_path() does this before it calls treat_directory() */ + if (path->buf[path->len - 1] != '/') + strbuf_addch(path, '/'); + if (cdir->ucd->check_only) + /* + * check_only is set as a result of treat_directory() getting + * to its bottom. Verify again the same set of directories + * with check_only set. + */ + return read_directory_recursive(dir, path->buf, path->len, + cdir->ucd, 1, simplify); + /* + * We get path_recurse in the first run when + * directory_exists_in_index() returns index_nonexistent. We + * are sure that new changes in the index does not impact the + * outcome. Return now. + */ + return path_recurse; +} + static enum path_treatment treat_path(struct dir_struct *dir, - struct dirent *de, + struct untracked_cache_dir *untracked, + struct cached_dir *cdir, struct strbuf *path, int baselen, const struct path_simplify *simplify) { int dtype; + struct dirent *de = cdir->de; + if (!de) + return treat_path_fast(dir, untracked, cdir, path, + baselen, simplify); if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git")) return path_none; strbuf_setlen(path, baselen); @@ -1221,7 +1558,121 @@ static enum path_treatment treat_path(struct dir_struct *dir, return path_none; dtype = DTYPE(de); - return treat_one_path(dir, path, simplify, dtype, de); + return treat_one_path(dir, untracked, path, simplify, dtype, de); +} + +static void add_untracked(struct untracked_cache_dir *dir, const char *name) +{ + if (!dir) + return; + ALLOC_GROW(dir->untracked, dir->untracked_nr + 1, + dir->untracked_alloc); + dir->untracked[dir->untracked_nr++] = xstrdup(name); +} + +static int valid_cached_dir(struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct strbuf *path, + int check_only) +{ + struct stat st; + + if (!untracked) + return 0; + + if (stat(path->len ? path->buf : ".", &st)) { + invalidate_directory(dir->untracked, untracked); + memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_stat_data_racy(&the_index, &untracked->stat_data, &st)) { + if (untracked->valid) + invalidate_directory(dir->untracked, untracked); + fill_stat_data(&untracked->stat_data, &st); + return 0; + } + + if (untracked->check_only != !!check_only) { + invalidate_directory(dir->untracked, untracked); + return 0; + } + + /* + * prep_exclude will be called eventually on this directory, + * but it's called much later in last_exclude_matching(). We + * need it now to determine the validity of the cache for this + * path. The next calls will be nearly no-op, the way + * prep_exclude() is designed. + */ + if (path->len && path->buf[path->len - 1] != '/') { + strbuf_addch(path, '/'); + prep_exclude(dir, path->buf, path->len); + strbuf_setlen(path, path->len - 1); + } else + prep_exclude(dir, path->buf, path->len); + + /* hopefully prep_exclude() haven't invalidated this entry... */ + return untracked->valid; +} + +static int open_cached_dir(struct cached_dir *cdir, + struct dir_struct *dir, + struct untracked_cache_dir *untracked, + struct strbuf *path, + int check_only) +{ + memset(cdir, 0, sizeof(*cdir)); + cdir->untracked = untracked; + if (valid_cached_dir(dir, untracked, path, check_only)) + return 0; + cdir->fdir = opendir(path->len ? path->buf : "."); + if (dir->untracked) + dir->untracked->dir_opened++; + if (!cdir->fdir) + return -1; + return 0; +} + +static int read_cached_dir(struct cached_dir *cdir) +{ + if (cdir->fdir) { + cdir->de = readdir(cdir->fdir); + if (!cdir->de) + return -1; + return 0; + } + while (cdir->nr_dirs < cdir->untracked->dirs_nr) { + struct untracked_cache_dir *d = cdir->untracked->dirs[cdir->nr_dirs]; + if (!d->recurse) { + cdir->nr_dirs++; + continue; + } + cdir->ucd = d; + cdir->nr_dirs++; + return 0; + } + cdir->ucd = NULL; + if (cdir->nr_files < cdir->untracked->untracked_nr) { + struct untracked_cache_dir *d = cdir->untracked; + cdir->file = d->untracked[cdir->nr_files++]; + return 0; + } + return -1; +} + +static void close_cached_dir(struct cached_dir *cdir) +{ + if (cdir->fdir) + closedir(cdir->fdir); + /* + * We have gone through this directory and found no untracked + * entries. Mark it valid. + */ + if (cdir->untracked) { + cdir->untracked->valid = 1; + cdir->untracked->recurse = 1; + } } /* @@ -1237,38 +1688,48 @@ static enum path_treatment treat_path(struct dir_struct *dir, */ static enum path_treatment read_directory_recursive(struct dir_struct *dir, const char *base, int baselen, - int check_only, + struct untracked_cache_dir *untracked, int check_only, const struct path_simplify *simplify) { - DIR *fdir; + struct cached_dir cdir; enum path_treatment state, subdir_state, dir_state = path_none; - struct dirent *de; struct strbuf path = STRBUF_INIT; strbuf_add(&path, base, baselen); - fdir = opendir(path.len ? path.buf : "."); - if (!fdir) + if (open_cached_dir(&cdir, dir, untracked, &path, check_only)) goto out; - while ((de = readdir(fdir)) != NULL) { + if (untracked) + untracked->check_only = !!check_only; + + while (!read_cached_dir(&cdir)) { /* check how the file or directory should be treated */ - state = treat_path(dir, de, &path, baselen, simplify); + state = treat_path(dir, untracked, &cdir, &path, baselen, simplify); + if (state > dir_state) dir_state = state; /* recurse into subdir if instructed by treat_path */ if (state == path_recurse) { - subdir_state = read_directory_recursive(dir, path.buf, - path.len, check_only, simplify); + struct untracked_cache_dir *ud; + ud = lookup_untracked(dir->untracked, untracked, + path.buf + baselen, + path.len - baselen); + subdir_state = + read_directory_recursive(dir, path.buf, path.len, + ud, check_only, simplify); if (subdir_state > dir_state) dir_state = subdir_state; } if (check_only) { /* abort early if maximum state has been reached */ - if (dir_state == path_untracked) + if (dir_state == path_untracked) { + if (cdir.fdir) + add_untracked(untracked, path.buf + baselen); break; + } /* skip the dir_add_* part */ continue; } @@ -1286,15 +1747,18 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, break; case path_untracked: - if (!(dir->flags & DIR_SHOW_IGNORED)) - dir_add_name(dir, path.buf, path.len); + if (dir->flags & DIR_SHOW_IGNORED) + break; + dir_add_name(dir, path.buf, path.len); + if (cdir.fdir) + add_untracked(untracked, path.buf + baselen); break; default: break; } } - closedir(fdir); + close_cached_dir(&cdir); out: strbuf_release(&path); @@ -1306,8 +1770,7 @@ static int cmp_name(const void *p1, const void *p2) const struct dir_entry *e1 = *(const struct dir_entry **)p1; const struct dir_entry *e2 = *(const struct dir_entry **)p2; - return cache_name_compare(e1->name, e1->len, - e2->name, e2->len); + return name_compare(e1->name, e1->len, e2->name, e2->len); } static struct path_simplify *create_simplify(const char **pathspec) @@ -1320,10 +1783,7 @@ static struct path_simplify *create_simplify(const char **pathspec) for (nr = 0 ; ; nr++) { const char *match; - if (nr >= alloc) { - alloc = alloc_nr(alloc); - simplify = xrealloc(simplify, alloc * sizeof(*simplify)); - } + ALLOC_GROW(simplify, nr + 1, alloc); match = *pathspec++; if (!match) break; @@ -1368,7 +1828,7 @@ static int treat_leading_path(struct dir_struct *dir, break; if (simplify_away(sb.buf, sb.len, simplify)) break; - if (treat_one_path(dir, &sb, simplify, + if (treat_one_path(dir, NULL, &sb, simplify, DT_DIR, NULL) == path_none) break; /* do not recurse into it */ if (len <= baselen) { @@ -1381,19 +1841,195 @@ static int treat_leading_path(struct dir_struct *dir, return rc; } -int read_directory(struct dir_struct *dir, const char *path, int len, const char **pathspec) +static const char *get_ident_string(void) +{ + static struct strbuf sb = STRBUF_INIT; + struct utsname uts; + + if (sb.len) + return sb.buf; + if (uname(&uts) < 0) + die_errno(_("failed to get kernel name and information")); + strbuf_addf(&sb, "Location %s, system %s %s %s", get_git_work_tree(), + uts.sysname, uts.release, uts.version); + return sb.buf; +} + +static int ident_in_untracked(const struct untracked_cache *uc) +{ + const char *end = uc->ident.buf + uc->ident.len; + const char *p = uc->ident.buf; + + for (p = uc->ident.buf; p < end; p += strlen(p) + 1) + if (!strcmp(p, get_ident_string())) + return 1; + return 0; +} + +void add_untracked_ident(struct untracked_cache *uc) +{ + if (ident_in_untracked(uc)) + return; + strbuf_addstr(&uc->ident, get_ident_string()); + /* this strbuf contains a list of strings, save NUL too */ + strbuf_addch(&uc->ident, 0); +} + +static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *dir, + int base_len, + const struct pathspec *pathspec) +{ + struct untracked_cache_dir *root; + int i; + + if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE")) + return NULL; + + /* + * We only support $GIT_DIR/info/exclude and core.excludesfile + * as the global ignore rule files. Any other additions + * (e.g. from command line) invalidate the cache. This + * condition also catches running setup_standard_excludes() + * before setting dir->untracked! + */ + if (dir->unmanaged_exclude_files) + return NULL; + + /* + * Optimize for the main use case only: whole-tree git + * status. More work involved in treat_leading_path() if we + * use cache on just a subset of the worktree. pathspec + * support could make the matter even worse. + */ + if (base_len || (pathspec && pathspec->nr)) + return NULL; + + /* Different set of flags may produce different results */ + if (dir->flags != dir->untracked->dir_flags || + /* + * See treat_directory(), case index_nonexistent. Without + * this flag, we may need to also cache .git file content + * for the resolve_gitlink_ref() call, which we don't. + */ + !(dir->flags & DIR_SHOW_OTHER_DIRECTORIES) || + /* We don't support collecting ignore files */ + (dir->flags & (DIR_SHOW_IGNORED | DIR_SHOW_IGNORED_TOO | + DIR_COLLECT_IGNORED))) + return NULL; + + /* + * If we use .gitignore in the cache and now you change it to + * .gitexclude, everything will go wrong. + */ + if (dir->exclude_per_dir != dir->untracked->exclude_per_dir && + strcmp(dir->exclude_per_dir, dir->untracked->exclude_per_dir)) + return NULL; + + /* + * EXC_CMDL is not considered in the cache. If people set it, + * skip the cache. + */ + if (dir->exclude_list_group[EXC_CMDL].nr) + return NULL; + + /* + * An optimization in prep_exclude() does not play well with + * CE_SKIP_WORKTREE. It's a rare case anyway, if a single + * entry has that bit set, disable the whole untracked cache. + */ + for (i = 0; i < active_nr; i++) + if (ce_skip_worktree(active_cache[i])) + return NULL; + + if (!ident_in_untracked(dir->untracked)) { + warning(_("Untracked cache is disabled on this system.")); + return NULL; + } + + if (!dir->untracked->root) { + const int len = sizeof(*dir->untracked->root); + dir->untracked->root = xmalloc(len); + memset(dir->untracked->root, 0, len); + } + + /* Validate $GIT_DIR/info/exclude and core.excludesfile */ + root = dir->untracked->root; + if (hashcmp(dir->ss_info_exclude.sha1, + dir->untracked->ss_info_exclude.sha1)) { + invalidate_gitignore(dir->untracked, root); + dir->untracked->ss_info_exclude = dir->ss_info_exclude; + } + if (hashcmp(dir->ss_excludes_file.sha1, + dir->untracked->ss_excludes_file.sha1)) { + invalidate_gitignore(dir->untracked, root); + dir->untracked->ss_excludes_file = dir->ss_excludes_file; + } + + /* Make sure this directory is not dropped out at saving phase */ + root->recurse = 1; + return root; +} + +int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec) { struct path_simplify *simplify; + struct untracked_cache_dir *untracked; + + /* + * Check out create_simplify() + */ + if (pathspec) + GUARD_PATHSPEC(pathspec, + PATHSPEC_FROMTOP | + PATHSPEC_MAXDEPTH | + PATHSPEC_LITERAL | + PATHSPEC_GLOB | + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); if (has_symlink_leading_path(path, len)) return dir->nr; - simplify = create_simplify(pathspec); + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * create_simplify(). + */ + simplify = create_simplify(pathspec ? pathspec->_raw : NULL); + untracked = validate_untracked_cache(dir, len, pathspec); + if (!untracked) + /* + * make sure untracked cache code path is disabled, + * e.g. prep_exclude() + */ + dir->untracked = NULL; if (!len || treat_leading_path(dir, path, len, simplify)) - read_directory_recursive(dir, path, len, 0, simplify); + read_directory_recursive(dir, path, len, untracked, 0, simplify); free_simplify(simplify); qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name); qsort(dir->ignored, dir->ignored_nr, sizeof(struct dir_entry *), cmp_name); + if (dir->untracked) { + static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS); + trace_printf_key(&trace_untracked_stats, + "node creation: %u\n" + "gitignore invalidation: %u\n" + "directory invalidation: %u\n" + "opendir: %u\n", + dir->untracked->dir_created, + dir->untracked->gitignore_invalidated, + dir->untracked->dir_invalidated, + dir->untracked->dir_opened); + if (dir->untracked == the_index.untracked && + (dir->untracked->dir_opened || + dir->untracked->gitignore_invalidated || + dir->untracked->dir_invalidated)) + the_index.cache_changed |= UNTRACKED_CHANGED; + if (dir->untracked != the_index.untracked) { + free(dir->untracked); + dir->untracked = NULL; + } + } return dir->nr; } @@ -1437,12 +2073,16 @@ int dir_inside_of(const char *subdir, const char *dir) int is_inside_dir(const char *dir) { - char cwd[PATH_MAX]; + char *cwd; + int rc; + if (!dir) return 0; - if (!getcwd(cwd, sizeof(cwd))) - die_errno("can't find the current directory"); - return dir_inside_of(cwd, dir) >= 0; + + cwd = xgetcwd(); + rc = (dir_inside_of(cwd, dir) >= 0); + free(cwd); + return rc; } int is_empty_dir(const char *path) @@ -1484,8 +2124,13 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) flag &= ~REMOVE_DIR_KEEP_TOPLEVEL; dir = opendir(path->buf); if (!dir) { - /* an empty dir could be removed even if it is unreadble */ - if (!keep_toplevel) + if (errno == ENOENT) + return keep_toplevel ? -1 : 0; + else if (errno == EACCES && !keep_toplevel) + /* + * An empty dir could be removable even if it + * is unreadable: + */ return rmdir(path->buf); else return -1; @@ -1501,13 +2146,21 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); - if (lstat(path->buf, &st)) - ; /* fall thru */ - else if (S_ISDIR(st.st_mode)) { + if (lstat(path->buf, &st)) { + if (errno == ENOENT) + /* + * file disappeared, which is what we + * wanted anyway + */ + continue; + /* fall thru */ + } else if (S_ISDIR(st.st_mode)) { if (!remove_dir_recurse(path, flag, &kept_down)) continue; /* happy */ - } else if (!only_empty && !unlink(path->buf)) + } else if (!only_empty && + (!unlink(path->buf) || errno == ENOENT)) { continue; /* happy, too */ + } /* path too long, stat fails, or non-directory still exists */ ret = -1; @@ -1517,7 +2170,7 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) strbuf_setlen(path, original_len); if (!ret && !keep_toplevel && !kept_down) - ret = rmdir(path->buf); + ret = (!rmdir(path->buf) || errno == ENOENT) ? 0 : -1; else if (kept_up) /* * report the uplevel that it is not an error that we @@ -1535,18 +2188,21 @@ int remove_dir_recursively(struct strbuf *path, int flag) void setup_standard_excludes(struct dir_struct *dir) { const char *path; - char *xdg_path; dir->exclude_per_dir = ".gitignore"; + + /* core.excludefile defaulting to $XDG_HOME/git/ignore */ + if (!excludes_file) + excludes_file = xdg_config_home("ignore"); + if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) + add_excludes_from_file_1(dir, excludes_file, + dir->untracked ? &dir->ss_excludes_file : NULL); + + /* per repository user preference */ path = git_path("info/exclude"); - if (!excludes_file) { - home_config_paths(NULL, &xdg_path, "ignore"); - excludes_file = xdg_path; - } if (!access_or_warn(path, R_OK, 0)) - add_excludes_from_file(dir, path); - if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) - add_excludes_from_file(dir, excludes_file); + add_excludes_from_file_1(dir, path, + dir->untracked ? &dir->ss_info_exclude : NULL); } int remove_path(const char *name) @@ -1568,71 +2224,6 @@ int remove_path(const char *name) return 0; } -static int pathspec_item_cmp(const void *a_, const void *b_) -{ - struct pathspec_item *a, *b; - - a = (struct pathspec_item *)a_; - b = (struct pathspec_item *)b_; - return strcmp(a->match, b->match); -} - -int init_pathspec(struct pathspec *pathspec, const char **paths) -{ - const char **p = paths; - int i; - - memset(pathspec, 0, sizeof(*pathspec)); - if (!p) - return 0; - while (*p) - p++; - pathspec->raw = paths; - pathspec->nr = p - paths; - if (!pathspec->nr) - return 0; - - pathspec->items = xmalloc(sizeof(struct pathspec_item)*pathspec->nr); - for (i = 0; i < pathspec->nr; i++) { - struct pathspec_item *item = pathspec->items+i; - const char *path = paths[i]; - - item->match = path; - item->len = strlen(path); - item->flags = 0; - if (limit_pathspec_to_literal()) { - item->nowildcard_len = item->len; - } else { - item->nowildcard_len = simple_length(path); - if (item->nowildcard_len < item->len) { - pathspec->has_wildcard = 1; - if (path[item->nowildcard_len] == '*' && - no_wildcard(path + item->nowildcard_len + 1)) - item->flags |= PATHSPEC_ONESTAR; - } - } - } - - qsort(pathspec->items, pathspec->nr, - sizeof(struct pathspec_item), pathspec_item_cmp); - - return 0; -} - -void free_pathspec(struct pathspec *pathspec) -{ - free(pathspec->items); - pathspec->items = NULL; -} - -int limit_pathspec_to_literal(void) -{ - static int flag = -1; - if (flag < 0) - flag = git_env_bool(GIT_LITERAL_PATHSPECS_ENVIRONMENT, 0); - return flag; -} - /* * Frees memory within dir which was allocated for exclude lists and * the exclude_stack. Does not free dir itself. @@ -1661,4 +2252,406 @@ void clear_directory(struct dir_struct *dir) free(stk); stk = prev; } + strbuf_release(&dir->basebuf); +} + +struct ondisk_untracked_cache { + struct stat_data info_exclude_stat; + struct stat_data excludes_file_stat; + uint32_t dir_flags; + unsigned char info_exclude_sha1[20]; + unsigned char excludes_file_sha1[20]; + char exclude_per_dir[FLEX_ARRAY]; +}; + +#define ouc_size(len) (offsetof(struct ondisk_untracked_cache, exclude_per_dir) + len + 1) + +struct write_data { + int index; /* number of written untracked_cache_dir */ + struct ewah_bitmap *check_only; /* from untracked_cache_dir */ + struct ewah_bitmap *valid; /* from untracked_cache_dir */ + struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */ + struct strbuf out; + struct strbuf sb_stat; + struct strbuf sb_sha1; +}; + +static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from) +{ + to->sd_ctime.sec = htonl(from->sd_ctime.sec); + to->sd_ctime.nsec = htonl(from->sd_ctime.nsec); + to->sd_mtime.sec = htonl(from->sd_mtime.sec); + to->sd_mtime.nsec = htonl(from->sd_mtime.nsec); + to->sd_dev = htonl(from->sd_dev); + to->sd_ino = htonl(from->sd_ino); + to->sd_uid = htonl(from->sd_uid); + to->sd_gid = htonl(from->sd_gid); + to->sd_size = htonl(from->sd_size); +} + +static void write_one_dir(struct untracked_cache_dir *untracked, + struct write_data *wd) +{ + struct stat_data stat_data; + struct strbuf *out = &wd->out; + unsigned char intbuf[16]; + unsigned int intlen, value; + int i = wd->index++; + + /* + * untracked_nr should be reset whenever valid is clear, but + * for safety.. + */ + if (!untracked->valid) { + untracked->untracked_nr = 0; + untracked->check_only = 0; + } + + if (untracked->check_only) + ewah_set(wd->check_only, i); + if (untracked->valid) { + ewah_set(wd->valid, i); + stat_data_to_disk(&stat_data, &untracked->stat_data); + strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data)); + } + if (!is_null_sha1(untracked->exclude_sha1)) { + ewah_set(wd->sha1_valid, i); + strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20); + } + + intlen = encode_varint(untracked->untracked_nr, intbuf); + strbuf_add(out, intbuf, intlen); + + /* skip non-recurse directories */ + for (i = 0, value = 0; i < untracked->dirs_nr; i++) + if (untracked->dirs[i]->recurse) + value++; + intlen = encode_varint(value, intbuf); + strbuf_add(out, intbuf, intlen); + + strbuf_add(out, untracked->name, strlen(untracked->name) + 1); + + for (i = 0; i < untracked->untracked_nr; i++) + strbuf_add(out, untracked->untracked[i], + strlen(untracked->untracked[i]) + 1); + + for (i = 0; i < untracked->dirs_nr; i++) + if (untracked->dirs[i]->recurse) + write_one_dir(untracked->dirs[i], wd); +} + +void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked) +{ + struct ondisk_untracked_cache *ouc; + struct write_data wd; + unsigned char varbuf[16]; + int len = 0, varint_len; + if (untracked->exclude_per_dir) + len = strlen(untracked->exclude_per_dir); + ouc = xmalloc(sizeof(*ouc) + len + 1); + stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat); + stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat); + hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1); + hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1); + ouc->dir_flags = htonl(untracked->dir_flags); + memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1); + + varint_len = encode_varint(untracked->ident.len, varbuf); + strbuf_add(out, varbuf, varint_len); + strbuf_add(out, untracked->ident.buf, untracked->ident.len); + + strbuf_add(out, ouc, ouc_size(len)); + free(ouc); + ouc = NULL; + + if (!untracked->root) { + varint_len = encode_varint(0, varbuf); + strbuf_add(out, varbuf, varint_len); + return; + } + + wd.index = 0; + wd.check_only = ewah_new(); + wd.valid = ewah_new(); + wd.sha1_valid = ewah_new(); + strbuf_init(&wd.out, 1024); + strbuf_init(&wd.sb_stat, 1024); + strbuf_init(&wd.sb_sha1, 1024); + write_one_dir(untracked->root, &wd); + + varint_len = encode_varint(wd.index, varbuf); + strbuf_add(out, varbuf, varint_len); + strbuf_addbuf(out, &wd.out); + ewah_serialize_strbuf(wd.valid, out); + ewah_serialize_strbuf(wd.check_only, out); + ewah_serialize_strbuf(wd.sha1_valid, out); + strbuf_addbuf(out, &wd.sb_stat); + strbuf_addbuf(out, &wd.sb_sha1); + strbuf_addch(out, '\0'); /* safe guard for string lists */ + + ewah_free(wd.valid); + ewah_free(wd.check_only); + ewah_free(wd.sha1_valid); + strbuf_release(&wd.out); + strbuf_release(&wd.sb_stat); + strbuf_release(&wd.sb_sha1); +} + +static void free_untracked(struct untracked_cache_dir *ucd) +{ + int i; + if (!ucd) + return; + for (i = 0; i < ucd->dirs_nr; i++) + free_untracked(ucd->dirs[i]); + for (i = 0; i < ucd->untracked_nr; i++) + free(ucd->untracked[i]); + free(ucd->untracked); + free(ucd->dirs); + free(ucd); +} + +void free_untracked_cache(struct untracked_cache *uc) +{ + if (uc) + free_untracked(uc->root); + free(uc); +} + +struct read_data { + int index; + struct untracked_cache_dir **ucd; + struct ewah_bitmap *check_only; + struct ewah_bitmap *valid; + struct ewah_bitmap *sha1_valid; + const unsigned char *data; + const unsigned char *end; +}; + +static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from) +{ + to->sd_ctime.sec = get_be32(&from->sd_ctime.sec); + to->sd_ctime.nsec = get_be32(&from->sd_ctime.nsec); + to->sd_mtime.sec = get_be32(&from->sd_mtime.sec); + to->sd_mtime.nsec = get_be32(&from->sd_mtime.nsec); + to->sd_dev = get_be32(&from->sd_dev); + to->sd_ino = get_be32(&from->sd_ino); + to->sd_uid = get_be32(&from->sd_uid); + to->sd_gid = get_be32(&from->sd_gid); + to->sd_size = get_be32(&from->sd_size); +} + +static int read_one_dir(struct untracked_cache_dir **untracked_, + struct read_data *rd) +{ + struct untracked_cache_dir ud, *untracked; + const unsigned char *next, *data = rd->data, *end = rd->end; + unsigned int value; + int i, len; + + memset(&ud, 0, sizeof(ud)); + + next = data; + value = decode_varint(&next); + if (next > end) + return -1; + ud.recurse = 1; + ud.untracked_alloc = value; + ud.untracked_nr = value; + if (ud.untracked_nr) + ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr); + data = next; + + next = data; + ud.dirs_alloc = ud.dirs_nr = decode_varint(&next); + if (next > end) + return -1; + ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr); + data = next; + + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + *untracked_ = untracked = xmalloc(sizeof(*untracked) + len); + memcpy(untracked, &ud, sizeof(ud)); + memcpy(untracked->name, data, len + 1); + data = next; + + for (i = 0; i < untracked->untracked_nr; i++) { + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + untracked->untracked[i] = xstrdup((const char*)data); + data = next; + } + + rd->ucd[rd->index++] = untracked; + rd->data = data; + + for (i = 0; i < untracked->dirs_nr; i++) { + len = read_one_dir(untracked->dirs + i, rd); + if (len < 0) + return -1; + } + return 0; +} + +static void set_check_only(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + ud->check_only = 1; +} + +static void read_stat(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + sizeof(struct stat_data) > rd->end) { + rd->data = rd->end + 1; + return; + } + stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data); + rd->data += sizeof(struct stat_data); + ud->valid = 1; +} + +static void read_sha1(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + 20 > rd->end) { + rd->data = rd->end + 1; + return; + } + hashcpy(ud->exclude_sha1, rd->data); + rd->data += 20; +} + +static void load_sha1_stat(struct sha1_stat *sha1_stat, + const struct stat_data *stat, + const unsigned char *sha1) +{ + stat_data_from_disk(&sha1_stat->stat, stat); + hashcpy(sha1_stat->sha1, sha1); + sha1_stat->valid = 1; +} + +struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz) +{ + const struct ondisk_untracked_cache *ouc; + struct untracked_cache *uc; + struct read_data rd; + const unsigned char *next = data, *end = (const unsigned char *)data + sz; + const char *ident; + int ident_len, len; + + if (sz <= 1 || end[-1] != '\0') + return NULL; + end--; + + ident_len = decode_varint(&next); + if (next + ident_len > end) + return NULL; + ident = (const char *)next; + next += ident_len; + + ouc = (const struct ondisk_untracked_cache *)next; + if (next + ouc_size(0) > end) + return NULL; + + uc = xcalloc(1, sizeof(*uc)); + strbuf_init(&uc->ident, ident_len); + strbuf_add(&uc->ident, ident, ident_len); + load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat, + ouc->info_exclude_sha1); + load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat, + ouc->excludes_file_sha1); + uc->dir_flags = get_be32(&ouc->dir_flags); + uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir); + /* NUL after exclude_per_dir is covered by sizeof(*ouc) */ + next += ouc_size(strlen(ouc->exclude_per_dir)); + if (next >= end) + goto done2; + + len = decode_varint(&next); + if (next > end || len == 0) + goto done2; + + rd.valid = ewah_new(); + rd.check_only = ewah_new(); + rd.sha1_valid = ewah_new(); + rd.data = next; + rd.end = end; + rd.index = 0; + rd.ucd = xmalloc(sizeof(*rd.ucd) * len); + + if (read_one_dir(&uc->root, &rd) || rd.index != len) + goto done; + + next = rd.data; + len = ewah_read_mmap(rd.valid, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.check_only, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.sha1_valid, next, end - next); + if (len < 0) + goto done; + + ewah_each_bit(rd.check_only, set_check_only, &rd); + rd.data = next + len; + ewah_each_bit(rd.valid, read_stat, &rd); + ewah_each_bit(rd.sha1_valid, read_sha1, &rd); + next = rd.data; + +done: + free(rd.ucd); + ewah_free(rd.valid); + ewah_free(rd.check_only); + ewah_free(rd.sha1_valid); +done2: + if (next != end) { + free_untracked_cache(uc); + uc = NULL; + } + return uc; +} + +void untracked_cache_invalidate_path(struct index_state *istate, + const char *path) +{ + const char *sep; + struct untracked_cache_dir *d; + if (!istate->untracked || !istate->untracked->root) + return; + sep = strrchr(path, '/'); + if (sep) + d = lookup_untracked(istate->untracked, + istate->untracked->root, + path, sep - path); + else + d = istate->untracked->root; + istate->untracked->dir_invalidated++; + d->valid = 0; + d->untracked_nr = 0; +} + +void untracked_cache_remove_from_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); +} + +void untracked_cache_add_to_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); } |