From b74e8cbd80c13257c8c0a68341e9aff9926d31f2 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 15 Jan 2007 22:56:34 -0800 Subject: git-fetch: split fetch_main into fetch_dumb and fetch_native Signed-off-by: Junio C Hamano --- git-fetch.sh | 174 +++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 105 insertions(+), 69 deletions(-) diff --git a/git-fetch.sh b/git-fetch.sh index ca984e739a..3e01265160 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -268,7 +268,101 @@ then fi fi -fetch_main () { +fetch_native () { + reflist="$1" + refs= + rref= + + for ref in $reflist + do + refs="$refs$LF$ref" + + # These are relative path from $GIT_DIR, typically starting at refs/ + # but may be HEAD + if expr "z$ref" : 'z\.' >/dev/null + then + not_for_merge=t + ref=$(expr "z$ref" : 'z\.\(.*\)') + else + not_for_merge= + fi + if expr "z$ref" : 'z+' >/dev/null + then + single_force=t + ref=$(expr "z$ref" : 'z+\(.*\)') + else + single_force= + fi + remote_name=$(expr "z$ref" : 'z\([^:]*\):') + local_name=$(expr "z$ref" : 'z[^:]*:\(.*\)') + + rref="$rref$LF$remote_name" + done + + ( : subshell because we muck with IFS + IFS=" $LF" + ( + git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || + echo failed "$remote" + ) | + ( + trap ' + if test -n "$keepfile" && test -f "$keepfile" + then + rm -f "$keepfile" + fi + ' 0 + + keepfile= + while read sha1 remote_name + do + case "$sha1" in + failed) + echo >&2 "Fetch failure: $remote" + exit 1 ;; + # special line coming from index-pack with the pack name + pack) + continue ;; + keep) + keepfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep" + continue ;; + esac + found= + single_force= + for ref in $refs + do + case "$ref" in + +$remote_name:*) + single_force=t + not_for_merge= + found="$ref" + break ;; + .+$remote_name:*) + single_force=t + not_for_merge=t + found="$ref" + break ;; + .$remote_name:*) + not_for_merge=t + found="$ref" + break ;; + $remote_name:*) + not_for_merge= + found="$ref" + break ;; + esac + done + local_name=$(expr "z$found" : 'z[^:]*:\(.*\)') + append_fetch_head "$sha1" "$remote" \ + "$remote_name" "$remote_nick" "$local_name" \ + "$not_for_merge" || exit + done + ) + ) || exit + +} + +fetch_dumb () { reflist="$1" refs= rref= @@ -360,9 +454,6 @@ fetch_main () { rsync_slurped_objects=t } ;; - *) - # We will do git native transport with just one call later. - continue ;; esac append_fetch_head "$head" "$remote" \ @@ -370,72 +461,17 @@ fetch_main () { done - case "$remote" in - http://* | https://* | ftp://* | rsync://* ) - ;; # we are already done. - *) - ( : subshell because we muck with IFS - IFS=" $LF" - ( - git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || - echo failed "$remote" - ) | - ( - trap ' - if test -n "$keepfile" && test -f "$keepfile" - then - rm -f "$keepfile" - fi - ' 0 - - keepfile= - while read sha1 remote_name - do - case "$sha1" in - failed) - echo >&2 "Fetch failure: $remote" - exit 1 ;; - # special line coming from index-pack with the pack name - pack) - continue ;; - keep) - keepfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep" - continue ;; - esac - found= - single_force= - for ref in $refs - do - case "$ref" in - +$remote_name:*) - single_force=t - not_for_merge= - found="$ref" - break ;; - .+$remote_name:*) - single_force=t - not_for_merge=t - found="$ref" - break ;; - .$remote_name:*) - not_for_merge=t - found="$ref" - break ;; - $remote_name:*) - not_for_merge= - found="$ref" - break ;; - esac - done - local_name=$(expr "z$found" : 'z[^:]*:\(.*\)') - append_fetch_head "$sha1" "$remote" \ - "$remote_name" "$remote_nick" "$local_name" \ - "$not_for_merge" || exit - done - ) - ) || exit ;; - esac +} +fetch_main () { + case "$remote" in + http://* | https://* | ftp://* | rsync://* ) + fetch_dumb "$@" + ;; + *) + fetch_native "$@" + ;; + esac } fetch_main "$reflist" || exit -- cgit v1.2.3 From d4289fff870a85b1b7e55ead2a5baf98847fc72a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 16 Jan 2007 00:23:24 -0800 Subject: git-fetch--tool: start rewriting parts of git-fetch in C. Signed-off-by: Junio C Hamano --- Makefile | 1 + builtin-fetch--tool.c | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++ builtin.h | 1 + git-fetch.sh | 134 +++---------------------------- git.c | 1 + 5 files changed, 227 insertions(+), 124 deletions(-) create mode 100644 builtin-fetch--tool.c diff --git a/Makefile b/Makefile index 40bdcff696..181ad94279 100644 --- a/Makefile +++ b/Makefile @@ -282,6 +282,7 @@ BUILTIN_OBJS = \ builtin-diff-index.o \ builtin-diff-stages.o \ builtin-diff-tree.o \ + builtin-fetch--tool.o \ builtin-fmt-merge-msg.o \ builtin-for-each-ref.o \ builtin-fsck.o \ diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c new file mode 100644 index 0000000000..3075ba04af --- /dev/null +++ b/builtin-fetch--tool.c @@ -0,0 +1,214 @@ +#include "cache.h" +#include "refs.h" +#include "commit.h" + +static void show_new(char *type, unsigned char *sha1_new) +{ + fprintf(stderr, " %s: %s\n", type, + find_unique_abbrev(sha1_new, DEFAULT_ABBREV)); +} + +static int update_ref(const char *action, + const char *refname, + unsigned char *sha1, + unsigned char *oldval) +{ + int len; + char msg[1024]; + char *rla = getenv("GIT_REFLOG_ACTION"); + static struct ref_lock *lock; + + if (!rla) + rla = "(reflog update)"; + len = snprintf(msg, sizeof(msg), "%s: %s", rla, action); + if (sizeof(msg) <= len) + die("insanely long action"); + lock = lock_any_ref_for_update(refname, oldval); + if (!lock) + return 1; + if (write_ref_sha1(lock, sha1, msg) < 0) + return 1; + return 0; +} + +static int update_local_ref(const char *name, + const char *new_head, + const char *note, + int verbose, int force) +{ + char type[20]; + unsigned char sha1_old[20], sha1_new[20]; + char oldh[41], newh[41]; + struct commit *current, *updated; + + if (get_sha1_hex(new_head, sha1_new)) + die("malformed object name %s", new_head); + if (sha1_object_info(sha1_new, type, NULL)) + die("object %s not found", new_head); + + if (!*name) { + /* Not storing */ + if (verbose) { + fprintf(stderr, "* fetched %s\n", note); + show_new(type, sha1_new); + } + return 0; + } + + if (get_sha1(name, sha1_old)) { + char *msg; + just_store: + /* new ref */ + if (!strncmp(name, "refs/tags/", 10)) + msg = "storing tag"; + else + msg = "storing head"; + fprintf(stderr, "* %s: storing %s\n", + name, note); + show_new(type, sha1_new); + return update_ref(msg, name, sha1_new, NULL); + } + + if (!hashcmp(sha1_old, sha1_new)) { + if (verbose) { + fprintf(stderr, "* %s: same as %s\n", name, note); + show_new(type, sha1_new); + } + return 0; + } + + if (!strncmp(name, "refs/tags/", 10)) { + fprintf(stderr, "* %s: updating with %s\n", name, note); + show_new(type, sha1_new); + return update_ref("updating tag", name, sha1_new, NULL); + } + + current = lookup_commit_reference(sha1_old); + updated = lookup_commit_reference(sha1_new); + if (!current || !updated) + goto just_store; + + strcpy(oldh, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV)); + strcpy(newh, find_unique_abbrev(sha1_new, DEFAULT_ABBREV)); + + if (in_merge_bases(current, &updated, 1)) { + fprintf(stderr, "* %s: fast forward to %s\n", + name, note); + fprintf(stderr, " old..new: %s..%s\n", oldh, newh); + return update_ref("fast forward", name, sha1_new, sha1_old); + } + if (!force) { + fprintf(stderr, + "* %s: not updating to non-fast forward %s\n", + name, note); + fprintf(stderr, + " old...new: %s...%s\n", oldh, newh); + return 1; + } + fprintf(stderr, + "* %s: forcing update to non-fast forward %s\n", + name, note); + fprintf(stderr, " old...new: %s...%s\n", oldh, newh); + return update_ref("forced-update", name, sha1_new, sha1_old); +} + +static int append_fetch_head(FILE *fp, + const char *head, const char *remote, + const char *remote_name, const char *remote_nick, + const char *local_name, int not_for_merge, + int verbose, int force) +{ + struct commit *commit; + int remote_len, i, note_len; + unsigned char sha1[20]; + char note[1024]; + const char *what, *kind; + + if (get_sha1(head, sha1)) + return error("Not a valid object name: %s", head); + commit = lookup_commit_reference(sha1); + if (!commit) + not_for_merge = 1; + + if (!strcmp(remote_name, "HEAD")) { + kind = ""; + what = ""; + } + else if (!strncmp(remote_name, "refs/heads/", 11)) { + kind = "branch"; + what = remote_name + 11; + } + else if (!strncmp(remote_name, "refs/tags/", 10)) { + kind = "tag"; + what = remote_name + 10; + } + else if (!strncmp(remote_name, "refs/remotes/", 13)) { + kind = "remote branch"; + what = remote_name + 13; + } + else { + kind = ""; + what = remote_name; + } + + remote_len = strlen(remote); + for (i = remote_len - 1; remote[i] == '/' && 0 <= i; i--) + ; + remote_len = i + 1; + if (4 < i && !strncmp(".git", remote + i - 3, 4)) + remote_len = i - 3; + note_len = sprintf(note, "%s\t%s\t", + sha1_to_hex(commit ? commit->object.sha1 : sha1), + not_for_merge ? "not-for-merge" : ""); + if (*what) { + if (*kind) + note_len += sprintf(note + note_len, "%s ", kind); + note_len += sprintf(note + note_len, "'%s' of ", what); + } + note_len += sprintf(note + note_len, "%.*s", remote_len, remote); + fprintf(fp, "%s\n", note); + return update_local_ref(local_name, head, note, verbose, force); +} + +int cmd_fetch__tool(int argc, const char **argv, const char *prefix) +{ + int verbose = 0; + int force = 0; + + while (1 < argc) { + const char *arg = argv[1]; + if (!strcmp("-v", arg)) + verbose = 1; + else if (!strcmp("-f", arg)) + force = 1; + else + break; + argc--; + argv++; + } + + if (argc <= 1) + return error("Missing subcommand"); + + if (!strcmp("append-fetch-head", argv[1])) { + int result; + FILE *fp; + + if (argc != 8) + return error("append-fetch-head takes 6 args"); + fp = fopen(git_path("FETCH_HEAD"), "a"); + result = append_fetch_head(fp, argv[2], argv[3], + argv[4], argv[5], + argv[6], !!argv[7][0], + verbose, force); + fclose(fp); + return result; + } + if (!strcmp("update-local-ref", argv[1])) { + if (argc != 5) + return error("update-local-ref takes 3 args"); + return update_local_ref(argv[2], argv[3], argv[4], + verbose, force); + } + return error("Unknown subcommand: %s", argv[1]); +} diff --git a/builtin.h b/builtin.h index 5108fd2d74..3cad4028d2 100644 --- a/builtin.h +++ b/builtin.h @@ -31,6 +31,7 @@ extern int cmd_diff_index(int argc, const char **argv, const char *prefix); extern int cmd_diff(int argc, const char **argv, const char *prefix); extern int cmd_diff_stages(int argc, const char **argv, const char *prefix); extern int cmd_diff_tree(int argc, const char **argv, const char *prefix); +extern int cmd_fetch__tool(int argc, const char **argv, const char *prefix); extern int cmd_fmt_merge_msg(int argc, const char **argv, const char *prefix); extern int cmd_for_each_ref(int argc, const char **argv, const char *prefix); extern int cmd_format_patch(int argc, const char **argv, const char *prefix); diff --git a/git-fetch.sh b/git-fetch.sh index 3e01265160..2aa34b3992 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -107,133 +107,19 @@ ls_remote_result=$(git ls-remote $exec "$remote") || die "Cannot get the repository state from $remote" append_fetch_head () { - head_="$1" - remote_="$2" - remote_name_="$3" - remote_nick_="$4" - local_name_="$5" - case "$6" in - t) not_for_merge_='not-for-merge' ;; - '') not_for_merge_= ;; - esac - - # remote-nick is the URL given on the command line (or a shorthand) - # remote-name is the $GIT_DIR relative refs/ path we computed - # for this refspec. - - # the $note_ variable will be fed to git-fmt-merge-msg for further - # processing. - case "$remote_name_" in - HEAD) - note_= ;; - refs/heads/*) - note_="$(expr "$remote_name_" : 'refs/heads/\(.*\)')" - note_="branch '$note_' of " ;; - refs/tags/*) - note_="$(expr "$remote_name_" : 'refs/tags/\(.*\)')" - note_="tag '$note_' of " ;; - refs/remotes/*) - note_="$(expr "$remote_name_" : 'refs/remotes/\(.*\)')" - note_="remote branch '$note_' of " ;; - *) - note_="$remote_name of " ;; - esac - remote_1_=$(expr "z$remote_" : 'z\(.*\)\.git/*$') && - remote_="$remote_1_" - note_="$note_$remote_" - - # 2.6.11-tree tag would not be happy to be fed to resolve. - if git-cat-file commit "$head_" >/dev/null 2>&1 - then - headc_=$(git-rev-parse --verify "$head_^0") || exit - echo "$headc_ $not_for_merge_ $note_" >>"$GIT_DIR/FETCH_HEAD" - else - echo "$head_ not-for-merge $note_" >>"$GIT_DIR/FETCH_HEAD" - fi - - update_local_ref "$local_name_" "$head_" "$note_" + flags= + test -n "$verbose" && flags="$flags -v" + test -n "$force" && flags="$flags -f" + GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ + git-fetch--tool append-fetch-head $flags "$@" } update_local_ref () { - # If we are storing the head locally make sure that it is - # a fast forward (aka "reverse push"). - - label_=$(git-cat-file -t $2) - newshort_=$(git-rev-parse --short $2) - if test -z "$1" ; then - [ "$verbose" ] && echo >&2 "* fetched $3" - [ "$verbose" ] && echo >&2 " $label_: $newshort_" - return 0 - fi - oldshort_=$(git show-ref --hash --abbrev "$1" 2>/dev/null) - - case "$1" in - refs/tags/*) - # Tags need not be pointing at commits so there - # is no way to guarantee "fast-forward" anyway. - if test -n "$oldshort_" - then - if now_=$(git show-ref --hash "$1") && test "$now_" = "$2" - then - [ "$verbose" ] && echo >&2 "* $1: same as $3" - [ "$verbose" ] && echo >&2 " $label_: $newshort_" ||: - else - echo >&2 "* $1: updating with $3" - echo >&2 " $label_: $newshort_" - git-update-ref -m "$GIT_REFLOG_ACTION: updating tag" "$1" "$2" - fi - else - echo >&2 "* $1: storing $3" - echo >&2 " $label_: $newshort_" - git-update-ref -m "$GIT_REFLOG_ACTION: storing tag" "$1" "$2" - fi - ;; - - refs/heads/* | refs/remotes/*) - # $1 is the ref being updated. - # $2 is the new value for the ref. - local=$(git-rev-parse --verify "$1^0" 2>/dev/null) - if test "$local" - then - # Require fast-forward. - mb=$(git-merge-base "$local" "$2") && - case "$2,$mb" in - $local,*) - if test -n "$verbose" - then - echo >&2 "* $1: same as $3" - echo >&2 " $label_: $newshort_" - fi - ;; - *,$local) - echo >&2 "* $1: fast forward to $3" - echo >&2 " old..new: $oldshort_..$newshort_" - git-update-ref -m "$GIT_REFLOG_ACTION: fast-forward" "$1" "$2" "$local" - ;; - *) - false - ;; - esac || { - case ",$force,$single_force," in - *,t,*) - echo >&2 "* $1: forcing update to non-fast forward $3" - echo >&2 " old...new: $oldshort_...$newshort_" - git-update-ref -m "$GIT_REFLOG_ACTION: forced-update" "$1" "$2" "$local" - ;; - *) - echo >&2 "* $1: not updating to non-fast forward $3" - echo >&2 " old...new: $oldshort_...$newshort_" - exit 1 - ;; - esac - } - else - echo >&2 "* $1: storing $3" - echo >&2 " $label_: $newshort_" - git-update-ref -m "$GIT_REFLOG_ACTION: storing head" "$1" "$2" - fi - ;; - esac + flags= + test -n "$verbose" && flags="$flags -v" + test -n "$force" && flags="$flags -f" + GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ + git-fetch--tool update-local-ref $flags "$@" } # updating the current HEAD with git-fetch in a bare diff --git a/git.c b/git.c index 45265f14d0..a167b1e42e 100644 --- a/git.c +++ b/git.c @@ -242,6 +242,7 @@ static void handle_internal_command(int argc, const char **argv, char **envp) { "diff-index", cmd_diff_index, RUN_SETUP }, { "diff-stages", cmd_diff_stages, RUN_SETUP }, { "diff-tree", cmd_diff_tree, RUN_SETUP }, + { "fetch--tool", cmd_fetch__tool, RUN_SETUP }, { "fmt-merge-msg", cmd_fmt_merge_msg, RUN_SETUP }, { "for-each-ref", cmd_for_each_ref, RUN_SETUP }, { "format-patch", cmd_format_patch, RUN_SETUP }, -- cgit v1.2.3 From fbe2687eba70385dab7e3d1f5cdcdfdc11dfe0ec Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 16 Jan 2007 01:53:29 -0800 Subject: git-fetch: move more code into C. This adds "native-store" subcommand to git-fetch--tool to move a huge loop implemented in shell into C. This shaves about 70% of the runtime to fetch and update 1000 tracking branches with a single fetch. Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++ git-fetch.sh | 57 +++-------------------- 2 files changed, 131 insertions(+), 51 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 3075ba04af..24343ac9b0 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -170,6 +170,119 @@ static int append_fetch_head(FILE *fp, return update_local_ref(local_name, head, note, verbose, force); } +static char *keep; +static void remove_keep(void) +{ + if (keep && *keep) + unlink(keep); +} + +static void remove_keep_on_signal(int signo) +{ + remove_keep(); + signal(SIGINT, SIG_DFL); + raise(signo); +} + +static char *find_local_name(const char *remote_name, const char *refs, + int *force_p, int *not_for_merge_p) +{ + const char *ref = refs; + int len = strlen(remote_name); + + while (ref) { + const char *next; + int single_force, not_for_merge; + + while (*ref == '\n') + ref++; + if (!*ref) + break; + next = strchr(ref, '\n'); + + single_force = not_for_merge = 0; + if (*ref == '+') { + single_force = 1; + ref++; + } + if (*ref == '.') { + not_for_merge = 1; + ref++; + if (*ref == '+') { + single_force = 1; + ref++; + } + } + if (!strncmp(remote_name, ref, len) && ref[len] == ':') { + const char *local_part = ref + len + 1; + char *ret; + int retlen; + + if (!next) + retlen = strlen(local_part); + else + retlen = next - local_part; + ret = xmalloc(retlen + 1); + memcpy(ret, local_part, retlen); + ret[retlen] = 0; + *force_p = single_force; + *not_for_merge_p = not_for_merge; + return ret; + } + ref = next; + } + return NULL; +} + +static int fetch_native_store(FILE *fp, + const char *remote, + const char *remote_nick, + const char *refs, + int verbose, int force) +{ + char buffer[1024]; + int err = 0; + + signal(SIGINT, remove_keep_on_signal); + atexit(remove_keep); + + while (fgets(buffer, sizeof(buffer), stdin)) { + int len; + char *cp; + char *local_name; + int single_force, not_for_merge; + + for (cp = buffer; *cp && !isspace(*cp); cp++) + ; + if (*cp) + *cp++ = 0; + len = strlen(cp); + if (len && cp[len-1] == '\n') + cp[--len] = 0; + if (!strcmp(buffer, "failed")) + die("Fetch failure: %s", remote); + if (!strcmp(buffer, "pack")) + continue; + if (!strcmp(buffer, "keep")) { + char *od = get_object_directory(); + int len = strlen(od) + strlen(cp) + 50; + keep = xmalloc(len); + sprintf(keep, "%s/pack/pack-%s.keep", od, cp); + continue; + } + + local_name = find_local_name(cp, refs, + &single_force, ¬_for_merge); + if (!local_name) + continue; + err |= append_fetch_head(fp, + buffer, remote, cp, remote_nick, + local_name, not_for_merge, + verbose, force || single_force); + } + return err; +} + int cmd_fetch__tool(int argc, const char **argv, const char *prefix) { int verbose = 0; @@ -210,5 +323,17 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) return update_local_ref(argv[2], argv[3], argv[4], verbose, force); } + if (!strcmp("native-store", argv[1])) { + int result; + FILE *fp; + + if (argc != 5) + return error("fetch-native-store takes 3 args"); + fp = fopen(git_path("FETCH_HEAD"), "a"); + result = fetch_native_store(fp, argv[2], argv[3], argv[4], + verbose, force); + fclose(fp); + return result; + } return error("Unknown subcommand: %s", argv[1]); } diff --git a/git-fetch.sh b/git-fetch.sh index 2aa34b3992..b74dd9a309 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -192,57 +192,12 @@ fetch_native () { echo failed "$remote" ) | ( - trap ' - if test -n "$keepfile" && test -f "$keepfile" - then - rm -f "$keepfile" - fi - ' 0 - - keepfile= - while read sha1 remote_name - do - case "$sha1" in - failed) - echo >&2 "Fetch failure: $remote" - exit 1 ;; - # special line coming from index-pack with the pack name - pack) - continue ;; - keep) - keepfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep" - continue ;; - esac - found= - single_force= - for ref in $refs - do - case "$ref" in - +$remote_name:*) - single_force=t - not_for_merge= - found="$ref" - break ;; - .+$remote_name:*) - single_force=t - not_for_merge=t - found="$ref" - break ;; - .$remote_name:*) - not_for_merge=t - found="$ref" - break ;; - $remote_name:*) - not_for_merge= - found="$ref" - break ;; - esac - done - local_name=$(expr "z$found" : 'z[^:]*:\(.*\)') - append_fetch_head "$sha1" "$remote" \ - "$remote_name" "$remote_nick" "$local_name" \ - "$not_for_merge" || exit - done + flags= + test -n "$verbose" && flags="$flags -v" + test -n "$force" && flags="$flags -f" + GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ + git-fetch--tool native-store \ + $flags "$remote" "$remote_nick" "$refs" ) ) || exit -- cgit v1.2.3 From d1e0ef6cc89e5ef2f914c37719b9c2327e534834 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 16 Jan 2007 02:31:36 -0800 Subject: git-fetch: rewrite another shell loop in C Move another shell loop that canonicalizes the list of refs for underlying git-fetch-pack and fetch-native-store into C. This seems to shave the runtime for the same 1000 branch repository from 30 seconds down to 15 seconds (it used to be 2 and half minutes with the original version). Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ git-fetch.sh | 30 ++---------------------------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 24343ac9b0..705a6649a9 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -283,6 +283,46 @@ static int fetch_native_store(FILE *fp, return err; } +static int parse_reflist(const char *reflist) +{ + const char *ref; + + printf("refs='"); + for (ref = reflist; ref; ) { + const char *next; + while (*ref && isspace(*ref)) + ref++; + if (!*ref) + break; + for (next = ref; *next && !isspace(*next); next++) + ; + printf("\n%.*s", (int)(next - ref), ref); + ref = next; + } + printf("'\n"); + + printf("rref='"); + for (ref = reflist; ref; ) { + const char *next, *colon; + while (*ref && isspace(*ref)) + ref++; + if (!*ref) + break; + for (next = ref; *next && !isspace(*next); next++) + ; + if (*ref == '.') + ref++; + if (*ref == '+') + ref++; + colon = strchr(ref, ':'); + putchar('\n'); + printf("%.*s", (int)((colon ? colon : next) - ref), ref); + ref = next; + } + printf("'\n"); + return 0; +} + int cmd_fetch__tool(int argc, const char **argv, const char *prefix) { int verbose = 0; @@ -335,5 +375,11 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) fclose(fp); return result; } + if (!strcmp("parse-reflist", argv[1])) { + if (argc != 3) + return error("parse-reflist takes 1 arg"); + return parse_reflist(argv[2]); + } + return error("Unknown subcommand: %s", argv[1]); } diff --git a/git-fetch.sh b/git-fetch.sh index b74dd9a309..3bed4091a3 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -155,35 +155,9 @@ then fi fetch_native () { - reflist="$1" - refs= - rref= - - for ref in $reflist - do - refs="$refs$LF$ref" - - # These are relative path from $GIT_DIR, typically starting at refs/ - # but may be HEAD - if expr "z$ref" : 'z\.' >/dev/null - then - not_for_merge=t - ref=$(expr "z$ref" : 'z\.\(.*\)') - else - not_for_merge= - fi - if expr "z$ref" : 'z+' >/dev/null - then - single_force=t - ref=$(expr "z$ref" : 'z+\(.*\)') - else - single_force= - fi - remote_name=$(expr "z$ref" : 'z\([^:]*\):') - local_name=$(expr "z$ref" : 'z[^:]*:\(.*\)') - rref="$rref$LF$remote_name" - done + eval=$(git-fetch--tool parse-reflist "$1") + eval "$eval" ( : subshell because we muck with IFS IFS=" $LF" -- cgit v1.2.3 From 86551586da8cba6c06ac04783a656843a4e47f35 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 16 Jan 2007 13:43:28 -0800 Subject: git-fetch: rewrite expand_ref_wildcard in C This does not seem to make measurable improvement when dealing with 1000 unpacked refs, but we would need something like it if we were to do a full rewrite in C somedaoy. Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++ git-parse-remote.sh | 46 +------------------------- 2 files changed, 91 insertions(+), 45 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 705a6649a9..3090ffea20 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -323,6 +323,91 @@ static int parse_reflist(const char *reflist) return 0; } +static int expand_refs_wildcard(const char *ls_remote_result, int numrefs, + const char **refs) +{ + int i, matchlen, replacelen; + int found_one = 0; + const char *remote = *refs++; + numrefs--; + + if (numrefs == 0) { + fprintf(stderr, "Nothing specified for fetching with remote.%s.fetch\n", + remote); + printf("empty\n"); + } + + for (i = 0; i < numrefs; i++) { + const char *ref = refs[i]; + const char *lref = ref; + const char *colon; + const char *tail; + const char *ls; + const char *next; + + if (*lref == '+') + lref++; + colon = strchr(lref, ':'); + tail = lref + strlen(lref); + if (!(colon && + 2 < colon - lref && + colon[-1] == '*' && + colon[-2] == '/' && + 2 < tail - (colon + 1) && + tail[-1] == '*' && + tail[-2] == '/')) { + /* not a glob */ + if (!found_one++) + printf("explicit\n"); + printf("%s\n", ref); + continue; + } + + /* glob */ + if (!found_one++) + printf("glob\n"); + + /* lref to colon-2 is remote hierarchy name; + * colon+1 to tail-2 is local. + */ + matchlen = (colon-1) - lref; + replacelen = (tail-1) - (colon+1); + for (ls = ls_remote_result; ls; ls = next) { + const char *eol; + unsigned char sha1[20]; + int namelen; + + while (*ls && isspace(*ls)) + ls++; + next = strchr(ls, '\n'); + eol = !next ? (ls + strlen(ls)) : next; + if (!memcmp("^{}", eol-3, 3)) + continue; + if (get_sha1_hex(ls, sha1)) + continue; + ls += 40; + while (ls < eol && isspace(*ls)) + ls++; + /* ls to next (or eol) is the name. + * is it identical to lref to colon-2? + */ + if ((eol - ls) <= matchlen || + strncmp(ls, lref, matchlen)) + continue; + + /* Yes, it is a match */ + namelen = eol - ls; + if (lref != ref) + putchar('+'); + printf("%.*s:%.*s%.*s\n", + namelen, ls, + replacelen, colon + 1, + namelen - matchlen, ls + matchlen); + } + } + return 0; +} + int cmd_fetch__tool(int argc, const char **argv, const char *prefix) { int verbose = 0; @@ -380,6 +465,11 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) return error("parse-reflist takes 1 arg"); return parse_reflist(argv[2]); } + if (!strcmp("expand-refs-wildcard", argv[1])) { + if (argc < 4) + return error("expand-refs-wildcard takes at least 2 args"); + return expand_refs_wildcard(argv[2], argc - 3, argv + 3); + } return error("Unknown subcommand: %s", argv[1]); } diff --git a/git-parse-remote.sh b/git-parse-remote.sh index 5208ee6ce0..9b19a21667 100755 --- a/git-parse-remote.sh +++ b/git-parse-remote.sh @@ -81,51 +81,7 @@ get_remote_default_refs_for_push () { # is to help prevent randomly "globbed" ref from being chosen as # a merge candidate expand_refs_wildcard () { - remote="$1" - shift - first_one=yes - if test "$#" = 0 - then - echo empty - echo >&2 "Nothing specified for fetching with remote.$remote.fetch" - fi - for ref - do - lref=${ref#'+'} - # a non glob pattern is given back as-is. - expr "z$lref" : 'zrefs/.*/\*:refs/.*/\*$' >/dev/null || { - if test -n "$first_one" - then - echo "explicit" - first_one= - fi - echo "$ref" - continue - } - - # glob - if test -n "$first_one" - then - echo "glob" - first_one= - fi - from=`expr "z$lref" : 'z\(refs/.*/\)\*:refs/.*/\*$'` - to=`expr "z$lref" : 'zrefs/.*/\*:\(refs/.*/\)\*$'` - local_force= - test "z$lref" = "z$ref" || local_force='+' - echo "$ls_remote_result" | - sed -e '/\^{}$/d' | - ( - IFS=' ' - while read sha1 name - do - # ignore the ones that do not start with $from - mapped=${name#"$from"} - test "z$name" = "z$mapped" && continue - echo "${local_force}${name}:${to}${mapped}" - done - ) - done + git fetch--tool expand-refs-wildcard "$ls_remote_result" "$@" } # Subroutine to canonicalize remote:local notation. -- cgit v1.2.3 From 46ce8b6d2a88b67a839fb53bfa0b8849215352b5 Mon Sep 17 00:00:00 2001 From: Julian Phillips Date: Tue, 13 Feb 2007 01:21:39 +0000 Subject: Allow fetch--tool to read from stdin If the reflist is "-" then read the reflist data from stdin instead, this will allow the passing of more than 128K of reflist data - which won't fit in the environment passed by execve. Signed-off-by: Julian Phillips Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 3090ffea20..48de08d858 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -2,6 +2,21 @@ #include "refs.h" #include "commit.h" +#define CHUNK_SIZE (1048576) + +static char *get_stdin(void) +{ + char *data = xmalloc(CHUNK_SIZE); + int offset = 0, read = 0; + read = xread(0, data, CHUNK_SIZE); + while (read == CHUNK_SIZE) { + offset += CHUNK_SIZE; + data = xrealloc(data, offset + CHUNK_SIZE); + read = xread(0, data + offset, CHUNK_SIZE); + } + return data; +} + static void show_new(char *type, unsigned char *sha1_new) { fprintf(stderr, " %s: %s\n", type, @@ -461,14 +476,22 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) return result; } if (!strcmp("parse-reflist", argv[1])) { + const char *reflist; if (argc != 3) return error("parse-reflist takes 1 arg"); - return parse_reflist(argv[2]); + reflist = argv[2]; + if (!strcmp(reflist, "-")) + reflist = get_stdin(); + return parse_reflist(reflist); } if (!strcmp("expand-refs-wildcard", argv[1])) { + const char *reflist; if (argc < 4) return error("expand-refs-wildcard takes at least 2 args"); - return expand_refs_wildcard(argv[2], argc - 3, argv + 3); + reflist = argv[2]; + if (!strcmp(reflist, "-")) + reflist = get_stdin(); + return expand_refs_wildcard(reflist, argc - 3, argv + 3); } return error("Unknown subcommand: %s", argv[1]); -- cgit v1.2.3 From 95339912b97279c29bd842fe036c70fca33d0d66 Mon Sep 17 00:00:00 2001 From: Julian Phillips Date: Tue, 13 Feb 2007 01:21:40 +0000 Subject: Use stdin reflist passing in parse-remote Use the new stdin reflist passing mechanism for the call to fetch--tool expand-refs-wildcard, allowing passing of more than ~128K of reflist data. Signed-off-by: Julian Phillips Signed-off-by: Junio C Hamano --- git-parse-remote.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git-parse-remote.sh b/git-parse-remote.sh index 9b19a21667..c46131f6d6 100755 --- a/git-parse-remote.sh +++ b/git-parse-remote.sh @@ -81,7 +81,8 @@ get_remote_default_refs_for_push () { # is to help prevent randomly "globbed" ref from being chosen as # a merge candidate expand_refs_wildcard () { - git fetch--tool expand-refs-wildcard "$ls_remote_result" "$@" + echo "$ls_remote_result" | + git fetch--tool expand-refs-wildcard "-" "$@" } # Subroutine to canonicalize remote:local notation. -- cgit v1.2.3 From 617669da4f80ecf05db7ef1976b56260e2124cc2 Mon Sep 17 00:00:00 2001 From: Julian Phillips Date: Tue, 13 Feb 2007 01:21:41 +0000 Subject: Use stdin reflist passing in git-fetch.sh Use the new stdin reflist passing mechanism for the call to fetch--tool parse-reflist, allowing passing of more than ~128K of reflist data. Signed-off-by: Julian Phillips Signed-off-by: Junio C Hamano --- git-fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-fetch.sh b/git-fetch.sh index 3bed4091a3..80f63c85f0 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -156,7 +156,7 @@ fi fetch_native () { - eval=$(git-fetch--tool parse-reflist "$1") + eval=$(echo "$1" | git-fetch--tool parse-reflist "-") eval "$eval" ( : subshell because we muck with IFS -- cgit v1.2.3 From fee7c2c71d9e35b2f54aa3631072bd7f73bb7b4c Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 24 Feb 2007 04:35:31 -0800 Subject: git-fetch--tool takes flags before the subcommand. Signed-off-by: Junio C Hamano --- git-fetch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git-fetch.sh b/git-fetch.sh index 80f63c85f0..f875e0f99e 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -111,7 +111,7 @@ append_fetch_head () { test -n "$verbose" && flags="$flags -v" test -n "$force" && flags="$flags -f" GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ - git-fetch--tool append-fetch-head $flags "$@" + git-fetch--tool $flags append-fetch-head "$@" } update_local_ref () { @@ -119,7 +119,7 @@ update_local_ref () { test -n "$verbose" && flags="$flags -v" test -n "$force" && flags="$flags -f" GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ - git-fetch--tool update-local-ref $flags "$@" + git-fetch--tool $flags update-local-ref "$@" } # updating the current HEAD with git-fetch in a bare @@ -170,8 +170,8 @@ fetch_native () { test -n "$verbose" && flags="$flags -v" test -n "$force" && flags="$flags -f" GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ - git-fetch--tool native-store \ - $flags "$remote" "$remote_nick" "$refs" + git-fetch--tool $flags native-store \ + "$remote" "$remote_nick" "$refs" ) ) || exit -- cgit v1.2.3 From dcf01c6e6b9f63d6f6239a6c6ff9f6373e4c5ff8 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 27 Feb 2007 02:39:51 -0800 Subject: builtin-fetch--tool: adjust to updated sha1_object_info(). Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 48de08d858..e9d16e6315 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -17,9 +17,9 @@ static char *get_stdin(void) return data; } -static void show_new(char *type, unsigned char *sha1_new) +static void show_new(enum object_type type, unsigned char *sha1_new) { - fprintf(stderr, " %s: %s\n", type, + fprintf(stderr, " %s: %s\n", typename(type), find_unique_abbrev(sha1_new, DEFAULT_ABBREV)); } @@ -51,14 +51,16 @@ static int update_local_ref(const char *name, const char *note, int verbose, int force) { - char type[20]; unsigned char sha1_old[20], sha1_new[20]; char oldh[41], newh[41]; struct commit *current, *updated; + enum object_type type; if (get_sha1_hex(new_head, sha1_new)) die("malformed object name %s", new_head); - if (sha1_object_info(sha1_new, type, NULL)) + + type = sha1_object_info(sha1_new, NULL); + if (type < 0) die("object %s not found", new_head); if (!*name) { -- cgit v1.2.3 From dec56c8cf1da06fe9ff4c9d3a26f74fb1ddd2fc6 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 26 Feb 2007 11:37:43 -0800 Subject: fetch--tool: fix uninitialized buffer when reading from stdin The original code allocates too much space and forgets to NUL terminate the string. Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index e9d16e6315..5301c3cb78 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -2,17 +2,24 @@ #include "refs.h" #include "commit.h" -#define CHUNK_SIZE (1048576) +#define CHUNK_SIZE 1024 static char *get_stdin(void) { + int offset = 0; char *data = xmalloc(CHUNK_SIZE); - int offset = 0, read = 0; - read = xread(0, data, CHUNK_SIZE); - while (read == CHUNK_SIZE) { - offset += CHUNK_SIZE; + + while (1) { + int cnt = xread(0, data + offset, CHUNK_SIZE); + if (cnt < 0) + die("error reading standard input: %s", + strerror(errno)); + if (cnt == 0) { + data[offset] = 0; + break; + } + offset += cnt; data = xrealloc(data, offset + CHUNK_SIZE); - read = xread(0, data + offset, CHUNK_SIZE); } return data; } -- cgit v1.2.3 From c7d68c80002090bddc1eb740d83818aa0a08bbbe Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 27 Feb 2007 23:51:48 -0800 Subject: builtin-fetch--tool: make sure not to overstep ls-remote-result buffer. Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 5301c3cb78..eeee0a5ebf 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -407,6 +407,8 @@ static int expand_refs_wildcard(const char *ls_remote_result, int numrefs, eol = !next ? (ls + strlen(ls)) : next; if (!memcmp("^{}", eol-3, 3)) continue; + if (eol - ls < 40) + continue; if (get_sha1_hex(ls, sha1)) continue; ls += 40; -- cgit v1.2.3 From e6eebbb3ae2f2831fe1319d5acdb6477b8abeadb Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 28 Feb 2007 17:01:00 -0800 Subject: git-fetch: retire update-local-ref which is not used anymore. Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 6 ------ git-fetch.sh | 8 -------- 2 files changed, 14 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index eeee0a5ebf..5261bf57fd 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -468,12 +468,6 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix) fclose(fp); return result; } - if (!strcmp("update-local-ref", argv[1])) { - if (argc != 5) - return error("update-local-ref takes 3 args"); - return update_local_ref(argv[2], argv[3], argv[4], - verbose, force); - } if (!strcmp("native-store", argv[1])) { int result; FILE *fp; diff --git a/git-fetch.sh b/git-fetch.sh index f438ac1ef2..4a8d8d6ef7 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -114,14 +114,6 @@ append_fetch_head () { git-fetch--tool $flags append-fetch-head "$@" } -update_local_ref () { - flags= - test -n "$verbose" && flags="$flags -v" - test -n "$force" && flags="$flags -f" - GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \ - git-fetch--tool $flags update-local-ref "$@" -} - # updating the current HEAD with git-fetch in a bare # repository is always fine. if test -z "$update_head_ok" && test $(is_bare_repository) = false -- cgit v1.2.3 From 855b34680ea334f05385b3ad58232c3ccdca51d5 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 28 Feb 2007 17:02:18 -0800 Subject: builtin-fetch--tool: fix reflog notes. Also the verbose output had unnecessary SHA1 and not-for-merge markers leaked because append_fetch_head() cheated Signed-off-by: Junio C Hamano --- builtin-fetch--tool.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c index 5261bf57fd..e9d6764550 100644 --- a/builtin-fetch--tool.c +++ b/builtin-fetch--tool.c @@ -181,16 +181,18 @@ static int append_fetch_head(FILE *fp, remote_len = i + 1; if (4 < i && !strncmp(".git", remote + i - 3, 4)) remote_len = i - 3; - note_len = sprintf(note, "%s\t%s\t", - sha1_to_hex(commit ? commit->object.sha1 : sha1), - not_for_merge ? "not-for-merge" : ""); + + note_len = 0; if (*what) { if (*kind) note_len += sprintf(note + note_len, "%s ", kind); note_len += sprintf(note + note_len, "'%s' of ", what); } note_len += sprintf(note + note_len, "%.*s", remote_len, remote); - fprintf(fp, "%s\n", note); + fprintf(fp, "%s\t%s\t%s\n", + sha1_to_hex(commit ? commit->object.sha1 : sha1), + not_for_merge ? "not-for-merge" : "", + note); return update_local_ref(local_name, head, note, verbose, force); } -- cgit v1.2.3 From f98ef68faf81f10f89d9168c21acae71dd50c69e Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 4 Mar 2007 15:36:08 -0800 Subject: .gitignore: add git-fetch--tool Signed-off-by: Junio C Hamano --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index eb8a1f8606..847f40a54e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ git-diff-tree git-describe git-fast-import git-fetch +git-fetch--tool git-fetch-pack git-findtags git-fmt-merge-msg -- cgit v1.2.3