Merge branch 'jk/credentials'

* jk/credentials:
  t: add test harness for external credential helpers
  credentials: add "store" helper
  strbuf: add strbuf_add*_urlencode
  Makefile: unix sockets may not available on some platforms
  credentials: add "cache" helper
  docs: end-user documentation for the credential subsystem
  credential: make relevance of http path configurable
  credential: add credential.*.username
  credential: apply helper config
  http: use credential API to get passwords
  credential: add function for parsing url components
  introduce credentials API
  t5550: fix typo
  test-lib: add test_config_global variant

Conflicts:
	strbuf.c

7 files changed, 665 insertions, 12 deletions

diff --git a/t/lib-credential.sh b/t/lib-credential.sh
new file mode 100755
index 0000000000..4a37cd79e5
--- /dev/null
+++ b/t/lib-credential.sh
@@ -0,0 +1,254 @@
+#!/bin/sh
+
+# Try a set of credential helpers; the expected stdin,
+# stdout and stderr should be provided on stdin,
+# separated by "--".
+check() {
+	read_chunk >stdin &&
+	read_chunk >expect-stdout &&
+	read_chunk >expect-stderr &&
+	test-credential "$@" <stdin >stdout 2>stderr &&
+	test_cmp expect-stdout stdout &&
+	test_cmp expect-stderr stderr
+}
+
+read_chunk() {
+	while read line; do
+		case "$line" in
+		--) break ;;
+		*) echo "$line" ;;
+		esac
+	done
+}
+
+# Clear any residual data from previous tests. We only
+# need this when testing third-party helpers which read and
+# write outside of our trash-directory sandbox.
+#
+# Don't bother checking for success here, as it is
+# outside the scope of tests and represents a best effort to
+# clean up after ourselves.
+helper_test_clean() {
+	reject $1 https example.com store-user
+	reject $1 https example.com user1
+	reject $1 https example.com user2
+	reject $1 http path.tld user
+	reject $1 https timeout.tld user
+}
+
+reject() {
+	(
+		echo protocol=$2
+		echo host=$3
+		echo username=$4
+	) | test-credential reject $1
+}
+
+helper_test() {
+	HELPER=$1
+
+	test_expect_success "helper ($HELPER) has no existing data" '
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''https://example.com'\'':
+		askpass: Password for '\''https://askpass-username@example.com'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) stores password" '
+		check approve $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		username=store-user
+		password=store-pass
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) can retrieve password" '
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		--
+		username=store-user
+		password=store-pass
+		--
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) requires matching protocol" '
+		check fill $HELPER <<-\EOF
+		protocol=http
+		host=example.com
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''http://example.com'\'':
+		askpass: Password for '\''http://askpass-username@example.com'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) requires matching host" '
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=other.tld
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''https://other.tld'\'':
+		askpass: Password for '\''https://askpass-username@other.tld'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) requires matching username" '
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		username=other
+		--
+		username=other
+		password=askpass-password
+		--
+		askpass: Password for '\''https://other@example.com'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) requires matching path" '
+		test_config credential.usehttppath true &&
+		check approve $HELPER <<-\EOF &&
+		protocol=http
+		host=path.tld
+		path=foo.git
+		username=user
+		password=pass
+		EOF
+		check fill $HELPER <<-\EOF
+		protocol=http
+		host=path.tld
+		path=bar.git
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''http://path.tld/bar.git'\'':
+		askpass: Password for '\''http://askpass-username@path.tld/bar.git'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) can forget host" '
+		check reject $HELPER <<-\EOF &&
+		protocol=https
+		host=example.com
+		EOF
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''https://example.com'\'':
+		askpass: Password for '\''https://askpass-username@example.com'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) can store multiple users" '
+		check approve $HELPER <<-\EOF &&
+		protocol=https
+		host=example.com
+		username=user1
+		password=pass1
+		EOF
+		check approve $HELPER <<-\EOF &&
+		protocol=https
+		host=example.com
+		username=user2
+		password=pass2
+		EOF
+		check fill $HELPER <<-\EOF &&
+		protocol=https
+		host=example.com
+		username=user1
+		--
+		username=user1
+		password=pass1
+		EOF
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		username=user2
+		--
+		username=user2
+		password=pass2
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) can forget user" '
+		check reject $HELPER <<-\EOF &&
+		protocol=https
+		host=example.com
+		username=user1
+		EOF
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		username=user1
+		--
+		username=user1
+		password=askpass-password
+		--
+		askpass: Password for '\''https://user1@example.com'\'':
+		EOF
+	'
+
+	test_expect_success "helper ($HELPER) remembers other user" '
+		check fill $HELPER <<-\EOF
+		protocol=https
+		host=example.com
+		username=user2
+		--
+		username=user2
+		password=pass2
+		EOF
+	'
+}
+
+helper_test_timeout() {
+	HELPER="$*"
+
+	test_expect_success "helper ($HELPER) times out" '
+		check approve "$HELPER" <<-\EOF &&
+		protocol=https
+		host=timeout.tld
+		username=user
+		password=pass
+		EOF
+		sleep 2 &&
+		check fill "$HELPER" <<-\EOF
+		protocol=https
+		host=timeout.tld
+		--
+		username=askpass-username
+		password=askpass-password
+		--
+		askpass: Username for '\''https://timeout.tld'\'':
+		askpass: Password for '\''https://askpass-username@timeout.tld'\'':
+		EOF
+	'
+}
+
+cat >askpass <<\EOF
+#!/bin/sh
+echo >&2 askpass: $*
+what=`echo $1 | cut -d" " -f1 | tr A-Z a-z | tr -cd a-z`
+echo "askpass-$what"
+EOF
+chmod +x askpass
+GIT_ASKPASS="$PWD/askpass"
+export GIT_ASKPASS
diff --git a/t/t0300-credentials.sh b/t/t0300-credentials.sh
new file mode 100755
index 0000000000..885af8fb62
--- /dev/null
+++ b/t/t0300-credentials.sh
@@ -0,0 +1,279 @@
+#!/bin/sh
+
+test_description='basic credential helper tests'
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-credential.sh
+
+test_expect_success 'setup helper scripts' '
+	cat >dump <<-\EOF &&
+	whoami=`echo $0 | sed s/.*git-credential-//`
+	echo >&2 "$whoami: $*"
+	while IFS== read key value; do
+		echo >&2 "$whoami: $key=$value"
+		eval "$key=$value"
+	done
+	EOF
+
+	cat >git-credential-useless <<-\EOF &&
+	#!/bin/sh
+	. ./dump
+	exit 0
+	EOF
+	chmod +x git-credential-useless &&
+
+	cat >git-credential-verbatim <<-\EOF &&
+	#!/bin/sh
+	user=$1; shift
+	pass=$1; shift
+	. ./dump
+	test -z "$user" || echo username=$user
+	test -z "$pass" || echo password=$pass
+	EOF
+	chmod +x git-credential-verbatim &&
+
+	PATH="$PWD:$PATH"
+'
+
+test_expect_success 'credential_fill invokes helper' '
+	check fill "verbatim foo bar" <<-\EOF
+	--
+	username=foo
+	password=bar
+	--
+	verbatim: get
+	EOF
+'
+
+test_expect_success 'credential_fill invokes multiple helpers' '
+	check fill useless "verbatim foo bar" <<-\EOF
+	--
+	username=foo
+	password=bar
+	--
+	useless: get
+	verbatim: get
+	EOF
+'
+
+test_expect_success 'credential_fill stops when we get a full response' '
+	check fill "verbatim one two" "verbatim three four" <<-\EOF
+	--
+	username=one
+	password=two
+	--
+	verbatim: get
+	EOF
+'
+
+test_expect_success 'credential_fill continues through partial response' '
+	check fill "verbatim one \"\"" "verbatim two three" <<-\EOF
+	--
+	username=two
+	password=three
+	--
+	verbatim: get
+	verbatim: get
+	verbatim: username=one
+	EOF
+'
+
+test_expect_success 'credential_fill passes along metadata' '
+	check fill "verbatim one two" <<-\EOF
+	protocol=ftp
+	host=example.com
+	path=foo.git
+	--
+	username=one
+	password=two
+	--
+	verbatim: get
+	verbatim: protocol=ftp
+	verbatim: host=example.com
+	verbatim: path=foo.git
+	EOF
+'
+
+test_expect_success 'credential_approve calls all helpers' '
+	check approve useless "verbatim one two" <<-\EOF
+	username=foo
+	password=bar
+	--
+	--
+	useless: store
+	useless: username=foo
+	useless: password=bar
+	verbatim: store
+	verbatim: username=foo
+	verbatim: password=bar
+	EOF
+'
+
+test_expect_success 'do not bother storing password-less credential' '
+	check approve useless <<-\EOF
+	username=foo
+	--
+	--
+	EOF
+'
+
+
+test_expect_success 'credential_reject calls all helpers' '
+	check reject useless "verbatim one two" <<-\EOF
+	username=foo
+	password=bar
+	--
+	--
+	useless: erase
+	useless: username=foo
+	useless: password=bar
+	verbatim: erase
+	verbatim: username=foo
+	verbatim: password=bar
+	EOF
+'
+
+test_expect_success 'usernames can be preserved' '
+	check fill "verbatim \"\" three" <<-\EOF
+	username=one
+	--
+	username=one
+	password=three
+	--
+	verbatim: get
+	verbatim: username=one
+	EOF
+'
+
+test_expect_success 'usernames can be overridden' '
+	check fill "verbatim two three" <<-\EOF
+	username=one
+	--
+	username=two
+	password=three
+	--
+	verbatim: get
+	verbatim: username=one
+	EOF
+'
+
+test_expect_success 'do not bother completing already-full credential' '
+	check fill "verbatim three four" <<-\EOF
+	username=one
+	password=two
+	--
+	username=one
+	password=two
+	--
+	EOF
+'
+
+# We can't test the basic terminal password prompt here because
+# getpass() tries too hard to find the real terminal. But if our
+# askpass helper is run, we know the internal getpass is working.
+test_expect_success 'empty helper list falls back to internal getpass' '
+	check fill <<-\EOF
+	--
+	username=askpass-username
+	password=askpass-password
+	--
+	askpass: Username:
+	askpass: Password:
+	EOF
+'
+
+test_expect_success 'internal getpass does not ask for known username' '
+	check fill <<-\EOF
+	username=foo
+	--
+	username=foo
+	password=askpass-password
+	--
+	askpass: Password:
+	EOF
+'
+
+HELPER="!f() {
+		cat >/dev/null
+		echo username=foo
+		echo password=bar
+	}; f"
+test_expect_success 'respect configured credentials' '
+	test_config credential.helper "$HELPER" &&
+	check fill <<-\EOF
+	--
+	username=foo
+	password=bar
+	--
+	EOF
+'
+
+test_expect_success 'match configured credential' '
+	test_config credential.https://example.com.helper "$HELPER" &&
+	check fill <<-\EOF
+	protocol=https
+	host=example.com
+	path=repo.git
+	--
+	username=foo
+	password=bar
+	--
+	EOF
+'
+
+test_expect_success 'do not match configured credential' '
+	test_config credential.https://foo.helper "$HELPER" &&
+	check fill <<-\EOF
+	protocol=https
+	host=bar
+	--
+	username=askpass-username
+	password=askpass-password
+	--
+	askpass: Username for '\''https://bar'\'':
+	askpass: Password for '\''https://askpass-username@bar'\'':
+	EOF
+'
+
+test_expect_success 'pull username from config' '
+	test_config credential.https://example.com.username foo &&
+	check fill <<-\EOF
+	protocol=https
+	host=example.com
+	--
+	username=foo
+	password=askpass-password
+	--
+	askpass: Password for '\''https://foo@example.com'\'':
+	EOF
+'
+
+test_expect_success 'http paths can be part of context' '
+	check fill "verbatim foo bar" <<-\EOF &&
+	protocol=https
+	host=example.com
+	path=foo.git
+	--
+	username=foo
+	password=bar
+	--
+	verbatim: get
+	verbatim: protocol=https
+	verbatim: host=example.com
+	EOF
+	test_config credential.https://example.com.useHttpPath true &&
+	check fill "verbatim foo bar" <<-\EOF
+	protocol=https
+	host=example.com
+	path=foo.git
+	--
+	username=foo
+	password=bar
+	--
+	verbatim: get
+	verbatim: protocol=https
+	verbatim: host=example.com
+	verbatim: path=foo.git
+	EOF
+'
+
+test_done
diff --git a/t/t0301-credential-cache.sh b/t/t0301-credential-cache.sh
new file mode 100755
index 0000000000..82c8411210
--- /dev/null
+++ b/t/t0301-credential-cache.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+test_description='credential-cache tests'
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-credential.sh
+
+test -z "$NO_UNIX_SOCKETS" || {
+	skip_all='skipping credential-cache tests, unix sockets not available'
+	test_done
+}
+
+# don't leave a stale daemon running
+trap 'code=$?; git credential-cache exit; (exit $code); die' EXIT
+
+helper_test cache
+helper_test_timeout cache --timeout=1
+
+# we can't rely on our "trap" above working after test_done,
+# as test_done will delete the trash directory containing
+# our socket, leaving us with no way to access the daemon.
+git credential-cache exit
+
+test_done
diff --git a/t/t0302-credential-store.sh b/t/t0302-credential-store.sh
new file mode 100755
index 0000000000..f61b40c69b
--- /dev/null
+++ b/t/t0302-credential-store.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+test_description='credential-store tests'
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-credential.sh
+
+helper_test store
+
+test_done
diff --git a/t/t0303-credential-external.sh b/t/t0303-credential-external.sh
new file mode 100755
index 0000000000..267f4c8ba3
--- /dev/null
+++ b/t/t0303-credential-external.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+test_description='external credential helper tests'
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-credential.sh
+
+pre_test() {
+	test -z "$GIT_TEST_CREDENTIAL_HELPER_SETUP" ||
+	eval "$GIT_TEST_CREDENTIAL_HELPER_SETUP"
+
+	# clean before the test in case there is cruft left
+	# over from a previous run that would impact results
+	helper_test_clean "$GIT_TEST_CREDENTIAL_HELPER"
+}
+
+post_test() {
+	# clean afterwards so that we are good citizens
+	# and don't leave cruft in the helper's storage, which
+	# might be long-term system storage
+	helper_test_clean "$GIT_TEST_CREDENTIAL_HELPER"
+}
+
+if test -z "$GIT_TEST_CREDENTIAL_HELPER"; then
+	say "# skipping external helper tests (set GIT_TEST_CREDENTIAL_HELPER)"
+else
+	pre_test
+	helper_test "$GIT_TEST_CREDENTIAL_HELPER"
+	post_test
+fi
+
+if test -z "$GIT_TEST_CREDENTIAL_HELPER_TIMEOUT"; then
+	say "# skipping external helper timeout tests"
+else
+	pre_test
+	helper_test_timeout "$GIT_TEST_CREDENTIAL_HELPER_TIMEOUT"
+	post_test
+fi
+
+test_done
diff --git a/t/t5550-http-fetch.sh b/t/t5550-http-fetch.sh
index 311a33ca84..95a133d697 100755
--- a/t/t5550-http-fetch.sh
+++ b/t/t5550-http-fetch.sh
@@ -49,40 +49,84 @@ test_expect_success 'setup askpass helpers' '
 	EOF
 	chmod +x askpass &&
 	GIT_ASKPASS="$PWD/askpass" &&
-	export GIT_ASKPASS &&
-	>askpass-expect-none &&
-	echo "askpass: Password for '\''$HTTPD_DEST'\'': " >askpass-expect-pass &&
-	{ echo "askpass: Username for '\''$HTTPD_DEST'\'': " &&
-	  cat askpass-expect-pass
-	} >askpass-expect-both
-'
+	export GIT_ASKPASS
+'
+
+expect_askpass() {
+	dest=$HTTPD_DEST
+	{
+		case "$1" in
+		none)
+			;;
+		pass)
+			echo "askpass: Password for 'http://$2@$dest': "
+			;;
+		both)
+			echo "askpass: Username for 'http://$dest': "
+			echo "askpass: Password for 'http://$2@$dest': "
+			;;
+		*)
+			false
+			;;
+		esac
+	} >askpass-expect &&
+	test_cmp askpass-expect askpass-query
+}
 
 test_expect_success 'cloning password-protected repository can fail' '
 	>askpass-query &&
 	echo wrong >askpass-response &&
 	test_must_fail git clone "$HTTPD_URL/auth/repo.git" clone-auth-fail &&
-	test_cmp askpass-expect-both askpass-query
+	expect_askpass both wrong
 '
 
 test_expect_success 'http auth can use user/pass in URL' '
 	>askpass-query &&
-	echo wrong >askpass-reponse &&
+	echo wrong >askpass-response &&
 	git clone "$HTTPD_URL_USER_PASS/auth/repo.git" clone-auth-none &&
-	test_cmp askpass-expect-none askpass-query
+	expect_askpass none
 '
 
 test_expect_success 'http auth can use just user in URL' '
 	>askpass-query &&
 	echo user@host >askpass-response &&
 	git clone "$HTTPD_URL_USER/auth/repo.git" clone-auth-pass &&
-	test_cmp askpass-expect-pass askpass-query
+	expect_askpass pass user@host
 '
 
 test_expect_success 'http auth can request both user and pass' '
 	>askpass-query &&
 	echo user@host >askpass-response &&
 	git clone "$HTTPD_URL/auth/repo.git" clone-auth-both &&
-	test_cmp askpass-expect-both askpass-query
+	expect_askpass both user@host
+'
+
+test_expect_success 'http auth respects credential helper config' '
+	test_config_global credential.helper "!f() {
+		cat >/dev/null
+		echo username=user@host
+		echo password=user@host
+	}; f" &&
+	>askpass-query &&
+	echo wrong >askpass-response &&
+	git clone "$HTTPD_URL/auth/repo.git" clone-auth-helper &&
+	expect_askpass none
+'
+
+test_expect_success 'http auth can get username from config' '
+	test_config_global "credential.$HTTPD_URL.username" user@host &&
+	>askpass-query &&
+	echo user@host >askpass-response &&
+	git clone "$HTTPD_URL/auth/repo.git" clone-auth-user &&
+	expect_askpass pass user@host
+'
+
+test_expect_success 'configured username does not override URL' '
+	test_config_global "credential.$HTTPD_URL.username" wrong &&
+	>askpass-query &&
+	echo user@host >askpass-response &&
+	git clone "$HTTPD_URL_USER/auth/repo.git" clone-auth-user2 &&
+	expect_askpass pass user@host
 '
 
 test_expect_success 'fetch changes via http' '
diff --git a/t/test-lib.sh b/t/test-lib.sh
index bdd9513b84..160479b81e 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -379,6 +379,11 @@ test_config () {
 	git config "$@"
 }
 
+test_config_global () {
+	test_when_finished "test_unconfig --global '$1'" &&
+	git config --global "$@"
+}
+
 # Use test_set_prereq to tell that a particular prerequisite is available.
 # The prerequisite can later be checked for in two ways:
 #