Merge branch 'jc/sha1'

* jc/sha1:
  A better-scheduled PPC SHA-1 implementation.
  test-sha1: test hashing large buffer
  Makefile: add framework to verify and bench sha1 implementations.

4 files changed, 331 insertions, 156 deletions

diff --git a/Makefile b/Makefile
index a78d05ddaa..7fa4a271bd 100644
--- a/Makefile
+++ b/Makefile
@@ -651,6 +651,12 @@ test-delta$X: test-delta.c diff-delta.o patch-delta.o
 test-dump-cache-tree$X: dump-cache-tree.o $(GITLIBS)
 	$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS)
 
+test-sha1$X: test-sha1.o $(GITLIBS)
+	$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS)
+
+check-sha1:: test-sha1$X
+	./test-sha1.sh
+
 check:
 	for i in *.c; do sparse $(ALL_CFLAGS) $(SPARSE_FLAGS) $$i || exit; done
 
diff --git a/ppc/sha1ppc.S b/ppc/sha1ppc.S
index e85611a4ef..f591d98b3f 100644
--- a/ppc/sha1ppc.S
+++ b/ppc/sha1ppc.S
@@ -3,183 +3,222 @@
  *
  * Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
  */
-#define FS	80
 
 /*
- * We roll the registers for T, A, B, C, D, E around on each
- * iteration; T on iteration t is A on iteration t+1, and so on.
- * We use registers 7 - 12 for this.
+ * PowerPC calling convention:
+ * %r0 - volatile temp
+ * %r1 - stack pointer.
+ * %r2 - reserved
+ * %r3-%r12 - Incoming arguments & return values; volatile.
+ * %r13-%r31 - Callee-save registers
+ * %lr - Return address, volatile
+ * %ctr - volatile
+ *
+ * Register usage in this routine:
+ * %r0 - temp
+ * %r3 - argument (pointer to 5 words of SHA state)
+ * %r4 - argument (pointer to data to hash)
+ * %r5 - Contant K in SHA round (initially number of blocks to hash)
+ * %r6-%r10 - Working copies of SHA variables A..E (actually E..A order)
+ * %r11-%r26 - Data being hashed W[].
+ * %r27-%r31 - Previous copies of A..E, for final add back.
+ * %ctr - loop count
+ */
+
+
+/*
+ * We roll the registers for A, B, C, D, E around on each
+ * iteration; E on iteration t is D on iteration t+1, and so on.
+ * We use registers 6 - 10 for this.  (Registers 27 - 31 hold
+ * the previous values.)
  */
-#define RT(t)	((((t)+5)%6)+7)
-#define RA(t)	((((t)+4)%6)+7)
-#define RB(t)	((((t)+3)%6)+7)
-#define RC(t)	((((t)+2)%6)+7)
-#define RD(t)	((((t)+1)%6)+7)
-#define RE(t)	((((t)+0)%6)+7)
-
-/* We use registers 16 - 31 for the W values */
-#define W(t)	(((t)%16)+16)
-
-#define STEPD0(t)				\
-	and	%r6,RB(t),RC(t);		\
-	andc	%r0,RD(t),RB(t);		\
-	rotlwi	RT(t),RA(t),5;			\
-	rotlwi	RB(t),RB(t),30;			\
-	or	%r6,%r6,%r0;			\
-	add	%r0,RE(t),%r15;			\
-	add	RT(t),RT(t),%r6;		\
-	add	%r0,%r0,W(t);			\
-	add	RT(t),RT(t),%r0
-
-#define STEPD1(t)				\
-	xor	%r6,RB(t),RC(t);		\
-	rotlwi	RT(t),RA(t),5;			\
-	rotlwi	RB(t),RB(t),30;			\
-	xor	%r6,%r6,RD(t);			\
-	add	%r0,RE(t),%r15;			\
-	add	RT(t),RT(t),%r6;		\
-	add	%r0,%r0,W(t);			\
-	add	RT(t),RT(t),%r0
-
-#define STEPD2(t)				\
-	and	%r6,RB(t),RC(t);		\
-	and	%r0,RB(t),RD(t);		\
-	rotlwi	RT(t),RA(t),5;			\
-	rotlwi	RB(t),RB(t),30;			\
-	or	%r6,%r6,%r0;			\
-	and	%r0,RC(t),RD(t);		\
-	or	%r6,%r6,%r0;			\
-	add	%r0,RE(t),%r15;			\
-	add	RT(t),RT(t),%r6;		\
-	add	%r0,%r0,W(t);			\
-	add	RT(t),RT(t),%r0
-
-#define LOADW(t)				\
-	lwz	W(t),(t)*4(%r4)
-
-#define UPDATEW(t)				\
-	xor	%r0,W((t)-3),W((t)-8);		\
-	xor	W(t),W((t)-16),W((t)-14);	\
-	xor	W(t),W(t),%r0;			\
-	rotlwi	W(t),W(t),1
-
-#define STEP0LD4(t)				\
-	STEPD0(t);   LOADW((t)+4);		\
-	STEPD0((t)+1); LOADW((t)+5);		\
-	STEPD0((t)+2); LOADW((t)+6);		\
-	STEPD0((t)+3); LOADW((t)+7)
-
-#define STEPUP4(t, fn)				\
-	STEP##fn(t);   UPDATEW((t)+4);		\
-	STEP##fn((t)+1); UPDATEW((t)+5);	\
-	STEP##fn((t)+2); UPDATEW((t)+6);	\
-	STEP##fn((t)+3); UPDATEW((t)+7)
-
-#define STEPUP20(t, fn)				\
-	STEPUP4(t, fn);				\
-	STEPUP4((t)+4, fn);			\
-	STEPUP4((t)+8, fn);			\
-	STEPUP4((t)+12, fn);			\
-	STEPUP4((t)+16, fn)
+#define RA(t)	(((t)+4)%5+6)
+#define RB(t)	(((t)+3)%5+6)
+#define RC(t)	(((t)+2)%5+6)
+#define RD(t)	(((t)+1)%5+6)
+#define RE(t)	(((t)+0)%5+6)
+
+/* We use registers 11 - 26 for the W values */
+#define W(t)	((t)%16+11)
+
+/* Register 5 is used for the constant k */
+
+/*
+ * The basic SHA-1 round function is:
+ * E += ROTL(A,5) + F(B,C,D) + W[i] + K;  B = ROTL(B,30)
+ * Then the variables are renamed: (A,B,C,D,E) = (E,A,B,C,D).
+ *
+ * Every 20 rounds, the function F() and the contant K changes:
+ * - 20 rounds of f0(b,c,d) = "bit wise b ? c : d" =  (^b & d) + (b & c)
+ * - 20 rounds of f1(b,c,d) = b^c^d = (b^d)^c
+ * - 20 rounds of f2(b,c,d) = majority(b,c,d) = (b&d) + ((b^d)&c)
+ * - 20 more rounds of f1(b,c,d)
+ *
+ * These are all scheduled for near-optimal performance on a G4.
+ * The G4 is a 3-issue out-of-order machine with 3 ALUs, but it can only
+ * *consider* starting the oldest 3 instructions per cycle.  So to get
+ * maximum performace out of it, you have to treat it as an in-order
+ * machine.  Which means interleaving the computation round t with the
+ * computation of W[t+4].
+ *
+ * The first 16 rounds use W values loaded directly from memory, while the
+ * remianing 64 use values computed from those first 16.  We preload
+ * 4 values before starting, so there are three kinds of rounds:
+ * - The first 12 (all f0) also load the W values from memory.
+ * - The next 64 compute W(i+4) in parallel. 8*f0, 20*f1, 20*f2, 16*f1.
+ * - The last 4 (all f1) do not do anything with W.
+ *
+ * Therefore, we have 6 different round functions:
+ * STEPD0_LOAD(t,s) - Perform round t and load W(s).  s < 16
+ * STEPD0_UPDATE(t,s) - Perform round t and compute W(s).  s >= 16.
+ * STEPD1_UPDATE(t,s)
+ * STEPD2_UPDATE(t,s)
+ * STEPD1(t) - Perform round t with no load or update.
+ *
+ * The G5 is more fully out-of-order, and can find the parallelism
+ * by itself.  The big limit is that it has a 2-cycle ALU latency, so
+ * even though it's 2-way, the code has to be scheduled as if it's
+ * 4-way, which can be a limit.  To help it, we try to schedule the
+ * read of RA(t) as late as possible so it doesn't stall waiting for
+ * the previous round's RE(t-1), and we try to rotate RB(t) as early
+ * as possible while reading RC(t) (= RB(t-1)) as late as possible.
+ */
+
+/* the initial loads. */
+#define LOADW(s) \
+	lwz	W(s),(s)*4(%r4)
+
+/*
+ * Perform a step with F0, and load W(s).  Uses W(s) as a temporary
+ * before loading it.
+ * This is actually 10 instructions, which is an awkward fit.
+ * It can execute grouped as listed, or delayed one instruction.
+ * (If delayed two instructions, there is a stall before the start of the
+ * second line.)  Thus, two iterations take 7 cycles, 3.5 cycles per round.
+ */
+#define STEPD0_LOAD(t,s) \
+add RE(t),RE(t),W(t); andc   %r0,RD(t),RB(t);  and    W(s),RC(t),RB(t); \
+add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;      rotlwi RB(t),RB(t),30;   \
+add RE(t),RE(t),W(s); add    %r0,%r0,%r5;      lwz    W(s),(s)*4(%r4);  \
+add RE(t),RE(t),%r0
+
+/*
+ * This is likewise awkward, 13 instructions.  However, it can also
+ * execute starting with 2 out of 3 possible moduli, so it does 2 rounds
+ * in 9 cycles, 4.5 cycles/round.
+ */
+#define STEPD0_UPDATE(t,s,loadk...) \
+add RE(t),RE(t),W(t); andc   %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
+add RE(t),RE(t),%r0;  and    %r0,RC(t),RB(t); xor    W(s),W(s),W((s)-8);      \
+add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     xor    W(s),W(s),W((s)-14);     \
+add RE(t),RE(t),%r5;  loadk; rotlwi RB(t),RB(t),30;  rotlwi W(s),W(s),1;     \
+add RE(t),RE(t),%r0
+
+/* Nicely optimal.  Conveniently, also the most common. */
+#define STEPD1_UPDATE(t,s,loadk...) \
+add RE(t),RE(t),W(t); xor    %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
+add RE(t),RE(t),%r5;  loadk; xor %r0,%r0,RC(t);  xor W(s),W(s),W((s)-8);      \
+add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     xor    W(s),W(s),W((s)-14);     \
+add RE(t),RE(t),%r0;  rotlwi RB(t),RB(t),30;  rotlwi W(s),W(s),1
+
+/*
+ * The naked version, no UPDATE, for the last 4 rounds.  3 cycles per.
+ * We could use W(s) as a temp register, but we don't need it.
+ */
+#define STEPD1(t) \
+                        add   RE(t),RE(t),W(t); xor    %r0,RD(t),RB(t); \
+rotlwi RB(t),RB(t),30;  add   RE(t),RE(t),%r5;  xor    %r0,%r0,RC(t);   \
+add    RE(t),RE(t),%r0; rotlwi %r0,RA(t),5;     /* spare slot */        \
+add    RE(t),RE(t),%r0
+
+/*
+ * 14 instructions, 5 cycles per.  The majority function is a bit
+ * awkward to compute.  This can execute with a 1-instruction delay,
+ * but it causes a 2-instruction delay, which triggers a stall.
+ */
+#define STEPD2_UPDATE(t,s,loadk...) \
+add RE(t),RE(t),W(t); and    %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
+add RE(t),RE(t),%r0;  xor    %r0,RD(t),RB(t); xor    W(s),W(s),W((s)-8);      \
+add RE(t),RE(t),%r5;  loadk; and %r0,%r0,RC(t);  xor W(s),W(s),W((s)-14);     \
+add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     rotlwi W(s),W(s),1;             \
+add RE(t),RE(t),%r0;  rotlwi RB(t),RB(t),30
+
+#define STEP0_LOAD4(t,s)		\
+	STEPD0_LOAD(t,s);		\
+	STEPD0_LOAD((t+1),(s)+1);	\
+	STEPD0_LOAD((t)+2,(s)+2);	\
+	STEPD0_LOAD((t)+3,(s)+3)
+
+#define STEPUP4(fn, t, s, loadk...)		\
+	STEP##fn##_UPDATE(t,s,);		\
+	STEP##fn##_UPDATE((t)+1,(s)+1,);	\
+	STEP##fn##_UPDATE((t)+2,(s)+2,);	\
+	STEP##fn##_UPDATE((t)+3,(s)+3,loadk)
+
+#define STEPUP20(fn, t, s, loadk...)	\
+	STEPUP4(fn, t, s,);		\
+	STEPUP4(fn, (t)+4, (s)+4,);	\
+	STEPUP4(fn, (t)+8, (s)+8,);	\
+	STEPUP4(fn, (t)+12, (s)+12,);	\
+	STEPUP4(fn, (t)+16, (s)+16, loadk)
 
 	.globl	sha1_core
 sha1_core:
-	stwu	%r1,-FS(%r1)
-	stw	%r15,FS-68(%r1)
-	stw	%r16,FS-64(%r1)
-	stw	%r17,FS-60(%r1)
-	stw	%r18,FS-56(%r1)
-	stw	%r19,FS-52(%r1)
-	stw	%r20,FS-48(%r1)
-	stw	%r21,FS-44(%r1)
-	stw	%r22,FS-40(%r1)
-	stw	%r23,FS-36(%r1)
-	stw	%r24,FS-32(%r1)
-	stw	%r25,FS-28(%r1)
-	stw	%r26,FS-24(%r1)
-	stw	%r27,FS-20(%r1)
-	stw	%r28,FS-16(%r1)
-	stw	%r29,FS-12(%r1)
-	stw	%r30,FS-8(%r1)
-	stw	%r31,FS-4(%r1)
+	stwu	%r1,-80(%r1)
+	stmw	%r13,4(%r1)
 
 	/* Load up A - E */
-	lwz	RA(0),0(%r3)	/* A */
-	lwz	RB(0),4(%r3)	/* B */
-	lwz	RC(0),8(%r3)	/* C */
-	lwz	RD(0),12(%r3)	/* D */
-	lwz	RE(0),16(%r3)	/* E */
+	lmw	%r27,0(%r3)
 
 	mtctr	%r5
 
-1:	LOADW(0)
+1:
+	LOADW(0)
+	lis	%r5,0x5a82
+	mr	RE(0),%r31
 	LOADW(1)
+	mr	RD(0),%r30
+	mr	RC(0),%r29
 	LOADW(2)
+	ori	%r5,%r5,0x7999	/* K0-19 */
+	mr	RB(0),%r28
 	LOADW(3)
+	mr	RA(0),%r27
+
+	STEP0_LOAD4(0, 4)
+	STEP0_LOAD4(4, 8)
+	STEP0_LOAD4(8, 12)
+	STEPUP4(D0, 12, 16,)
+	STEPUP4(D0, 16, 20, lis %r5,0x6ed9)
 
-	lis	%r15,0x5a82	/* K0-19 */
-	ori	%r15,%r15,0x7999
-	STEP0LD4(0)
-	STEP0LD4(4)
-	STEP0LD4(8)
-	STEPUP4(12, D0)
-	STEPUP4(16, D0)
-
-	lis	%r15,0x6ed9	/* K20-39 */
-	ori	%r15,%r15,0xeba1
-	STEPUP20(20, D1)
-
-	lis	%r15,0x8f1b	/* K40-59 */
-	ori	%r15,%r15,0xbcdc
-	STEPUP20(40, D2)
-
-	lis	%r15,0xca62	/* K60-79 */
-	ori	%r15,%r15,0xc1d6
-	STEPUP4(60, D1)
-	STEPUP4(64, D1)
-	STEPUP4(68, D1)
-	STEPUP4(72, D1)
+	ori	%r5,%r5,0xeba1	/* K20-39 */
+	STEPUP20(D1, 20, 24, lis %r5,0x8f1b)
+
+	ori	%r5,%r5,0xbcdc	/* K40-59 */
+	STEPUP20(D2, 40, 44, lis %r5,0xca62)
+
+	ori	%r5,%r5,0xc1d6	/* K60-79 */
+	STEPUP4(D1, 60, 64,)
+	STEPUP4(D1, 64, 68,)
+	STEPUP4(D1, 68, 72,)
+	STEPUP4(D1, 72, 76,)
+	addi	%r4,%r4,64
 	STEPD1(76)
 	STEPD1(77)
 	STEPD1(78)
 	STEPD1(79)
 
-	lwz	%r20,16(%r3)
-	lwz	%r19,12(%r3)
-	lwz	%r18,8(%r3)
-	lwz	%r17,4(%r3)
-	lwz	%r16,0(%r3)
-	add	%r20,RE(80),%r20
-	add	RD(0),RD(80),%r19
-	add	RC(0),RC(80),%r18
-	add	RB(0),RB(80),%r17
-	add	RA(0),RA(80),%r16
-	mr	RE(0),%r20
-	stw	RA(0),0(%r3)
-	stw	RB(0),4(%r3)
-	stw	RC(0),8(%r3)
-	stw	RD(0),12(%r3)
-	stw	RE(0),16(%r3)
+	/* Add results to original values */
+	add	%r31,%r31,RE(0)
+	add	%r30,%r30,RD(0)
+	add	%r29,%r29,RC(0)
+	add	%r28,%r28,RB(0)
+	add	%r27,%r27,RA(0)
 
-	addi	%r4,%r4,64
 	bdnz	1b
 
-	lwz	%r15,FS-68(%r1)
-	lwz	%r16,FS-64(%r1)
-	lwz	%r17,FS-60(%r1)
-	lwz	%r18,FS-56(%r1)
-	lwz	%r19,FS-52(%r1)
-	lwz	%r20,FS-48(%r1)
-	lwz	%r21,FS-44(%r1)
-	lwz	%r22,FS-40(%r1)
-	lwz	%r23,FS-36(%r1)
-	lwz	%r24,FS-32(%r1)
-	lwz	%r25,FS-28(%r1)
-	lwz	%r26,FS-24(%r1)
-	lwz	%r27,FS-20(%r1)
-	lwz	%r28,FS-16(%r1)
-	lwz	%r29,FS-12(%r1)
-	lwz	%r30,FS-8(%r1)
-	lwz	%r31,FS-4(%r1)
-	addi	%r1,%r1,FS
+	/* Save final hash, restore registers, and return */
+	stmw	%r27,0(%r3)
+	lmw	%r13,4(%r1)
+	addi	%r1,%r1,80
 	blr
diff --git a/test-sha1.c b/test-sha1.c
new file mode 100644
index 0000000000..78d7e983a7
--- /dev/null
+++ b/test-sha1.c
@@ -0,0 +1,47 @@
+#include "cache.h"
+
+int main(int ac, char **av)
+{
+	SHA_CTX ctx;
+	unsigned char sha1[20];
+	unsigned bufsz = 8192;
+	char *buffer;
+
+	if (ac == 2)
+		bufsz = strtoul(av[1], NULL, 10) * 1024 * 1024;
+
+	if (!bufsz)
+		bufsz = 8192;
+
+	while ((buffer = malloc(bufsz)) == NULL) {
+		fprintf(stderr, "bufsz %u is too big, halving...\n", bufsz);
+		bufsz /= 2;
+		if (bufsz < 1024)
+			die("OOPS");
+	}
+
+	SHA1_Init(&ctx);
+
+	while (1) {
+		ssize_t sz, this_sz;
+		char *cp = buffer;
+		unsigned room = bufsz;
+		this_sz = 0;
+		while (room) {
+			sz = xread(0, cp, room);
+			if (sz == 0)
+				break;
+			if (sz < 0)
+				die("test-sha1: %s", strerror(errno));
+			this_sz += sz;
+			cp += sz;
+			room -= sz;
+		}
+		if (this_sz == 0)
+			break;
+		SHA1_Update(&ctx, buffer, this_sz);
+	}
+	SHA1_Final(sha1, &ctx);
+	puts(sha1_to_hex(sha1));
+	exit(0);
+}
diff --git a/test-sha1.sh b/test-sha1.sh
new file mode 100755
index 0000000000..640856af5a
--- /dev/null
+++ b/test-sha1.sh
@@ -0,0 +1,83 @@
+#!/bin/sh
+
+dd if=/dev/zero bs=1048576 count=100 2>/dev/null |
+/usr/bin/time ./test-sha1 >/dev/null
+
+while read expect cnt pfx
+do
+	case "$expect" in '#'*) continue ;; esac
+	actual=`
+		{
+			test -z "$pfx" || echo "$pfx"
+			dd if=/dev/zero bs=1048576 count=$cnt 2>/dev/null |
+			tr '[\0]' '[g]'
+		} | ./test-sha1 $cnt
+	`
+	if test "$expect" = "$actual"
+	then
+		echo "OK: $expect $cnt $pfx"
+	else
+		echo >&2 "OOPS: $cnt"
+		echo >&2 "expect: $expect"
+		echo >&2 "actual: $actual"
+		exit 1
+	fi
+done <<EOF
+da39a3ee5e6b4b0d3255bfef95601890afd80709 0
+3f786850e387550fdab836ed7e6dc881de23001b 0 a
+5277cbb45a15902137d332d97e89cf8136545485 0 ab
+03cfd743661f07975fa2f1220c5194cbaff48451 0 abc
+3330b4373640f9e4604991e73c7e86bfd8da2dc3 0 abcd
+ec11312386ad561674f724b8cca7cf1796e26d1d 0 abcde
+bdc37c074ec4ee6050d68bc133c6b912f36474df 0 abcdef
+69bca99b923859f2dc486b55b87f49689b7358c7 0 abcdefg
+e414af7161c9554089f4106d6f1797ef14a73666 0 abcdefgh
+0707f2970043f9f7c22029482db27733deaec029 0 abcdefghi
+a4dd8aa74a5636728fe52451636e2e17726033aa 1
+9986b45e2f4d7086372533bb6953a8652fa3644a 1 frotz
+23d8d4f788e8526b4877548a32577543cbaaf51f 10
+8cd23f822ab44c7f481b8c92d591f6d1fcad431c 10 frotz
+f3b5604a4e604899c1233edb3bf1cc0ede4d8c32 512
+b095bd837a371593048136e429e9ac4b476e1bb3 512 frotz
+08fa81d6190948de5ccca3966340cc48c10cceac 1200 xyzzy
+e33a291f42c30a159733dd98b8b3e4ff34158ca0 4090 4G
+#a3bf783bc20caa958f6cb24dd140a7b21984838d 9999 nitfol
+EOF
+
+exit
+
+# generating test vectors
+# inputs are number of megabytes followed by some random string to prefix.
+
+while read cnt pfx
+do
+	actual=`
+		{
+			test -z "$pfx" || echo "$pfx"
+			dd if=/dev/zero bs=1048576 count=$cnt 2>/dev/null |
+			tr '[\0]' '[g]'
+		} | sha1sum |
+		sed -e 's/ .*//'
+	`
+	echo "$actual $cnt $pfx"
+done <<EOF
+0
+0 a
+0 ab
+0 abc
+0 abcd
+0 abcde
+0 abcdef
+0 abcdefg
+0 abcdefgh
+0 abcdefghi
+1
+1 frotz
+10
+10 frotz
+512
+512 frotz
+1200 xyzzy
+4090 4G
+9999 nitfol
+EOF