Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt

3 files changed, 96 insertions, 21 deletions

diff --git a/testrig/db.go b/testrig/db.go
index 01cf93934..fe38c3164 100644
--- a/testrig/db.go
+++ b/testrig/db.go
@@ -65,7 +65,14 @@ func NewTestDB() db.DB {
 }
 
 // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests.
-func StandardDBSetup(db db.DB) {
+//
+// The accounts parameter is provided in case the db should be populated with a certain set of accounts.
+// If accounts is nil, then the standard test accounts will be used.
+//
+// When testing http signatures, you should pass into this function the same accounts map that you generated
+// signatures with, otherwise this function will randomly generate new keys for accounts and signature
+// verification will fail.
+func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 	for _, m := range testModels {
 		if err := db.CreateTable(m); err != nil {
 			panic(err)
@@ -96,9 +103,17 @@ func StandardDBSetup(db db.DB) {
 		}
 	}
 
-	for _, v := range NewTestAccounts() {
-		if err := db.Put(v); err != nil {
-			panic(err)
+	if accounts == nil {
+		for _, v := range NewTestAccounts() {
+			if err := db.Put(v); err != nil {
+				panic(err)
+			}
+		}
+	} else {
+		for _, v := range accounts {
+			if err := db.Put(v); err != nil {
+				panic(err)
+			}
 		}
 	}
 
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index 1934170d2..da5cbe7af 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -36,9 +36,9 @@ import (
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@@ -443,9 +443,9 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			FeaturedCollectionURI: "http://fossbros-anonymous.io/users/foss_satan/collections/featured",
 			ActorType:             gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:           "",
-			PrivateKey:            nil,
+			PrivateKey:            &rsa.PrivateKey{},
 			PublicKey:             &rsa.PublicKey{},
-			PublicKeyURI:          "http://fossbros-anonymous.io/users/foss_satan#main-key",
+			PublicKeyURI:          "http://fossbros-anonymous.io/users/foss_satan/main-key",
 			SensitizedAt:          time.Time{},
 			SilencedAt:            time.Time{},
 			SuspendedAt:           time.Time{},
@@ -1033,6 +1033,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			},
 			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
 		},
+		"local_account_2_status_5": {
+			ID:                       "01FCQSQ667XHJ9AV9T27SJJSX5",
+			URI:                      "http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5",
+			URL:                      "http://localhost:8080/@1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5",
+			Content:                  "🐢 hi zork! 🐢",
+			CreatedAt:                time.Now().Add(-1 * time.Minute),
+			UpdatedAt:                time.Now().Add(-1 * time.Minute),
+			Local:                    true,
+			AccountID:                "01F8MH5NBDF2MV7CTC4Q5128HF",
+			InReplyToID:              "01F8MHAMCHF6Y650WCRSCP4WMY",
+			InReplyToAccountID:       "01F8MH1H7YV1Z7D2C8K2730QBF",
+			InReplyToURI:             "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
+			BoostOfID:                "",
+			ContentWarning:           "",
+			Visibility:               gtsmodel.VisibilityPublic,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
+			VisibilityAdvanced: &gtsmodel.VisibilityAdvanced{
+				Federated: true,
+				Boostable: true,
+				Replyable: true,
+				Likeable:  true,
+			},
+			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
+		},
 	}
 }
 
@@ -1155,14 +1181,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit
 }
 
 // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on.
-func NewTestFediPeople() map[string]typeutils.Accountable {
+func NewTestFediPeople() map[string]ap.Accountable {
 	newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
 		panic(err)
 	}
 	newPerson1Pub := &newPerson1Priv.PublicKey
 
-	return map[string]typeutils.Accountable{
+	return map[string]ap.Accountable{
 		"new_person_1": newPerson(
 			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
 			URLMustParse("https://unknown-instance.com/users/brand_new_person/following"),
@@ -1187,13 +1213,47 @@ func NewTestFediPeople() map[string]typeutils.Accountable {
 
 // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures.
 func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature {
-	sig, digest, date := getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].URI))
+	var sig, digest, date string
+	var target *url.URL
+	statuses := NewTestStatuses()
+
+	target = URLMustParse(accounts["local_account_1"].URI)
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceZork := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1Replies := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1RepliesNext := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
 	return map[string]ActivityWithSignature{
-		"foss_satan_dereference_zork": {
-			SignatureHeader: sig,
-			DigestHeader:    digest,
-			DateHeader:      date,
-		},
+		"foss_satan_dereference_zork":                                  fossSatanDereferenceZork,
+		"foss_satan_dereference_local_account_1_status_1_replies":      fossSatanDereferenceLocalAccount1Status1Replies,
+		"foss_satan_dereference_local_account_1_status_1_replies_next": fossSatanDereferenceLocalAccount1Status1RepliesNext,
+		"foss_satan_dereference_local_account_1_status_1_replies_last": fossSatanDereferenceLocalAccount1Status1RepliesLast,
 	}
 }
 
@@ -1215,7 +1275,7 @@ func getSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
 	}
 
 	// use the client to create a new transport
-	c := NewTestTransportController(client)
+	c := NewTestTransportController(client, NewTestDB())
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1247,7 +1307,6 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 	client := &mockHTTPClient{
 		do: func(req *http.Request) (*http.Response, error) {
 			signatureHeader = req.Header.Get("Signature")
-			digestHeader = req.Header.Get("Digest")
 			dateHeader = req.Header.Get("Date")
 			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
 			return &http.Response{
@@ -1258,7 +1317,7 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 	}
 
 	// use the client to create a new transport
-	c := NewTestTransportController(client)
+	c := NewTestTransportController(client, NewTestDB())
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1290,7 +1349,7 @@ func newPerson(
 	avatarURL *url.URL,
 	avatarContentType string,
 	headerURL *url.URL,
-	headerContentType string) typeutils.Accountable {
+	headerContentType string) ap.Accountable {
 	person := streams.NewActivityStreamsPerson()
 
 	// id should be the activitypub URI of this user
diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go
index f2b5b93f7..ec591a9b1 100644
--- a/testrig/transportcontroller.go
+++ b/testrig/transportcontroller.go
@@ -24,6 +24,7 @@ import (
 	"net/http"
 
 	"github.com/go-fed/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -37,8 +38,8 @@ import (
 // Unlike the other test interfaces provided in this package, you'll probably want to call this function
 // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
 // basis.
-func NewTestTransportController(client pub.HttpClient) transport.Controller {
-	return transport.NewController(NewTestConfig(), &federation.Clock{}, client, NewTestLog())
+func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
+	return transport.NewController(NewTestConfig(), db, &federation.Clock{}, client, NewTestLog())
 }
 
 // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,