Ap (#14)

Big restructuring and initial work on activitypub

9 files changed, 669 insertions, 98 deletions

diff --git a/testrig/actions.go b/testrig/actions.go
index 1caa18581..7ed75b18f 100644
--- a/testrig/actions.go
+++ b/testrig/actions.go
@@ -19,24 +19,26 @@
 package testrig
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/action"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/account"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/admin"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/app"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/auth"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver"
-	mediaModule "github.com/superseriousbusiness/gotosocial/internal/apimodule/media"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/security"
-	"github.com/superseriousbusiness/gotosocial/internal/apimodule/status"
-	"github.com/superseriousbusiness/gotosocial/internal/cache"
+	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/app"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/auth"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver"
+	mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
+	"github.com/superseriousbusiness/gotosocial/internal/api/security"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
@@ -44,33 +46,39 @@ import (
 
 // Run creates and starts a gotosocial testrig server
 var Run action.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error {
+	c := NewTestConfig()
 	dbService := NewTestDB()
 	router := NewTestRouter()
 	storageBackend := NewTestStorage()
-	mediaHandler := NewTestMediaHandler(dbService, storageBackend)
-	oauthServer := NewTestOauthServer(dbService)
-	distributor := NewTestDistributor()
-	if err := distributor.Start(); err != nil {
-		return fmt.Errorf("error starting distributor: %s", err)
-	}
-	mastoConverter := NewTestMastoConverter(dbService)
 
-	c := NewTestConfig()
+	typeConverter := NewTestTypeConverter(dbService)
+	transportController := NewTestTransportController(NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
+		return &http.Response{
+			StatusCode: 200,
+			Body:       r,
+		}, nil
+	}))
+	federator := federation.NewFederator(dbService, transportController, c, log, typeConverter)
+	processor := NewTestProcessor(dbService, storageBackend, federator)
+	if err := processor.Start(); err != nil {
+		return fmt.Errorf("error starting processor: %s", err)
+	}
 
 	StandardDBSetup(dbService)
 	StandardStorageSetup(storageBackend, "./testrig/media")
 
 	// build client api modules
-	authModule := auth.New(oauthServer, dbService, log)
-	accountModule := account.New(c, dbService, oauthServer, mediaHandler, mastoConverter, log)
-	appsModule := app.New(oauthServer, dbService, mastoConverter, log)
-	mm := mediaModule.New(dbService, mediaHandler, mastoConverter, c, log)
-	fileServerModule := fileserver.New(c, dbService, storageBackend, log)
-	adminModule := admin.New(c, dbService, mediaHandler, mastoConverter, log)
-	statusModule := status.New(c, dbService, mediaHandler, mastoConverter, distributor, log)
+	authModule := auth.New(c, dbService, NewTestOauthServer(dbService), log)
+	accountModule := account.New(c, processor, log)
+	appsModule := app.New(c, processor, log)
+	mm := mediaModule.New(c, processor, log)
+	fileServerModule := fileserver.New(c, processor, log)
+	adminModule := admin.New(c, processor, log)
+	statusModule := status.New(c, processor, log)
 	securityModule := security.New(c, log)
 
-	apiModules := []apimodule.ClientAPIModule{
+	apis := []api.ClientModule{
 		// modules with middleware go first
 		securityModule,
 		authModule,
@@ -84,20 +92,13 @@ var Run action.GTSAction = func(ctx context.Context, _ *config.Config, log *logr
 		statusModule,
 	}
 
-	for _, m := range apiModules {
+	for _, m := range apis {
 		if err := m.Route(router); err != nil {
 			return fmt.Errorf("routing error: %s", err)
 		}
-		if err := m.CreateTables(dbService); err != nil {
-			return fmt.Errorf("table creation error: %s", err)
-		}
 	}
 
-	// if err := dbService.CreateInstanceAccount(); err != nil {
-	// 	return fmt.Errorf("error creating instance account: %s", err)
-	// }
-
-	gts, err := gotosocial.New(dbService, &cache.MockCache{}, router, federation.New(dbService, log), c)
+	gts, err := gotosocial.New(dbService, router, federator, c)
 	if err != nil {
 		return fmt.Errorf("error creating gotosocial service: %s", err)
 	}
diff --git a/testrig/db.go b/testrig/db.go
index 5974eae69..4d22ab3c8 100644
--- a/testrig/db.go
+++ b/testrig/db.go
@@ -23,7 +23,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 )
 
@@ -54,7 +54,7 @@ func NewTestDB() db.DB {
 	config := NewTestConfig()
 	l := logrus.New()
 	l.SetLevel(logrus.TraceLevel)
-	testDB, err := db.New(context.Background(), config, l)
+	testDB, err := db.NewPostgresService(context.Background(), config, l)
 	if err != nil {
 		panic(err)
 	}
diff --git a/testrig/distributor.go b/testrig/federator.go
index a7206e5ea..63ad520db 100644
--- a/testrig/distributor.go
+++ b/testrig/federator.go
@@ -18,9 +18,12 @@
 
 package testrig
 
-import "github.com/superseriousbusiness/gotosocial/internal/distributor"
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+)
 
-// NewTestDistributor returns a Distributor suitable for testing purposes
-func NewTestDistributor() distributor.Distributor {
-	return distributor.New(NewTestLog())
+func NewTestFederator(db db.DB, tc transport.Controller) federation.Federator {
+	return federation.NewFederator(db, tc, NewTestConfig(), NewTestLog(), NewTestTypeConverter(db))
 }
diff --git a/testrig/media/test-jpeg.jpg b/testrig/media/test-jpeg.jpg
new file mode 100644
index 000000000..a9ab154d4
--- /dev/null
+++ b/testrig/media/test-jpeg.jpg
diff --git a/testrig/processor.go b/testrig/processor.go
new file mode 100644
index 000000000..9aa8e2509
--- /dev/null
+++ b/testrig/processor.go
@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package testrig
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/message"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
+)
+
+// NewTestProcessor returns a Processor suitable for testing purposes
+func NewTestProcessor(db db.DB, storage storage.Storage, federator federation.Federator) message.Processor {
+	return message.NewProcessor(NewTestConfig(), NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, db, NewTestLog())
+}
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index 0d95ef21d..e550c66f7 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -19,13 +19,26 @@
 package testrig
 
 import (
+	"bytes"
+	"context"
+	"crypto"
 	"crypto/rand"
 	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"io/ioutil"
 	"net"
+	"net/http"
+	"net/url"
 	"time"
 
-	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/go-fed/activity/pub"
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@@ -274,15 +287,16 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			URI:                     "http://localhost:8080/users/weed_lord420",
 			URL:                     "http://localhost:8080/@weed_lord420",
 			LastWebfingeredAt:       time.Time{},
-			InboxURL:                "http://localhost:8080/users/weed_lord420/inbox",
-			OutboxURL:               "http://localhost:8080/users/weed_lord420/outbox",
-			SharedInboxURL:          "",
-			FollowersURL:            "http://localhost:8080/users/weed_lord420/followers",
-			FeaturedCollectionURL:   "http://localhost:8080/users/weed_lord420/collections/featured",
+			InboxURI:                "http://localhost:8080/users/weed_lord420/inbox",
+			OutboxURI:               "http://localhost:8080/users/weed_lord420/outbox",
+			FollowersURI:            "http://localhost:8080/users/weed_lord420/followers",
+			FollowingURI:            "http://localhost:8080/users/weed_lord420/following",
+			FeaturedCollectionURI:   "http://localhost:8080/users/weed_lord420/collections/featured",
 			ActorType:               gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:             "",
 			PrivateKey:              &rsa.PrivateKey{},
 			PublicKey:               &rsa.PublicKey{},
+			PublicKeyURI:            "http://localhost:8080/users/weed_lord420#main-key",
 			SensitizedAt:            time.Time{},
 			SilencedAt:              time.Time{},
 			SuspendedAt:             time.Time{},
@@ -310,12 +324,13 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Language:                "en",
 			URI:                     "http://localhost:8080/users/admin",
 			URL:                     "http://localhost:8080/@admin",
+			PublicKeyURI:            "http://localhost:8080/users/admin#main-key",
 			LastWebfingeredAt:       time.Time{},
-			InboxURL:                "http://localhost:8080/users/admin/inbox",
-			OutboxURL:               "http://localhost:8080/users/admin/outbox",
-			SharedInboxURL:          "",
-			FollowersURL:            "http://localhost:8080/users/admin/followers",
-			FeaturedCollectionURL:   "http://localhost:8080/users/admin/collections/featured",
+			InboxURI:                "http://localhost:8080/users/admin/inbox",
+			OutboxURI:               "http://localhost:8080/users/admin/outbox",
+			FollowersURI:            "http://localhost:8080/users/admin/followers",
+			FollowingURI:            "http://localhost:8080/users/admin/following",
+			FeaturedCollectionURI:   "http://localhost:8080/users/admin/collections/featured",
 			ActorType:               gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:             "",
 			PrivateKey:              &rsa.PrivateKey{},
@@ -348,15 +363,16 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			URI:                     "http://localhost:8080/users/the_mighty_zork",
 			URL:                     "http://localhost:8080/@the_mighty_zork",
 			LastWebfingeredAt:       time.Time{},
-			InboxURL:                "http://localhost:8080/users/the_mighty_zork/inbox",
-			OutboxURL:               "http://localhost:8080/users/the_mighty_zork/outbox",
-			SharedInboxURL:          "",
-			FollowersURL:            "http://localhost:8080/users/the_mighty_zork/followers",
-			FeaturedCollectionURL:   "http://localhost:8080/users/the_mighty_zork/collections/featured",
+			InboxURI:                "http://localhost:8080/users/the_mighty_zork/inbox",
+			OutboxURI:               "http://localhost:8080/users/the_mighty_zork/outbox",
+			FollowersURI:            "http://localhost:8080/users/the_mighty_zork/followers",
+			FollowingURI:            "http://localhost:8080/users/the_mighty_zork/following",
+			FeaturedCollectionURI:   "http://localhost:8080/users/the_mighty_zork/collections/featured",
 			ActorType:               gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:             "",
 			PrivateKey:              &rsa.PrivateKey{},
 			PublicKey:               &rsa.PublicKey{},
+			PublicKeyURI:            "http://localhost:8080/users/the_mighty_zork#main-key",
 			SensitizedAt:            time.Time{},
 			SilencedAt:              time.Time{},
 			SuspendedAt:             time.Time{},
@@ -385,15 +401,16 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			URI:                     "http://localhost:8080/users/1happyturtle",
 			URL:                     "http://localhost:8080/@1happyturtle",
 			LastWebfingeredAt:       time.Time{},
-			InboxURL:                "http://localhost:8080/users/1happyturtle/inbox",
-			OutboxURL:               "http://localhost:8080/users/1happyturtle/outbox",
-			SharedInboxURL:          "",
-			FollowersURL:            "http://localhost:8080/users/1happyturtle/followers",
-			FeaturedCollectionURL:   "http://localhost:8080/users/1happyturtle/collections/featured",
+			InboxURI:                "http://localhost:8080/users/1happyturtle/inbox",
+			OutboxURI:               "http://localhost:8080/users/1happyturtle/outbox",
+			FollowersURI:            "http://localhost:8080/users/1happyturtle/followers",
+			FollowingURI:            "http://localhost:8080/users/1happyturtle/following",
+			FeaturedCollectionURI:   "http://localhost:8080/users/1happyturtle/collections/featured",
 			ActorType:               gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:             "",
 			PrivateKey:              &rsa.PrivateKey{},
 			PublicKey:               &rsa.PublicKey{},
+			PublicKeyURI:            "http://localhost:8080/users/1happyturtle#main-key",
 			SensitizedAt:            time.Time{},
 			SilencedAt:              time.Time{},
 			SuspendedAt:             time.Time{},
@@ -426,18 +443,19 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			Discoverable:          true,
 			Sensitive:             false,
 			Language:              "en",
-			URI:                   "https://fossbros-anonymous.io/users/foss_satan",
-			URL:                   "https://fossbros-anonymous.io/@foss_satan",
+			URI:                   "http://fossbros-anonymous.io/users/foss_satan",
+			URL:                   "http://fossbros-anonymous.io/@foss_satan",
 			LastWebfingeredAt:     time.Time{},
-			InboxURL:              "https://fossbros-anonymous.io/users/foss_satan/inbox",
-			OutboxURL:             "https://fossbros-anonymous.io/users/foss_satan/outbox",
-			SharedInboxURL:        "",
-			FollowersURL:          "https://fossbros-anonymous.io/users/foss_satan/followers",
-			FeaturedCollectionURL: "https://fossbros-anonymous.io/users/foss_satan/collections/featured",
+			InboxURI:              "http://fossbros-anonymous.io/users/foss_satan/inbox",
+			OutboxURI:             "http://fossbros-anonymous.io/users/foss_satan/outbox",
+			FollowersURI:          "http://fossbros-anonymous.io/users/foss_satan/followers",
+			FollowingURI:          "http://fossbros-anonymous.io/users/foss_satan/following",
+			FeaturedCollectionURI: "http://fossbros-anonymous.io/users/foss_satan/collections/featured",
 			ActorType:             gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:           "",
-			PrivateKey:            &rsa.PrivateKey{},
-			PublicKey:             nil,
+			PrivateKey:            nil,
+			PublicKey:             &rsa.PublicKey{},
+			PublicKeyURI:          "http://fossbros-anonymous.io/users/foss_satan#main-key",
 			SensitizedAt:          time.Time{},
 			SilencedAt:            time.Time{},
 			SuspendedAt:           time.Time{},
@@ -468,10 +486,10 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 		}
 		pub := &priv.PublicKey
 
-		// only local accounts get a private key
-		if v.Domain == "" {
-			v.PrivateKey = priv
-		}
+		// normally only local accounts get a private key (obviously)
+		// but for testing purposes and signing requests, we'll give
+		// remote accounts a private key as well
+		v.PrivateKey = priv
 		v.PublicKey = pub
 	}
 	return accounts
@@ -676,25 +694,26 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 func NewTestEmojis() map[string]*gtsmodel.Emoji {
 	return map[string]*gtsmodel.Emoji{
 		"rainbow": {
-			ID:                   "a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
-			Shortcode:            "rainbow",
-			Domain:               "",
-			CreatedAt:            time.Now(),
-			UpdatedAt:            time.Now(),
-			ImageRemoteURL:       "",
-			ImageStaticRemoteURL: "",
-			ImageURL:             "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
-			ImagePath:            "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
-			ImageStaticURL:       "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
-			ImageStaticPath:      "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
-			ImageContentType:     "image/png",
-			ImageFileSize:        36702,
-			ImageStaticFileSize:  10413,
-			ImageUpdatedAt:       time.Now(),
-			Disabled:             false,
-			URI:                  "http://localhost:8080/emoji/a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
-			VisibleInPicker:      true,
-			CategoryID:           "",
+			ID:                     "a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
+			Shortcode:              "rainbow",
+			Domain:                 "",
+			CreatedAt:              time.Now(),
+			UpdatedAt:              time.Now(),
+			ImageRemoteURL:         "",
+			ImageStaticRemoteURL:   "",
+			ImageURL:               "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImagePath:              "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageStaticURL:         "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageStaticPath:        "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageContentType:       "image/png",
+			ImageStaticContentType: "image/png",
+			ImageFileSize:          36702,
+			ImageStaticFileSize:    10413,
+			ImageUpdatedAt:         time.Now(),
+			Disabled:               false,
+			URI:                    "http://localhost:8080/emoji/a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
+			VisibleInPicker:        true,
+			CategoryID:             "",
 		},
 	}
 }
@@ -993,3 +1012,436 @@ func NewTestFaves() map[string]*gtsmodel.StatusFave {
 		},
 	}
 }
+
+type ActivityWithSignature struct {
+	Activity        pub.Activity
+	SignatureHeader string
+	DigestHeader    string
+	DateHeader      string
+}
+
+// NewTestActivities returns a bunch of pub.Activity types for use in testing the federation protocols.
+// A struct of accounts needs to be passed in because the activities will also be bundled along with
+// their requesting signatures.
+func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature {
+	dmForZork := newNote(
+		URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6"),
+		URLMustParse("https://fossbros-anonymous.io/@foss_satan/5424b153-4553-4f30-9358-7b92f7cd42f6"),
+		"hey zork here's a new private note for you",
+		"new note for zork",
+		URLMustParse("https://fossbros-anonymous.io/users/foss_satan"),
+		[]*url.URL{URLMustParse("http://localhost:8080/users/the_mighty_zork")},
+		nil,
+		true)
+	createDmForZork := wrapNoteInCreate(
+		URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6/activity"),
+		URLMustParse("https://fossbros-anonymous.io/users/foss_satan"),
+		time.Now(),
+		dmForZork)
+	sig, digest, date := getSignatureForActivity(createDmForZork, accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].InboxURI))
+
+	return map[string]ActivityWithSignature{
+		"dm_for_zork": {
+			Activity:        createDmForZork,
+			SignatureHeader: sig,
+			DigestHeader:    digest,
+			DateHeader:      date,
+		},
+	}
+}
+
+// NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on.
+func NewTestFediPeople() map[string]typeutils.Accountable {
+	new_person_1priv, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(err)
+	}
+	new_person_1pub := &new_person_1priv.PublicKey
+
+	return map[string]typeutils.Accountable{
+		"new_person_1": newPerson(
+			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/following"),
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/followers"),
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/inbox"),
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/outbox"),
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/collections/featured"),
+			"brand_new_person",
+			"Geoff Brando New Personson",
+			"hey I'm a new person, your instance hasn't seen me yet uwu",
+			URLMustParse("https://unknown-instance.com/@brand_new_person"),
+			true,
+			URLMustParse("https://unknown-instance.com/users/brand_new_person#main-key"),
+			new_person_1pub,
+			URLMustParse("https://unknown-instance.com/media/some_avatar_filename.jpeg"),
+			"image/jpeg",
+			URLMustParse("https://unknown-instance.com/media/some_header_filename.jpeg"),
+			"image/png",
+		),
+	}
+}
+
+func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature {
+	sig, digest, date := getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].URI))
+	return map[string]ActivityWithSignature{
+		"foss_satan_dereference_zork": {
+			SignatureHeader: sig,
+			DigestHeader:    digest,
+			DateHeader:      date,
+		},
+	}
+}
+
+// getSignatureForActivity does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
+// the HTTP Signature for the given activity, public key ID, private key, and destination.
+func getSignatureForActivity(activity pub.Activity, pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
+	// create a client that basically just pulls the signature out of the request and sets it
+	client := &mockHTTPClient{
+		do: func(req *http.Request) (*http.Response, error) {
+			signatureHeader = req.Header.Get("Signature")
+			digestHeader = req.Header.Get("Digest")
+			dateHeader = req.Header.Get("Date")
+			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
+			return &http.Response{
+				StatusCode: 200,
+				Body:       r,
+			}, nil
+		},
+	}
+
+	// use the client to create a new transport
+	c := NewTestTransportController(client)
+	tp, err := c.NewTransport(pubKeyID, privkey)
+	if err != nil {
+		panic(err)
+	}
+
+	// convert the activity into json bytes
+	m, err := activity.Serialize()
+	if err != nil {
+		panic(err)
+	}
+	bytes, err := json.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+
+	// trigger the delivery function, which will trigger the 'do' function of the recorder above
+	if err := tp.Deliver(context.Background(), bytes, destination); err != nil {
+		panic(err)
+	}
+
+	// headers should now be populated
+	return
+}
+
+// getSignatureForDereference does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
+// the HTTP Signature for the given derefence GET request using public key ID, private key, and destination.
+func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
+	// create a client that basically just pulls the signature out of the request and sets it
+	client := &mockHTTPClient{
+		do: func(req *http.Request) (*http.Response, error) {
+			signatureHeader = req.Header.Get("Signature")
+			digestHeader = req.Header.Get("Digest")
+			dateHeader = req.Header.Get("Date")
+			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
+			return &http.Response{
+				StatusCode: 200,
+				Body:       r,
+			}, nil
+		},
+	}
+
+	// use the client to create a new transport
+	c := NewTestTransportController(client)
+	tp, err := c.NewTransport(pubKeyID, privkey)
+	if err != nil {
+		panic(err)
+	}
+
+	// trigger the delivery function, which will trigger the 'do' function of the recorder above
+	if _, err := tp.Dereference(context.Background(), destination); err != nil {
+		panic(err)
+	}
+
+	// headers should now be populated
+	return
+}
+
+func newPerson(
+	profileIDURI *url.URL,
+	followingURI *url.URL,
+	followersURI *url.URL,
+	inboxURI *url.URL,
+	outboxURI *url.URL,
+	featuredURI *url.URL,
+	username string,
+	displayName string,
+	note string,
+	profileURL *url.URL,
+	discoverable bool,
+	publicKeyURI *url.URL,
+	pkey *rsa.PublicKey,
+	avatarURL *url.URL,
+	avatarContentType string,
+	headerURL *url.URL,
+	headerContentType string) typeutils.Accountable {
+	person := streams.NewActivityStreamsPerson()
+
+	// id should be the activitypub URI of this user
+	// something like https://example.org/users/example_user
+	idProp := streams.NewJSONLDIdProperty()
+	idProp.SetIRI(profileIDURI)
+	person.SetJSONLDId(idProp)
+
+	// following
+	// The URI for retrieving a list of accounts this user is following
+	followingProp := streams.NewActivityStreamsFollowingProperty()
+	followingProp.SetIRI(followingURI)
+	person.SetActivityStreamsFollowing(followingProp)
+
+	// followers
+	// The URI for retrieving a list of this user's followers
+	followersProp := streams.NewActivityStreamsFollowersProperty()
+	followersProp.SetIRI(followersURI)
+	person.SetActivityStreamsFollowers(followersProp)
+
+	// inbox
+	// the activitypub inbox of this user for accepting messages
+	inboxProp := streams.NewActivityStreamsInboxProperty()
+	inboxProp.SetIRI(inboxURI)
+	person.SetActivityStreamsInbox(inboxProp)
+
+	// outbox
+	// the activitypub outbox of this user for serving messages
+	outboxProp := streams.NewActivityStreamsOutboxProperty()
+	outboxProp.SetIRI(outboxURI)
+	person.SetActivityStreamsOutbox(outboxProp)
+
+	// featured posts
+	// Pinned posts.
+	featuredProp := streams.NewTootFeaturedProperty()
+	featuredProp.SetIRI(featuredURI)
+	person.SetTootFeatured(featuredProp)
+
+	// featuredTags
+	// NOT IMPLEMENTED
+
+	// preferredUsername
+	// Used for Webfinger lookup. Must be unique on the domain, and must correspond to a Webfinger acct: URI.
+	preferredUsernameProp := streams.NewActivityStreamsPreferredUsernameProperty()
+	preferredUsernameProp.SetXMLSchemaString(username)
+	person.SetActivityStreamsPreferredUsername(preferredUsernameProp)
+
+	// name
+	// Used as profile display name.
+	nameProp := streams.NewActivityStreamsNameProperty()
+	if displayName != "" {
+		nameProp.AppendXMLSchemaString(displayName)
+	} else {
+		nameProp.AppendXMLSchemaString(username)
+	}
+	person.SetActivityStreamsName(nameProp)
+
+	// summary
+	// Used as profile bio.
+	if note != "" {
+		summaryProp := streams.NewActivityStreamsSummaryProperty()
+		summaryProp.AppendXMLSchemaString(note)
+		person.SetActivityStreamsSummary(summaryProp)
+	}
+
+	// url
+	// Used as profile link.
+	urlProp := streams.NewActivityStreamsUrlProperty()
+	urlProp.AppendIRI(profileURL)
+	person.SetActivityStreamsUrl(urlProp)
+
+	// manuallyApprovesFollowers
+	// Will be shown as a locked account.
+	// TODO: NOT IMPLEMENTED **YET** -- this needs to be added as an activitypub extension to https://github.com/go-fed/activity, see https://github.com/go-fed/activity/tree/master/astool
+
+	// discoverable
+	// Will be shown in the profile directory.
+	discoverableProp := streams.NewTootDiscoverableProperty()
+	discoverableProp.Set(discoverable)
+	person.SetTootDiscoverable(discoverableProp)
+
+	// devices
+	// NOT IMPLEMENTED, probably won't implement
+
+	// alsoKnownAs
+	// Required for Move activity.
+	// TODO: NOT IMPLEMENTED **YET** -- this needs to be added as an activitypub extension to https://github.com/go-fed/activity, see https://github.com/go-fed/activity/tree/master/astool
+
+	// publicKey
+	// Required for signatures.
+	publicKeyProp := streams.NewW3IDSecurityV1PublicKeyProperty()
+
+	// create the public key
+	publicKey := streams.NewW3IDSecurityV1PublicKey()
+
+	// set ID for the public key
+	publicKeyIDProp := streams.NewJSONLDIdProperty()
+	publicKeyIDProp.SetIRI(publicKeyURI)
+	publicKey.SetJSONLDId(publicKeyIDProp)
+
+	// set owner for the public key
+	publicKeyOwnerProp := streams.NewW3IDSecurityV1OwnerProperty()
+	publicKeyOwnerProp.SetIRI(profileIDURI)
+	publicKey.SetW3IDSecurityV1Owner(publicKeyOwnerProp)
+
+	// set the pem key itself
+	encodedPublicKey, err := x509.MarshalPKIXPublicKey(pkey)
+	if err != nil {
+		panic(err)
+	}
+	publicKeyBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "PUBLIC KEY",
+		Bytes: encodedPublicKey,
+	})
+	publicKeyPEMProp := streams.NewW3IDSecurityV1PublicKeyPemProperty()
+	publicKeyPEMProp.Set(string(publicKeyBytes))
+	publicKey.SetW3IDSecurityV1PublicKeyPem(publicKeyPEMProp)
+
+	// append the public key to the public key property
+	publicKeyProp.AppendW3IDSecurityV1PublicKey(publicKey)
+
+	// set the public key property on the Person
+	person.SetW3IDSecurityV1PublicKey(publicKeyProp)
+
+	// tag
+	// TODO: Any tags used in the summary of this profile
+
+	// attachment
+	// Used for profile fields.
+	// TODO: The PropertyValue type has to be added: https://schema.org/PropertyValue
+
+	// endpoints
+	// NOT IMPLEMENTED -- this is for shared inbox which we don't use
+
+	// icon
+	// Used as profile avatar.
+	iconProperty := streams.NewActivityStreamsIconProperty()
+	iconImage := streams.NewActivityStreamsImage()
+	mediaType := streams.NewActivityStreamsMediaTypeProperty()
+	mediaType.Set(avatarContentType)
+	iconImage.SetActivityStreamsMediaType(mediaType)
+	avatarURLProperty := streams.NewActivityStreamsUrlProperty()
+	avatarURLProperty.AppendIRI(avatarURL)
+	iconImage.SetActivityStreamsUrl(avatarURLProperty)
+	iconProperty.AppendActivityStreamsImage(iconImage)
+	person.SetActivityStreamsIcon(iconProperty)
+
+	// image
+	// Used as profile header.
+	headerProperty := streams.NewActivityStreamsImageProperty()
+	headerImage := streams.NewActivityStreamsImage()
+	headerMediaType := streams.NewActivityStreamsMediaTypeProperty()
+	mediaType.Set(headerContentType)
+	headerImage.SetActivityStreamsMediaType(headerMediaType)
+	headerURLProperty := streams.NewActivityStreamsUrlProperty()
+	headerURLProperty.AppendIRI(headerURL)
+	headerImage.SetActivityStreamsUrl(headerURLProperty)
+	headerProperty.AppendActivityStreamsImage(headerImage)
+
+	return person
+}
+
+// newNote returns a new activity streams note for the given parameters
+func newNote(
+	noteID *url.URL,
+	noteURL *url.URL,
+	noteContent string,
+	noteSummary string,
+	noteAttributedTo *url.URL,
+	noteTo []*url.URL,
+	noteCC []*url.URL,
+	noteSensitive bool) vocab.ActivityStreamsNote {
+
+	// create the note itself
+	note := streams.NewActivityStreamsNote()
+
+	// set id
+	if noteID != nil {
+		id := streams.NewJSONLDIdProperty()
+		id.Set(noteID)
+		note.SetJSONLDId(id)
+	}
+
+	// set noteURL
+	if noteURL != nil {
+		url := streams.NewActivityStreamsUrlProperty()
+		url.AppendIRI(noteURL)
+		note.SetActivityStreamsUrl(url)
+	}
+
+	// set noteContent
+	if noteContent != "" {
+		content := streams.NewActivityStreamsContentProperty()
+		content.AppendXMLSchemaString(noteContent)
+		note.SetActivityStreamsContent(content)
+	}
+
+	// set noteSummary (aka content warning)
+	if noteSummary != "" {
+		summary := streams.NewActivityStreamsSummaryProperty()
+		summary.AppendXMLSchemaString(noteSummary)
+		note.SetActivityStreamsSummary(summary)
+	}
+
+	// set noteAttributedTo (the url of the author of the note)
+	if noteAttributedTo != nil {
+		attributedTo := streams.NewActivityStreamsAttributedToProperty()
+		attributedTo.AppendIRI(noteAttributedTo)
+		note.SetActivityStreamsAttributedTo(attributedTo)
+	}
+
+	return note
+}
+
+// wrapNoteInCreate wraps the given activity streams note in a Create activity streams action
+func wrapNoteInCreate(createID *url.URL, createActor *url.URL, createPublished time.Time, createNote vocab.ActivityStreamsNote) vocab.ActivityStreamsCreate {
+	// create the.... create
+	create := streams.NewActivityStreamsCreate()
+
+	// set createID
+	if createID != nil {
+		id := streams.NewJSONLDIdProperty()
+		id.Set(createID)
+		create.SetJSONLDId(id)
+	}
+
+	// set createActor
+	if createActor != nil {
+		actor := streams.NewActivityStreamsActorProperty()
+		actor.AppendIRI(createActor)
+		create.SetActivityStreamsActor(actor)
+	}
+
+	// set createPublished (time)
+	if !createPublished.IsZero() {
+		published := streams.NewActivityStreamsPublishedProperty()
+		published.Set(createPublished)
+		create.SetActivityStreamsPublished(published)
+	}
+
+	// setCreateTo
+	if createNote.GetActivityStreamsTo() != nil {
+		create.SetActivityStreamsTo(createNote.GetActivityStreamsTo())
+	}
+
+	// setCreateCC
+	if createNote.GetActivityStreamsCc() != nil {
+		create.SetActivityStreamsCc(createNote.GetActivityStreamsCc())
+	}
+
+	// set createNote
+	if createNote != nil {
+		note := streams.NewActivityStreamsObjectProperty()
+		note.AppendActivityStreamsNote(createNote)
+		create.SetActivityStreamsObject(note)
+	}
+
+	return create
+}
diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go
new file mode 100644
index 000000000..f2b5b93f7
--- /dev/null
+++ b/testrig/transportcontroller.go
@@ -0,0 +1,73 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package testrig
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/go-fed/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+)
+
+// NewTestTransportController returns a test transport controller with the given http client.
+//
+// Obviously for testing purposes you should not be making actual http calls to other servers.
+// To obviate this, use the function NewMockHTTPClient in this package to return a mock http
+// client that doesn't make any remote calls but just returns whatever you tell it to.
+//
+// Unlike the other test interfaces provided in this package, you'll probably want to call this function
+// PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
+// basis.
+func NewTestTransportController(client pub.HttpClient) transport.Controller {
+	return transport.NewController(NewTestConfig(), &federation.Clock{}, client, NewTestLog())
+}
+
+// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,
+// but will always just execute the given `do` function, allowing responses to be mocked.
+//
+// If 'do' is nil, then a no-op function will be used instead, that just returns status 200.
+//
+// Note that you should never ever make ACTUAL http calls with this thing.
+func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error)) pub.HttpClient {
+	if do == nil {
+		return &mockHTTPClient{
+			do: func(req *http.Request) (*http.Response, error) {
+				r := ioutil.NopCloser(bytes.NewReader([]byte{}))
+				return &http.Response{
+					StatusCode: 200,
+					Body:       r,
+				}, nil
+			},
+		}
+	}
+	return &mockHTTPClient{
+		do: do,
+	}
+}
+
+type mockHTTPClient struct {
+	do func(req *http.Request) (*http.Response, error)
+}
+
+func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	return m.do(req)
+}
diff --git a/testrig/mastoconverter.go b/testrig/typeconverter.go
index 10bdbdc95..9d49e6c99 100644
--- a/testrig/mastoconverter.go
+++ b/testrig/typeconverter.go
@@ -20,10 +20,10 @@ package testrig
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
-// NewTestMastoConverter returned a mastotypes converter with the given db and the default test config
-func NewTestMastoConverter(db db.DB) mastotypes.Converter {
-	return mastotypes.New(NewTestConfig(), db)
+// NewTestTypeConverter returned a type converter with the given db and the default test config
+func NewTestTypeConverter(db db.DB) typeutils.TypeConverter {
+	return typeutils.NewConverter(NewTestConfig(), db)
 }
diff --git a/testrig/util.go b/testrig/util.go
index 96a979342..0fb8aa887 100644
--- a/testrig/util.go
+++ b/testrig/util.go
@@ -22,6 +22,7 @@ import (
 	"bytes"
 	"io"
 	"mime/multipart"
+	"net/url"
 	"os"
 )
 
@@ -62,3 +63,13 @@ func CreateMultipartFormData(fieldName string, fileName string, extraFields map[
 	}
 	return b, w, nil
 }
+
+// URLMustParse tries to parse the given URL and panics if it can't.
+// Should only be used in tests.
+func URLMustParse(stringURL string) *url.URL {
+	u, err := url.Parse(stringURL)
+	if err != nil {
+		panic(err)
+	}
+	return u
+}