[chore] Webfinger rework (#627)

* move finger to dereferencer

* totally break GetRemoteAccount

* start reworking finger func a bit

* start reworking getRemoteAccount a bit

* move mention parts to namestring

* rework webfingerget

* use util function to extract webfinger parts

* use accountDomain

* rework finger again, final form

* just a real nasty commit, the worst

* remove refresh from account

* use new ASRepToAccount signature

* fix incorrect debug call

* fix for new getRemoteAccount

* rework GetRemoteAccount

* start updating tests to remove repetition

* break a lot of tests
Move shared test logic into the testrig,
rather than having it scattered all over
the place. This allows us to just mock
the transport controller once, and have
all tests use it (unless they need not to
for some other reason).

* fix up tests to use main mock httpclient

* webfinger only if necessary

* cheeky linting with the lads

* update mentionName regex
recognize instance accounts

* don't finger instance accounts

* test webfinger part extraction

* increase default worker count to 4 per cpu

* don't repeat regex parsing

* final search for discovered accountDomain

* be more permissive in namestring lookup

* add more extraction tests

* simplify GetParseMentionFunc

* skip long search if local account

* fix broken test

17 files changed, 148 insertions, 312 deletions

diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
index d9ce68cc0..8ea1665ac 100644
--- a/internal/processing/account/account_test.go
+++ b/internal/processing/account/account_test.go
@@ -23,7 +23,6 @@ import (
 
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -48,7 +47,6 @@ type AccountStandardTestSuite struct {
 	mediaManager        media.Manager
 	oauthServer         oauth.Server
 	fromClientAPIChan   chan messages.FromClientAPI
-	httpClient          pub.HttpClient
 	transportController transport.Controller
 	federator           federation.Federator
 	emailSender         email.Sender
@@ -97,8 +95,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
-	suite.httpClient = testrig.NewMockHTTPClient(nil)
-	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
+	suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go
index 70c1cd9fe..d6f1aa71f 100644
--- a/internal/processing/account/get.go
+++ b/internal/processing/account/get.go
@@ -26,6 +26,7 @@ import (
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -80,7 +81,13 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod
 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err))
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername:    requestingAccount.Username,
+			RemoteAccountID:       targetAccountURI,
+			RemoteAccountHost:     targetAccount.Domain,
+			RemoteAccountUsername: targetAccount.Username,
+			Blocking:              true,
+		})
 		if err == nil {
 			targetAccount = a
 		}
diff --git a/internal/processing/account_test.go b/internal/processing/account_test.go
index f974f0b9d..3fd7d83f4 100644
--- a/internal/processing/account_test.go
+++ b/internal/processing/account_test.go
@@ -60,7 +60,7 @@ func (suite *AccountTestSuite) TestAccountDeleteLocal() {
 	time.Sleep(1 * time.Second) // wait a sec for the delete to process
 
 	// the delete should be federated outwards to the following account's inbox
-	sent, ok := suite.sentHTTPRequests[followingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[followingAccount.InboxURI]
 	suite.True(ok)
 	delete := &struct {
 		Actor  string `json:"actor"`
diff --git a/internal/processing/federation/getfollowers.go b/internal/processing/federation/getfollowers.go
index 6c54dbd50..30ec1c58f 100644
--- a/internal/processing/federation/getfollowers.go
+++ b/internal/processing/federation/getfollowers.go
@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}
diff --git a/internal/processing/federation/getfollowing.go b/internal/processing/federation/getfollowing.go
index 6b1afae92..13291939a 100644
--- a/internal/processing/federation/getfollowing.go
+++ b/internal/processing/federation/getfollowing.go
@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}
diff --git a/internal/processing/federation/getoutbox.go b/internal/processing/federation/getoutbox.go
index 4e428c1ae..b2e618b98 100644
--- a/internal/processing/federation/getoutbox.go
+++ b/internal/processing/federation/getoutbox.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -41,7 +42,10 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}
diff --git a/internal/processing/federation/getstatus.go b/internal/processing/federation/getstatus.go
index ef77d6c4f..f61c87d1d 100644
--- a/internal/processing/federation/getstatus.go
+++ b/internal/processing/federation/getstatus.go
@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}
diff --git a/internal/processing/federation/getstatusreplies.go b/internal/processing/federation/getstatusreplies.go
index 3a2c9d944..fad986ea5 100644
--- a/internal/processing/federation/getstatusreplies.go
+++ b/internal/processing/federation/getstatusreplies.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -42,7 +43,10 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}
diff --git a/internal/processing/federation/getuser.go b/internal/processing/federation/getuser.go
index 63c83f3c5..372a3a57b 100644
--- a/internal/processing/federation/getuser.go
+++ b/internal/processing/federation/getuser.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
@@ -52,7 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque
 
 		// if we're not already handshaking/dereferencing a remote account, dereference it now
 		if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) {
-			requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+			requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+				RequestingUsername: requestedUsername,
+				RemoteAccountID:    requestingAccountURI,
+			})
 			if err != nil {
 				return nil, gtserror.NewErrorUnauthorized(err)
 			}
diff --git a/internal/processing/followrequest_test.go b/internal/processing/followrequest_test.go
index 957e501fc..41e4f686e 100644
--- a/internal/processing/followrequest_test.go
+++ b/internal/processing/followrequest_test.go
@@ -57,7 +57,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestAccept() {
 	time.Sleep(1 * time.Second)
 
 	// accept should be sent to some_user
-	sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
 	suite.True(ok)
 
 	accept := &struct {
@@ -109,7 +109,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestReject() {
 	time.Sleep(1 * time.Second)
 
 	// reject should be sent to some_user
-	sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
 	suite.True(ok)
 
 	reject := &struct {
diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go
index a2eaa9bd1..60f5cc787 100644
--- a/internal/processing/fromfederator.go
+++ b/internal/processing/fromfederator.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
@@ -130,7 +131,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -172,7 +177,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -210,7 +219,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -267,7 +280,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -332,7 +349,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder
 		return err
 	}
 
-	if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil {
+	if _, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: federatorMsg.ReceivingAccount.Username,
+		RemoteAccountID:    incomingAccountURL,
+		Blocking:           true,
+	}); err != nil {
 		return fmt.Errorf("error enriching updated account from federator: %s", err)
 	}
 
diff --git a/internal/processing/fromfederator_test.go b/internal/processing/fromfederator_test.go
index 6028fd065..4d339cb18 100644
--- a/internal/processing/fromfederator_test.go
+++ b/internal/processing/fromfederator_test.go
@@ -423,7 +423,7 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestLocked() {
 	suite.Equal(originAccount.ID, notif.Account.ID)
 
 	// no messages should have been sent out, since we didn't need to federate an accept
-	suite.Empty(suite.sentHTTPRequests)
+	suite.Empty(suite.httpClient.SentMessages)
 }
 
 func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
@@ -474,8 +474,8 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
 	suite.Equal(originAccount.ID, notif.Account.ID)
 
 	// an accept message should be sent to satan's inbox
-	suite.Len(suite.sentHTTPRequests, 1)
-	acceptBytes := suite.sentHTTPRequests[originAccount.InboxURI]
+	suite.Len(suite.httpClient.SentMessages, 1)
+	acceptBytes := suite.httpClient.SentMessages[originAccount.InboxURI]
 	accept := &struct {
 		Actor  string `json:"actor"`
 		ID     string `json:"id"`
diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go
index 1149f2646..c8c8736be 100644
--- a/internal/processing/media/media_test.go
+++ b/internal/processing/media/media_test.go
@@ -19,12 +19,7 @@
 package media_test
 
 import (
-	"bytes"
-	"io"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -79,7 +74,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.transportController = suite.mockTransportController()
+	suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1))
 	suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
@@ -89,40 +84,3 @@ func (suite *MediaStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
 }
-
-func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
-	do := func(req *http.Request) (*http.Response, error) {
-		logrus.Debugf("received request for %s", req.URL)
-
-		responseBytes := []byte{}
-		responseType := ""
-		responseLength := 0
-
-		if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
-			responseBytes = attachment.Data
-			responseType = attachment.ContentType
-		}
-
-		if len(responseBytes) != 0 {
-			// we found something, so print what we're going to return
-			logrus.Debugf("returning response %s", string(responseBytes))
-		}
-		responseLength = len(responseBytes)
-
-		reader := bytes.NewReader(responseBytes)
-		readCloser := io.NopCloser(reader)
-		response := &http.Response{
-			StatusCode:    200,
-			Body:          readCloser,
-			ContentLength: int64(responseLength),
-			Header: http.Header{
-				"content-type": {responseType},
-			},
-		}
-
-		return response, nil
-	}
-	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	mockClient := testrig.NewMockHTTPClient(do)
-	return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
-}
diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
index 7032f8e28..c60ff5c97 100644
--- a/internal/processing/processor_test.go
+++ b/internal/processing/processor_test.go
@@ -19,16 +19,8 @@
 package processing_test
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -50,6 +42,7 @@ type ProcessingStandardTestSuite struct {
 	storage             *kv.KVStore
 	mediaManager        media.Manager
 	typeconverter       typeutils.TypeConverter
+	httpClient          *testrig.MockHTTPClient
 	transportController transport.Controller
 	federator           federation.Federator
 	oauthServer         oauth.Server
@@ -69,8 +62,6 @@ type ProcessingStandardTestSuite struct {
 	testBlocks       map[string]*gtsmodel.Block
 	testActivities   map[string]testrig.ActivityWithSignature
 
-	sentHTTPRequests map[string][]byte
-
 	processor processing.Processor
 }
 
@@ -102,123 +93,12 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 	suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
 	suite.storage = testrig.NewTestStorage()
 	suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
-
-	// make an http client that stores POST requests it receives into a map,
-	// and also responds to correctly to dereference requests
-	suite.sentHTTPRequests = make(map[string][]byte)
-	httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		if req.Method == http.MethodPost && req.Body != nil {
-			requestBytes, err := ioutil.ReadAll(req.Body)
-			if err != nil {
-				panic(err)
-			}
-			if err := req.Body.Close(); err != nil {
-				panic(err)
-			}
-			suite.sentHTTPRequests[req.URL.String()] = requestBytes
-		}
-
-		if req.URL.String() == suite.testAccounts["remote_account_1"].URI {
-			// the request is for remote account 1
-			satan := suite.testAccounts["remote_account_1"]
-
-			satanAS, err := suite.typeconverter.AccountToAS(context.Background(), satan)
-			if err != nil {
-				panic(err)
-			}
-
-			satanI, err := streams.Serialize(satanAS)
-			if err != nil {
-				panic(err)
-			}
-			satanJson, err := json.Marshal(satanI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(satanJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(satanJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		if req.URL.String() == suite.testAccounts["remote_account_2"].URI {
-			// the request is for remote account 2
-			someAccount := suite.testAccounts["remote_account_2"]
-
-			someAccountAS, err := suite.typeconverter.AccountToAS(context.Background(), someAccount)
-			if err != nil {
-				panic(err)
-			}
-
-			someAccountI, err := streams.Serialize(someAccountAS)
-			if err != nil {
-				panic(err)
-			}
-			someAccountJson, err := json.Marshal(someAccountI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(someAccountJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(someAccountJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		if req.URL.String() == "http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1" {
-			// the request is for the forwarded message
-			message := suite.testActivities["forwarded_message"].Activity.GetActivityStreamsObject().At(0).GetActivityStreamsNote()
-			messageI, err := streams.Serialize(message)
-			if err != nil {
-				panic(err)
-			}
-			messageJson, err := json.Marshal(messageI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(messageJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(messageJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	})
+	suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")
 
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
+	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
diff --git a/internal/processing/search.go b/internal/processing/search.go
index 1b0813fd9..d4ff94a21 100644
--- a/internal/processing/search.go
+++ b/internal/processing/search.go
@@ -20,6 +20,7 @@ package processing
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -28,53 +29,70 @@ import (
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
+func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
 	l := logrus.WithFields(logrus.Fields{
 		"func":  "SearchGet",
-		"query": searchQuery.Query,
+		"query": search.Query,
 	})
 
-	results := &apimodel.SearchResult{
+	// tidy up the query and make sure it wasn't just spaces
+	query := strings.TrimSpace(search.Query)
+	if query == "" {
+		err := errors.New("search query was empty string after trimming space")
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	searchResult := &apimodel.SearchResult{
 		Accounts: []apimodel.Account{},
 		Statuses: []apimodel.Status{},
 		Hashtags: []apimodel.Tag{},
 	}
 	foundAccounts := []*gtsmodel.Account{}
 	foundStatuses := []*gtsmodel.Status{}
-	// foundHashtags := []*gtsmodel.Tag{}
-
-	// convert the query to lowercase and trim leading/trailing spaces
-	query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
 
 	var foundOne bool
-	// check if the query is something like @whatever_username@example.org -- this means it's a remote account
-	if _, domain, err := util.ExtractMentionParts(searchQuery.Query); err == nil && domain != "" {
-		l.Debug("search term is a mention, looking it up...")
-		foundAccount, err := p.searchAccountByMention(ctx, authed, searchQuery.Query, searchQuery.Resolve)
-		if err == nil && foundAccount != nil {
+
+	/*
+		SEARCH BY MENTION
+		check if the query is something like @whatever_username@example.org -- this means it's a remote account
+	*/
+	maybeNamestring := query
+	if maybeNamestring[0] != '@' {
+		maybeNamestring = "@" + maybeNamestring
+	}
+
+	if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil {
+		l.Debugf("search term %s is a mention, looking it up...", maybeNamestring)
+		if foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve); err == nil && foundAccount != nil {
 			foundAccounts = append(foundAccounts, foundAccount)
 			foundOne = true
 			l.Debug("got an account by searching by mention")
+		} else if err != nil {
+			l.Debugf("error looking up account %s: %s", maybeNamestring, err)
 		}
 	}
 
-	// check if the query is a URI with a recognizable scheme and just do a lookup for that, straight up
+	/*
+		SEARCH BY URI
+		check if the query is a URI with a recognizable scheme and dereference it
+	*/
 	if !foundOne {
 		if uri, err := url.Parse(query); err == nil && (uri.Scheme == "https" || uri.Scheme == "http") {
 			// 1. check if it's a status
-			if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundStatus != nil {
+			if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, search.Resolve); err == nil && foundStatus != nil {
 				foundStatuses = append(foundStatuses, foundStatus)
 				l.Debug("got a status by searching by URI")
 			}
 
 			// 2. check if it's an account
-			if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundAccount != nil {
+			if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, search.Resolve); err == nil && foundAccount != nil {
 				foundAccounts = append(foundAccounts, foundAccount)
 				l.Debug("got an account by searching by URI")
 			}
@@ -90,7 +108,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
 		if blocked, err := p.db.IsBlocked(ctx, authed.Account.ID, foundAccount.ID, true); err == nil && !blocked {
 			// all good, convert it and add it to the results
 			if apiAcct, err := p.tc.AccountToAPIAccountPublic(ctx, foundAccount); err == nil && apiAcct != nil {
-				results.Accounts = append(results.Accounts, *apiAcct)
+				searchResult.Accounts = append(searchResult.Accounts, *apiAcct)
 			}
 		}
 	}
@@ -105,10 +123,10 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
 			continue
 		}
 
-		results.Statuses = append(results.Statuses, *apiStatus)
+		searchResult.Statuses = append(searchResult.Statuses, *apiStatus)
 	}
 
-	return results, nil
+	return searchResult, nil
 }
 
 func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
@@ -147,7 +165,10 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
 
 	if resolve {
 		// we don't have it locally so try and dereference it
-		account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true)
+		account, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: authed.Account.Username,
+			RemoteAccountID:    uri,
+		})
 		if err != nil {
 			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
 		}
@@ -156,18 +177,14 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
 	return nil, nil
 }
 
-func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, mention string, resolve bool) (*gtsmodel.Account, error) {
-	// query is for a remote account
-	username, domain, err := util.ExtractMentionParts(mention)
-	if err != nil {
-		return nil, fmt.Errorf("searchAccountByMention: error extracting mention parts: %s", err)
-	}
+func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
+	maybeAcct := &gtsmodel.Account{}
+	var err error
 
 	// if it's a local account we can skip a whole bunch of stuff
-	maybeAcct := &gtsmodel.Account{}
-	if domain == config.GetHost() {
+	if domain == config.GetHost() || domain == config.GetAccountDomain() || domain == "" {
 		maybeAcct, err = p.db.GetLocalAccountByUsername(ctx, username)
-		if err != nil {
+		if err != nil && err != db.ErrNoEntries {
 			return nil, fmt.Errorf("searchAccountByMention: error getting local account by username: %s", err)
 		}
 		return maybeAcct, nil
@@ -191,22 +208,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
 
 	// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
 	if resolve {
-		// we're allowed to resolve it so let's try
-		// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
-		acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain)
+		maybeAcct, err = p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername:    authed.Account.Username,
+			RemoteAccountUsername: username,
+			RemoteAccountHost:     domain,
+		})
 		if err != nil {
-			// something went wrong doing the webfinger lookup so we can't process the request
-			return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
-		}
-
-		if acctURI.Scheme == "https" || acctURI.Scheme == "http" {
-			acct, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
-			if err != nil {
-				logrus.Debugf("could not get remote account by mention %s with uri %s: %s", mention, acctURI, err)
-				return nil, err
-			}
-			return acct, nil
+			return nil, fmt.Errorf("searchAccountByMention: error getting remote account: %s", err)
 		}
+		return maybeAcct, nil
 	}
 
 	return nil, nil
diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go
index 17c68c0b6..f8834e2c2 100644
--- a/internal/processing/status/status_test.go
+++ b/internal/processing/status/status_test.go
@@ -80,7 +80,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
 	suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
-	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)
diff --git a/internal/processing/util.go b/internal/processing/util.go
index 7ae7b8604..76d1187f9 100644
--- a/internal/processing/util.go
+++ b/internal/processing/util.go
@@ -21,12 +21,14 @@ package processing
 import (
 	"context"
 	"fmt"
-	"strings"
 
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc {
@@ -37,98 +39,35 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.
 			return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID)
 		}
 
-		// A mentioned account looks like "@test@example.org" or just "@test" for a local account
-		// -- we can guarantee this from the regex that targetAccounts should have been derived from.
-		// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
-
-		// 1.  trim off the first @
-		trimmed := strings.TrimPrefix(targetAccount, "@")
-
-		// 2. split the username and domain
-		split := strings.Split(trimmed, "@")
-
-		// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
-
-		var local bool
-		switch len(split) {
-		case 1:
-			local = true
-		case 2:
-			local = false
-		default:
-			return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount)
-		}
-
-		var username, domain string
-		username = split[0]
-		if !local {
-			domain = split[1]
-		}
-
-		// 4. check we now have a proper username and domain
-		if username == "" || (!local && domain == "") {
-			return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount)
+		username, domain, err := util.ExtractNamestringParts(targetAccount)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't extract namestring parts from %s: %s", targetAccount, err)
 		}
 
 		var mentionedAccount *gtsmodel.Account
-
-		if local {
+		if domain == "" || domain == config.GetHost() || domain == config.GetAccountDomain() {
 			localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username)
 			if err != nil {
 				return nil, err
 			}
 			mentionedAccount = localAccount
 		} else {
-			remoteAccount := &gtsmodel.Account{}
-
-			where := []db.Where{
-				{
-					Key:             "username",
-					Value:           username,
-					CaseInsensitive: true,
-				},
-				{
-					Key:             "domain",
-					Value:           domain,
-					CaseInsensitive: true,
-				},
+			var requestingUsername string
+			if originAccount.Domain == "" {
+				requestingUsername = originAccount.Username
+			}
+			remoteAccount, err := federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+				RequestingUsername:    requestingUsername,
+				RemoteAccountUsername: username,
+				RemoteAccountHost:     domain,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error dereferencing account: %s", err)
 			}
 
-			err := dbConn.GetWhere(ctx, where, remoteAccount)
-			if err == nil {
-				// the account was already in the database
-				mentionedAccount = remoteAccount
-			} else {
-				// we couldn't get it from the database
-				if err != db.ErrNoEntries {
-					// a serious error has happened so bail
-					return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
-				}
-
-				// We just don't have the account, so try webfingering it.
-				//
-				// If the mention originates from our instance we should use the username of the origin account to do the dereferencing,
-				// otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use
-				// a remote account to do remote dereferencing!
-				var fingeringUsername string
-				if originAccount.Domain == "" {
-					fingeringUsername = originAccount.Username
-				}
-
-				acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain)
-				if err != nil {
-					// something went wrong doing the webfinger lookup so we can't process the request
-					return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
-				}
-
-				resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true)
-				if err != nil {
-					return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err)
-				}
+			// we were able to resolve it!
+			mentionedAccount = remoteAccount
 
-				// we were able to resolve it!
-				mentionedAccount = resolvedAccount
-			}
 		}
 
 		mentionID, err := id.NewRandomULID()