Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt

18 files changed, 1312 insertions, 692 deletions

diff --git a/internal/federation/authenticate.go b/internal/federation/authenticate.go
index 0cb8db6dc..699691ca6 100644
--- a/internal/federation/authenticate.go
+++ b/internal/federation/authenticate.go
@@ -147,6 +147,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 	if strings.EqualFold(requestingHost, f.config.Host) {
 		// LOCAL ACCOUNT REQUEST
 		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
+		l.Tracef("proceeding without dereference for local public key %s", requestingPublicKeyID)
 		if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil {
 			return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err)
 		}
@@ -158,6 +159,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 	} else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil {
 		// REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY
 		// this is a remote account and we already have the public key for it so use that
+		l.Tracef("proceeding without dereference for cached public key %s", requestingPublicKeyID)
 		publicKey = requestingRemoteAccount.PublicKey
 		pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI)
 		if err != nil {
@@ -167,7 +169,8 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 		// REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY
 		// the request is remote and we don't have the public key yet,
 		// so we need to authenticate the request properly by dereferencing the remote key
-		transport, err := f.GetTransportForUser(requestedUsername)
+		l.Tracef("proceeding with dereference for uncached public key %s", requestingPublicKeyID)
+		transport, err := f.transportController.NewTransportForUsername(requestedUsername)
 		if err != nil {
 			return nil, false, fmt.Errorf("transport err: %s", err)
 		}
@@ -209,15 +212,28 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 		}
 		pkOwnerURI = pkOwnerProp.GetIRI()
 	}
+
+	// after all that, public key should be defined
 	if publicKey == nil {
 		return nil, false, errors.New("returned public key was empty")
 	}
 
 	// do the actual authentication here!
-	algo := httpsig.RSA_SHA256 // TODO: make this more robust
-	if err := verifier.Verify(publicKey, algo); err != nil {
-		return nil, false, nil
+	algos := []httpsig.Algorithm{
+		httpsig.RSA_SHA512,
+		httpsig.RSA_SHA256,
+		httpsig.ED25519,
+	}
+
+	for _, algo := range algos {
+		l.Tracef("trying algo: %s", algo)
+		if err := verifier.Verify(publicKey, algo); err == nil {
+			l.Tracef("authentication for %s PASSED with algorithm %s", pkOwnerURI, algo)
+			return pkOwnerURI, true, nil
+		}
+		l.Tracef("authentication for %s NOT PASSED with algorithm %s: %s", pkOwnerURI, algo, err)
 	}
 
-	return pkOwnerURI, true, nil
+	l.Infof("authentication not passed for %s", pkOwnerURI)
+	return nil, false, nil
 }
diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go
index b87462acd..8975d6c0c 100644
--- a/internal/federation/dereference.go
+++ b/internal/federation/dereference.go
@@ -1,526 +1,32 @@
 package federation
 
 import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
 	"net/url"
 
-	"github.com/go-fed/activity/streams"
-	"github.com/go-fed/activity/streams/vocab"
-	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
-func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) {
-	f.startHandshake(username, remoteAccountID)
-	defer f.stopHandshake(username, remoteAccountID)
-
-	if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil {
-		return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	b, err := transport.Dereference(context.Background(), remoteAccountID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err)
-	}
-
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	switch t.GetTypeName() {
-	case string(gtsmodel.ActivityStreamsPerson):
-		p, ok := t.(vocab.ActivityStreamsPerson)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams person")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsApplication):
-		p, ok := t.(vocab.ActivityStreamsApplication)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams application")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsService):
-		p, ok := t.(vocab.ActivityStreamsService)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams service")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
+func (f *federator) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
+	return f.dereferencer.GetRemoteAccount(username, remoteAccountID, refresh)
 }
 
-func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
-	if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil {
-		return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	b, err := transport.Dereference(context.Background(), remoteStatusID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err)
-	}
-
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
-	switch t.GetTypeName() {
-	case gtsmodel.ActivityStreamsArticle:
-		p, ok := t.(vocab.ActivityStreamsArticle)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsArticle")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsDocument:
-		p, ok := t.(vocab.ActivityStreamsDocument)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsDocument")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsImage:
-		p, ok := t.(vocab.ActivityStreamsImage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsImage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsVideo:
-		p, ok := t.(vocab.ActivityStreamsVideo)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsVideo")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsNote:
-		p, ok := t.(vocab.ActivityStreamsNote)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsNote")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPage:
-		p, ok := t.(vocab.ActivityStreamsPage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsEvent:
-		p, ok := t.(vocab.ActivityStreamsEvent)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsEvent")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPlace:
-		p, ok := t.(vocab.ActivityStreamsPlace)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPlace")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsProfile:
-		p, ok := t.(vocab.ActivityStreamsProfile)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsProfile")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
+func (f *federator) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
+	return f.dereferencer.GetRemoteStatus(username, remoteStatusID, refresh)
 }
 
-func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
-	if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil {
-		return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
+func (f *federator) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) {
+	return f.dereferencer.EnrichRemoteStatus(username, status)
 }
 
-// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
-// federated status, back in the federating db's Create function.
-//
-// When a status comes in from the federation API, there are certain fields that
-// haven't been dereferenced yet, because we needed to provide a snappy synchronous
-// response to the caller. By the time it reaches this function though, it's being
-// processed asynchronously, so we have all the time in the world to fetch the various
-// bits and bobs that are attached to the status, and properly flesh it out, before we
-// send the status to any timelines and notify people.
-//
-// Things to dereference and fetch here:
-//
-// 1. Media attachments.
-// 2. Hashtags.
-// 3. Emojis.
-// 4. Mentions.
-// 5. Posting account.
-// 6. Replied-to-status.
-//
-// SIDE EFFECTS:
-// This function will deference all of the above, insert them in the database as necessary,
-// and attach them to the status. The status itself will not be added to the database yet,
-// that's up the caller to do.
-func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
-	l := f.log.WithFields(logrus.Fields{
-		"func":   "dereferenceStatusFields",
-		"status": fmt.Sprintf("%+v", status),
-	})
-	l.Debug("entering function")
-
-	statusURI, err := url.Parse(status.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
-	}
-	if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
-	}
-
-	t, err := f.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error creating transport: %s", err)
-	}
-
-	// the status should have an ID by now, but just in case it doesn't let's generate one here
-	// because we'll need it further down
-	if status.ID == "" {
-		newID, err := id.NewULIDFromTime(status.CreatedAt)
-		if err != nil {
-			return err
-		}
-		status.ID = newID
-	}
-
-	// 1. Media attachments.
-	//
-	// At this point we should know:
-	// * the media type of the file we're looking for (a.File.ContentType)
-	// * the blurhash (a.Blurhash)
-	// * the file type (a.Type)
-	// * the remote URL (a.RemoteURL)
-	// This should be enough to pass along to the media processor.
-	attachmentIDs := []string{}
-	for _, a := range status.GTSMediaAttachments {
-		l.Debugf("dereferencing attachment: %+v", a)
-
-		// it might have been processed elsewhere so check first if it's already in the database or not
-		maybeAttachment := &gtsmodel.MediaAttachment{}
-		err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
-		if err == nil {
-			// we already have it in the db, dereferenced, no need to do it again
-			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
-			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
-			continue
-		}
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			// we have a real error
-			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		// it just doesn't exist yet so carry on
-		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
-		deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
-		if err != nil {
-			l.Errorf("error dereferencing status attachment: %s", err)
-			continue
-		}
-		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
-		deferencedAttachment.StatusID = status.ID
-		deferencedAttachment.Description = a.Description
-		if err := f.db.Put(deferencedAttachment); err != nil {
-			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
-	}
-	status.Attachments = attachmentIDs
-
-	// 2. Hashtags
-
-	// 3. Emojis
-
-	// 4. Mentions
-	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
-	//
-	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
-	mentions := []string{}
-	for _, m := range status.GTSMentions {
-		if m.ID == "" {
-			mID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			m.ID = mID
-		}
-
-		uri, err := url.Parse(m.MentionedAccountURI)
-		if err != nil {
-			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
-			continue
-		}
-
-		m.StatusID = status.ID
-		m.OriginAccountID = status.GTSAuthorAccount.ID
-		m.OriginAccountURI = status.GTSAuthorAccount.URI
-
-		targetAccount := &gtsmodel.Account{}
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
-			// proper error
-			if _, ok := err.(db.ErrNoEntries); !ok {
-				return fmt.Errorf("db error checking for account with uri %s", uri.String())
-			}
-
-			// we just don't have it yet, so we should go get it....
-			accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri)
-			if err != nil {
-				// we can't dereference it so just skip it
-				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false)
-			if err != nil {
-				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccountID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			targetAccount.ID = targetAccountID
-
-			if err := f.db.Put(targetAccount); err != nil {
-				return fmt.Errorf("db error inserting account with uri %s", uri.String())
-			}
-		}
-
-		// by this point, we know the targetAccount exists in our database with an ID :)
-		m.TargetAccountID = targetAccount.ID
-		if err := f.db.Put(m); err != nil {
-			return fmt.Errorf("error creating mention: %s", err)
-		}
-		mentions = append(mentions, m.ID)
-	}
-	status.Mentions = mentions
-
-	return nil
+func (f *federator) DereferenceRemoteThread(username string, statusIRI *url.URL) error {
+	return f.dereferencer.DereferenceThread(username, statusIRI)
 }
 
-func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
-	l := f.log.WithFields(logrus.Fields{
-		"func":               "dereferenceAccountFields",
-		"requestingUsername": requestingUsername,
-	})
-
-	accountURI, err := url.Parse(account.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err)
-	}
-	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host)
-	}
-
-	t, err := f.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error getting transport for user: %s", err)
-	}
-
-	// fetch the header and avatar
-	if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
-		// if this doesn't work, just skip it -- we can do it later
-		l.Debugf("error fetching header/avi for account: %s", err)
-	}
-
-	if err := f.db.UpdateByID(account.ID, account); err != nil {
-		return fmt.Errorf("error updating account in database: %s", err)
-	}
-
-	return nil
+func (f *federator) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+	return f.dereferencer.GetRemoteInstance(username, remoteInstanceURI)
 }
 
 func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
-	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
-		// we can't do anything unfortunately
-		return errors.New("DereferenceAnnounce: no URI to dereference")
-	}
-
-	boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-	if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host)
-	}
-
-	// check if we already have the boosted status in the database
-	boostedStatus := &gtsmodel.Status{}
-	err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
-	if err == nil {
-		// nice, we already have it so we don't actually need to dereference it from remote
-		announce.Content = boostedStatus.Content
-		announce.ContentWarning = boostedStatus.ContentWarning
-		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-		announce.Sensitive = boostedStatus.Sensitive
-		announce.Language = boostedStatus.Language
-		announce.Text = boostedStatus.Text
-		announce.BoostOfID = boostedStatus.ID
-		announce.BoostOfAccountID = boostedStatus.AccountID
-		announce.Visibility = boostedStatus.Visibility
-		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-		announce.GTSBoostedStatus = boostedStatus
-		return nil
-	}
-
-	// we don't have it so we need to dereference it
-	statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// make sure we have the author account in the db
-	attributedToProp := statusable.GetActivityStreamsAttributedTo()
-	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
-		accountURI := iter.GetIRI()
-		if accountURI == nil {
-			continue
-		}
-
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
-			// we already have it, fine
-			continue
-		}
-
-		// we don't have the boosted status author account yet so dereference it
-		accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
-		}
-		account, err := f.typeConverter.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
-		}
-
-		accountID, err := id.NewRandomULID()
-		if err != nil {
-			return err
-		}
-		account.ID = accountID
-
-		if err := f.db.Put(account); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
-		}
-
-		if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
-		}
-	}
-
-	// now convert the statusable into something we can understand
-	boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
-	if err != nil {
-		return nil
-	}
-	boostedStatus.ID = boostedStatusID
-
-	if err := f.db.Put(boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// now dereference additional fields straight away (we're already async here so we have time)
-	if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// update with the newly dereferenced fields
-	if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
-	}
-
-	// we have everything we need!
-	announce.Content = boostedStatus.Content
-	announce.ContentWarning = boostedStatus.ContentWarning
-	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-	announce.Sensitive = boostedStatus.Sensitive
-	announce.Language = boostedStatus.Language
-	announce.Text = boostedStatus.Text
-	announce.BoostOfID = boostedStatus.ID
-	announce.BoostOfAccountID = boostedStatus.AccountID
-	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-	announce.GTSBoostedStatus = boostedStatus
-	return nil
-}
-
-// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
-// on behalf of requestingUsername.
-//
-// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
-//
-// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
-// to reflect the creation of these new attachments.
-func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
-	accountURI, err := url.Parse(targetAccount.URI)
-	if err != nil {
-		return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
-	}
-	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
-		return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
-	}
-
-	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
-		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.AvatarRemoteURL,
-			Avatar:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing avatar for user: %s", err)
-		}
-		targetAccount.AvatarMediaAttachmentID = a.ID
-	}
-
-	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
-		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.HeaderRemoteURL,
-			Header:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing header for user: %s", err)
-		}
-		targetAccount.HeaderMediaAttachmentID = a.ID
-	}
-	return nil
+	return f.dereferencer.DereferenceAnnounce(announce, requestingUsername)
 }
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
new file mode 100644
index 000000000..c403ec66f
--- /dev/null
+++ b/internal/federation/dereferencing/account.go
@@ -0,0 +1,243 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+)
+
+// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form,
+// and populates it with additional fields, media, etc.
+//
+// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by
+// the federatingDB's Create function, or during the federated authorization flow.
+func (d *deref) EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) {
+	if err := d.populateAccountFields(account, username, false); err != nil {
+		return nil, err
+	}
+
+	if err := d.db.UpdateByID(account.ID, account); err != nil {
+		return nil, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err)
+	}
+
+	return account, nil
+}
+
+// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
+// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new
+// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before,
+// it will be false.
+//
+// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling
+// the remote instance again.
+//
+// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true).
+func (d *deref) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
+	new := true
+
+	// check if we already have the account in our db
+	maybeAccount := &gtsmodel.Account{}
+	if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteAccountID.String()}}, maybeAccount); err == nil {
+		// we've seen this account before so it's not new
+		new = false
+
+		// if we're not being asked to refresh, we can just return the maybeAccount as-is and avoid doing any external calls
+		if !refresh {
+			return maybeAccount, new, nil
+		}
+	}
+
+	accountable, err := d.dereferenceAccountable(username, remoteAccountID)
+	if err != nil {
+		return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err)
+	}
+
+	gtsAccount, err := d.typeConverter.ASRepresentationToAccount(accountable, false)
+	if err != nil {
+		return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err)
+	}
+
+	if new {
+		// generate a new id since we haven't seen this account before, and do a put
+		ulid, err := id.NewRandomULID()
+		if err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err)
+		}
+		gtsAccount.ID = ulid
+
+		if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
+		}
+
+		if err := d.db.Put(gtsAccount); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err)
+		}
+	} else {
+		// take the id we already have and do an update
+		gtsAccount.ID = maybeAccount.ID
+
+		if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
+		}
+
+		if err := d.db.UpdateByID(gtsAccount.ID, gtsAccount); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error updating existing account: %s", err)
+		}
+	}
+
+	return gtsAccount, new, nil
+}
+
+// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
+// it finds as something that an account model can be constructed out of.
+//
+// Will work for Person, Application, or Service models.
+func (d *deref) dereferenceAccountable(username string, remoteAccountID *url.URL) (ap.Accountable, error) {
+	d.startHandshake(username, remoteAccountID)
+	defer d.stopHandshake(username, remoteAccountID)
+
+	if blocked, err := d.blockedDomain(remoteAccountID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteAccountID)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %s", remoteAccountID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %s", err)
+	}
+
+	switch t.GetTypeName() {
+	case string(gtsmodel.ActivityStreamsPerson):
+		p, ok := t.(vocab.ActivityStreamsPerson)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsApplication):
+		p, ok := t.(vocab.ActivityStreamsApplication)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsService):
+		p, ok := t.(vocab.ActivityStreamsService)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName())
+}
+
+// populateAccountFields populates any fields on the given account that weren't populated by the initial
+// dereferencing. This includes things like header and avatar etc.
+func (d *deref) populateAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":               "PopulateAccountFields",
+		"requestingUsername": requestingUsername,
+	})
+
+	accountURI, err := url.Parse(account.URI)
+	if err != nil {
+		return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err)
+	}
+	if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host)
+	}
+
+	t, err := d.transportController.NewTransportForUsername(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err)
+	}
+
+	// fetch the header and avatar
+	if err := d.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
+		// if this doesn't work, just skip it -- we can do it later
+		l.Debugf("error fetching header/avi for account: %s", err)
+	}
+
+	return nil
+}
+
+// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
+// on behalf of requestingUsername.
+//
+// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
+//
+// SIDE EFFECTS: remote header and avatar will be stored in local storage.
+func (d *deref) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
+	accountURI, err := url.Parse(targetAccount.URI)
+	if err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
+	}
+	if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
+	}
+
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+		a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.AvatarRemoteURL,
+			Avatar:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing avatar for user: %s", err)
+		}
+		targetAccount.AvatarMediaAttachmentID = a.ID
+	}
+
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+		a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.HeaderRemoteURL,
+			Header:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing header for user: %s", err)
+		}
+		targetAccount.HeaderMediaAttachmentID = a.ID
+	}
+	return nil
+}
diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go
new file mode 100644
index 000000000..2522a4034
--- /dev/null
+++ b/internal/federation/dereferencing/announce.go
@@ -0,0 +1,65 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("DereferenceAnnounce: no URI to dereference")
+	}
+
+	boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+	if blocked, err := d.blockedDomain(boostedStatusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host)
+	}
+
+	// dereference statuses in the thread of the boosted status
+	if err := d.DereferenceThread(requestingUsername, boostedStatusURI); err != nil {
+		return fmt.Errorf("DereferenceAnnounce: error dereferencing thread of boosted status: %s", err)
+	}
+
+	boostedStatus, _, _, err := d.GetRemoteStatus(requestingUsername, boostedStatusURI, false)
+	if err != nil {
+		return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.BoostOfAccountID = boostedStatus.AccountID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}
diff --git a/internal/federation/dereferencing/blocked.go b/internal/federation/dereferencing/blocked.go
new file mode 100644
index 000000000..a66afbb60
--- /dev/null
+++ b/internal/federation/dereferencing/blocked.go
@@ -0,0 +1,41 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) blockedDomain(host string) (bool, error) {
+	b := &gtsmodel.DomainBlock{}
+	err := d.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b)
+	if err == nil {
+		// block exists
+		return true, nil
+	}
+
+	if _, ok := err.(db.ErrNoEntries); ok {
+		// there are no entries so there's no block
+		return false, nil
+	}
+
+	// there's an actual error
+	return false, err
+}
diff --git a/internal/federation/dereferencing/collectionpage.go b/internal/federation/dereferencing/collectionpage.go
new file mode 100644
index 000000000..5feadc1ad
--- /dev/null
+++ b/internal/federation/dereferencing/collectionpage.go
@@ -0,0 +1,70 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
+func (d *deref) DereferenceCollectionPage(username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
+	if blocked, err := d.blockedDomain(pageIRI.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error creating transport: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), pageIRI)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error deferencing %s: %s", pageIRI.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error resolving json into ap vocab type: %s", err)
+	}
+
+	if t.GetTypeName() != gtsmodel.ActivityStreamsCollectionPage {
+		return nil, fmt.Errorf("DereferenceCollectionPage: type name %s not supported", t.GetTypeName())
+	}
+
+	p, ok := t.(vocab.ActivityStreamsCollectionPage)
+	if !ok {
+		return nil, errors.New("DereferenceCollectionPage: error resolving type as activitystreams collection page")
+	}
+
+	return p, nil
+}
diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go
new file mode 100644
index 000000000..03b90569a
--- /dev/null
+++ b/internal/federation/dereferencing/dereferencer.go
@@ -0,0 +1,73 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"net/url"
+	"sync"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
+type Dereferencer interface {
+	GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
+	EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error)
+
+	GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error)
+	EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error)
+
+	GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+
+	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
+	DereferenceThread(username string, statusIRI *url.URL) error
+
+	Handshaking(username string, remoteAccountID *url.URL) bool
+}
+
+type deref struct {
+	log                 *logrus.Logger
+	db                  db.DB
+	typeConverter       typeutils.TypeConverter
+	transportController transport.Controller
+	mediaHandler        media.Handler
+	config              *config.Config
+	handshakes          map[string][]*url.URL
+	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
+}
+
+// NewDereferencer returns a Dereferencer initialized with the given parameters.
+func NewDereferencer(config *config.Config, db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler, log *logrus.Logger) Dereferencer {
+	return &deref{
+		log:                 log,
+		db:                  db,
+		typeConverter:       typeConverter,
+		transportController: transportController,
+		mediaHandler:        mediaHandler,
+		config:              config,
+		handshakeSync:       &sync.Mutex{},
+	}
+}
diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go
new file mode 100644
index 000000000..cda8eafd0
--- /dev/null
+++ b/internal/federation/dereferencing/handshake.go
@@ -0,0 +1,98 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import "net/url"
+
+func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	if d.handshakes == nil {
+		// handshakes isn't even initialized yet so we can't be handshaking with anyone
+		return false
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// user isn't handshaking with anyone, bail
+		return false
+	}
+
+	for _, id := range remoteIDs {
+		if id.String() == remoteAccountID.String() {
+			// we are currently handshaking with the remote account, yep
+			return true
+		}
+	}
+
+	// didn't find it which means we're not handshaking
+	return false
+}
+
+func (d *deref) startHandshake(username string, remoteAccountID *url.URL) {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	// lazily initialize handshakes
+	if d.handshakes == nil {
+		d.handshakes = make(map[string][]*url.URL)
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// there was nothing in there yet, so just add this entry and return
+		d.handshakes[username] = []*url.URL{remoteAccountID}
+		return
+	}
+
+	// add the remote ID to the slice
+	remoteIDs = append(remoteIDs, remoteAccountID)
+	d.handshakes[username] = remoteIDs
+}
+
+func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	if d.handshakes == nil {
+		return
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// there was nothing in there yet anyway so just bail
+		return
+	}
+
+	newRemoteIDs := []*url.URL{}
+	for _, id := range remoteIDs {
+		if id.String() != remoteAccountID.String() {
+			newRemoteIDs = append(newRemoteIDs, id)
+		}
+	}
+
+	if len(newRemoteIDs) == 0 {
+		// there are no handshakes so just remove this user entry from the map and save a few bytes
+		delete(d.handshakes, username)
+	} else {
+		// there are still other handshakes ongoing
+		d.handshakes[username] = newRemoteIDs
+	}
+}
diff --git a/internal/federation/dereferencing/instance.go b/internal/federation/dereferencing/instance.go
new file mode 100644
index 000000000..80f626662
--- /dev/null
+++ b/internal/federation/dereferencing/instance.go
@@ -0,0 +1,40 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+	if blocked, err := d.blockedDomain(remoteInstanceURI.Host); blocked || err != nil {
+		return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
+}
diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
new file mode 100644
index 000000000..b05f6e72c
--- /dev/null
+++ b/internal/federation/dereferencing/status.go
@@ -0,0 +1,369 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form,
+// and populates it with additional fields, media, etc.
+//
+// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by
+// the federatingDB's Create function, but additional dereferencing is needed on it.
+func (d *deref) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) {
+	if err := d.populateStatusFields(status, username); err != nil {
+		return nil, err
+	}
+
+	if err := d.db.UpdateByID(status.ID, status); err != nil {
+		return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err)
+	}
+
+	return status, nil
+}
+
+// GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status,
+// puts it in the database, and returns it to a caller. The boolean indicates whether the status is new
+// to us or not. If we haven't seen the status before, bool will be true. If we have seen the status before,
+// it will be false.
+//
+// If refresh is true, then even if we have the status in our database already, it will be dereferenced from its
+// remote representation, as will its owner.
+//
+// If a dereference was performed, then the function also returns the ap.Statusable representation for further processing.
+//
+// SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored.
+func (d *deref) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
+	new := true
+
+	// check if we already have the status in our db
+	maybeStatus := &gtsmodel.Status{}
+	if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteStatusID.String()}}, maybeStatus); err == nil {
+		// we've seen this status before so it's not new
+		new = false
+
+		// if we're not being asked to refresh, we can just return the maybeStatus as-is and avoid doing any external calls
+		if !refresh {
+			return maybeStatus, nil, new, nil
+		}
+	}
+
+	statusable, err := d.dereferenceStatusable(username, remoteStatusID)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error dereferencing statusable: %s", err)
+	}
+
+	accountURI, err := ap.ExtractAttributedTo(statusable)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err)
+	}
+
+	// do this so we know we have the remote account of the status in the db
+	_, _, err = d.GetRemoteAccount(username, accountURI, false)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err)
+	}
+
+	gtsStatus, err := d.typeConverter.ASStatusToStatus(statusable)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err)
+	}
+
+	if new {
+		ulid, err := id.NewULIDFromTime(gtsStatus.CreatedAt)
+		if err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err)
+		}
+		gtsStatus.ID = ulid
+
+		if err := d.populateStatusFields(gtsStatus, username); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
+		}
+
+		if err := d.db.Put(gtsStatus); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error putting new status: %s", err)
+		}
+	} else {
+		gtsStatus.ID = maybeStatus.ID
+
+		if err := d.populateStatusFields(gtsStatus, username); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
+		}
+
+		if err := d.db.UpdateByID(gtsStatus.ID, gtsStatus); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error updating status: %s", err)
+		}
+	}
+
+	return gtsStatus, statusable, new, nil
+}
+
+func (d *deref) dereferenceStatusable(username string, remoteStatusID *url.URL) (ap.Statusable, error) {
+	if blocked, err := d.blockedDomain(remoteStatusID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteStatusID)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error deferencing %s: %s", remoteStatusID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error resolving json into ap vocab type: %s", err)
+	}
+
+	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+	switch t.GetTypeName() {
+	case gtsmodel.ActivityStreamsArticle:
+		p, ok := t.(vocab.ActivityStreamsArticle)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsArticle")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsDocument:
+		p, ok := t.(vocab.ActivityStreamsDocument)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsDocument")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsImage:
+		p, ok := t.(vocab.ActivityStreamsImage)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsImage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsVideo:
+		p, ok := t.(vocab.ActivityStreamsVideo)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsVideo")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsNote:
+		p, ok := t.(vocab.ActivityStreamsNote)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsNote")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPage:
+		p, ok := t.(vocab.ActivityStreamsPage)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsEvent:
+		p, ok := t.(vocab.ActivityStreamsEvent)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsEvent")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPlace:
+		p, ok := t.(vocab.ActivityStreamsPlace)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPlace")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsProfile:
+		p, ok := t.(vocab.ActivityStreamsProfile)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsProfile")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("DereferenceStatusable: type name %s not supported", t.GetTypeName())
+}
+
+// populateStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (d *deref) populateStatusFields(status *gtsmodel.Status, requestingUsername string) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	// make sure we have a status URI and that the domain in question isn't blocked
+	statusURI, err := url.Parse(status.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
+	}
+	if blocked, err := d.blockedDomain(statusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
+	}
+
+	// we can continue -- create a new transport here because we'll probably need it
+	t, err := d.transportController.NewTransportForUsername(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one
+	if status.ID == "" {
+		newID, err := id.NewULIDFromTime(status.CreatedAt)
+		if err != nil {
+			return err
+		}
+		status.ID = newID
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Tracef("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := d.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Tracef("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			l.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
+		if err := d.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+
+		if m.ID != "" {
+			continue
+			// we've already populated this mention, since it has an ID
+		}
+
+		mID, err := id.NewRandomULID()
+		if err != nil {
+			return err
+		}
+		m.ID = mID
+
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAuthorAccount.ID
+		m.OriginAccountURI = status.GTSAuthorAccount.URI
+
+		targetAccount, _, err := d.GetRemoteAccount(requestingUsername, uri, false)
+		if err != nil {
+			continue
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := d.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	// status has replyToURI but we don't have an ID yet for the status it replies to
+	if status.InReplyToURI != "" && status.InReplyToID == "" {
+		replyToStatus := &gtsmodel.Status{}
+		if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: status.InReplyToURI}}, replyToStatus); err == nil {
+			// we have the status
+			status.InReplyToID = replyToStatus.ID
+			status.InReplyToAccountID = replyToStatus.AccountID
+		}
+	}
+
+	return nil
+}
diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go
new file mode 100644
index 000000000..2a407f923
--- /dev/null
+++ b/internal/federation/dereferencing/thread.go
@@ -0,0 +1,250 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo),
+// and dereferences statusables in the conversation.
+//
+// This process involves working up and down the chain of replies, and parsing through the collections of IDs
+// presented by remote instances as part of their replies collections, and will likely involve making several calls to
+// multiple different hosts.
+func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "DereferenceThread",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering DereferenceThread")
+
+	// if it's our status we already have everything stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	// first make sure we have this status in our db
+	_, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true)
+	if err != nil {
+		return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err)
+	}
+
+	// first iterate up through ancestors, dereferencing if necessary as we go
+	if err := d.iterateAncestors(username, *statusIRI); err != nil {
+		return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err)
+	}
+
+	// now iterate down through descendants, again dereferencing as we go
+	if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil {
+		return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err)
+	}
+
+	return nil
+}
+
+// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way.
+func (d *deref) iterateAncestors(username string, statusIRI url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateAncestors",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateAncestors")
+
+	// if it's our status we don't need to dereference anything so we can immediately move up the chain
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, moving up to next ancestor")
+
+		// since this is our status, we know we can extract the id from the status path
+		_, id, err := util.ParseStatusesPath(&statusIRI)
+		if err != nil {
+			return err
+		}
+
+		status := &gtsmodel.Status{}
+		if err := d.db.GetByID(id, status); err != nil {
+			return err
+		}
+
+		if status.InReplyToURI == "" {
+			// status doesn't reply to anything
+			return nil
+		}
+		nextIRI, err := url.Parse(status.URI)
+		if err != nil {
+			return err
+		}
+		return d.iterateAncestors(username, *nextIRI)
+	}
+
+	// If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus
+	// We call it with refresh to true because we want the statusable representation to parse inReplyTo from.
+	status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true)
+	if err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	inReplyTo := ap.ExtractInReplyToURI(statusable)
+	if inReplyTo == nil || inReplyTo.String() == "" {
+		// status doesn't reply to anything
+		return nil
+	}
+
+	// get the ancestor status into our database if we don't have it yet
+	if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	// now enrich the current status, since we should have the ancestor in the db
+	if _, err := d.EnrichRemoteStatus(username, status); err != nil {
+		l.Debugf("error enriching remote status: %s", err)
+		return nil
+	}
+
+	// now move up to the next ancestor
+	return d.iterateAncestors(username, *inReplyTo)
+}
+
+func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateDescendants",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateDescendants")
+
+	// if it's our status we already have descendants stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	replies := statusable.GetActivityStreamsReplies()
+	if replies == nil || !replies.IsActivityStreamsCollection() {
+		l.Debug("no replies, bailing")
+		return nil
+	}
+
+	repliesCollection := replies.GetActivityStreamsCollection()
+	if repliesCollection == nil {
+		l.Debug("replies collection is nil, bailing")
+		return nil
+	}
+
+	first := repliesCollection.GetActivityStreamsFirst()
+	if first == nil {
+		l.Debug("replies collection has no first, bailing")
+		return nil
+	}
+
+	firstPage := first.GetActivityStreamsCollectionPage()
+	if firstPage == nil {
+		l.Debug("first has no collection page, bailing")
+		return nil
+	}
+
+	firstPageNext := firstPage.GetActivityStreamsNext()
+	if firstPageNext == nil || !firstPageNext.IsIRI() {
+		l.Debug("next is not an iri, bailing")
+		return nil
+	}
+
+	var foundReplies int
+	currentPageIRI := firstPageNext.GetIRI()
+
+pageLoop:
+	for {
+		l.Debugf("dereferencing page %s", currentPageIRI)
+		nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI)
+		if err != nil {
+			return nil
+		}
+
+		// next items could be either a list of URLs or a list of statuses
+
+		nextItems := nextPage.GetActivityStreamsItems()
+		if nextItems.Len() == 0 {
+			// no items on this page, which means we're done
+			break pageLoop
+		}
+
+		// have a look through items and see what we can find
+		for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() {
+			// We're looking for a url to feed to GetRemoteStatus.
+			// Items can be either an IRI, or a Note.
+			// If a note, we grab the ID from it and call it, rather than parsing the note.
+
+			var itemURI *url.URL
+			if iter.IsIRI() {
+				// iri, easy
+				itemURI = iter.GetIRI()
+			} else if iter.IsActivityStreamsNote() {
+				// note, get the id from it to use as iri
+				n := iter.GetActivityStreamsNote()
+				id := n.GetJSONLDId()
+				if id != nil && id.IsIRI() {
+					itemURI = id.GetIRI()
+				}
+			} else {
+				// if it's not an iri or a note, we don't know how to process it
+				continue
+			}
+
+			if itemURI.Host == d.config.Host {
+				// skip if the reply is from us -- we already have it then
+				continue
+			}
+
+			// we can confidently say now that we found something
+			foundReplies = foundReplies + 1
+
+			// get the remote statusable and put it in the db
+			_, statusable, new, err := d.GetRemoteStatus(username, itemURI, false)
+			if new && err == nil && statusable != nil {
+				// now iterate descendants of *that* status
+				if err := d.iterateDescendants(username, *itemURI, statusable); err != nil {
+					continue
+				}
+			}
+		}
+
+		next := nextPage.GetActivityStreamsNext()
+		if next != nil && next.IsIRI() {
+			l.Debug("setting next page")
+			currentPageIRI = next.GetIRI()
+		} else {
+			l.Debug("no next page, bailing")
+			break pageLoop
+		}
+	}
+
+	l.Debugf("foundReplies %d", foundReplies)
+	return nil
+}
diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go
index e4a4920c8..3f4e3e413 100644
--- a/internal/federation/federatingdb/update.go
+++ b/internal/federation/federatingdb/update.go
@@ -9,8 +9,8 @@ import (
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -78,7 +78,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		typeName == gtsmodel.ActivityStreamsPerson ||
 		typeName == gtsmodel.ActivityStreamsService {
 		// it's an UPDATE to some kind of account
-		var accountable typeutils.Accountable
+		var accountable ap.Accountable
 
 		switch asType.GetTypeName() {
 		case gtsmodel.ActivityStreamsApplication:
diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go
index 1acdb6cb1..9e21b43bf 100644
--- a/internal/federation/federatingprotocol.go
+++ b/internal/federation/federatingprotocol.go
@@ -31,7 +31,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -139,7 +138,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 
 		// we don't have an entry for this instance yet so dereference it
-		i, err = f.DereferenceRemoteInstance(username, &url.URL{
+		i, err = f.GetRemoteInstance(username, &url.URL{
 			Scheme: publicKeyOwnerURI.Scheme,
 			Host:   publicKeyOwnerURI.Host,
 		})
@@ -153,51 +152,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 	}
 
-	requestingAccount := &gtsmodel.Account{}
-	if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil {
-		// there's been a proper error so return it
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		// we don't know this account (yet) so let's dereference it right now
-		person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI)
-		if err != nil {
-			return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		a, err := f.typeConverter.ASRepresentationToAccount(person, false)
-		if err != nil {
-			return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		aID, err := id.NewRandomULID()
-		if err != nil {
-			return ctx, false, err
-		}
-		a.ID = aID
-
-		if err := f.db.Put(a); err != nil {
-			l.Errorf("error inserting dereferenced remote account: %s", err)
-		}
-
-		requestingAccount = a
-
-		// send the newly dereferenced account into the processor channel for further async processing
-		fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
-		if fromFederatorChanI == nil {
-			l.Error("from federator channel wasn't set on context")
-		}
-		fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator)
-		if !ok {
-			l.Error("from federator channel was set on context but couldn't be parsed")
-		}
-
-		fromFederatorChan <- gtsmodel.FromFederator{
-			APObjectType:   gtsmodel.ActivityStreamsProfile,
-			APActivityType: gtsmodel.ActivityStreamsCreate,
-			GTSModel:       requestingAccount,
-		}
+	requestingAccount, _, err := f.GetRemoteAccount(username, publicKeyOwnerURI, false)
+	if err != nil {
+		return nil, false, fmt.Errorf("couldn't get remote account: %s", err)
 	}
 
 	withRequester := context.WithValue(ctx, util.APRequestingAccount, requestingAccount)
diff --git a/internal/federation/federator.go b/internal/federation/federator.go
index a5ffb3de8..ea9e61831 100644
--- a/internal/federation/federator.go
+++ b/internal/federation/federator.go
@@ -21,12 +21,13 @@ package federation
 import (
 	"context"
 	"net/url"
-	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -40,6 +41,7 @@ type Federator interface {
 	FederatingActor() pub.FederatingActor
 	// FederatingDB returns the underlying FederatingDB interface.
 	FederatingDB() federatingdb.DB
+
 	// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	//
@@ -49,29 +51,21 @@ type Federator interface {
 	//
 	// If something goes wrong during authentication, nil, false, and an error will be returned.
 	AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error)
+
 	// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
 	// account, or an error if it doesn't exist or can't be retrieved.
 	FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
-	// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
-	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
-	DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)
-	// DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI).
-	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
-	DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error)
-	// DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then
-	// does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo
-	DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
-	// DereferenceStatusFields does further dereferencing on a status.
-	DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error
-	// DereferenceAccountFields does further dereferencing on an account.
-	DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error
-	// DereferenceAnnounce does further dereferencing on an announce.
+
+	DereferenceRemoteThread(username string, statusURI *url.URL) error
 	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
-	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
-	// This can be used for making signed http requests.
-	//
-	// If username is an empty string, our instance user's credentials will be used instead.
-	GetTransportForUser(username string) (transport.Transport, error)
+
+	GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
+
+	GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error)
+	EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error)
+
+	GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+
 	// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
 	Handshaking(username string, remoteAccountID *url.URL) bool
 	pub.CommonBehavior
@@ -85,16 +79,17 @@ type federator struct {
 	clock               pub.Clock
 	typeConverter       typeutils.TypeConverter
 	transportController transport.Controller
+	dereferencer        dereferencing.Dereferencer
 	mediaHandler        media.Handler
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
-	handshakes          map[string][]*url.URL
-	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
 }
 
 // NewFederator returns a new federator
 func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator {
 
+	dereferencer := dereferencing.NewDereferencer(config, db, typeConverter, transportController, mediaHandler, log)
+
 	clock := &Clock{}
 	f := &federator{
 		config:              config,
@@ -103,9 +98,9 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		clock:               &Clock{},
 		typeConverter:       typeConverter,
 		transportController: transportController,
+		dereferencer:        dereferencer,
 		mediaHandler:        mediaHandler,
 		log:                 log,
-		handshakeSync:       &sync.Mutex{},
 	}
 	actor := newFederatingActor(f, f, federatingDB, clock)
 	f.actor = actor
diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go
index 4ba0796cd..d74070487 100644
--- a/internal/federation/federator_test.go
+++ b/internal/federation/federator_test.go
@@ -69,7 +69,7 @@ func (suite *ProtocolTestSuite) SetupSuite() {
 }
 
 func (suite *ProtocolTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, suite.accounts)
 
 }
 
@@ -87,7 +87,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 	// setup transport controller with a no-op client so we don't make external calls
 	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
 		return nil, nil
-	}))
+	}), suite.db)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
@@ -152,7 +152,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
 			StatusCode: 200,
 			Body:       r,
 		}, nil
-	}))
+	}), suite.db)
 
 	// now setup module being tested, with the mock transport controller
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
diff --git a/internal/federation/finger.go b/internal/federation/finger.go
index 6c6e9f6dc..0ffc60e5a 100644
--- a/internal/federation/finger.go
+++ b/internal/federation/finger.go
@@ -34,7 +34,7 @@ func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsernam
 		return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain)
 	}
 
-	t, err := f.GetTransportForUser(requestingUsername)
+	t, err := f.transportController.NewTransportForUsername(requestingUsername)
 	if err != nil {
 		return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
 	}
diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go
index 511e3e174..47c8a6c84 100644
--- a/internal/federation/handshake.go
+++ b/internal/federation/handshake.go
@@ -3,78 +3,5 @@ package federation
 import "net/url"
 
 func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool {
-	f.handshakeSync.Lock()
-	defer f.handshakeSync.Unlock()
-
-	if f.handshakes == nil {
-		// handshakes isn't even initialized yet so we can't be handshaking with anyone
-		return false
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// user isn't handshaking with anyone, bail
-		return false
-	}
-
-	for _, id := range remoteIDs {
-		if id.String() == remoteAccountID.String() {
-			// we are currently handshaking with the remote account, yep
-			return true
-		}
-	}
-
-	// didn't find it which means we're not handshaking
-	return false
-}
-
-func (f *federator) startHandshake(username string, remoteAccountID *url.URL) {
-	f.handshakeSync.Lock()
-	defer f.handshakeSync.Unlock()
-
-	// lazily initialize handshakes
-	if f.handshakes == nil {
-		f.handshakes = make(map[string][]*url.URL)
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// there was nothing in there yet, so just add this entry and return
-		f.handshakes[username] = []*url.URL{remoteAccountID}
-		return
-	}
-
-	// add the remote ID to the slice
-	remoteIDs = append(remoteIDs, remoteAccountID)
-	f.handshakes[username] = remoteIDs
-}
-
-func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) {
-	f.handshakeSync.Lock()
-	defer f.handshakeSync.Unlock()
-
-	if f.handshakes == nil {
-		return
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// there was nothing in there yet anyway so just bail
-		return
-	}
-
-	newRemoteIDs := []*url.URL{}
-	for _, id := range remoteIDs {
-		if id.String() != remoteAccountID.String() {
-			newRemoteIDs = append(newRemoteIDs, id)
-		}
-	}
-
-	if len(newRemoteIDs) == 0 {
-		// there are no handshakes so just remove this user entry from the map and save a few bytes
-		delete(f.handshakes, username)
-	} else {
-		// there are still other handshakes ongoing
-		f.handshakes[username] = newRemoteIDs
-	}
+	return f.dereferencer.Handshaking(username, remoteAccountID)
 }
diff --git a/internal/federation/transport.go b/internal/federation/transport.go
index a92f66d25..ed28749a1 100644
--- a/internal/federation/transport.go
+++ b/internal/federation/transport.go
@@ -6,8 +6,6 @@ import (
 	"net/url"
 
 	"github.com/go-fed/activity/pub"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -35,7 +33,6 @@ import (
 // returned Transport so that any private credentials are able to be
 // garbage collected.
 func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) {
-
 	var username string
 	var err error
 
@@ -53,32 +50,5 @@ func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofe
 		return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String())
 	}
 
-	account := &gtsmodel.Account{}
-	if err := f.db.GetLocalAccountByUsername(username, account); err != nil {
-		return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err)
-	}
-
-	return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey)
-}
-
-func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
-	// We need an account to use to create a transport for dereferecing something.
-	// If a username has been given, we can fetch the account with that username and use it.
-	// Otherwise, we can take the instance account and use those credentials to make the request.
-	ourAccount := &gtsmodel.Account{}
-	var u string
-	if username == "" {
-		u = f.config.Host
-	} else {
-		u = username
-	}
-	if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil {
-		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
-	}
-
-	transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey)
-	if err != nil {
-		return nil, fmt.Errorf("error creating transport for user %s: %s", username, err)
-	}
-	return transport, nil
+	return f.transportController.NewTransportForUsername(username)
 }