[feature] status refetch support (#1690)

* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* first draft of status refetching logic

Signed-off-by: kim <grufwub@gmail.com>

* fix testrig to use new federation alloc func signature

Signed-off-by: kim <grufwub@gmail.com>

* fix log format directive

Signed-off-by: kim <grufwub@gmail.com>

* add status fetched_at migration

Signed-off-by: kim <grufwub@gmail.com>

* remove unused / unchecked for error types

Signed-off-by: kim <grufwub@gmail.com>

* add back the used type...

Signed-off-by: kim <grufwub@gmail.com>

* add separate internal getStatus() function for derefThread() that doesn't recurse

Signed-off-by: kim <grufwub@gmail.com>

* improved mention and media attachment error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix log and error format directives

Signed-off-by: kim <grufwub@gmail.com>

* update account deref to match status deref changes

Signed-off-by: kim <grufwub@gmail.com>

* very small code formatting change to make things clearer

Signed-off-by: kim <grufwub@gmail.com>

* add more code comments

Signed-off-by: kim <grufwub@gmail.com>

* improved code commenting

Signed-off-by: kim <grufwub@gmail.com>

* only check for required further derefs if needed

Signed-off-by: kim <grufwub@gmail.com>

* improved cache invalidation

Signed-off-by: kim <grufwub@gmail.com>

* tweak cache restarting to use a (very small) backoff

Signed-off-by: kim <grufwub@gmail.com>

* small readability changes and fixes

Signed-off-by: kim <grufwub@gmail.com>

* fix account sync issues

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts + update account enrichment to accept already-passed accountable

Signed-off-by: kim <grufwub@gmail.com>

* remove secondary function declaration

Signed-off-by: kim <grufwub@gmail.com>

* normalise dereferencer get status / account behaviour, fix remaining tests

Signed-off-by: kim <grufwub@gmail.com>

* fix remaining rebase conflicts, finish commenting code

Signed-off-by: kim <grufwub@gmail.com>

* appease the linter

Signed-off-by: kim <grufwub@gmail.com>

* add source file header

Signed-off-by: kim <grufwub@gmail.com>

* update to use TIMESTAMPTZ column type instead of just TIMESTAMP

Signed-off-by: kim <grufwub@gmail.com>

* don't pass in 'updated_at' to UpdateEmoji()

Signed-off-by: kim <grufwub@gmail.com>

* use new ap.Resolve{Account,Status}able() functions

Signed-off-by: kim <grufwub@gmail.com>

* remove the somewhat confusing rescoping of the same variable names

Signed-off-by: kim <grufwub@gmail.com>

* update migration file name, improved database delete error returns

Signed-off-by: kim <grufwub@gmail.com>

* formatting

Signed-off-by: kim <grufwub@gmail.com>

* improved multi-delete database functions to minimise DB calls

Signed-off-by: kim <grufwub@gmail.com>

* remove unused type

Signed-off-by: kim <grufwub@gmail.com>

* fix delete statements

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>

1 files changed, 31 insertions, 26 deletions

diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go
index 82e08060c..eccdbb894 100644
--- a/internal/processing/fromfederator.go
+++ b/internal/processing/fromfederator.go
@@ -110,17 +110,30 @@ func (p *Processor) ProcessFromFederator(ctx context.Context, federatorMsg messa
 func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federatorMsg messages.FromFederator) error {
 	// check for either an IRI that we still need to dereference, OR an already dereferenced
 	// and converted status pinned to the message.
-	var status *gtsmodel.Status
+	var (
+		status *gtsmodel.Status
+		err    error
+	)
 
 	if federatorMsg.GTSModel != nil {
-		// there's a gts model already pinned to the message, it should be a status
 		var ok bool
+
+		// there's a gts model already pinned to the message, it should be a status
 		if status, ok = federatorMsg.GTSModel.(*gtsmodel.Status); !ok {
 			return errors.New("ProcessFromFederator: note was not parseable as *gtsmodel.Status")
 		}
 
-		var err error
-		status, err = p.federator.EnrichRemoteStatus(ctx, federatorMsg.ReceivingAccount.Username, status, true)
+		// Since this was a create originating AP object
+		// statusable may have been set on message (no problem if not).
+		statusable, _ := federatorMsg.APObjectModel.(ap.Statusable)
+
+		// Call refresh on status to deref if necessary etc.
+		status, _, err = p.federator.RefreshStatus(ctx,
+			federatorMsg.ReceivingAccount.Username,
+			status,
+			statusable,
+			false,
+		)
 		if err != nil {
 			return err
 		}
@@ -129,38 +142,29 @@ func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federa
 		if federatorMsg.APIri == nil {
 			return errors.New("ProcessFromFederator: status was not pinned to federatorMsg, and neither was an IRI for us to dereference")
 		}
-		var err error
-		status, _, err = p.federator.GetStatus(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri, false, false)
-		if err != nil {
-			return err
-		}
-	}
 
-	// make sure the account is pinned
-	if status.Account == nil {
-		a, err := p.state.DB.GetAccountByID(ctx, status.AccountID)
+		status, _, err = p.federator.GetStatusByURI(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri)
 		if err != nil {
 			return err
 		}
-		status.Account = a
 	}
 
-	// Get the remote account to make sure the avi and header are cached.
-	if status.Account.Domain != "" {
-		remoteAccountID, err := url.Parse(status.Account.URI)
+	if status.Account == nil || status.Account.IsRemote() {
+		// Either no account attached yet, or a remote account.
+		// Both situations we need to parse account URI to fetch it.
+		remoteAccURI, err := url.Parse(status.AccountURI)
 		if err != nil {
 			return err
 		}
 
-		a, err := p.federator.GetAccountByURI(ctx,
+		// Ensure that account for this status has been deref'd.
+		status.Account, _, err = p.federator.GetAccountByURI(ctx,
 			federatorMsg.ReceivingAccount.Username,
-			remoteAccountID,
+			remoteAccURI,
 		)
 		if err != nil {
 			return err
 		}
-
-		status.Account = a
 	}
 
 	if err := p.timelineAndNotifyStatus(ctx, status); err != nil {
@@ -193,7 +197,7 @@ func (p *Processor) processCreateFaveFromFederator(ctx context.Context, federato
 			return err
 		}
 
-		a, err := p.federator.GetAccountByURI(ctx,
+		a, _, err := p.federator.GetAccountByURI(ctx,
 			federatorMsg.ReceivingAccount.Username,
 			remoteAccountID,
 		)
@@ -234,7 +238,7 @@ func (p *Processor) processCreateFollowRequestFromFederator(ctx context.Context,
 			return err
 		}
 
-		a, err := p.federator.GetAccountByURI(ctx,
+		a, _, err := p.federator.GetAccountByURI(ctx,
 			federatorMsg.ReceivingAccount.Username,
 			remoteAccountID,
 		)
@@ -294,7 +298,7 @@ func (p *Processor) processCreateAnnounceFromFederator(ctx context.Context, fede
 			return err
 		}
 
-		a, err := p.federator.GetAccountByURI(ctx,
+		a, _, err := p.federator.GetAccountByURI(ctx,
 			federatorMsg.ReceivingAccount.Username,
 			remoteAccountID,
 		)
@@ -376,11 +380,12 @@ func (p *Processor) processUpdateAccountFromFederator(ctx context.Context, feder
 	}
 
 	// Call RefreshAccount to fetch up-to-date bio, avatar, header, etc.
-	updatedAccount, err := p.federator.RefreshAccount(
+	updatedAccount, _, err := p.federator.RefreshAccount(
 		ctx,
 		federatorMsg.ReceivingAccount.Username,
-		incomingAccountable,
 		incomingAccount,
+		incomingAccountable,
+		true,
 	)
 	if err != nil {
 		return fmt.Errorf("error enriching updated account from federator: %s", err)