From 6c9d8e78ebbd81928abab1bbb49a3c4daa7d2a3a Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Fri, 12 May 2023 10:15:54 +0100 Subject: [feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim * fix recipient looping issue Signed-off-by: kim * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim * first draft of status refetching logic Signed-off-by: kim * fix testrig to use new federation alloc func signature Signed-off-by: kim * fix log format directive Signed-off-by: kim * add status fetched_at migration Signed-off-by: kim * remove unused / unchecked for error types Signed-off-by: kim * add back the used type... Signed-off-by: kim * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim * improved mention and media attachment error handling Signed-off-by: kim * fix log and error format directives Signed-off-by: kim * update account deref to match status deref changes Signed-off-by: kim * very small code formatting change to make things clearer Signed-off-by: kim * add more code comments Signed-off-by: kim * improved code commenting Signed-off-by: kim * only check for required further derefs if needed Signed-off-by: kim * improved cache invalidation Signed-off-by: kim * tweak cache restarting to use a (very small) backoff Signed-off-by: kim * small readability changes and fixes Signed-off-by: kim * fix account sync issues Signed-off-by: kim * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim * remove secondary function declaration Signed-off-by: kim * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim * appease the linter Signed-off-by: kim * add source file header Signed-off-by: kim * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim * update migration file name, improved database delete error returns Signed-off-by: kim * formatting Signed-off-by: kim * improved multi-delete database functions to minimise DB calls Signed-off-by: kim * remove unused type Signed-off-by: kim * fix delete statements Signed-off-by: kim --------- Signed-off-by: kim --- internal/processing/fromfederator.go | 57 ++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 26 deletions(-) (limited to 'internal/processing/fromfederator.go') diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 82e08060c..eccdbb894 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -110,17 +110,30 @@ func (p *Processor) ProcessFromFederator(ctx context.Context, federatorMsg messa func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federatorMsg messages.FromFederator) error { // check for either an IRI that we still need to dereference, OR an already dereferenced // and converted status pinned to the message. - var status *gtsmodel.Status + var ( + status *gtsmodel.Status + err error + ) if federatorMsg.GTSModel != nil { - // there's a gts model already pinned to the message, it should be a status var ok bool + + // there's a gts model already pinned to the message, it should be a status if status, ok = federatorMsg.GTSModel.(*gtsmodel.Status); !ok { return errors.New("ProcessFromFederator: note was not parseable as *gtsmodel.Status") } - var err error - status, err = p.federator.EnrichRemoteStatus(ctx, federatorMsg.ReceivingAccount.Username, status, true) + // Since this was a create originating AP object + // statusable may have been set on message (no problem if not). + statusable, _ := federatorMsg.APObjectModel.(ap.Statusable) + + // Call refresh on status to deref if necessary etc. + status, _, err = p.federator.RefreshStatus(ctx, + federatorMsg.ReceivingAccount.Username, + status, + statusable, + false, + ) if err != nil { return err } @@ -129,38 +142,29 @@ func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federa if federatorMsg.APIri == nil { return errors.New("ProcessFromFederator: status was not pinned to federatorMsg, and neither was an IRI for us to dereference") } - var err error - status, _, err = p.federator.GetStatus(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri, false, false) - if err != nil { - return err - } - } - // make sure the account is pinned - if status.Account == nil { - a, err := p.state.DB.GetAccountByID(ctx, status.AccountID) + status, _, err = p.federator.GetStatusByURI(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri) if err != nil { return err } - status.Account = a } - // Get the remote account to make sure the avi and header are cached. - if status.Account.Domain != "" { - remoteAccountID, err := url.Parse(status.Account.URI) + if status.Account == nil || status.Account.IsRemote() { + // Either no account attached yet, or a remote account. + // Both situations we need to parse account URI to fetch it. + remoteAccURI, err := url.Parse(status.AccountURI) if err != nil { return err } - a, err := p.federator.GetAccountByURI(ctx, + // Ensure that account for this status has been deref'd. + status.Account, _, err = p.federator.GetAccountByURI(ctx, federatorMsg.ReceivingAccount.Username, - remoteAccountID, + remoteAccURI, ) if err != nil { return err } - - status.Account = a } if err := p.timelineAndNotifyStatus(ctx, status); err != nil { @@ -193,7 +197,7 @@ func (p *Processor) processCreateFaveFromFederator(ctx context.Context, federato return err } - a, err := p.federator.GetAccountByURI(ctx, + a, _, err := p.federator.GetAccountByURI(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, ) @@ -234,7 +238,7 @@ func (p *Processor) processCreateFollowRequestFromFederator(ctx context.Context, return err } - a, err := p.federator.GetAccountByURI(ctx, + a, _, err := p.federator.GetAccountByURI(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, ) @@ -294,7 +298,7 @@ func (p *Processor) processCreateAnnounceFromFederator(ctx context.Context, fede return err } - a, err := p.federator.GetAccountByURI(ctx, + a, _, err := p.federator.GetAccountByURI(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, ) @@ -376,11 +380,12 @@ func (p *Processor) processUpdateAccountFromFederator(ctx context.Context, feder } // Call RefreshAccount to fetch up-to-date bio, avatar, header, etc. - updatedAccount, err := p.federator.RefreshAccount( + updatedAccount, _, err := p.federator.RefreshAccount( ctx, federatorMsg.ReceivingAccount.Username, - incomingAccountable, incomingAccount, + incomingAccountable, + true, ) if err != nil { return fmt.Errorf("error enriching updated account from federator: %s", err) -- cgit v1.2.3