From c9c0773f2c2363dcfa37e675b83ec3f0b49bd0d9 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:50:46 +0100 Subject: [performance] update remaining worker pools to use queues (#2865) * start replacing client + federator + media workers with new worker + queue types * refactor federatingDB.Delete(), drop queued messages when deleting account / status * move all queue purging to the processor workers * undo toolchain updates * code comments, ensure dereferencer worker pool gets started * update gruf libraries in readme * start the job scheduler separately to the worker pools * reshuffle ordering or server.go + remove duplicate worker start / stop * update go-list version * fix vendoring * move queue invalidation to before wipeing / deletion, to ensure queued work not dropped * add logging to worker processing functions in testrig, don't start workers in unexpected places * update go-structr to add (+then rely on) QueueCtx{} type * ensure more worker pools get started properly in tests * fix remaining broken tests relying on worker queue logic * fix account test suite queue popping logic, ensure noop workers do not pull from queue * move back accidentally shuffled account deletion order * ensure error (non nil!!) gets passed in refactored federatingDB{}.Delete() * silently drop deletes from accounts not permitted to * don't warn log on forwarded deletes * make if else clauses easier to parse * use getFederatorMsg() * improved code comment * improved code comment re: requesting account delete checks * remove boolean result from worker start / stop since false = already running or already stopped * remove optional passed-in http.client * remove worker starting from the admin CLI commands (we don't need to handle side-effects) * update prune cli to start scheduler but not all of the workers * fix rebase issues * remove redundant return statements * i'm sorry sir linter --- internal/processing/account/delete.go | 42 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 22 deletions(-) (limited to 'internal/processing/account/delete.go') diff --git a/internal/processing/account/delete.go b/internal/processing/account/delete.go index a900c566d..3f051edf0 100644 --- a/internal/processing/account/delete.go +++ b/internal/processing/account/delete.go @@ -102,16 +102,13 @@ func (p *Processor) Delete( // and the above Delete function will be called afterwards from the processor, to clear // out the account's bits and bobs, and stubbify it. func (p *Processor) DeleteSelf(ctx context.Context, account *gtsmodel.Account) gtserror.WithCode { - fromClientAPIMessage := messages.FromClientAPI{ + // Process the delete side effects asynchronously. + p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{ APObjectType: ap.ActorPerson, APActivityType: ap.ActivityDelete, - OriginAccount: account, - TargetAccount: account, - } - - // Process the delete side effects asynchronously. - p.state.Workers.EnqueueClientAPI(ctx, fromClientAPIMessage) - + Origin: account, + Target: account, + }) return nil } @@ -193,7 +190,8 @@ func (p *Processor) deleteAccountFollows(ctx context.Context, account *gtsmodel. var ( // Use this slice to batch unfollow messages. - msgs = []messages.FromClientAPI{} + msgs = []*messages.FromClientAPI{} + // To avoid checking if account is local over + over // inside the subsequent loops, just generate static // side effects function once now. @@ -214,7 +212,7 @@ func (p *Processor) deleteAccountFollows(ctx context.Context, account *gtsmodel. } if msg := unfollowSideEffects(ctx, account, follow); msg != nil { // There was a side effect to process. - msgs = append(msgs, *msg) + msgs = append(msgs, msg) } } @@ -244,13 +242,13 @@ func (p *Processor) deleteAccountFollows(ctx context.Context, account *gtsmodel. if msg := unfollowSideEffects(ctx, account, follow); msg != nil { // There was a side effect to process. - msgs = append(msgs, *msg) + msgs = append(msgs, msg) } } // Process accreted messages in serial. for _, msg := range msgs { - if err := p.state.Workers.ProcessFromClientAPI(ctx, msg); err != nil { + if err := p.state.Workers.Client.Process(ctx, msg); err != nil { log.Errorf( ctx, "error processing %s of %s during Delete of account %s: %v", @@ -306,8 +304,8 @@ func (p *Processor) unfollowSideEffectsFunc(local bool) func( APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityUndo, GTSModel: follow, - OriginAccount: account, - TargetAccount: follow.TargetAccount, + Origin: account, + Target: follow.TargetAccount, } } } @@ -337,7 +335,7 @@ func (p *Processor) deleteAccountStatuses( statuses []*gtsmodel.Status err error maxID string - msgs = []messages.FromClientAPI{} + msgs = []*messages.FromClientAPI{} ) statusLoop: @@ -404,29 +402,29 @@ statusLoop: continue } - msgs = append(msgs, messages.FromClientAPI{ + msgs = append(msgs, &messages.FromClientAPI{ APObjectType: ap.ActivityAnnounce, APActivityType: ap.ActivityUndo, GTSModel: status, - OriginAccount: boost.Account, - TargetAccount: account, + Origin: boost.Account, + Target: account, }) } // Now prepare to Delete status. - msgs = append(msgs, messages.FromClientAPI{ + msgs = append(msgs, &messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityDelete, GTSModel: status, - OriginAccount: account, - TargetAccount: account, + Origin: account, + Target: account, }) } } // Process accreted messages in serial. for _, msg := range msgs { - if err := p.state.Workers.ProcessFromClientAPI(ctx, msg); err != nil { + if err := p.state.Workers.Client.Process(ctx, msg); err != nil { log.Errorf( ctx, "error processing %s of %s during Delete of account %s: %v", -- cgit v1.2.3