From 6a6a4993338262f87df34c9be051bfaac75c1829 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sat, 26 Apr 2025 09:56:15 +0000 Subject: [performance] rewrite timelines to rely on new timeline cache type (#3941) * start work rewriting timeline cache type * further work rewriting timeline caching * more work integration new timeline code * remove old code * add local timeline, fix up merge conflicts * remove old use of go-bytes * implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr * remove old timeline package, add local timeline cache * remove references to old timeline types that needed starting up in tests * start adding page validation * fix test-identified timeline cache package issues * fix up more tests, fix missing required changes, etc * add exclusion for test.out in gitignore * clarify some things better in code comments * tweak cache size limits * fix list timeline cache fetching * further list timeline fixes * linter, ssssssssshhhhhhhhhhhh please * fix linter hints * reslice the output if it's beyond length of 'lim' * remove old timeline initialization code, bump go-structr to v0.9.4 * continued from previous commit * improved code comments * don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts * finish writing more code comments * some variable renaming, for ease of following * change the way we update lo,hi paging values during timeline load * improved code comments for updated / returned lo , hi paging values * finish writing code comments for the StatusTimeline{} type itself * fill in more code comments * update go-structr version to latest with changed timeline unique indexing logic * have a local and public timeline *per user* * rewrite calls to public / local timeline calls * remove the zero length check, as lo, hi values might still be set * simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions * swap the lo, hi values :facepalm: * add (now) missing slice reverse of tag timeline statuses when paging ASC * remove local / public caches (is out of scope for this work), share more timeline code * remove unnecessary change * again, remove more unused code * remove unused function to appease the linter * move boost checking to prepare function * fix use of timeline.lastOrder, fix incorrect range functions used * remove comments for repeat code * remove the boost logic from prepare function * do a maximum of 5 loads, not 10 * add repeat boost filtering logic, update go-structr, general improvements * more code comments * add important note * fix timeline tests now that timelines are returned in page order * remove unused field * add StatusTimeline{} tests * add more status timeline tests * start adding preloading support * ensure repeat boosts are marked in preloaded entries * share a bunch of the database load code in timeline cache, don't clear timelines on relationship change * add logic to allow dynamic clear / preloading of timelines * comment-out unused functions, but leave in place as we might end-up using them * fix timeline preload state check * much improved status timeline code comments * more code comments, don't bother inserting statuses if timeline not preloaded * shift around some logic to make sure things aren't accidentally left set * finish writing code comments * remove trim-after-insert behaviour * fix-up some comments referring to old logic * remove unsetting of lo, hi * fix preload repeatBoost checking logic * don't return on status filter errors, these are usually transient * better concurrency safety in Clear() and Done() * fix test broken due to addition of preloader * fix repeatBoost logic that doesn't account for already-hidden repeatBoosts * ensure edit submodels are dropped on cache insertion * update code-comment to expand CAS accronym * use a plus1hULID() instead of 24h * remove unused functions * add note that public / local timeline requester can be nil * fix incorrect visibility filtering of tag timeline statuses * ensure we filter home timeline statuses on local only * some small re-orderings to confirm query params in correct places * fix the local only home timeline filter func --- internal/processing/workers/fromclientapi.go | 86 +++++++++++++++++++--------- 1 file changed, 58 insertions(+), 28 deletions(-) (limited to 'internal/processing/workers/fromclientapi.go') diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go index 28a2b37b9..661fea866 100644 --- a/internal/processing/workers/fromclientapi.go +++ b/internal/processing/workers/fromclientapi.go @@ -371,7 +371,7 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA if status.InReplyToID != "" { // Interaction counts changed on the replied status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, status.InReplyToID) + p.surface.invalidateStatusFromTimelines(status.InReplyToID) } return nil @@ -413,7 +413,7 @@ func (p *clientAPI) CreatePollVote(ctx context.Context, cMsg *messages.FromClien } // Interaction counts changed on the source status, uncache from timelines. - p.surface.invalidateStatusFromTimelines(ctx, vote.Poll.StatusID) + p.surface.invalidateStatusFromTimelines(vote.Poll.StatusID) return nil } @@ -565,7 +565,7 @@ func (p *clientAPI) CreateLike(ctx context.Context, cMsg *messages.FromClientAPI // Interaction counts changed on the faved status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, fave.StatusID) + p.surface.invalidateStatusFromTimelines(fave.StatusID) return nil } @@ -671,7 +671,7 @@ func (p *clientAPI) CreateAnnounce(ctx context.Context, cMsg *messages.FromClien // Interaction counts changed on the boosted status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, boost.BoostOfID) + p.surface.invalidateStatusFromTimelines(boost.BoostOfID) return nil } @@ -682,22 +682,20 @@ func (p *clientAPI) CreateBlock(ctx context.Context, cMsg *messages.FromClientAP return gtserror.Newf("%T not parseable as *gtsmodel.Block", cMsg.GTSModel) } - // Remove blockee's statuses from blocker's timeline. - if err := p.state.Timelines.Home.WipeItemsFromAccountID( - ctx, - block.AccountID, - block.TargetAccountID, - ); err != nil { - return gtserror.Newf("error wiping timeline items for block: %w", err) + if block.Account.IsLocal() { + // Remove posts by target from origin's timelines. + p.surface.removeRelationshipFromTimelines(ctx, + block.AccountID, + block.TargetAccountID, + ) } - // Remove blocker's statuses from blockee's timeline. - if err := p.state.Timelines.Home.WipeItemsFromAccountID( - ctx, - block.TargetAccountID, - block.AccountID, - ); err != nil { - return gtserror.Newf("error wiping timeline items for block: %w", err) + if block.TargetAccount.IsLocal() { + // Remove posts by origin from target's timelines. + p.surface.removeRelationshipFromTimelines(ctx, + block.TargetAccountID, + block.AccountID, + ) } // TODO: same with notifications? @@ -737,7 +735,7 @@ func (p *clientAPI) UpdateStatus(ctx context.Context, cMsg *messages.FromClientA } // Status representation has changed, invalidate from timelines. - p.surface.invalidateStatusFromTimelines(ctx, status.ID) + p.surface.invalidateStatusFromTimelines(status.ID) return nil } @@ -858,6 +856,22 @@ func (p *clientAPI) UndoFollow(ctx context.Context, cMsg *messages.FromClientAPI log.Errorf(ctx, "error updating account stats: %v", err) } + if follow.Account.IsLocal() { + // Remove posts by target from origin's timelines. + p.surface.removeRelationshipFromTimelines(ctx, + follow.AccountID, + follow.TargetAccountID, + ) + } + + if follow.TargetAccount.IsLocal() { + // Remove posts by origin from target's timelines. + p.surface.removeRelationshipFromTimelines(ctx, + follow.TargetAccountID, + follow.AccountID, + ) + } + if err := p.federate.UndoFollow(ctx, follow); err != nil { log.Errorf(ctx, "error federating follow undo: %v", err) } @@ -890,7 +904,7 @@ func (p *clientAPI) UndoFave(ctx context.Context, cMsg *messages.FromClientAPI) // Interaction counts changed on the faved status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, statusFave.StatusID) + p.surface.invalidateStatusFromTimelines(statusFave.StatusID) return nil } @@ -910,9 +924,8 @@ func (p *clientAPI) UndoAnnounce(ctx context.Context, cMsg *messages.FromClientA log.Errorf(ctx, "error updating account stats: %v", err) } - if err := p.surface.deleteStatusFromTimelines(ctx, status.ID); err != nil { - log.Errorf(ctx, "error removing timelined status: %v", err) - } + // Delete the boost wrapper status from timelines. + p.surface.deleteStatusFromTimelines(ctx, status.ID) if err := p.federate.UndoAnnounce(ctx, status); err != nil { log.Errorf(ctx, "error federating announce undo: %v", err) @@ -920,7 +933,7 @@ func (p *clientAPI) UndoAnnounce(ctx context.Context, cMsg *messages.FromClientA // Interaction counts changed on the boosted status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, status.BoostOfID) + p.surface.invalidateStatusFromTimelines(status.BoostOfID) return nil } @@ -983,7 +996,7 @@ func (p *clientAPI) DeleteStatus(ctx context.Context, cMsg *messages.FromClientA if status.InReplyToID != "" { // Interaction counts changed on the replied status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, status.InReplyToID) + p.surface.invalidateStatusFromTimelines(status.InReplyToID) } return nil @@ -1026,6 +1039,23 @@ func (p *clientAPI) DeleteAccountOrUser(ctx context.Context, cMsg *messages.From p.state.Workers.Federator.Queue.Delete("Receiving.ID", account.ID) p.state.Workers.Federator.Queue.Delete("TargetURI", account.URI) + // Remove any entries authored by account from timelines. + p.surface.removeTimelineEntriesByAccount(account.ID) + + // Remove any of their cached timelines. + p.state.Caches.Timelines.Home.Delete(account.ID) + + // Get the IDs of all the lists owned by the given account ID. + listIDs, err := p.state.DB.GetListIDsByAccountID(ctx, account.ID) + if err != nil { + log.Errorf(ctx, "error getting lists for account %s: %v", account.ID, err) + } + + // Remove list timelines of account. + for _, listID := range listIDs { + p.state.Caches.Timelines.List.Delete(listID) + } + if err := p.federate.DeleteAccount(ctx, cMsg.Target); err != nil { log.Errorf(ctx, "error federating account delete: %v", err) } @@ -1169,7 +1199,7 @@ func (p *clientAPI) AcceptLike(ctx context.Context, cMsg *messages.FromClientAPI // Interaction counts changed on the faved status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, req.Like.StatusID) + p.surface.invalidateStatusFromTimelines(req.Like.StatusID) return nil } @@ -1202,7 +1232,7 @@ func (p *clientAPI) AcceptReply(ctx context.Context, cMsg *messages.FromClientAP // Interaction counts changed on the replied status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, reply.InReplyToID) + p.surface.invalidateStatusFromTimelines(reply.InReplyToID) return nil } @@ -1240,7 +1270,7 @@ func (p *clientAPI) AcceptAnnounce(ctx context.Context, cMsg *messages.FromClien // Interaction counts changed on the original status; // uncache the prepared version from all timelines. - p.surface.invalidateStatusFromTimelines(ctx, boost.BoostOfID) + p.surface.invalidateStatusFromTimelines(boost.BoostOfID) return nil } -- cgit v1.2.3