From 6a6a4993338262f87df34c9be051bfaac75c1829 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sat, 26 Apr 2025 09:56:15 +0000 Subject: [performance] rewrite timelines to rely on new timeline cache type (#3941) * start work rewriting timeline cache type * further work rewriting timeline caching * more work integration new timeline code * remove old code * add local timeline, fix up merge conflicts * remove old use of go-bytes * implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr * remove old timeline package, add local timeline cache * remove references to old timeline types that needed starting up in tests * start adding page validation * fix test-identified timeline cache package issues * fix up more tests, fix missing required changes, etc * add exclusion for test.out in gitignore * clarify some things better in code comments * tweak cache size limits * fix list timeline cache fetching * further list timeline fixes * linter, ssssssssshhhhhhhhhhhh please * fix linter hints * reslice the output if it's beyond length of 'lim' * remove old timeline initialization code, bump go-structr to v0.9.4 * continued from previous commit * improved code comments * don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts * finish writing more code comments * some variable renaming, for ease of following * change the way we update lo,hi paging values during timeline load * improved code comments for updated / returned lo , hi paging values * finish writing code comments for the StatusTimeline{} type itself * fill in more code comments * update go-structr version to latest with changed timeline unique indexing logic * have a local and public timeline *per user* * rewrite calls to public / local timeline calls * remove the zero length check, as lo, hi values might still be set * simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions * swap the lo, hi values :facepalm: * add (now) missing slice reverse of tag timeline statuses when paging ASC * remove local / public caches (is out of scope for this work), share more timeline code * remove unnecessary change * again, remove more unused code * remove unused function to appease the linter * move boost checking to prepare function * fix use of timeline.lastOrder, fix incorrect range functions used * remove comments for repeat code * remove the boost logic from prepare function * do a maximum of 5 loads, not 10 * add repeat boost filtering logic, update go-structr, general improvements * more code comments * add important note * fix timeline tests now that timelines are returned in page order * remove unused field * add StatusTimeline{} tests * add more status timeline tests * start adding preloading support * ensure repeat boosts are marked in preloaded entries * share a bunch of the database load code in timeline cache, don't clear timelines on relationship change * add logic to allow dynamic clear / preloading of timelines * comment-out unused functions, but leave in place as we might end-up using them * fix timeline preload state check * much improved status timeline code comments * more code comments, don't bother inserting statuses if timeline not preloaded * shift around some logic to make sure things aren't accidentally left set * finish writing code comments * remove trim-after-insert behaviour * fix-up some comments referring to old logic * remove unsetting of lo, hi * fix preload repeatBoost checking logic * don't return on status filter errors, these are usually transient * better concurrency safety in Clear() and Done() * fix test broken due to addition of preloader * fix repeatBoost logic that doesn't account for already-hidden repeatBoosts * ensure edit submodels are dropped on cache insertion * update code-comment to expand CAS accronym * use a plus1hULID() instead of 24h * remove unused functions * add note that public / local timeline requester can be nil * fix incorrect visibility filtering of tag timeline statuses * ensure we filter home timeline statuses on local only * some small re-orderings to confirm query params in correct places * fix the local only home timeline filter func --- internal/processing/timeline/timeline.go | 135 +++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) (limited to 'internal/processing/timeline/timeline.go') diff --git a/internal/processing/timeline/timeline.go b/internal/processing/timeline/timeline.go index 5966fe864..54ea2cccd 100644 --- a/internal/processing/timeline/timeline.go +++ b/internal/processing/timeline/timeline.go @@ -18,9 +18,33 @@ package timeline import ( + "context" + "errors" + "net/http" + "net/url" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + timelinepkg "github.com/superseriousbusiness/gotosocial/internal/cache/timeline" + "github.com/superseriousbusiness/gotosocial/internal/db" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" + "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/util/xslices" +) + +var ( + // pre-prepared URL values to be passed in to + // paging response forms. The paging package always + // copies values before any modifications so it's + // safe to only use a single map variable for these. + localOnlyTrue = url.Values{"local": {"true"}} + localOnlyFalse = url.Values{"local": {"false"}} ) type Processor struct { @@ -36,3 +60,114 @@ func New(state *state.State, converter *typeutils.Converter, visFilter *visibili visFilter: visFilter, } } + +func (p *Processor) getStatusTimeline( + ctx context.Context, + requester *gtsmodel.Account, + timeline *timelinepkg.StatusTimeline, + page *paging.Page, + pagePath string, + pageQuery url.Values, + filterCtx statusfilter.FilterContext, + loadPage func(*paging.Page) (statuses []*gtsmodel.Status, err error), + filter func(*gtsmodel.Status) (delete bool), + postFilter func(*gtsmodel.Status) (remove bool), +) ( + *apimodel.PageableResponse, + gtserror.WithCode, +) { + var err error + var filters []*gtsmodel.Filter + var mutes *usermute.CompiledUserMuteList + + if requester != nil { + // Fetch all filters relevant for requesting account. + filters, err = p.state.DB.GetFiltersForAccountID(ctx, + requester.ID, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("error getting account filters: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Get a list of all account mutes for requester. + allMutes, err := p.state.DB.GetAccountMutes(ctx, + requester.ID, + nil, // i.e. all + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("error getting account mutes: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Compile all account mutes to useable form. + mutes = usermute.NewCompiledUserMuteList(allMutes) + } + + // Ensure we have valid + // input paging cursor. + id.ValidatePage(page) + + // Load status page via timeline cache, also + // getting lo, hi values for next, prev pages. + // + // NOTE: this safely handles the case of a nil + // input timeline, i.e. uncached timeline type. + apiStatuses, lo, hi, err := timeline.Load(ctx, + + // Status page + // to load. + page, + + // Caller provided database + // status page loading function. + loadPage, + + // Status load function for cached timeline entries. + func(ids []string) ([]*gtsmodel.Status, error) { + return p.state.DB.GetStatusesByIDs(ctx, ids) + }, + + // Call provided status + // filtering function. + filter, + + // Frontend API model preparation function. + func(status *gtsmodel.Status) (*apimodel.Status, error) { + + // Check if status needs filtering OUTSIDE of caching stage. + // TODO: this will be moved to separate postFilter hook when + // all filtering has been removed from the type converter. + if postFilter != nil && postFilter(status) { + return nil, nil + } + + // Finally, pass status to get converted to API model. + apiStatus, err := p.converter.StatusToAPIStatus(ctx, + status, + requester, + filterCtx, + filters, + mutes, + ) + if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + return nil, err + } + return apiStatus, nil + }, + ) + + if err != nil { + err := gtserror.Newf("error loading timeline: %w", err) + return nil, gtserror.WrapWithCode(http.StatusInternalServerError, err) + } + + // Package returned API statuses as pageable response. + return paging.PackageResponse(paging.ResponseParams{ + Items: xslices.ToAny(apiStatuses), + Path: pagePath, + Next: page.Next(lo, hi), + Prev: page.Prev(lo, hi), + Query: pageQuery, + }), nil +} -- cgit v1.2.3