[performance] rewrite timelines to rely on new timeline cache type (#3941)

* start work rewriting timeline cache type

* further work rewriting timeline caching

* more work integration new timeline code

* remove old code

* add local timeline, fix up merge conflicts

* remove old use of go-bytes

* implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr

* remove old timeline package, add local timeline cache

* remove references to old timeline types that needed starting up in tests

* start adding page validation

* fix test-identified timeline cache package issues

* fix up more tests, fix missing required changes, etc

* add exclusion for test.out in gitignore

* clarify some things better in code comments

* tweak cache size limits

* fix list timeline cache fetching

* further list timeline fixes

* linter, ssssssssshhhhhhhhhhhh please

* fix linter hints

* reslice the output if it's beyond length of 'lim'

* remove old timeline initialization code, bump go-structr to v0.9.4

* continued from previous commit

* improved code comments

* don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts

* finish writing more code comments

* some variable renaming, for ease of following

* change the way we update lo,hi paging values during timeline load

* improved code comments for updated / returned lo , hi paging values

* finish writing code comments for the StatusTimeline{} type itself

* fill in more code comments

* update go-structr version to latest with changed timeline unique indexing logic

* have a local and public timeline *per user*

* rewrite calls to public / local timeline calls

* remove the zero length check, as lo, hi values might still be set

* simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions

* swap the lo, hi values :facepalm:

* add (now) missing slice reverse of tag timeline statuses when paging ASC

* remove local / public caches (is out of scope for this work), share more timeline code

* remove unnecessary change

* again, remove more unused code

* remove unused function to appease the linter

* move boost checking to prepare function

* fix use of timeline.lastOrder, fix incorrect range functions used

* remove comments for repeat code

* remove the boost logic from prepare function

* do a maximum of 5 loads, not 10

* add repeat boost filtering logic, update go-structr, general improvements

* more code comments

* add important note

* fix timeline tests now that timelines are returned in page order

* remove unused field

* add StatusTimeline{} tests

* add more status timeline tests

* start adding preloading support

* ensure repeat boosts are marked in preloaded entries

* share a bunch of the database load code in timeline cache, don't clear timelines on relationship change

* add logic to allow dynamic clear / preloading of timelines

* comment-out unused functions, but leave in place as we might end-up using them

* fix timeline preload state check

* much improved status timeline code comments

* more code comments, don't bother inserting statuses if timeline not preloaded

* shift around some logic to make sure things aren't accidentally left set

* finish writing code comments

* remove trim-after-insert behaviour

* fix-up some comments referring to old logic

* remove unsetting of lo, hi

* fix preload repeatBoost checking logic

* don't return on status filter errors, these are usually transient

* better concurrency safety in Clear() and Done()

* fix test broken due to addition of preloader

* fix repeatBoost logic that doesn't account for already-hidden repeatBoosts

* ensure edit submodels are dropped on cache insertion

* update code-comment to expand CAS accronym

* use a plus1hULID() instead of 24h

* remove unused functions

* add note that public / local timeline requester can be nil

* fix incorrect visibility filtering of tag timeline statuses

* ensure we filter home timeline statuses on local only

* some small re-orderings to confirm query params in correct places

* fix the local only home timeline filter func

1 files changed, 135 insertions, 0 deletions

diff --git a/internal/processing/timeline/timeline.go b/internal/processing/timeline/timeline.go
index 5966fe864..54ea2cccd 100644
--- a/internal/processing/timeline/timeline.go
+++ b/internal/processing/timeline/timeline.go
@@ -18,9 +18,33 @@
 package timeline
 
 import (
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	timelinepkg "github.com/superseriousbusiness/gotosocial/internal/cache/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/usermute"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/paging"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/util/xslices"
+)
+
+var (
+	// pre-prepared URL values to be passed in to
+	// paging response forms. The paging package always
+	// copies values before any modifications so it's
+	// safe to only use a single map variable for these.
+	localOnlyTrue  = url.Values{"local": {"true"}}
+	localOnlyFalse = url.Values{"local": {"false"}}
 )
 
 type Processor struct {
@@ -36,3 +60,114 @@ func New(state *state.State, converter *typeutils.Converter, visFilter *visibili
 		visFilter: visFilter,
 	}
 }
+
+func (p *Processor) getStatusTimeline(
+	ctx context.Context,
+	requester *gtsmodel.Account,
+	timeline *timelinepkg.StatusTimeline,
+	page *paging.Page,
+	pagePath string,
+	pageQuery url.Values,
+	filterCtx statusfilter.FilterContext,
+	loadPage func(*paging.Page) (statuses []*gtsmodel.Status, err error),
+	filter func(*gtsmodel.Status) (delete bool),
+	postFilter func(*gtsmodel.Status) (remove bool),
+) (
+	*apimodel.PageableResponse,
+	gtserror.WithCode,
+) {
+	var err error
+	var filters []*gtsmodel.Filter
+	var mutes *usermute.CompiledUserMuteList
+
+	if requester != nil {
+		// Fetch all filters relevant for requesting account.
+		filters, err = p.state.DB.GetFiltersForAccountID(ctx,
+			requester.ID,
+		)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			err := gtserror.Newf("error getting account filters: %w", err)
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		// Get a list of all account mutes for requester.
+		allMutes, err := p.state.DB.GetAccountMutes(ctx,
+			requester.ID,
+			nil, // i.e. all
+		)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			err := gtserror.Newf("error getting account mutes: %w", err)
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		// Compile all account mutes to useable form.
+		mutes = usermute.NewCompiledUserMuteList(allMutes)
+	}
+
+	// Ensure we have valid
+	// input paging cursor.
+	id.ValidatePage(page)
+
+	// Load status page via timeline cache, also
+	// getting lo, hi values for next, prev pages.
+	//
+	// NOTE: this safely handles the case of a nil
+	// input timeline, i.e. uncached timeline type.
+	apiStatuses, lo, hi, err := timeline.Load(ctx,
+
+		// Status page
+		// to load.
+		page,
+
+		// Caller provided database
+		// status page loading function.
+		loadPage,
+
+		// Status load function for cached timeline entries.
+		func(ids []string) ([]*gtsmodel.Status, error) {
+			return p.state.DB.GetStatusesByIDs(ctx, ids)
+		},
+
+		// Call provided status
+		// filtering function.
+		filter,
+
+		// Frontend API model preparation function.
+		func(status *gtsmodel.Status) (*apimodel.Status, error) {
+
+			// Check if status needs filtering OUTSIDE of caching stage.
+			// TODO: this will be moved to separate postFilter hook when
+			// all filtering has been removed from the type converter.
+			if postFilter != nil && postFilter(status) {
+				return nil, nil
+			}
+
+			// Finally, pass status to get converted to API model.
+			apiStatus, err := p.converter.StatusToAPIStatus(ctx,
+				status,
+				requester,
+				filterCtx,
+				filters,
+				mutes,
+			)
+			if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
+				return nil, err
+			}
+			return apiStatus, nil
+		},
+	)
+
+	if err != nil {
+		err := gtserror.Newf("error loading timeline: %w", err)
+		return nil, gtserror.WrapWithCode(http.StatusInternalServerError, err)
+	}
+
+	// Package returned API statuses as pageable response.
+	return paging.PackageResponse(paging.ResponseParams{
+		Items: xslices.ToAny(apiStatuses),
+		Path:  pagePath,
+		Next:  page.Next(lo, hi),
+		Prev:  page.Prev(lo, hi),
+		Query: pageQuery,
+	}), nil
+}