From 6a6a4993338262f87df34c9be051bfaac75c1829 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sat, 26 Apr 2025 09:56:15 +0000 Subject: [performance] rewrite timelines to rely on new timeline cache type (#3941) * start work rewriting timeline cache type * further work rewriting timeline caching * more work integration new timeline code * remove old code * add local timeline, fix up merge conflicts * remove old use of go-bytes * implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr * remove old timeline package, add local timeline cache * remove references to old timeline types that needed starting up in tests * start adding page validation * fix test-identified timeline cache package issues * fix up more tests, fix missing required changes, etc * add exclusion for test.out in gitignore * clarify some things better in code comments * tweak cache size limits * fix list timeline cache fetching * further list timeline fixes * linter, ssssssssshhhhhhhhhhhh please * fix linter hints * reslice the output if it's beyond length of 'lim' * remove old timeline initialization code, bump go-structr to v0.9.4 * continued from previous commit * improved code comments * don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts * finish writing more code comments * some variable renaming, for ease of following * change the way we update lo,hi paging values during timeline load * improved code comments for updated / returned lo , hi paging values * finish writing code comments for the StatusTimeline{} type itself * fill in more code comments * update go-structr version to latest with changed timeline unique indexing logic * have a local and public timeline *per user* * rewrite calls to public / local timeline calls * remove the zero length check, as lo, hi values might still be set * simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions * swap the lo, hi values :facepalm: * add (now) missing slice reverse of tag timeline statuses when paging ASC * remove local / public caches (is out of scope for this work), share more timeline code * remove unnecessary change * again, remove more unused code * remove unused function to appease the linter * move boost checking to prepare function * fix use of timeline.lastOrder, fix incorrect range functions used * remove comments for repeat code * remove the boost logic from prepare function * do a maximum of 5 loads, not 10 * add repeat boost filtering logic, update go-structr, general improvements * more code comments * add important note * fix timeline tests now that timelines are returned in page order * remove unused field * add StatusTimeline{} tests * add more status timeline tests * start adding preloading support * ensure repeat boosts are marked in preloaded entries * share a bunch of the database load code in timeline cache, don't clear timelines on relationship change * add logic to allow dynamic clear / preloading of timelines * comment-out unused functions, but leave in place as we might end-up using them * fix timeline preload state check * much improved status timeline code comments * more code comments, don't bother inserting statuses if timeline not preloaded * shift around some logic to make sure things aren't accidentally left set * finish writing code comments * remove trim-after-insert behaviour * fix-up some comments referring to old logic * remove unsetting of lo, hi * fix preload repeatBoost checking logic * don't return on status filter errors, these are usually transient * better concurrency safety in Clear() and Done() * fix test broken due to addition of preloader * fix repeatBoost logic that doesn't account for already-hidden repeatBoosts * ensure edit submodels are dropped on cache insertion * update code-comment to expand CAS accronym * use a plus1hULID() instead of 24h * remove unused functions * add note that public / local timeline requester can be nil * fix incorrect visibility filtering of tag timeline statuses * ensure we filter home timeline statuses on local only * some small re-orderings to confirm query params in correct places * fix the local only home timeline filter func --- internal/db/bundb/timeline_test.go | 237 +++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 129 deletions(-) (limited to 'internal/db/bundb/timeline_test.go') diff --git a/internal/db/bundb/timeline_test.go b/internal/db/bundb/timeline_test.go index 4988ab362..9652df33a 100644 --- a/internal/db/bundb/timeline_test.go +++ b/internal/db/bundb/timeline_test.go @@ -20,14 +20,12 @@ package bundb_test import ( "context" "testing" - "time" - "codeberg.org/gruf/go-kv" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -35,38 +33,6 @@ type TimelineTestSuite struct { BunDBStandardTestSuite } -func getFutureStatus() *gtsmodel.Status { - theDistantFuture := time.Now().Add(876600 * time.Hour) - id := id.NewULIDFromTime(theDistantFuture) - - return >smodel.Status{ - ID: id, - URI: "http://localhost:8080/users/admin/statuses/" + id, - URL: "http://localhost:8080/@admin/statuses/" + id, - Content: "it's the future, wooooooooooooooooooooooooooooooooo", - Text: "it's the future, wooooooooooooooooooooooooooooooooo", - ContentType: gtsmodel.StatusContentTypePlain, - AttachmentIDs: []string{}, - TagIDs: []string{}, - MentionIDs: []string{}, - EmojiIDs: []string{}, - CreatedAt: theDistantFuture, - Local: util.Ptr(true), - AccountURI: "http://localhost:8080/users/admin", - AccountID: "01F8MH17FWEB39HZJ76B6VXSKF", - InReplyToID: "", - BoostOfID: "", - ContentWarning: "", - Visibility: gtsmodel.VisibilityPublic, - Sensitive: util.Ptr(false), - Language: "en", - CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", - Federated: util.Ptr(true), - InteractionPolicy: gtsmodel.DefaultInteractionPolicyPublic(), - ActivityStreamsType: ap.ObjectNote, - } -} - func (suite *TimelineTestSuite) publicCount() int { var publicCount int for _, status := range suite.testStatuses { @@ -92,7 +58,7 @@ func (suite *TimelineTestSuite) localCount() int { return localCount } -func (suite *TimelineTestSuite) checkStatuses(statuses []*gtsmodel.Status, maxID string, minID string, expectedLength int) { +func (suite *TimelineTestSuite) checkStatuses(statuses []*gtsmodel.Status, maxID string, minID string, expectedOrder paging.Order, expectedLength int) { if l := len(statuses); l != expectedLength { suite.FailNowf("", "expected %d statuses in slice, got %d", expectedLength, l) } else if l == 0 { @@ -100,74 +66,73 @@ func (suite *TimelineTestSuite) checkStatuses(statuses []*gtsmodel.Status, maxID return } - // Check ordering + bounds of statuses. - highest := statuses[0].ID - for _, status := range statuses { - id := status.ID + if expectedOrder.Ascending() { + // Check ordering + bounds of statuses. + lowest := statuses[0].ID + for _, status := range statuses { + id := status.ID - if id >= maxID { - suite.FailNowf("", "%s greater than maxID %s", id, maxID) - } + if id >= maxID { + suite.FailNowf("", "%s greater than maxID %s", id, maxID) + } - if id <= minID { - suite.FailNowf("", "%s smaller than minID %s", id, minID) - } + if id <= minID { + suite.FailNowf("", "%s smaller than minID %s", id, minID) + } - if id > highest { - suite.FailNowf("", "statuses in slice were not ordered highest -> lowest ID") - } + if id < lowest { + suite.FailNowf("", "statuses in slice were not ordered lowest -> highest ID") + } - highest = id - } -} + lowest = id + } + } else { + // Check ordering + bounds of statuses. + highest := statuses[0].ID + for _, status := range statuses { + id := status.ID -func (suite *TimelineTestSuite) TestGetPublicTimeline() { - ctx := context.Background() + if id >= maxID { + suite.FailNowf("", "%s greater than maxID %s", id, maxID) + } - s, err := suite.db.GetPublicTimeline(ctx, "", "", "", 20, false) - if err != nil { - suite.FailNow(err.Error()) - } + if id <= minID { + suite.FailNowf("", "%s smaller than minID %s", id, minID) + } - suite.T().Log(kv.Field{ - K: "statuses", V: s, - }) + if id > highest { + suite.FailNowf("", "statuses in slice were not ordered highest -> lowest ID") + } - suite.checkStatuses(s, id.Highest, id.Lowest, suite.publicCount()) + highest = id + } + } } -func (suite *TimelineTestSuite) TestGetPublicTimelineLocal() { +func (suite *TimelineTestSuite) TestGetPublicTimeline() { ctx := context.Background() - s, err := suite.db.GetPublicTimeline(ctx, "", "", "", 20, true) + page := toPage("", "", "", 20) + + s, err := suite.db.GetPublicTimeline(ctx, page) if err != nil { suite.FailNow(err.Error()) } - suite.T().Log(kv.Field{ - K: "statuses", V: s, - }) - - suite.checkStatuses(s, id.Highest, id.Lowest, suite.localCount()) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), suite.publicCount()) } -func (suite *TimelineTestSuite) TestGetPublicTimelineWithFutureStatus() { +func (suite *TimelineTestSuite) TestGetPublicTimelineLocal() { ctx := context.Background() - // Insert a status set far in the - // future, it shouldn't be retrieved. - futureStatus := getFutureStatus() - if err := suite.db.PutStatus(ctx, futureStatus); err != nil { - suite.FailNow(err.Error()) - } + page := toPage("", "", "", 20) - s, err := suite.db.GetPublicTimeline(ctx, "", "", "", 20, false) + s, err := suite.db.GetLocalTimeline(ctx, page) if err != nil { suite.FailNow(err.Error()) } - suite.NotContains(s, futureStatus) - suite.checkStatuses(s, id.Highest, id.Lowest, suite.publicCount()) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), suite.localCount()) } func (suite *TimelineTestSuite) TestGetHomeTimeline() { @@ -176,12 +141,14 @@ func (suite *TimelineTestSuite) TestGetHomeTimeline() { viewingAccount = suite.testAccounts["local_account_1"] ) - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", "", 20, false) + page := toPage("", "", "", 20) + + s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 20) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 20) } func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { @@ -201,13 +168,15 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { suite.FailNow(err.Error()) } + page := toPage("", "", "", 20) + // First try with list just set to exclusive. // We should only get zork's own statuses. - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", "", 20, false) + s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 9) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 9) // Remove admin account from the exclusive list. listEntry := suite.testListEntries["local_account_1_list_1_entry_2"] @@ -217,11 +186,11 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { // Zork should only see their own // statuses and admin's statuses now. - s, err = suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", "", 20, false) + s, err = suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 13) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 13) } func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { @@ -246,36 +215,16 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { } } + page := toPage("", "", "", 20) + // Query should work fine; though far // fewer statuses will be returned ofc. - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", "", 20, false) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(s, id.Highest, id.Lowest, 9) -} - -func (suite *TimelineTestSuite) TestGetHomeTimelineWithFutureStatus() { - var ( - ctx = context.Background() - viewingAccount = suite.testAccounts["local_account_1"] - ) - - // Insert a status set far in the - // future, it shouldn't be retrieved. - futureStatus := getFutureStatus() - if err := suite.db.PutStatus(ctx, futureStatus); err != nil { - suite.FailNow(err.Error()) - } - - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", "", 20, false) + s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.NotContains(s, futureStatus) - suite.checkStatuses(s, id.Highest, id.Lowest, 20) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 9) } func (suite *TimelineTestSuite) TestGetHomeTimelineBackToFront() { @@ -284,14 +233,16 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineBackToFront() { viewingAccount = suite.testAccounts["local_account_1"] ) - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, "", "", id.Lowest, 5, false) + page := toPage("", "", id.Lowest, 5) + + s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01F8MHAYFKS4KMXF8K5Y1C0KRN", s[0].ID) - suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[len(s)-1].ID) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 5) + suite.Equal("01F8MHAYFKS4KMXF8K5Y1C0KRN", s[len(s)-1].ID) + suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[0].ID) } func (suite *TimelineTestSuite) TestGetHomeTimelineFromHighest() { @@ -300,12 +251,14 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineFromHighest() { viewingAccount = suite.testAccounts["local_account_1"] ) - s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, id.Highest, "", "", 5, false) + page := toPage(id.Highest, "", "", 5) + + s, err := suite.db.GetHomeTimeline(ctx, viewingAccount.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 5) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 5) suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) suite.Equal("01HEN2RZ8BG29Y5Z9VJC73HZW7", s[len(s)-1].ID) } @@ -316,12 +269,14 @@ func (suite *TimelineTestSuite) TestGetListTimelineNoParams() { list = suite.testLists["local_account_1_list_1"] ) - s, err := suite.db.GetListTimeline(ctx, list.ID, "", "", "", 20) + page := toPage("", "", "", 20) + + s, err := suite.db.GetListTimeline(ctx, list.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 13) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 13) } func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { @@ -330,12 +285,14 @@ func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { list = suite.testLists["local_account_1_list_1"] ) - s, err := suite.db.GetListTimeline(ctx, list.ID, id.Highest, "", "", 5) + page := toPage(id.Highest, "", "", 5) + + s, err := suite.db.GetListTimeline(ctx, list.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 5) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 5) suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) suite.Equal("01FN3VJGFH10KR7S2PB0GFJZYG", s[len(s)-1].ID) } @@ -346,14 +303,16 @@ func (suite *TimelineTestSuite) TestGetListTimelineMinID() { list = suite.testLists["local_account_1_list_1"] ) - s, err := suite.db.GetListTimeline(ctx, list.ID, "", "", id.Lowest, 5) + page := toPage("", "", id.Lowest, 5) + + s, err := suite.db.GetListTimeline(ctx, list.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01F8MHC8VWDRBQR0N1BATDDEM5", s[0].ID) - suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[len(s)-1].ID) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 5) + suite.Equal("01F8MHC8VWDRBQR0N1BATDDEM5", s[len(s)-1].ID) + suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[0].ID) } func (suite *TimelineTestSuite) TestGetListTimelineMinIDPagingUp() { @@ -362,14 +321,16 @@ func (suite *TimelineTestSuite) TestGetListTimelineMinIDPagingUp() { list = suite.testLists["local_account_1_list_1"] ) - s, err := suite.db.GetListTimeline(ctx, list.ID, "", "", "01F8MHC8VWDRBQR0N1BATDDEM5", 5) + page := toPage("", "", "01F8MHC8VWDRBQR0N1BATDDEM5", 5) + + s, err := suite.db.GetListTimeline(ctx, list.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, "01F8MHC8VWDRBQR0N1BATDDEM5", 5) - suite.Equal("01G20ZM733MGN8J344T4ZDDFY1", s[0].ID) - suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", s[len(s)-1].ID) + suite.checkStatuses(s, id.Highest, "01F8MHC8VWDRBQR0N1BATDDEM5", page.Order(), 5) + suite.Equal("01G20ZM733MGN8J344T4ZDDFY1", s[len(s)-1].ID) + suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", s[0].ID) } func (suite *TimelineTestSuite) TestGetTagTimelineNoParams() { @@ -378,15 +339,33 @@ func (suite *TimelineTestSuite) TestGetTagTimelineNoParams() { tag = suite.testTags["welcome"] ) - s, err := suite.db.GetTagTimeline(ctx, tag.ID, "", "", "", 1) + page := toPage("", "", "", 1) + + s, err := suite.db.GetTagTimeline(ctx, tag.ID, page) if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 1) + suite.checkStatuses(s, id.Highest, id.Lowest, page.Order(), 1) suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[0].ID) } func TestTimelineTestSuite(t *testing.T) { suite.Run(t, new(TimelineTestSuite)) } + +// toPage is a helper function to wrap a series of paging arguments in paging.Page{}. +func toPage(maxID, sinceID, minID string, limit int) *paging.Page { + var pg paging.Page + pg.Limit = limit + + if maxID != "" { + pg.Max = paging.MaxID(maxID) + } + + if sinceID != "" || minID != "" { + pg.Min = paging.EitherMinID(minID, sinceID) + } + + return &pg +} -- cgit v1.2.3