From 84279f6a6a0201c90a6747fe8b82c38d5b4e49e2 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:46:09 +0000 Subject: [performance] cache more database calls, reduce required database calls overall (#3290) * improvements to caching for lists and relationship to accounts / follows * fix nil panic in AddToList() * ensure list related caches are correctly invalidated * ensure returned ID lists are ordered correctly * bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded) * remove zero checks in uncached key load functions (go-structr now handles this) * fix issues after rebase on upstream/main * update the expected return order of CSV exports (since list entries are now down by entry creation date) * rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests * use returning statements on delete to get cache invalidation info * fixes to recent database delete changes * fix broken list entries delete sql * remove unused db function * update remainder of delete functions to behave in similar way, some other small tweaks * fix delete user sql, allow returning on err no entries * uncomment + fix list database tests * update remaining list tests * update envparsing test * add comments to each specific key being invalidated * add more cache invalidation explanatory comments * whoops; actually delete poll votes from database in the DeletePollByID() func * remove added but-commented-out field * improved comment regarding paging being disabled * make cache invalidation comments match what's actually happening * fix up delete query comments to match what is happening * rename function to read a bit better * don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call) * update function name in test * move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic() * use the poll_id column in poll_votes for selecting votes in poll ID * fix function name --- internal/typeutils/csv.go | 71 ++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 44 deletions(-) (limited to 'internal/typeutils/csv.go') diff --git a/internal/typeutils/csv.go b/internal/typeutils/csv.go index 063e31d54..b1e35ef1a 100644 --- a/internal/typeutils/csv.go +++ b/internal/typeutils/csv.go @@ -34,16 +34,14 @@ func (c *Converter) AccountToExportStats( a *gtsmodel.Account, ) (*apimodel.AccountExportStats, error) { // Ensure account stats populated. - if a.Stats == nil { - if err := c.state.DB.PopulateAccountStats(ctx, a); err != nil { - return nil, gtserror.Newf( - "error getting stats for account %s: %w", - a.ID, err, - ) - } + if err := c.state.DB.PopulateAccountStats(ctx, a); err != nil { + return nil, gtserror.Newf( + "error getting stats for account %s: %w", + a.ID, err, + ) } - listsCount, err := c.state.DB.CountListsForAccountID(ctx, a.ID) + listsCount, err := c.state.DB.CountListsByAccountID(ctx, a.ID) if err != nil { return nil, gtserror.Newf( "error counting lists for account %s: %w", @@ -202,6 +200,7 @@ func (c *Converter) ListsToCSV( ctx context.Context, lists []*gtsmodel.List, ) ([][]string, error) { + // We need to know our own domain for this. // Try account domain, fall back to host. thisDomain := config.GetAccountDomain() @@ -215,41 +214,23 @@ func (c *Converter) ListsToCSV( // For each item, add a record. for _, list := range lists { - for _, entry := range list.ListEntries { - if entry.Follow == nil { - // Retrieve follow. - var err error - entry.Follow, err = c.state.DB.GetFollowByID( - ctx, - entry.FollowID, - ) - if err != nil { - return nil, gtserror.Newf( - "db error getting follow for list entry %s: %w", - entry.ID, err, - ) - } - } - if entry.Follow.TargetAccount == nil { - // Retrieve account. - var err error - entry.Follow.TargetAccount, err = c.state.DB.GetAccountByID( - // Barebones is fine here. - gtscontext.SetBarebones(ctx), - entry.Follow.TargetAccountID, - ) - if err != nil { - return nil, gtserror.Newf( - "db error getting target account for list entry %s: %w", - entry.ID, err, - ) - } - } + // Get all follows contained with this list. + follows, err := c.state.DB.GetFollowsInList(ctx, + list.ID, + nil, + ) + if err != nil { + err := gtserror.Newf("db error getting follows for list: %w", err) + return nil, err + } + // Append each follow as CSV record. + for _, follow := range follows { var ( - username = entry.Follow.TargetAccount.Username - domain = entry.Follow.TargetAccount.Domain + // Extract username / domain from target. + username = follow.TargetAccount.Username + domain = follow.TargetAccount.Domain ) if domain == "" { @@ -259,14 +240,16 @@ func (c *Converter) ListsToCSV( } records = append(records, []string{ - // List title: eg., Very cool list + // List title: e.g. + // Very cool list list.Title, - // Account address: eg., someone@example.org - // -- NOTE: without the leading '@'! + + // Account address: e.g., + // someone@example.org + // NOTE: without the leading '@'! username + "@" + domain, }) } - } return records, nil -- cgit v1.2.3