[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

1 files changed, 27 insertions, 44 deletions

diff --git a/internal/typeutils/csv.go b/internal/typeutils/csv.go
index 063e31d54..b1e35ef1a 100644
--- a/internal/typeutils/csv.go
+++ b/internal/typeutils/csv.go
@@ -34,16 +34,14 @@ func (c *Converter) AccountToExportStats(
 	a *gtsmodel.Account,
 ) (*apimodel.AccountExportStats, error) {
 	// Ensure account stats populated.
-	if a.Stats == nil {
-		if err := c.state.DB.PopulateAccountStats(ctx, a); err != nil {
-			return nil, gtserror.Newf(
-				"error getting stats for account %s: %w",
-				a.ID, err,
-			)
-		}
+	if err := c.state.DB.PopulateAccountStats(ctx, a); err != nil {
+		return nil, gtserror.Newf(
+			"error getting stats for account %s: %w",
+			a.ID, err,
+		)
 	}
 
-	listsCount, err := c.state.DB.CountListsForAccountID(ctx, a.ID)
+	listsCount, err := c.state.DB.CountListsByAccountID(ctx, a.ID)
 	if err != nil {
 		return nil, gtserror.Newf(
 			"error counting lists for account %s: %w",
@@ -202,6 +200,7 @@ func (c *Converter) ListsToCSV(
 	ctx context.Context,
 	lists []*gtsmodel.List,
 ) ([][]string, error) {
+
 	// We need to know our own domain for this.
 	// Try account domain, fall back to host.
 	thisDomain := config.GetAccountDomain()
@@ -215,41 +214,23 @@ func (c *Converter) ListsToCSV(
 
 	// For each item, add a record.
 	for _, list := range lists {
-		for _, entry := range list.ListEntries {
-			if entry.Follow == nil {
-				// Retrieve follow.
-				var err error
-				entry.Follow, err = c.state.DB.GetFollowByID(
-					ctx,
-					entry.FollowID,
-				)
-				if err != nil {
-					return nil, gtserror.Newf(
-						"db error getting follow for list entry %s: %w",
-						entry.ID, err,
-					)
-				}
-			}
 
-			if entry.Follow.TargetAccount == nil {
-				// Retrieve account.
-				var err error
-				entry.Follow.TargetAccount, err = c.state.DB.GetAccountByID(
-					// Barebones is fine here.
-					gtscontext.SetBarebones(ctx),
-					entry.Follow.TargetAccountID,
-				)
-				if err != nil {
-					return nil, gtserror.Newf(
-						"db error getting target account for list entry %s: %w",
-						entry.ID, err,
-					)
-				}
-			}
+		// Get all follows contained with this list.
+		follows, err := c.state.DB.GetFollowsInList(ctx,
+			list.ID,
+			nil,
+		)
+		if err != nil {
+			err := gtserror.Newf("db error getting follows for list: %w", err)
+			return nil, err
+		}
 
+		// Append each follow as CSV record.
+		for _, follow := range follows {
 			var (
-				username = entry.Follow.TargetAccount.Username
-				domain   = entry.Follow.TargetAccount.Domain
+				// Extract username / domain from target.
+				username = follow.TargetAccount.Username
+				domain   = follow.TargetAccount.Domain
 			)
 
 			if domain == "" {
@@ -259,14 +240,16 @@ func (c *Converter) ListsToCSV(
 			}
 
 			records = append(records, []string{
-				// List title: eg., Very cool list
+				// List title: e.g.
+				// Very cool list
 				list.Title,
-				// Account address: eg., someone@example.org
-				// -- NOTE: without the leading '@'!
+
+				// Account address: e.g.,
+				// someone@example.org
+				// NOTE: without the leading '@'!
 				username + "@" + domain,
 			})
 		}
-
 	}
 
 	return records, nil