[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

1 files changed, 12 insertions, 31 deletions

diff --git a/internal/processing/account/lists.go b/internal/processing/account/lists.go
index 1d92bee82..04cf4ca73 100644
--- a/internal/processing/account/lists.go
+++ b/internal/processing/account/lists.go
@@ -30,8 +30,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
-var noLists = make([]*apimodel.List, 0)
-
 // ListsGet returns all lists owned by requestingAccount, which contain a follow for targetAccountID.
 func (p *Processor) ListsGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string) ([]*apimodel.List, gtserror.WithCode) {
 	targetAccount, err := p.state.DB.GetAccountByID(ctx, targetAccountID)
@@ -54,52 +52,35 @@ func (p *Processor) ListsGet(ctx context.Context, requestingAccount *gtsmodel.Ac
 	// Requester has to follow targetAccount
 	// for them to be in any of their lists.
 	follow, err := p.state.DB.GetFollow(
+
 		// Don't populate follow.
 		gtscontext.SetBarebones(ctx),
 		requestingAccount.ID,
 		targetAccountID,
 	)
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %w", err))
+		err := gtserror.Newf("error getting follow: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	if follow == nil {
-		return noLists, nil // by definition we know they're in no lists
-	}
-
-	listEntries, err := p.state.DB.GetListEntriesForFollowID(
-		// Don't populate entries.
-		gtscontext.SetBarebones(ctx),
-		follow.ID,
-	)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %w", err))
+		return []*apimodel.List{}, nil
 	}
 
-	count := len(listEntries)
-	if count == 0 {
-		return noLists, nil
+	// Get all lists that this follow is an entry within.
+	lists, err := p.state.DB.GetListsContainingFollowID(ctx, follow.ID)
+	if err != nil {
+		err := gtserror.Newf("error getting lists for follow: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	apiLists := make([]*apimodel.List, 0, count)
-	for _, listEntry := range listEntries {
-		list, err := p.state.DB.GetListByID(
-			// Don't populate list.
-			gtscontext.SetBarebones(ctx),
-			listEntry.ListID,
-		)
-
-		if err != nil {
-			log.Debugf(ctx, "skipping list %s due to error %q", listEntry.ListID, err)
-			continue
-		}
-
+	apiLists := make([]*apimodel.List, 0, len(lists))
+	for _, list := range lists {
 		apiList, err := p.converter.ListToAPIList(ctx, list)
 		if err != nil {
-			log.Debugf(ctx, "skipping list %s due to error %q", listEntry.ListID, err)
+			log.Errorf(ctx, "error converting list: %v", err)
 			continue
 		}
-
 		apiLists = append(apiLists, apiList)
 	}