[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

1 files changed, 33 insertions, 105 deletions

diff --git a/internal/processing/list/get.go b/internal/processing/list/get.go
index cdd3c6e0c..b98678eef 100644
--- a/internal/processing/list/get.go
+++ b/internal/processing/list/get.go
@@ -20,7 +20,6 @@ package list
 import (
 	"context"
 	"errors"
-	"fmt"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -28,7 +27,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/util"
+	"github.com/superseriousbusiness/gotosocial/internal/paging"
 )
 
 // Get returns the api model of one list with the given ID.
@@ -49,16 +48,14 @@ func (p *Processor) Get(ctx context.Context, account *gtsmodel.Account, id strin
 
 // GetAll returns multiple lists created by the given account, sorted by list ID DESC (newest first).
 func (p *Processor) GetAll(ctx context.Context, account *gtsmodel.Account) ([]*apimodel.List, gtserror.WithCode) {
-	lists, err := p.state.DB.GetListsForAccountID(
+	lists, err := p.state.DB.GetListsByAccountID(
+
 		// Use barebones ctx; no embedded
 		// structs necessary for simple GET.
 		gtscontext.SetBarebones(ctx),
 		account.ID,
 	)
-	if err != nil {
-		if errors.Is(err, db.ErrNoEntries) {
-			return nil, nil
-		}
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
@@ -68,66 +65,23 @@ func (p *Processor) GetAll(ctx context.Context, account *gtsmodel.Account) ([]*a
 		if errWithCode != nil {
 			return nil, errWithCode
 		}
-
 		apiLists = append(apiLists, apiList)
 	}
 
 	return apiLists, nil
 }
 
-// GetAllListAccounts returns all accounts that are in the given list,
-// owned by the given account. There's no pagination for this endpoint.
-//
-// See https://docs.joinmastodon.org/methods/lists/#query-parameters:
-//
-//	Limit: Integer. Maximum number of results. Defaults to 40 accounts.
-//	Max 80 accounts. Set to 0 in order to get all accounts without pagination.
-func (p *Processor) GetAllListAccounts(
-	ctx context.Context,
-	account *gtsmodel.Account,
-	listID string,
-) ([]*apimodel.Account, gtserror.WithCode) {
-	// Ensure list exists + is owned by requesting account.
-	_, errWithCode := p.getList(
-		// Use barebones ctx; no embedded
-		// structs necessary for this call.
-		gtscontext.SetBarebones(ctx),
-		account.ID,
-		listID,
-	)
-	if errWithCode != nil {
-		return nil, errWithCode
-	}
-
-	// Get all entries for this list.
-	listEntries, err := p.state.DB.GetListEntries(ctx, listID, "", "", "", 0)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		err = gtserror.Newf("error getting list entries: %w", err)
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	// Extract accounts from list entries + add them to response.
-	accounts := make([]*apimodel.Account, 0, len(listEntries))
-	p.accountsFromListEntries(ctx, listEntries, func(acc *apimodel.Account) {
-		accounts = append(accounts, acc)
-	})
-
-	return accounts, nil
-}
-
 // GetListAccounts returns accounts that are in the given list, owned by the given account.
-// The additional parameters can be used for paging.
+// The additional parameters can be used for paging. Nil page param returns all accounts.
 func (p *Processor) GetListAccounts(
 	ctx context.Context,
 	account *gtsmodel.Account,
 	listID string,
-	maxID string,
-	sinceID string,
-	minID string,
-	limit int,
+	page *paging.Page,
 ) (*apimodel.PageableResponse, gtserror.WithCode) {
 	// Ensure list exists + is owned by requesting account.
 	_, errWithCode := p.getList(
+
 		// Use barebones ctx; no embedded
 		// structs necessary for this call.
 		gtscontext.SetBarebones(ctx),
@@ -138,71 +92,45 @@ func (p *Processor) GetListAccounts(
 		return nil, errWithCode
 	}
 
-	// To know which accounts are in the list,
-	// we need to first get requested list entries.
-	listEntries, err := p.state.DB.GetListEntries(ctx, listID, maxID, sinceID, minID, limit)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		err = fmt.Errorf("GetListAccounts: error getting list entries: %w", err)
+	// Get all accounts contained within list.
+	accounts, err := p.state.DB.GetAccountsInList(ctx,
+		listID,
+		page,
+	)
+	if err != nil {
+		err := gtserror.Newf("db error getting accounts in list: %w", err)
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	count := len(listEntries)
+	// Check for any accounts.
+	count := len(accounts)
 	if count == 0 {
-		// No list entries means no accounts.
-		return util.EmptyPageableResponse(), nil
+		return paging.EmptyResponse(), nil
 	}
 
 	var (
+		// Preallocate expected frontend items.
 		items = make([]interface{}, 0, count)
 
-		// Set next + prev values before filtering and API
-		// converting, so caller can still page properly.
-		nextMaxIDValue = listEntries[count-1].ID
-		prevMinIDValue = listEntries[0].ID
+		// Set paging low / high IDs.
+		lo = accounts[count-1].ID
+		hi = accounts[0].ID
 	)
 
-	// Extract accounts from list entries + add them to response.
-	p.accountsFromListEntries(ctx, listEntries, func(acc *apimodel.Account) {
-		items = append(items, acc)
-	})
-
-	return util.PackagePageableResponse(util.PageableResponseParams{
-		Items:          items,
-		Path:           "/api/v1/lists/" + listID + "/accounts",
-		NextMaxIDValue: nextMaxIDValue,
-		PrevMinIDValue: prevMinIDValue,
-		Limit:          limit,
-	})
-}
-
-func (p *Processor) accountsFromListEntries(
-	ctx context.Context,
-	listEntries []*gtsmodel.ListEntry,
-	appendAcc func(*apimodel.Account),
-) {
-	// For each list entry, we want the account it points to.
-	// To get this, we need to first get the follow that the
-	// list entry pertains to, then extract the target account
-	// from that follow.
-	//
-	// We do paging not by account ID, but by list entry ID.
-	for _, listEntry := range listEntries {
-		if err := p.state.DB.PopulateListEntry(ctx, listEntry); err != nil {
-			log.Errorf(ctx, "error populating list entry: %v", err)
-			continue
-		}
-
-		if err := p.state.DB.PopulateFollow(ctx, listEntry.Follow); err != nil {
-			log.Errorf(ctx, "error populating follow: %v", err)
-			continue
-		}
-
-		apiAccount, err := p.converter.AccountToAPIAccountPublic(ctx, listEntry.Follow.TargetAccount)
+	// Convert accounts to frontend.
+	for _, account := range accounts {
+		apiAccount, err := p.converter.AccountToAPIAccountPublic(ctx, account)
 		if err != nil {
-			log.Errorf(ctx, "error converting to public api account: %v", err)
+			log.Errorf(ctx, "error converting to api account: %v", err)
 			continue
 		}
-
-		appendAcc(apiAccount)
+		items = append(items, apiAccount)
 	}
+
+	return paging.PackageResponse(paging.ResponseParams{
+		Items: items,
+		Path:  "/api/v1/lists/" + listID + "/accounts",
+		Next:  page.Next(lo, hi),
+		Prev:  page.Prev(lo, hi),
+	}), nil
 }