[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

1 files changed, 111 insertions, 103 deletions

diff --git a/internal/db/bundb/relationship_follow.go b/internal/db/bundb/relationship_follow.go
index 413f3a2af..042d12f37 100644
--- a/internal/db/bundb/relationship_follow.go
+++ b/internal/db/bundb/relationship_follow.go
@@ -20,7 +20,6 @@ package bundb
 import (
 	"context"
 	"errors"
-	"fmt"
 	"slices"
 	"time"
 
@@ -82,15 +81,8 @@ func (r *relationshipDB) GetFollowsByIDs(ctx context.Context, ids []string) ([]*
 	follows, err := r.state.Caches.DB.Follow.LoadIDs("ID",
 		ids,
 		func(uncached []string) ([]*gtsmodel.Follow, error) {
-			// Avoid querying
-			// if none uncached.
-			count := len(uncached)
-			if count == 0 {
-				return nil, nil
-			}
-
 			// Preallocate expected length of uncached follows.
-			follows := make([]*gtsmodel.Follow, 0, count)
+			follows := make([]*gtsmodel.Follow, 0, len(uncached))
 
 			// Perform database query scanning
 			// the remaining (uncached) IDs.
@@ -252,139 +244,155 @@ func (r *relationshipDB) UpdateFollow(ctx context.Context, follow *gtsmodel.Foll
 	})
 }
 
-func (r *relationshipDB) deleteFollow(ctx context.Context, id string) error {
-	// Delete the follow itself using the given ID.
+func (r *relationshipDB) DeleteFollow(
+	ctx context.Context,
+	sourceAccountID string,
+	targetAccountID string,
+) error {
+
+	// Gather necessary fields from
+	// deleted for cache invaliation.
+	var deleted gtsmodel.Follow
+	deleted.AccountID = sourceAccountID
+	deleted.TargetAccountID = targetAccountID
+
+	// Delete follow from origin
+	// account, to targeting account,
+	// returning the deleted models.
 	if _, err := r.db.NewDelete().
-		Table("follows").
-		Where("? = ?", bun.Ident("id"), id).
-		Exec(ctx); err != nil {
+		Model(&deleted).
+		Where("? = ?", bun.Ident("account_id"), sourceAccountID).
+		Where("? = ?", bun.Ident("target_account_id"), targetAccountID).
+		Returning("?", bun.Ident("id")).
+		Exec(ctx); err != nil &&
+		!errors.Is(err, db.ErrNoEntries) {
 		return err
 	}
 
-	// Delete every list entry that used this followID.
-	if err := r.state.DB.DeleteListEntriesForFollowID(ctx, id); err != nil {
-		return fmt.Errorf("deleteFollow: error deleting list entries: %w", err)
+	// Invalidate cached follow with source / target account IDs,
+	// manually calling invalidate hook in case it isn't cached.
+	r.state.Caches.DB.Follow.Invalidate("AccountID,TargetAccountID",
+		sourceAccountID, targetAccountID)
+	r.state.Caches.OnInvalidateFollow(&deleted)
+
+	// Delete every list entry that was created targetting this follow ID.
+	if err := r.state.DB.DeleteAllListEntriesByFollows(ctx, deleted.ID); err != nil {
+		return gtserror.Newf("error deleting list entries: %w", err)
 	}
 
 	return nil
 }
 
-func (r *relationshipDB) DeleteFollow(ctx context.Context, sourceAccountID string, targetAccountID string) error {
-	// Load follow into cache before attempting a delete,
-	// as we need it cached in order to trigger the invalidate
-	// callback. This in turn invalidates others.
-	follow, err := r.GetFollow(
-		gtscontext.SetBarebones(ctx),
-		sourceAccountID,
-		targetAccountID,
-	)
-	if err != nil {
-		if errors.Is(err, db.ErrNoEntries) {
-			// Already gone.
-			return nil
-		}
+func (r *relationshipDB) DeleteFollowByID(ctx context.Context, id string) error {
+	// Gather necessary fields from
+	// deleted for cache invaliation.
+	var deleted gtsmodel.Follow
+	deleted.ID = id
+
+	// Delete follow with given ID,
+	// returning the deleted models.
+	if _, err := r.db.NewDelete().
+		Model(&deleted).
+		Where("? = ?", bun.Ident("id"), id).
+		Returning("?, ?",
+			bun.Ident("account_id"),
+			bun.Ident("target_account_id"),
+		).
+		Exec(ctx); err != nil &&
+		!errors.Is(err, db.ErrNoEntries) {
 		return err
 	}
 
-	// Drop this now-cached follow on return after delete.
-	defer r.state.Caches.DB.Follow.Invalidate("AccountID,TargetAccountID", sourceAccountID, targetAccountID)
-
-	// Finally delete follow from DB.
-	return r.deleteFollow(ctx, follow.ID)
-}
+	// Invalidate cached follow with ID, manually
+	// call invalidate hook in case not cached.
+	r.state.Caches.DB.Follow.Invalidate("ID", id)
+	r.state.Caches.OnInvalidateFollow(&deleted)
 
-func (r *relationshipDB) DeleteFollowByID(ctx context.Context, id string) error {
-	// Load follow into cache before attempting a delete,
-	// as we need it cached in order to trigger the invalidate
-	// callback. This in turn invalidates others.
-	follow, err := r.GetFollowByID(gtscontext.SetBarebones(ctx), id)
-	if err != nil {
-		if errors.Is(err, db.ErrNoEntries) {
-			// Already gone.
-			return nil
-		}
-		return err
+	// Delete every list entry that was created targetting this follow ID.
+	if err := r.state.DB.DeleteAllListEntriesByFollows(ctx, id); err != nil {
+		return gtserror.Newf("error deleting list entries: %w", err)
 	}
 
-	// Drop this now-cached follow on return after delete.
-	defer r.state.Caches.DB.Follow.Invalidate("ID", id)
-
-	// Finally delete follow from DB.
-	return r.deleteFollow(ctx, follow.ID)
+	return nil
 }
 
 func (r *relationshipDB) DeleteFollowByURI(ctx context.Context, uri string) error {
-	// Load follow into cache before attempting a delete,
-	// as we need it cached in order to trigger the invalidate
-	// callback. This in turn invalidates others.
-	follow, err := r.GetFollowByURI(gtscontext.SetBarebones(ctx), uri)
-	if err != nil {
-		if errors.Is(err, db.ErrNoEntries) {
-			// Already gone.
-			return nil
-		}
+	// Gather necessary fields from
+	// deleted for cache invaliation.
+	var deleted gtsmodel.Follow
+
+	// Delete follow with given URI,
+	// returning the deleted models.
+	if _, err := r.db.NewDelete().
+		Model(&deleted).
+		Where("? = ?", bun.Ident("uri"), uri).
+		Returning("?, ?, ?",
+			bun.Ident("id"),
+			bun.Ident("account_id"),
+			bun.Ident("target_account_id"),
+		).
+		Exec(ctx); err != nil &&
+		!errors.Is(err, db.ErrNoEntries) {
 		return err
 	}
 
-	// Drop this now-cached follow on return after delete.
-	defer r.state.Caches.DB.Follow.Invalidate("URI", uri)
+	// Invalidate cached follow with URI, manually
+	// call invalidate hook in case not cached.
+	r.state.Caches.DB.Follow.Invalidate("URI", uri)
+	r.state.Caches.OnInvalidateFollow(&deleted)
 
-	// Finally delete follow from DB.
-	return r.deleteFollow(ctx, follow.ID)
+	// Delete every list entry that was created targetting this follow ID.
+	if err := r.state.DB.DeleteAllListEntriesByFollows(ctx, deleted.ID); err != nil {
+		return gtserror.Newf("error deleting list entries: %w", err)
+	}
+
+	return nil
 }
 
 func (r *relationshipDB) DeleteAccountFollows(ctx context.Context, accountID string) error {
-	var followIDs []string
+	// Gather necessary fields from
+	// deleted for cache invaliation.
+	var deleted []*gtsmodel.Follow
 
-	// Get full list of IDs.
-	if _, err := r.db.
-		NewSelect().
-		Column("id").
-		Table("follows").
+	// Delete all follows either from
+	// account, or targeting account,
+	// returning the deleted models.
+	if _, err := r.db.NewDelete().
+		Model(&deleted).
 		WhereOr("? = ? OR ? = ?",
 			bun.Ident("account_id"),
 			accountID,
 			bun.Ident("target_account_id"),
 			accountID,
 		).
-		Exec(ctx, &followIDs); err != nil {
+		Returning("?, ?, ?",
+			bun.Ident("id"),
+			bun.Ident("account_id"),
+			bun.Ident("target_account_id"),
+		).
+		Exec(ctx); err != nil &&
+		!errors.Is(err, db.ErrNoEntries) {
 		return err
 	}
 
-	if len(followIDs) == 0 {
-		// Nothing
-		// to delete.
-		return nil
-	}
-
-	defer func() {
-		// Invalidate all account's incoming / outoing follows on return.
-		r.state.Caches.DB.Follow.Invalidate("AccountID", accountID)
-		r.state.Caches.DB.Follow.Invalidate("TargetAccountID", accountID)
-	}()
+	// Gather the follow IDs that were deleted for removing related list entries.
+	followIDs := util.Gather(nil, deleted, func(follow *gtsmodel.Follow) string {
+		return follow.ID
+	})
 
-	// Load all follows into cache, this *really* isn't great
-	// but it is the only way we can ensure we invalidate all
-	// related caches correctly (e.g. visibility).
-	_, err := r.GetAccountFollows(ctx, accountID, nil)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		return err
+	// Delete every list entry that was created targetting any of these follow IDs.
+	if err := r.state.DB.DeleteAllListEntriesByFollows(ctx, followIDs...); err != nil {
+		return gtserror.Newf("error deleting list entries: %w", err)
 	}
 
-	// Delete all follows from DB.
-	_, err = r.db.NewDelete().
-		Table("follows").
-		Where("? IN (?)", bun.Ident("id"), bun.In(followIDs)).
-		Exec(ctx)
-	if err != nil {
-		return err
-	}
+	// Invalidate all account's incoming / outoing follows.
+	r.state.Caches.DB.Follow.Invalidate("AccountID", accountID)
+	r.state.Caches.DB.Follow.Invalidate("TargetAccountID", accountID)
 
-	for _, id := range followIDs {
-		// Finally, delete all list entries associated with each follow ID.
-		if err := r.state.DB.DeleteListEntriesForFollowID(ctx, id); err != nil {
-			return err
-		}
+	// In case not all follow were in
+	// cache, manually call invalidate hooks.
+	for _, follow := range deleted {
+		r.state.Caches.OnInvalidateFollow(follow)
 	}
 
 	return nil