[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

4 files changed, 172 insertions, 94 deletions

diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 8291dec5a..09e505ff5 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -62,7 +62,6 @@ func (c *Caches) Init() {
 	log.Infof(nil, "init: %p", c)
 
 	c.initAccount()
-	c.initAccountIDsFollowingTag()
 	c.initAccountNote()
 	c.initAccountSettings()
 	c.initAccountStats()
@@ -84,11 +83,13 @@ func (c *Caches) Init() {
 	c.initFollowIDs()
 	c.initFollowRequest()
 	c.initFollowRequestIDs()
+	c.initFollowingTagIDs()
 	c.initInReplyToIDs()
 	c.initInstance()
 	c.initInteractionRequest()
 	c.initList()
-	c.initListEntry()
+	c.initListIDs()
+	c.initListedIDs()
 	c.initMarker()
 	c.initMedia()
 	c.initMention()
@@ -105,7 +106,6 @@ func (c *Caches) Init() {
 	c.initStatusFave()
 	c.initStatusFaveIDs()
 	c.initTag()
-	c.initTagIDsFollowedByAccount()
 	c.initThreadMute()
 	c.initToken()
 	c.initTombstone()
@@ -148,7 +148,6 @@ func (c *Caches) Stop() {
 // significant overhead to all cache writes.
 func (c *Caches) Sweep(threshold float64) {
 	c.DB.Account.Trim(threshold)
-	c.DB.AccountIDsFollowingTag.Trim(threshold)
 	c.DB.AccountNote.Trim(threshold)
 	c.DB.AccountSettings.Trim(threshold)
 	c.DB.AccountStats.Trim(threshold)
@@ -168,11 +167,13 @@ func (c *Caches) Sweep(threshold float64) {
 	c.DB.FollowIDs.Trim(threshold)
 	c.DB.FollowRequest.Trim(threshold)
 	c.DB.FollowRequestIDs.Trim(threshold)
+	c.DB.FollowingTagIDs.Trim(threshold)
 	c.DB.InReplyToIDs.Trim(threshold)
 	c.DB.Instance.Trim(threshold)
 	c.DB.InteractionRequest.Trim(threshold)
 	c.DB.List.Trim(threshold)
-	c.DB.ListEntry.Trim(threshold)
+	c.DB.ListIDs.Trim(threshold)
+	c.DB.ListedIDs.Trim(threshold)
 	c.DB.Marker.Trim(threshold)
 	c.DB.Media.Trim(threshold)
 	c.DB.Mention.Trim(threshold)
@@ -189,7 +190,6 @@ func (c *Caches) Sweep(threshold float64) {
 	c.DB.StatusFave.Trim(threshold)
 	c.DB.StatusFaveIDs.Trim(threshold)
 	c.DB.Tag.Trim(threshold)
-	c.DB.TagIDsFollowedByAccount.Trim(threshold)
 	c.DB.ThreadMute.Trim(threshold)
 	c.DB.Token.Trim(threshold)
 	c.DB.Tombstone.Trim(threshold)
diff --git a/internal/cache/db.go b/internal/cache/db.go
index 7f54ee8c5..fe9085613 100644
--- a/internal/cache/db.go
+++ b/internal/cache/db.go
@@ -29,9 +29,6 @@ type DBCaches struct {
 	// Account provides access to the gtsmodel Account database cache.
 	Account StructCache[*gtsmodel.Account]
 
-	// AccountIDsFollowingTag caches account IDs following a given tag ID.
-	AccountIDsFollowingTag SliceCache[string]
-
 	// AccountNote provides access to the gtsmodel Note database cache.
 	AccountNote StructCache[*gtsmodel.AccountNote]
 
@@ -88,10 +85,23 @@ type DBCaches struct {
 
 	// FollowIDs provides access to the follower / following IDs database cache.
 	// THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{accountID} WHERE PREFIX IS:
-	// - '>'  for following IDs
-	// - 'l>' for local following IDs
-	// - '<'  for follower IDs
-	// - 'l<' for local follower IDs
+	//
+	// - '>{$accountID}'  for following IDs
+	//   e.g. FollowIDs.Load(">" + account.ID, func() {})
+	//   which will load a slice of follows IDs FROM account.
+	//
+	// - 'l>{$accountID}' for local following IDs
+	//   e.g. FollowIDs.Load("l>" + account.ID, func() {})
+	//   which will load a slice of LOCAL follows IDs FROM account.
+	//
+	// - '<{$accountID}' for follower IDs
+	//   e.g. FollowIDs.Load("<" + account.ID, func() {})
+	//   which will load a slice of follows IDs TARGETTING account.
+	//
+	// - 'l<{$accountID}' for local follower IDs
+	//   e.g. FollowIDs.Load("l<" + account.ID, func() {})
+	//   which will load a slice of LOCAL follows IDs TARGETTING account.
+	//
 	FollowIDs SliceCache[string]
 
 	// FollowRequest provides access to the gtsmodel FollowRequest database cache.
@@ -99,10 +109,30 @@ type DBCaches struct {
 
 	// FollowRequestIDs provides access to the follow requester / requesting IDs database
 	// cache. THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{accountID} WHERE PREFIX IS:
-	// - '>'  for following IDs
-	// - '<'  for follower IDs
+	//
+	// - '>{$accountID}'  for follow request IDs
+	//   e.g. FollowRequestIDs.Load(">" + account.ID, func() {})
+	//   which will load a slice of follow request IDs TARGETTING account.
+	//
+	// - '<{$accountID}'  for follow request IDs
+	//   e.g. FollowRequestIDs.Load("<" + account.ID, func() {})
+	//   which will load a slice of follow request IDs FROM account.
+	//
 	FollowRequestIDs SliceCache[string]
 
+	// FollowingTagIDs provides access to account IDs following / tag IDs followed by
+	// account db cache. THIS CACHE IS KEYED AS THE FOLLOWING {prefix}{id} WHERE:
+	//
+	// - '>{$accountID}' for tag IDs followed by account
+	//   e.g. FollowingTagIDs.Load(">" + account.ID, func() {})
+	//   which will load a slice of tag IDs followed by account.
+	//
+	// - '<{$tagIDs}' for account IDs following tag
+	//   e.g. FollowingTagIDs.Load("<" + tag.ID, func() {})
+	//   which will load a slice of account IDs following tag.
+	//
+	FollowingTagIDs SliceCache[string]
+
 	// Instance provides access to the gtsmodel Instance database cache.
 	Instance StructCache[*gtsmodel.Instance]
 
@@ -115,8 +145,31 @@ type DBCaches struct {
 	// List provides access to the gtsmodel List database cache.
 	List StructCache[*gtsmodel.List]
 
-	// ListEntry provides access to the gtsmodel ListEntry database cache.
-	ListEntry StructCache[*gtsmodel.ListEntry]
+	// ListIDs provides access to the list IDs owned by account / list IDs follow
+	// contained in db cache. THIS CACHE IS KEYED AS FOLLOWING {prefix}{id} WHERE:
+	//
+	// - 'a{$accountID}' for list IDs owned by account
+	//   e.g. ListIDs.Load("a" + account.ID, func() {})
+	//   which will load a slice of list IDs owned by account.
+	//
+	// - 'f{$followID}' for list IDs follow contained in
+	//   e.g. ListIDs.Load("f" + follow.ID, func() {})
+	//   which will load a slice of list IDs containing follow.
+	//
+	ListIDs SliceCache[string]
+
+	// ListedIDs provides access to the account IDs in list / follow IDs in
+	// list db cache. THIS CACHE IS KEYED AS FOLLOWING {prefix}{id} WHERE:
+	//
+	// - 'a{listID}' for account IDs in list ID
+	//   e.g. ListedIDs.Load("a" + list.ID, func() {})
+	//   which will load a slice of account IDs in list.
+	//
+	// - 'f{listID}' for follow IDs in list ID
+	//   e.g. ListedIDs.Load("f" + list.ID, func() {})
+	//   which will load a slice of follow IDs in list.
+	//
+	ListedIDs SliceCache[string]
 
 	// Marker provides access to the gtsmodel Marker database cache.
 	Marker StructCache[*gtsmodel.Marker]
@@ -151,10 +204,10 @@ type DBCaches struct {
 	// Status provides access to the gtsmodel Status database cache.
 	Status StructCache[*gtsmodel.Status]
 
-	// StatusBookmark ...
+	// StatusBookmark provides access to the gtsmodel StatusBookmark database cache.
 	StatusBookmark StructCache[*gtsmodel.StatusBookmark]
 
-	// StatusBookmarkIDs ...
+	// StatusBookmarkIDs provides access to the status bookmark IDs list database cache.
 	StatusBookmarkIDs SliceCache[string]
 
 	// StatusFave provides access to the gtsmodel StatusFave database cache.
@@ -166,9 +219,6 @@ type DBCaches struct {
 	// Tag provides access to the gtsmodel Tag database cache.
 	Tag StructCache[*gtsmodel.Tag]
 
-	// TagIDsFollowedByAccount caches tag IDs followed by a given account ID.
-	TagIDsFollowedByAccount SliceCache[string]
-
 	// ThreadMute provides access to the gtsmodel ThreadMute database cache.
 	ThreadMute StructCache[*gtsmodel.ThreadMute]
 
@@ -243,17 +293,6 @@ func (c *Caches) initAccount() {
 	})
 }
 
-func (c *Caches) initAccountIDsFollowingTag() {
-	// Calculate maximum cache size.
-	cap := calculateSliceCacheMax(
-		config.GetCacheAccountIDsFollowingTagMemRatio(),
-	)
-
-	log.Infof(nil, "cache size = %d", cap)
-
-	c.DB.AccountIDsFollowingTag.Init(0, cap)
-}
-
 func (c *Caches) initAccountNote() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(
@@ -761,6 +800,17 @@ func (c *Caches) initFollowRequestIDs() {
 	c.DB.FollowRequestIDs.Init(0, cap)
 }
 
+func (c *Caches) initFollowingTagIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheFollowingTagIDsMemRatio(),
+	)
+
+	log.Infof(nil, "cache size = %d", cap)
+
+	c.DB.FollowingTagIDs.Init(0, cap)
+}
+
 func (c *Caches) initInReplyToIDs() {
 	// Calculate maximum cache size.
 	cap := calculateSliceCacheMax(
@@ -860,7 +910,6 @@ func (c *Caches) initList() {
 		// will be populated separately.
 		// See internal/db/bundb/list.go.
 		l2.Account = nil
-		l2.ListEntries = nil
 
 		return l2
 	}
@@ -876,37 +925,26 @@ func (c *Caches) initList() {
 	})
 }
 
-func (c *Caches) initListEntry() {
+func (c *Caches) initListIDs() {
 	// Calculate maximum cache size.
-	cap := calculateResultCacheMax(
-		sizeofListEntry(), // model in-mem size.
-		config.GetCacheListEntryMemRatio(),
+	cap := calculateSliceCacheMax(
+		config.GetCacheListIDsMemRatio(),
 	)
 
 	log.Infof(nil, "cache size = %d", cap)
 
-	copyF := func(l1 *gtsmodel.ListEntry) *gtsmodel.ListEntry {
-		l2 := new(gtsmodel.ListEntry)
-		*l2 = *l1
+	c.DB.ListIDs.Init(0, cap)
+}
 
-		// Don't include ptr fields that
-		// will be populated separately.
-		// See internal/db/bundb/list.go.
-		l2.Follow = nil
+func (c *Caches) initListedIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheListedIDsMemRatio(),
+	)
 
-		return l2
-	}
+	log.Infof(nil, "cache size = %d", cap)
 
-	c.DB.ListEntry.Init(structr.CacheConfig[*gtsmodel.ListEntry]{
-		Indices: []structr.IndexConfig{
-			{Fields: "ID"},
-			{Fields: "ListID", Multiple: true},
-			{Fields: "FollowID", Multiple: true},
-		},
-		MaxSize:   cap,
-		IgnoreErr: ignoreErrors,
-		Copy:      copyF,
-	})
+	c.DB.ListedIDs.Init(0, cap)
 }
 
 func (c *Caches) initMarker() {
@@ -1368,17 +1406,6 @@ func (c *Caches) initTag() {
 	})
 }
 
-func (c *Caches) initTagIDsFollowedByAccount() {
-	// Calculate maximum cache size.
-	cap := calculateSliceCacheMax(
-		config.GetCacheTagIDsFollowedByAccountMemRatio(),
-	)
-
-	log.Infof(nil, "cache size = %d", cap)
-
-	c.DB.TagIDsFollowedByAccount.Init(0, cap)
-}
-
 func (c *Caches) initThreadMute() {
 	cap := calculateResultCacheMax(
 		sizeofThreadMute(), // model in-mem size.
diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go
index ac326eda3..ca12e412c 100644
--- a/internal/cache/invalidate.go
+++ b/internal/cache/invalidate.go
@@ -97,9 +97,6 @@ func (c *Caches) OnInvalidateFollow(follow *gtsmodel.Follow) {
 	// Invalidate follow request with this same ID.
 	c.DB.FollowRequest.Invalidate("ID", follow.ID)
 
-	// Invalidate any related list entries.
-	c.DB.ListEntry.Invalidate("FollowID", follow.ID)
-
 	// Invalidate follow origin account ID cached visibility.
 	c.Visibility.Invalidate("ItemID", follow.AccountID)
 	c.Visibility.Invalidate("RequesterID", follow.AccountID)
@@ -108,18 +105,47 @@ func (c *Caches) OnInvalidateFollow(follow *gtsmodel.Follow) {
 	c.Visibility.Invalidate("ItemID", follow.TargetAccountID)
 	c.Visibility.Invalidate("RequesterID", follow.TargetAccountID)
 
-	// Invalidate source account's following
-	// lists, and destination's follwer lists.
-	// (see FollowIDs() comment for details).
+	// Invalidate ID slice cache.
 	c.DB.FollowIDs.Invalidate(
+
+		// Invalidate follow ID lists
+		// TARGETTING origin account
+		// (including local-only follows).
 		">"+follow.AccountID,
 		"l>"+follow.AccountID,
+
+		// Invalidate follow ID lists
+		// FROM the origin account
+		// (including local-only follows).
 		"<"+follow.AccountID,
 		"l<"+follow.AccountID,
-		"<"+follow.TargetAccountID,
-		"l<"+follow.TargetAccountID,
+
+		// Invalidate follow ID lists
+		// TARGETTING the target account
+		// (including local-only follows).
 		">"+follow.TargetAccountID,
 		"l>"+follow.TargetAccountID,
+
+		// Invalidate follow ID lists
+		// FROM the target account
+		// (including local-only follows).
+		"<"+follow.TargetAccountID,
+		"l<"+follow.TargetAccountID,
+	)
+
+	// Invalidate ID slice cache.
+	c.DB.ListIDs.Invalidate(
+
+		// Invalidate source
+		// account's owned lists.
+		"a"+follow.AccountID,
+
+		// Invalidate target account's.
+		"a"+follow.TargetAccountID,
+
+		// Invalidate lists containing
+		// list entries for follow.
+		"f"+follow.ID,
 	)
 }
 
@@ -127,20 +153,48 @@ func (c *Caches) OnInvalidateFollowRequest(followReq *gtsmodel.FollowRequest) {
 	// Invalidate follow with this same ID.
 	c.DB.Follow.Invalidate("ID", followReq.ID)
 
-	// Invalidate source account's followreq
-	// lists, and destinations follow req lists.
-	// (see FollowRequestIDs() comment for details).
+	// Invalidate ID slice cache.
 	c.DB.FollowRequestIDs.Invalidate(
+
+		// Invalidate follow request ID
+		// lists TARGETTING origin account
+		// (including local-only follows).
 		">"+followReq.AccountID,
+
+		// Invalidate follow request ID
+		// lists FROM the origin account
+		// (including local-only follows).
 		"<"+followReq.AccountID,
+
+		// Invalidate follow request ID
+		// lists TARGETTING target account
+		// (including local-only follows).
 		">"+followReq.TargetAccountID,
+
+		// Invalidate follow request ID
+		// lists FROM the target account
+		// (including local-only follows).
 		"<"+followReq.TargetAccountID,
 	)
 }
 
 func (c *Caches) OnInvalidateList(list *gtsmodel.List) {
-	// Invalidate all cached entries of this list.
-	c.DB.ListEntry.Invalidate("ListID", list.ID)
+	// Invalidate list IDs cache.
+	c.DB.ListIDs.Invalidate(
+		"a" + list.AccountID,
+	)
+
+	// Invalidate ID slice cache.
+	c.DB.ListedIDs.Invalidate(
+
+		// Invalidate list of
+		// account IDs in list.
+		"a"+list.ID,
+
+		// Invalidate list of
+		// follow IDs in list.
+		"f"+list.ID,
+	)
 }
 
 func (c *Caches) OnInvalidateMedia(media *gtsmodel.MediaAttachment) {
@@ -184,7 +238,7 @@ func (c *Caches) OnInvalidateStatus(status *gtsmodel.Status) {
 	// the media IDs in use before the media table is
 	// aware of the status ID they are linked to.
 	//
-	// c.DB.Media().Invalidate("StatusID") will not work.
+	// c.DB.Media.Invalidate("StatusID") will not work.
 	c.DB.Media.InvalidateIDs("ID", status.AttachmentIDs)
 
 	if status.BoostOfID != "" {
diff --git a/internal/cache/size.go b/internal/cache/size.go
index 49c2f4318..8367e4c46 100644
--- a/internal/cache/size.go
+++ b/internal/cache/size.go
@@ -166,6 +166,7 @@ func calculateCacheMax(keySz, valSz uintptr, ratio float64) int {
 
 // totalOfRatios returns the total of all cache ratios added together.
 func totalOfRatios() float64 {
+
 	// NOTE: this is not performant calculating
 	// this every damn time (mainly the mutex unlocks
 	// required to access each config var). fortunately
@@ -189,11 +190,13 @@ func totalOfRatios() float64 {
 		config.GetCacheFollowIDsMemRatio() +
 		config.GetCacheFollowRequestMemRatio() +
 		config.GetCacheFollowRequestIDsMemRatio() +
+		config.GetCacheFollowingTagIDsMemRatio() +
+		config.GetCacheInReplyToIDsMemRatio() +
 		config.GetCacheInstanceMemRatio() +
 		config.GetCacheInteractionRequestMemRatio() +
-		config.GetCacheInReplyToIDsMemRatio() +
 		config.GetCacheListMemRatio() +
-		config.GetCacheListEntryMemRatio() +
+		config.GetCacheListIDsMemRatio() +
+		config.GetCacheListedIDsMemRatio() +
 		config.GetCacheMarkerMemRatio() +
 		config.GetCacheMediaMemRatio() +
 		config.GetCacheMentionMemRatio() +
@@ -201,7 +204,9 @@ func totalOfRatios() float64 {
 		config.GetCacheNotificationMemRatio() +
 		config.GetCachePollMemRatio() +
 		config.GetCachePollVoteMemRatio() +
+		config.GetCachePollVoteIDsMemRatio() +
 		config.GetCacheReportMemRatio() +
+		config.GetCacheSinBinStatusMemRatio() +
 		config.GetCacheStatusMemRatio() +
 		config.GetCacheStatusBookmarkMemRatio() +
 		config.GetCacheStatusBookmarkIDsMemRatio() +
@@ -212,6 +217,8 @@ func totalOfRatios() float64 {
 		config.GetCacheTokenMemRatio() +
 		config.GetCacheTombstoneMemRatio() +
 		config.GetCacheUserMemRatio() +
+		config.GetCacheUserMuteMemRatio() +
+		config.GetCacheUserMuteIDsMemRatio() +
 		config.GetCacheWebfingerMemRatio() +
 		config.GetCacheVisibilityMemRatio()
 }
@@ -466,16 +473,6 @@ func sizeofList() uintptr {
 	}))
 }
 
-func sizeofListEntry() uintptr {
-	return uintptr(size.Of(&gtsmodel.ListEntry{
-		ID:        exampleID,
-		CreatedAt: exampleTime,
-		UpdatedAt: exampleTime,
-		ListID:    exampleID,
-		FollowID:  exampleID,
-	}))
-}
-
 func sizeofMarker() uintptr {
 	return uintptr(size.Of(&gtsmodel.Marker{
 		AccountID:  exampleID,