[performance] refactoring + add fave / follow / request / visibility caching (#1607)

* refactor visibility checking, add caching for visibility

* invalidate visibility cache items on account / status deletes

* fix requester ID passed to visibility cache nil ptr

* de-interface caches, fix home / public timeline caching + visibility

* finish adding code comments for visibility filter

* fix angry goconst linter warnings

* actually finish adding filter visibility code comments for timeline functions

* move home timeline status author check to after visibility

* remove now-unused code

* add more code comments

* add TODO code comment, update printed cache start names

* update printed cache names on stop

* start adding separate follow(request) delete db functions, add specific visibility cache tests

* add relationship type caching

* fix getting local account follows / followed-bys, other small codebase improvements

* simplify invalidation using cache hooks, add more GetAccountBy___() functions

* fix boosting to return 404 if not boostable but no error (to not leak status ID)

* remove dead code

* improved placement of cache invalidation

* update license headers

* add example follow, follow-request config entries

* add example visibility cache configuration to config file

* use specific PutFollowRequest() instead of just Put()

* add tests for all GetAccountBy()

* add GetBlockBy() tests

* update block to check primitive fields

* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests

* fix copy-pasted code

* update envparsing test

* whitespace

* fix bun struct tag

* add license header to gtscontext

* fix old license header

* improved error creation to not use fmt.Errorf() when not needed

* fix various rebase conflicts, fix account test

* remove commented-out code, fix-up mention caching

* fix mention select bun statement

* ensure mention target account populated, pass in context to customrenderer logging

* remove more uncommented code, fix typeutil test

* add statusfave database model caching

* add status fave cache configuration

* add status fave cache example config

* woops, catch missed error. nice catch linter!

* add back testrig panic on nil db

* update example configuration to match defaults, slight tweak to cache configuration defaults

* update envparsing test with new defaults

* fetch followingget to use the follow target account

* use accounnt.IsLocal() instead of empty domain check

* use constants for the cache visibility type check

* use bun.In() for notification type restriction in db query

* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())

* use bun query building for nested select statements to ensure working with postgres

* update public timeline future status checks to match visibility filter

* same as previous, for home timeline

* update public timeline tests to dynamically check for appropriate statuses

* migrate accounts to allow unique constraint on public_key

* provide minimal account with publicKey

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

1 files changed, 302 insertions, 0 deletions

diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 6fc195ad0..e35eb0665 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -2501,6 +2501,158 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration {
 // SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field
 func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) }
 
+// GetCacheGTSFollowMaxSize safely fetches the Configuration value for state's 'Cache.GTS.FollowMaxSize' field
+func (st *ConfigState) GetCacheGTSFollowMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowMaxSize safely sets the Configuration value for state's 'Cache.GTS.FollowMaxSize' field
+func (st *ConfigState) SetCacheGTSFollowMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowMaxSizeFlag returns the flag name for the 'Cache.GTS.FollowMaxSize' field
+func CacheGTSFollowMaxSizeFlag() string { return "cache-gts-follow-max-size" }
+
+// GetCacheGTSFollowMaxSize safely fetches the value for global configuration 'Cache.GTS.FollowMaxSize' field
+func GetCacheGTSFollowMaxSize() int { return global.GetCacheGTSFollowMaxSize() }
+
+// SetCacheGTSFollowMaxSize safely sets the value for global configuration 'Cache.GTS.FollowMaxSize' field
+func SetCacheGTSFollowMaxSize(v int) { global.SetCacheGTSFollowMaxSize(v) }
+
+// GetCacheGTSFollowTTL safely fetches the Configuration value for state's 'Cache.GTS.FollowTTL' field
+func (st *ConfigState) GetCacheGTSFollowTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowTTL safely sets the Configuration value for state's 'Cache.GTS.FollowTTL' field
+func (st *ConfigState) SetCacheGTSFollowTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowTTLFlag returns the flag name for the 'Cache.GTS.FollowTTL' field
+func CacheGTSFollowTTLFlag() string { return "cache-gts-follow-ttl" }
+
+// GetCacheGTSFollowTTL safely fetches the value for global configuration 'Cache.GTS.FollowTTL' field
+func GetCacheGTSFollowTTL() time.Duration { return global.GetCacheGTSFollowTTL() }
+
+// SetCacheGTSFollowTTL safely sets the value for global configuration 'Cache.GTS.FollowTTL' field
+func SetCacheGTSFollowTTL(v time.Duration) { global.SetCacheGTSFollowTTL(v) }
+
+// GetCacheGTSFollowSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.FollowSweepFreq' field
+func (st *ConfigState) GetCacheGTSFollowSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowSweepFreq safely sets the Configuration value for state's 'Cache.GTS.FollowSweepFreq' field
+func (st *ConfigState) SetCacheGTSFollowSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowSweepFreqFlag returns the flag name for the 'Cache.GTS.FollowSweepFreq' field
+func CacheGTSFollowSweepFreqFlag() string { return "cache-gts-follow-sweep-freq" }
+
+// GetCacheGTSFollowSweepFreq safely fetches the value for global configuration 'Cache.GTS.FollowSweepFreq' field
+func GetCacheGTSFollowSweepFreq() time.Duration { return global.GetCacheGTSFollowSweepFreq() }
+
+// SetCacheGTSFollowSweepFreq safely sets the value for global configuration 'Cache.GTS.FollowSweepFreq' field
+func SetCacheGTSFollowSweepFreq(v time.Duration) { global.SetCacheGTSFollowSweepFreq(v) }
+
+// GetCacheGTSFollowRequestMaxSize safely fetches the Configuration value for state's 'Cache.GTS.FollowRequestMaxSize' field
+func (st *ConfigState) GetCacheGTSFollowRequestMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowRequestMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowRequestMaxSize safely sets the Configuration value for state's 'Cache.GTS.FollowRequestMaxSize' field
+func (st *ConfigState) SetCacheGTSFollowRequestMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowRequestMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowRequestMaxSizeFlag returns the flag name for the 'Cache.GTS.FollowRequestMaxSize' field
+func CacheGTSFollowRequestMaxSizeFlag() string { return "cache-gts-follow-request-max-size" }
+
+// GetCacheGTSFollowRequestMaxSize safely fetches the value for global configuration 'Cache.GTS.FollowRequestMaxSize' field
+func GetCacheGTSFollowRequestMaxSize() int { return global.GetCacheGTSFollowRequestMaxSize() }
+
+// SetCacheGTSFollowRequestMaxSize safely sets the value for global configuration 'Cache.GTS.FollowRequestMaxSize' field
+func SetCacheGTSFollowRequestMaxSize(v int) { global.SetCacheGTSFollowRequestMaxSize(v) }
+
+// GetCacheGTSFollowRequestTTL safely fetches the Configuration value for state's 'Cache.GTS.FollowRequestTTL' field
+func (st *ConfigState) GetCacheGTSFollowRequestTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowRequestTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowRequestTTL safely sets the Configuration value for state's 'Cache.GTS.FollowRequestTTL' field
+func (st *ConfigState) SetCacheGTSFollowRequestTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowRequestTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowRequestTTLFlag returns the flag name for the 'Cache.GTS.FollowRequestTTL' field
+func CacheGTSFollowRequestTTLFlag() string { return "cache-gts-follow-request-ttl" }
+
+// GetCacheGTSFollowRequestTTL safely fetches the value for global configuration 'Cache.GTS.FollowRequestTTL' field
+func GetCacheGTSFollowRequestTTL() time.Duration { return global.GetCacheGTSFollowRequestTTL() }
+
+// SetCacheGTSFollowRequestTTL safely sets the value for global configuration 'Cache.GTS.FollowRequestTTL' field
+func SetCacheGTSFollowRequestTTL(v time.Duration) { global.SetCacheGTSFollowRequestTTL(v) }
+
+// GetCacheGTSFollowRequestSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.FollowRequestSweepFreq' field
+func (st *ConfigState) GetCacheGTSFollowRequestSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.FollowRequestSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSFollowRequestSweepFreq safely sets the Configuration value for state's 'Cache.GTS.FollowRequestSweepFreq' field
+func (st *ConfigState) SetCacheGTSFollowRequestSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.FollowRequestSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSFollowRequestSweepFreqFlag returns the flag name for the 'Cache.GTS.FollowRequestSweepFreq' field
+func CacheGTSFollowRequestSweepFreqFlag() string { return "cache-gts-follow-request-sweep-freq" }
+
+// GetCacheGTSFollowRequestSweepFreq safely fetches the value for global configuration 'Cache.GTS.FollowRequestSweepFreq' field
+func GetCacheGTSFollowRequestSweepFreq() time.Duration {
+	return global.GetCacheGTSFollowRequestSweepFreq()
+}
+
+// SetCacheGTSFollowRequestSweepFreq safely sets the value for global configuration 'Cache.GTS.FollowRequestSweepFreq' field
+func SetCacheGTSFollowRequestSweepFreq(v time.Duration) { global.SetCacheGTSFollowRequestSweepFreq(v) }
+
 // GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
 func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
 	st.mutex.Lock()
@@ -2878,6 +3030,81 @@ func GetCacheGTSStatusSweepFreq() time.Duration { return global.GetCacheGTSStatu
 // SetCacheGTSStatusSweepFreq safely sets the value for global configuration 'Cache.GTS.StatusSweepFreq' field
 func SetCacheGTSStatusSweepFreq(v time.Duration) { global.SetCacheGTSStatusSweepFreq(v) }
 
+// GetCacheGTSStatusFaveMaxSize safely fetches the Configuration value for state's 'Cache.GTS.StatusFaveMaxSize' field
+func (st *ConfigState) GetCacheGTSStatusFaveMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.StatusFaveMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSStatusFaveMaxSize safely sets the Configuration value for state's 'Cache.GTS.StatusFaveMaxSize' field
+func (st *ConfigState) SetCacheGTSStatusFaveMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.StatusFaveMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSStatusFaveMaxSizeFlag returns the flag name for the 'Cache.GTS.StatusFaveMaxSize' field
+func CacheGTSStatusFaveMaxSizeFlag() string { return "cache-gts-status-fave-max-size" }
+
+// GetCacheGTSStatusFaveMaxSize safely fetches the value for global configuration 'Cache.GTS.StatusFaveMaxSize' field
+func GetCacheGTSStatusFaveMaxSize() int { return global.GetCacheGTSStatusFaveMaxSize() }
+
+// SetCacheGTSStatusFaveMaxSize safely sets the value for global configuration 'Cache.GTS.StatusFaveMaxSize' field
+func SetCacheGTSStatusFaveMaxSize(v int) { global.SetCacheGTSStatusFaveMaxSize(v) }
+
+// GetCacheGTSStatusFaveTTL safely fetches the Configuration value for state's 'Cache.GTS.StatusFaveTTL' field
+func (st *ConfigState) GetCacheGTSStatusFaveTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.StatusFaveTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSStatusFaveTTL safely sets the Configuration value for state's 'Cache.GTS.StatusFaveTTL' field
+func (st *ConfigState) SetCacheGTSStatusFaveTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.StatusFaveTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSStatusFaveTTLFlag returns the flag name for the 'Cache.GTS.StatusFaveTTL' field
+func CacheGTSStatusFaveTTLFlag() string { return "cache-gts-status-fave-ttl" }
+
+// GetCacheGTSStatusFaveTTL safely fetches the value for global configuration 'Cache.GTS.StatusFaveTTL' field
+func GetCacheGTSStatusFaveTTL() time.Duration { return global.GetCacheGTSStatusFaveTTL() }
+
+// SetCacheGTSStatusFaveTTL safely sets the value for global configuration 'Cache.GTS.StatusFaveTTL' field
+func SetCacheGTSStatusFaveTTL(v time.Duration) { global.SetCacheGTSStatusFaveTTL(v) }
+
+// GetCacheGTSStatusFaveSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.StatusFaveSweepFreq' field
+func (st *ConfigState) GetCacheGTSStatusFaveSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.StatusFaveSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSStatusFaveSweepFreq safely sets the Configuration value for state's 'Cache.GTS.StatusFaveSweepFreq' field
+func (st *ConfigState) SetCacheGTSStatusFaveSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.StatusFaveSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSStatusFaveSweepFreqFlag returns the flag name for the 'Cache.GTS.StatusFaveSweepFreq' field
+func CacheGTSStatusFaveSweepFreqFlag() string { return "cache-gts-status-fave-sweep-freq" }
+
+// GetCacheGTSStatusFaveSweepFreq safely fetches the value for global configuration 'Cache.GTS.StatusFaveSweepFreq' field
+func GetCacheGTSStatusFaveSweepFreq() time.Duration { return global.GetCacheGTSStatusFaveSweepFreq() }
+
+// SetCacheGTSStatusFaveSweepFreq safely sets the value for global configuration 'Cache.GTS.StatusFaveSweepFreq' field
+func SetCacheGTSStatusFaveSweepFreq(v time.Duration) { global.SetCacheGTSStatusFaveSweepFreq(v) }
+
 // GetCacheGTSTombstoneMaxSize safely fetches the Configuration value for state's 'Cache.GTS.TombstoneMaxSize' field
 func (st *ConfigState) GetCacheGTSTombstoneMaxSize() (v int) {
 	st.mutex.Lock()
@@ -3103,6 +3330,81 @@ func GetCacheGTSWebfingerSweepFreq() time.Duration { return global.GetCacheGTSWe
 // SetCacheGTSWebfingerSweepFreq safely sets the value for global configuration 'Cache.GTS.WebfingerSweepFreq' field
 func SetCacheGTSWebfingerSweepFreq(v time.Duration) { global.SetCacheGTSWebfingerSweepFreq(v) }
 
+// GetCacheVisibilityMaxSize safely fetches the Configuration value for state's 'Cache.VisibilityMaxSize' field
+func (st *ConfigState) GetCacheVisibilityMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.VisibilityMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheVisibilityMaxSize safely sets the Configuration value for state's 'Cache.VisibilityMaxSize' field
+func (st *ConfigState) SetCacheVisibilityMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.VisibilityMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheVisibilityMaxSizeFlag returns the flag name for the 'Cache.VisibilityMaxSize' field
+func CacheVisibilityMaxSizeFlag() string { return "cache-visibility-max-size" }
+
+// GetCacheVisibilityMaxSize safely fetches the value for global configuration 'Cache.VisibilityMaxSize' field
+func GetCacheVisibilityMaxSize() int { return global.GetCacheVisibilityMaxSize() }
+
+// SetCacheVisibilityMaxSize safely sets the value for global configuration 'Cache.VisibilityMaxSize' field
+func SetCacheVisibilityMaxSize(v int) { global.SetCacheVisibilityMaxSize(v) }
+
+// GetCacheVisibilityTTL safely fetches the Configuration value for state's 'Cache.VisibilityTTL' field
+func (st *ConfigState) GetCacheVisibilityTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.VisibilityTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheVisibilityTTL safely sets the Configuration value for state's 'Cache.VisibilityTTL' field
+func (st *ConfigState) SetCacheVisibilityTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.VisibilityTTL = v
+	st.reloadToViper()
+}
+
+// CacheVisibilityTTLFlag returns the flag name for the 'Cache.VisibilityTTL' field
+func CacheVisibilityTTLFlag() string { return "cache-visibility-ttl" }
+
+// GetCacheVisibilityTTL safely fetches the value for global configuration 'Cache.VisibilityTTL' field
+func GetCacheVisibilityTTL() time.Duration { return global.GetCacheVisibilityTTL() }
+
+// SetCacheVisibilityTTL safely sets the value for global configuration 'Cache.VisibilityTTL' field
+func SetCacheVisibilityTTL(v time.Duration) { global.SetCacheVisibilityTTL(v) }
+
+// GetCacheVisibilitySweepFreq safely fetches the Configuration value for state's 'Cache.VisibilitySweepFreq' field
+func (st *ConfigState) GetCacheVisibilitySweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.VisibilitySweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheVisibilitySweepFreq safely sets the Configuration value for state's 'Cache.VisibilitySweepFreq' field
+func (st *ConfigState) SetCacheVisibilitySweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.VisibilitySweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheVisibilitySweepFreqFlag returns the flag name for the 'Cache.VisibilitySweepFreq' field
+func CacheVisibilitySweepFreqFlag() string { return "cache-visibility-sweep-freq" }
+
+// GetCacheVisibilitySweepFreq safely fetches the value for global configuration 'Cache.VisibilitySweepFreq' field
+func GetCacheVisibilitySweepFreq() time.Duration { return global.GetCacheVisibilitySweepFreq() }
+
+// SetCacheVisibilitySweepFreq safely sets the value for global configuration 'Cache.VisibilitySweepFreq' field
+func SetCacheVisibilitySweepFreq(v time.Duration) { global.SetCacheVisibilitySweepFreq(v) }
+
 // GetAdminAccountUsername safely fetches the Configuration value for state's 'AdminAccountUsername' field
 func (st *ConfigState) GetAdminAccountUsername() (v string) {
 	st.mutex.Lock()