[performance] cache media attachments (#1525)

* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly

* add media attachment caching, slightly tweak default cache config

* further tweak default cache config values

* replace other media attachment db calls to go through cache

* update envparsing test

* fix delete media attachment sql

* fix media sql query

* invalidate cached media entries during status create / update

* fix envparsing test

* fix typo in panic log message...

* add 'updated_at' column during UpdateAttachment

* remove unused func

---------

Signed-off-by: kim <grufwub@gmail.com>

3 files changed, 94 insertions, 11 deletions

diff --git a/internal/config/config.go b/internal/config/config.go
index 5673b76dd..fdfda8583 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -177,6 +177,10 @@ type GTSCacheConfiguration struct {
 	EmojiCategoryTTL       time.Duration `name:"emoji-category-ttl"`
 	EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
 
+	MediaMaxSize   int           `name:"media-max-size"`
+	MediaTTL       time.Duration `name:"media-ttl"`
+	MediaSweepFreq time.Duration `name:"media-sweep-freq"`
+
 	MentionMaxSize   int           `name:"mention-max-size"`
 	MentionTTL       time.Duration `name:"mention-ttl"`
 	MentionSweepFreq time.Duration `name:"mention-sweep-freq"`
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 528419e97..e9dd2b743 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -116,13 +116,13 @@ var Defaults = Configuration{
 
 	Cache: CacheConfiguration{
 		GTS: GTSCacheConfiguration{
-			AccountMaxSize:   100,
+			AccountMaxSize:   500,
 			AccountTTL:       time.Minute * 5,
-			AccountSweepFreq: time.Second * 10,
+			AccountSweepFreq: time.Second * 30,
 
 			BlockMaxSize:   100,
 			BlockTTL:       time.Minute * 5,
-			BlockSweepFreq: time.Second * 10,
+			BlockSweepFreq: time.Second * 30,
 
 			DomainBlockMaxSize:   1000,
 			DomainBlockTTL:       time.Hour * 24,
@@ -130,35 +130,39 @@ var Defaults = Configuration{
 
 			EmojiMaxSize:   500,
 			EmojiTTL:       time.Minute * 5,
-			EmojiSweepFreq: time.Second * 10,
+			EmojiSweepFreq: time.Second * 30,
 
 			EmojiCategoryMaxSize:   100,
 			EmojiCategoryTTL:       time.Minute * 5,
-			EmojiCategorySweepFreq: time.Second * 10,
+			EmojiCategorySweepFreq: time.Second * 30,
+
+			MediaMaxSize:   500,
+			MediaTTL:       time.Minute * 5,
+			MediaSweepFreq: time.Second * 30,
 
 			MentionMaxSize:   500,
 			MentionTTL:       time.Minute * 5,
-			MentionSweepFreq: time.Second * 10,
+			MentionSweepFreq: time.Second * 30,
 
 			NotificationMaxSize:   500,
 			NotificationTTL:       time.Minute * 5,
-			NotificationSweepFreq: time.Second * 10,
+			NotificationSweepFreq: time.Second * 30,
 
 			ReportMaxSize:   100,
 			ReportTTL:       time.Minute * 5,
-			ReportSweepFreq: time.Second * 10,
+			ReportSweepFreq: time.Second * 30,
 
 			StatusMaxSize:   500,
 			StatusTTL:       time.Minute * 5,
-			StatusSweepFreq: time.Second * 10,
+			StatusSweepFreq: time.Second * 30,
 
 			TombstoneMaxSize:   100,
 			TombstoneTTL:       time.Minute * 5,
-			TombstoneSweepFreq: time.Second * 10,
+			TombstoneSweepFreq: time.Second * 30,
 
 			UserMaxSize:   100,
 			UserTTL:       time.Minute * 5,
-			UserSweepFreq: time.Second * 10,
+			UserSweepFreq: time.Second * 30,
 		},
 	},
 
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 41c56a571..5ea7b61b6 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -2426,6 +2426,81 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration {
 // SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field
 func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) }
 
+// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
+func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaMaxSize safely sets the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
+func (st *ConfigState) SetCacheGTSMediaMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaMaxSizeFlag returns the flag name for the 'Cache.GTS.MediaMaxSize' field
+func CacheGTSMediaMaxSizeFlag() string { return "cache-gts-media-max-size" }
+
+// GetCacheGTSMediaMaxSize safely fetches the value for global configuration 'Cache.GTS.MediaMaxSize' field
+func GetCacheGTSMediaMaxSize() int { return global.GetCacheGTSMediaMaxSize() }
+
+// SetCacheGTSMediaMaxSize safely sets the value for global configuration 'Cache.GTS.MediaMaxSize' field
+func SetCacheGTSMediaMaxSize(v int) { global.SetCacheGTSMediaMaxSize(v) }
+
+// GetCacheGTSMediaTTL safely fetches the Configuration value for state's 'Cache.GTS.MediaTTL' field
+func (st *ConfigState) GetCacheGTSMediaTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaTTL safely sets the Configuration value for state's 'Cache.GTS.MediaTTL' field
+func (st *ConfigState) SetCacheGTSMediaTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaTTLFlag returns the flag name for the 'Cache.GTS.MediaTTL' field
+func CacheGTSMediaTTLFlag() string { return "cache-gts-media-ttl" }
+
+// GetCacheGTSMediaTTL safely fetches the value for global configuration 'Cache.GTS.MediaTTL' field
+func GetCacheGTSMediaTTL() time.Duration { return global.GetCacheGTSMediaTTL() }
+
+// SetCacheGTSMediaTTL safely sets the value for global configuration 'Cache.GTS.MediaTTL' field
+func SetCacheGTSMediaTTL(v time.Duration) { global.SetCacheGTSMediaTTL(v) }
+
+// GetCacheGTSMediaSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
+func (st *ConfigState) GetCacheGTSMediaSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
+func (st *ConfigState) SetCacheGTSMediaSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaSweepFreqFlag returns the flag name for the 'Cache.GTS.MediaSweepFreq' field
+func CacheGTSMediaSweepFreqFlag() string { return "cache-gts-media-sweep-freq" }
+
+// GetCacheGTSMediaSweepFreq safely fetches the value for global configuration 'Cache.GTS.MediaSweepFreq' field
+func GetCacheGTSMediaSweepFreq() time.Duration { return global.GetCacheGTSMediaSweepFreq() }
+
+// SetCacheGTSMediaSweepFreq safely sets the value for global configuration 'Cache.GTS.MediaSweepFreq' field
+func SetCacheGTSMediaSweepFreq(v time.Duration) { global.SetCacheGTSMediaSweepFreq(v) }
+
 // GetCacheGTSMentionMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MentionMaxSize' field
 func (st *ConfigState) GetCacheGTSMentionMaxSize() (v int) {
 	st.mutex.Lock()