From 84279f6a6a0201c90a6747fe8b82c38d5b4e49e2 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:46:09 +0000 Subject: [performance] cache more database calls, reduce required database calls overall (#3290) * improvements to caching for lists and relationship to accounts / follows * fix nil panic in AddToList() * ensure list related caches are correctly invalidated * ensure returned ID lists are ordered correctly * bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded) * remove zero checks in uncached key load functions (go-structr now handles this) * fix issues after rebase on upstream/main * update the expected return order of CSV exports (since list entries are now down by entry creation date) * rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests * use returning statements on delete to get cache invalidation info * fixes to recent database delete changes * fix broken list entries delete sql * remove unused db function * update remainder of delete functions to behave in similar way, some other small tweaks * fix delete user sql, allow returning on err no entries * uncomment + fix list database tests * update remaining list tests * update envparsing test * add comments to each specific key being invalidated * add more cache invalidation explanatory comments * whoops; actually delete poll votes from database in the DeletePollByID() func * remove added but-commented-out field * improved comment regarding paging being disabled * make cache invalidation comments match what's actually happening * fix up delete query comments to match what is happening * rename function to read a bit better * don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call) * update function name in test * move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic() * use the poll_id column in poll_votes for selecting votes in poll ID * fix function name --- internal/config/helpers.gen.go | 136 +++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 74 deletions(-) (limited to 'internal/config/helpers.gen.go') diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 75231d37b..d25d6cca8 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -2850,37 +2850,6 @@ func GetCacheAccountMemRatio() float64 { return global.GetCacheAccountMemRatio() // SetCacheAccountMemRatio safely sets the value for global configuration 'Cache.AccountMemRatio' field func SetCacheAccountMemRatio(v float64) { global.SetCacheAccountMemRatio(v) } -// GetCacheAccountIDsFollowingTagMemRatio safely fetches the Configuration value for state's 'Cache.AccountIDsFollowingTagMemRatio' field -func (st *ConfigState) GetCacheAccountIDsFollowingTagMemRatio() (v float64) { - st.mutex.RLock() - v = st.config.Cache.AccountIDsFollowingTagMemRatio - st.mutex.RUnlock() - return -} - -// SetCacheAccountIDsFollowingTagMemRatio safely sets the Configuration value for state's 'Cache.AccountIDsFollowingTagMemRatio' field -func (st *ConfigState) SetCacheAccountIDsFollowingTagMemRatio(v float64) { - st.mutex.Lock() - defer st.mutex.Unlock() - st.config.Cache.AccountIDsFollowingTagMemRatio = v - st.reloadToViper() -} - -// CacheAccountIDsFollowingTagMemRatioFlag returns the flag name for the 'Cache.AccountIDsFollowingTagMemRatio' field -func CacheAccountIDsFollowingTagMemRatioFlag() string { - return "cache-account-ids-following-tag-mem-ratio" -} - -// GetCacheAccountIDsFollowingTagMemRatio safely fetches the value for global configuration 'Cache.AccountIDsFollowingTagMemRatio' field -func GetCacheAccountIDsFollowingTagMemRatio() float64 { - return global.GetCacheAccountIDsFollowingTagMemRatio() -} - -// SetCacheAccountIDsFollowingTagMemRatio safely sets the value for global configuration 'Cache.AccountIDsFollowingTagMemRatio' field -func SetCacheAccountIDsFollowingTagMemRatio(v float64) { - global.SetCacheAccountIDsFollowingTagMemRatio(v) -} - // GetCacheAccountNoteMemRatio safely fetches the Configuration value for state's 'Cache.AccountNoteMemRatio' field func (st *ConfigState) GetCacheAccountNoteMemRatio() (v float64) { st.mutex.RLock() @@ -3362,6 +3331,31 @@ func GetCacheFollowRequestIDsMemRatio() float64 { return global.GetCacheFollowRe // SetCacheFollowRequestIDsMemRatio safely sets the value for global configuration 'Cache.FollowRequestIDsMemRatio' field func SetCacheFollowRequestIDsMemRatio(v float64) { global.SetCacheFollowRequestIDsMemRatio(v) } +// GetCacheFollowingTagIDsMemRatio safely fetches the Configuration value for state's 'Cache.FollowingTagIDsMemRatio' field +func (st *ConfigState) GetCacheFollowingTagIDsMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.FollowingTagIDsMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheFollowingTagIDsMemRatio safely sets the Configuration value for state's 'Cache.FollowingTagIDsMemRatio' field +func (st *ConfigState) SetCacheFollowingTagIDsMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.FollowingTagIDsMemRatio = v + st.reloadToViper() +} + +// CacheFollowingTagIDsMemRatioFlag returns the flag name for the 'Cache.FollowingTagIDsMemRatio' field +func CacheFollowingTagIDsMemRatioFlag() string { return "cache-following-tag-ids-mem-ratio" } + +// GetCacheFollowingTagIDsMemRatio safely fetches the value for global configuration 'Cache.FollowingTagIDsMemRatio' field +func GetCacheFollowingTagIDsMemRatio() float64 { return global.GetCacheFollowingTagIDsMemRatio() } + +// SetCacheFollowingTagIDsMemRatio safely sets the value for global configuration 'Cache.FollowingTagIDsMemRatio' field +func SetCacheFollowingTagIDsMemRatio(v float64) { global.SetCacheFollowingTagIDsMemRatio(v) } + // GetCacheInReplyToIDsMemRatio safely fetches the Configuration value for state's 'Cache.InReplyToIDsMemRatio' field func (st *ConfigState) GetCacheInReplyToIDsMemRatio() (v float64) { st.mutex.RLock() @@ -3462,30 +3456,55 @@ func GetCacheListMemRatio() float64 { return global.GetCacheListMemRatio() } // SetCacheListMemRatio safely sets the value for global configuration 'Cache.ListMemRatio' field func SetCacheListMemRatio(v float64) { global.SetCacheListMemRatio(v) } -// GetCacheListEntryMemRatio safely fetches the Configuration value for state's 'Cache.ListEntryMemRatio' field -func (st *ConfigState) GetCacheListEntryMemRatio() (v float64) { +// GetCacheListIDsMemRatio safely fetches the Configuration value for state's 'Cache.ListIDsMemRatio' field +func (st *ConfigState) GetCacheListIDsMemRatio() (v float64) { st.mutex.RLock() - v = st.config.Cache.ListEntryMemRatio + v = st.config.Cache.ListIDsMemRatio st.mutex.RUnlock() return } -// SetCacheListEntryMemRatio safely sets the Configuration value for state's 'Cache.ListEntryMemRatio' field -func (st *ConfigState) SetCacheListEntryMemRatio(v float64) { +// SetCacheListIDsMemRatio safely sets the Configuration value for state's 'Cache.ListIDsMemRatio' field +func (st *ConfigState) SetCacheListIDsMemRatio(v float64) { st.mutex.Lock() defer st.mutex.Unlock() - st.config.Cache.ListEntryMemRatio = v + st.config.Cache.ListIDsMemRatio = v st.reloadToViper() } -// CacheListEntryMemRatioFlag returns the flag name for the 'Cache.ListEntryMemRatio' field -func CacheListEntryMemRatioFlag() string { return "cache-list-entry-mem-ratio" } +// CacheListIDsMemRatioFlag returns the flag name for the 'Cache.ListIDsMemRatio' field +func CacheListIDsMemRatioFlag() string { return "cache-list-ids-mem-ratio" } -// GetCacheListEntryMemRatio safely fetches the value for global configuration 'Cache.ListEntryMemRatio' field -func GetCacheListEntryMemRatio() float64 { return global.GetCacheListEntryMemRatio() } +// GetCacheListIDsMemRatio safely fetches the value for global configuration 'Cache.ListIDsMemRatio' field +func GetCacheListIDsMemRatio() float64 { return global.GetCacheListIDsMemRatio() } -// SetCacheListEntryMemRatio safely sets the value for global configuration 'Cache.ListEntryMemRatio' field -func SetCacheListEntryMemRatio(v float64) { global.SetCacheListEntryMemRatio(v) } +// SetCacheListIDsMemRatio safely sets the value for global configuration 'Cache.ListIDsMemRatio' field +func SetCacheListIDsMemRatio(v float64) { global.SetCacheListIDsMemRatio(v) } + +// GetCacheListedIDsMemRatio safely fetches the Configuration value for state's 'Cache.ListedIDsMemRatio' field +func (st *ConfigState) GetCacheListedIDsMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.ListedIDsMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheListedIDsMemRatio safely sets the Configuration value for state's 'Cache.ListedIDsMemRatio' field +func (st *ConfigState) SetCacheListedIDsMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.ListedIDsMemRatio = v + st.reloadToViper() +} + +// CacheListedIDsMemRatioFlag returns the flag name for the 'Cache.ListedIDsMemRatio' field +func CacheListedIDsMemRatioFlag() string { return "cache-listed-ids-mem-ratio" } + +// GetCacheListedIDsMemRatio safely fetches the value for global configuration 'Cache.ListedIDsMemRatio' field +func GetCacheListedIDsMemRatio() float64 { return global.GetCacheListedIDsMemRatio() } + +// SetCacheListedIDsMemRatio safely sets the value for global configuration 'Cache.ListedIDsMemRatio' field +func SetCacheListedIDsMemRatio(v float64) { global.SetCacheListedIDsMemRatio(v) } // GetCacheMarkerMemRatio safely fetches the Configuration value for state's 'Cache.MarkerMemRatio' field func (st *ConfigState) GetCacheMarkerMemRatio() (v float64) { @@ -3887,37 +3906,6 @@ func GetCacheTagMemRatio() float64 { return global.GetCacheTagMemRatio() } // SetCacheTagMemRatio safely sets the value for global configuration 'Cache.TagMemRatio' field func SetCacheTagMemRatio(v float64) { global.SetCacheTagMemRatio(v) } -// GetCacheTagIDsFollowedByAccountMemRatio safely fetches the Configuration value for state's 'Cache.TagIDsFollowedByAccountMemRatio' field -func (st *ConfigState) GetCacheTagIDsFollowedByAccountMemRatio() (v float64) { - st.mutex.RLock() - v = st.config.Cache.TagIDsFollowedByAccountMemRatio - st.mutex.RUnlock() - return -} - -// SetCacheTagIDsFollowedByAccountMemRatio safely sets the Configuration value for state's 'Cache.TagIDsFollowedByAccountMemRatio' field -func (st *ConfigState) SetCacheTagIDsFollowedByAccountMemRatio(v float64) { - st.mutex.Lock() - defer st.mutex.Unlock() - st.config.Cache.TagIDsFollowedByAccountMemRatio = v - st.reloadToViper() -} - -// CacheTagIDsFollowedByAccountMemRatioFlag returns the flag name for the 'Cache.TagIDsFollowedByAccountMemRatio' field -func CacheTagIDsFollowedByAccountMemRatioFlag() string { - return "cache-tag-ids-followed-by-account-mem-ratio" -} - -// GetCacheTagIDsFollowedByAccountMemRatio safely fetches the value for global configuration 'Cache.TagIDsFollowedByAccountMemRatio' field -func GetCacheTagIDsFollowedByAccountMemRatio() float64 { - return global.GetCacheTagIDsFollowedByAccountMemRatio() -} - -// SetCacheTagIDsFollowedByAccountMemRatio safely sets the value for global configuration 'Cache.TagIDsFollowedByAccountMemRatio' field -func SetCacheTagIDsFollowedByAccountMemRatio(v float64) { - global.SetCacheTagIDsFollowedByAccountMemRatio(v) -} - // GetCacheThreadMuteMemRatio safely fetches the Configuration value for state's 'Cache.ThreadMuteMemRatio' field func (st *ConfigState) GetCacheThreadMuteMemRatio() (v float64) { st.mutex.RLock() -- cgit v1.2.3