From 00adf18c2470a69c255ea75990bbbae6e57eea89 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:34:35 +0100 Subject: [feature] simpler cache size configuration (#2051) * add automatic cache max size generation based on ratios of a singular fixed memory target Signed-off-by: kim * remove now-unused cache max-size config variables Signed-off-by: kim * slight ratio tweak Signed-off-by: kim * remove unused visibility config var Signed-off-by: kim * add secret little ratio config trick Signed-off-by: kim * fixed a word Signed-off-by: kim * update cache library to remove use of TTL in result caches + slice cache Signed-off-by: kim * update other cache usages to use correct interface Signed-off-by: kim * update example config to explain the cache memory target Signed-off-by: kim * update env parsing test with new config values Signed-off-by: kim * do some ratio twiddling Signed-off-by: kim * add missing header * update envparsing with latest defaults Signed-off-by: kim * update size calculations to take into account result cache, simple cache and extra map overheads Signed-off-by: kim * tweak the ratios some more Signed-off-by: kim * more nan rampaging Signed-off-by: kim * fix envparsing script Signed-off-by: kim * update cache library, add sweep function to keep caches trim Signed-off-by: kim * sweep caches once a minute Signed-off-by: kim * add a regular job to sweep caches and keep under 80% utilisation Signed-off-by: kim * remove dead code Signed-off-by: kim * add new size library used to libraries section of readme Signed-off-by: kim * add better explanations for the mem-ratio numbers Signed-off-by: kim * update go-cache Signed-off-by: kim * library version bump Signed-off-by: kim * update cache.result{} size model estimation Signed-off-by: kim --------- Signed-off-by: kim --- internal/config/defaults.go | 149 +++++++++++++------------------------------- 1 file changed, 44 insertions(+), 105 deletions(-) (limited to 'internal/config/defaults.go') diff --git a/internal/config/defaults.go b/internal/config/defaults.go index e8cb39325..e740d1c98 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -126,111 +126,50 @@ var Defaults = Configuration{ AdvancedSenderMultiplier: 2, // 2 senders per CPU Cache: CacheConfiguration{ - GTS: GTSCacheConfiguration{ - AccountMaxSize: 2000, - AccountTTL: time.Minute * 30, - AccountSweepFreq: time.Minute, - - AccountNoteMaxSize: 1000, - AccountNoteTTL: time.Minute * 30, - AccountNoteSweepFreq: time.Minute, - - BlockMaxSize: 1000, - BlockTTL: time.Minute * 30, - BlockSweepFreq: time.Minute, - - BlockIDsMaxSize: 500, - BlockIDsTTL: time.Minute * 30, - BlockIDsSweepFreq: time.Minute, - - DomainBlockMaxSize: 2000, - DomainBlockTTL: time.Hour * 24, - DomainBlockSweepFreq: time.Minute, - - EmojiMaxSize: 2000, - EmojiTTL: time.Minute * 30, - EmojiSweepFreq: time.Minute, - - EmojiCategoryMaxSize: 100, - EmojiCategoryTTL: time.Minute * 30, - EmojiCategorySweepFreq: time.Minute, - - FollowMaxSize: 2000, - FollowTTL: time.Minute * 30, - FollowSweepFreq: time.Minute, - - FollowIDsMaxSize: 500, - FollowIDsTTL: time.Minute * 30, - FollowIDsSweepFreq: time.Minute, - - FollowRequestMaxSize: 2000, - FollowRequestTTL: time.Minute * 30, - FollowRequestSweepFreq: time.Minute, - - FollowRequestIDsMaxSize: 500, - FollowRequestIDsTTL: time.Minute * 30, - FollowRequestIDsSweepFreq: time.Minute, - - InstanceMaxSize: 2000, - InstanceTTL: time.Minute * 30, - InstanceSweepFreq: time.Minute, - - ListMaxSize: 2000, - ListTTL: time.Minute * 30, - ListSweepFreq: time.Minute, - - ListEntryMaxSize: 2000, - ListEntryTTL: time.Minute * 30, - ListEntrySweepFreq: time.Minute, - - MarkerMaxSize: 2000, - MarkerTTL: time.Hour * 6, - MarkerSweepFreq: time.Minute, - - MediaMaxSize: 1000, - MediaTTL: time.Minute * 30, - MediaSweepFreq: time.Minute, - - MentionMaxSize: 2000, - MentionTTL: time.Minute * 30, - MentionSweepFreq: time.Minute, - - NotificationMaxSize: 1000, - NotificationTTL: time.Minute * 30, - NotificationSweepFreq: time.Minute, - - ReportMaxSize: 100, - ReportTTL: time.Minute * 30, - ReportSweepFreq: time.Minute, - - StatusMaxSize: 2000, - StatusTTL: time.Minute * 30, - StatusSweepFreq: time.Minute, - - StatusFaveMaxSize: 2000, - StatusFaveTTL: time.Minute * 30, - StatusFaveSweepFreq: time.Minute, - - TagMaxSize: 2000, - TagTTL: time.Minute * 30, - TagSweepFreq: time.Minute, - - TombstoneMaxSize: 500, - TombstoneTTL: time.Minute * 30, - TombstoneSweepFreq: time.Minute, - - UserMaxSize: 500, - UserTTL: time.Minute * 30, - UserSweepFreq: time.Minute, - - WebfingerMaxSize: 250, - WebfingerTTL: time.Hour * 24, - WebfingerSweepFreq: time.Minute * 15, - }, - - VisibilityMaxSize: 2000, - VisibilityTTL: time.Minute * 30, - VisibilitySweepFreq: time.Minute, + // Rough memory target that the total + // size of all State.Caches will attempt + // to remain with. Emphasis on *rough*. + MemoryTarget: 200 * bytesize.MiB, + + // These ratios signal what percentage + // of the available cache target memory + // is allocated to each object type's + // cache. + // + // These are weighted by a totally + // assorted mixture of priority, and + // manual twiddling to get the generated + // cache capacity ratios within normal + // amounts dependent size of the models. + // + // when TODO items in the size.go source + // file have been addressed, these should + // be able to make some more sense :D + AccountMemRatio: 18, + AccountNoteMemRatio: 0.1, + BlockMemRatio: 3, + BlockIDsMemRatio: 3, + EmojiMemRatio: 3, + EmojiCategoryMemRatio: 0.1, + FollowMemRatio: 4, + FollowIDsMemRatio: 4, + FollowRequestMemRatio: 2, + FollowRequestIDsMemRatio: 2, + InstanceMemRatio: 1, + ListMemRatio: 3, + ListEntryMemRatio: 3, + MarkerMemRatio: 0.5, + MediaMemRatio: 4, + MentionMemRatio: 5, + NotificationMemRatio: 5, + ReportMemRatio: 1, + StatusMemRatio: 18, + StatusFaveMemRatio: 5, + TagMemRatio: 3, + TombstoneMemRatio: 2, + UserMemRatio: 0.1, + WebfingerMemRatio: 0.1, + VisibilityMemRatio: 2, }, HTTPClient: HTTPClientConfiguration{ -- cgit v1.2.3