From 00adf18c2470a69c255ea75990bbbae6e57eea89 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:34:35 +0100 Subject: [feature] simpler cache size configuration (#2051) * add automatic cache max size generation based on ratios of a singular fixed memory target Signed-off-by: kim * remove now-unused cache max-size config variables Signed-off-by: kim * slight ratio tweak Signed-off-by: kim * remove unused visibility config var Signed-off-by: kim * add secret little ratio config trick Signed-off-by: kim * fixed a word Signed-off-by: kim * update cache library to remove use of TTL in result caches + slice cache Signed-off-by: kim * update other cache usages to use correct interface Signed-off-by: kim * update example config to explain the cache memory target Signed-off-by: kim * update env parsing test with new config values Signed-off-by: kim * do some ratio twiddling Signed-off-by: kim * add missing header * update envparsing with latest defaults Signed-off-by: kim * update size calculations to take into account result cache, simple cache and extra map overheads Signed-off-by: kim * tweak the ratios some more Signed-off-by: kim * more nan rampaging Signed-off-by: kim * fix envparsing script Signed-off-by: kim * update cache library, add sweep function to keep caches trim Signed-off-by: kim * sweep caches once a minute Signed-off-by: kim * add a regular job to sweep caches and keep under 80% utilisation Signed-off-by: kim * remove dead code Signed-off-by: kim * add new size library used to libraries section of readme Signed-off-by: kim * add better explanations for the mem-ratio numbers Signed-off-by: kim * update go-cache Signed-off-by: kim * library version bump Signed-off-by: kim * update cache.result{} size model estimation Signed-off-by: kim --------- Signed-off-by: kim --- test/envparsing.sh | 107 +++++++++++++---------------------------------------- 1 file changed, 26 insertions(+), 81 deletions(-) (limited to 'test') diff --git a/test/envparsing.sh b/test/envparsing.sh index 14b265b32..f75b6fd3f 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECT=$(cat <<"EOF" +EXPECT=$(cat << "EOF" { "account-domain": "peepee", "accounts-allow-custom-css": true, @@ -18,86 +18,31 @@ EXPECT=$(cat <<"EOF" "application-name": "gts", "bind-address": "127.0.0.1", "cache": { - "gts": { - "account-max-size": 99, - "account-note-max-size": 1000, - "account-note-sweep-freq": 60000000000, - "account-note-ttl": 1800000000000, - "account-sweep-freq": 1000000000, - "account-ttl": 10800000000000, - "block-ids-max-size": 500, - "block-ids-sweep-freq": 60000000000, - "block-ids-ttl": 1800000000000, - "block-max-size": 1000, - "block-sweep-freq": 60000000000, - "block-ttl": 1800000000000, - "domain-block-max-size": 2000, - "domain-block-sweep-freq": 60000000000, - "domain-block-ttl": 86400000000000, - "emoji-category-max-size": 100, - "emoji-category-sweep-freq": 60000000000, - "emoji-category-ttl": 1800000000000, - "emoji-max-size": 2000, - "emoji-sweep-freq": 60000000000, - "emoji-ttl": 1800000000000, - "follow-ids-max-size": 500, - "follow-ids-sweep-freq": 60000000000, - "follow-ids-ttl": 1800000000000, - "follow-max-size": 2000, - "follow-request-ids-max-size": 500, - "follow-request-ids-sweep-freq": 60000000000, - "follow-request-ids-ttl": 1800000000000, - "follow-request-max-size": 2000, - "follow-request-sweep-freq": 60000000000, - "follow-request-ttl": 1800000000000, - "follow-sweep-freq": 60000000000, - "follow-ttl": 1800000000000, - "instance-max-size": 2000, - "instance-sweep-freq": 60000000000, - "instance-ttl": 1800000000000, - "list-entry-max-size": 2000, - "list-entry-sweep-freq": 60000000000, - "list-entry-ttl": 1800000000000, - "list-max-size": 2000, - "list-sweep-freq": 60000000000, - "list-ttl": 1800000000000, - "marker-max-size": 2000, - "marker-sweep-freq": 60000000000, - "marker-ttl": 21600000000000, - "media-max-size": 1000, - "media-sweep-freq": 60000000000, - "media-ttl": 1800000000000, - "mention-max-size": 2000, - "mention-sweep-freq": 60000000000, - "mention-ttl": 1800000000000, - "notification-max-size": 1000, - "notification-sweep-freq": 60000000000, - "notification-ttl": 1800000000000, - "report-max-size": 100, - "report-sweep-freq": 60000000000, - "report-ttl": 1800000000000, - "status-fave-max-size": 2000, - "status-fave-sweep-freq": 60000000000, - "status-fave-ttl": 1800000000000, - "status-max-size": 2000, - "status-sweep-freq": 60000000000, - "status-ttl": 1800000000000, - "tag-max-size": 2000, - "tag-sweep-freq": 60000000000, - "tag-ttl": 1800000000000, - "tombstone-max-size": 500, - "tombstone-sweep-freq": 60000000000, - "tombstone-ttl": 1800000000000, - "user-max-size": 500, - "user-sweep-freq": 60000000000, - "user-ttl": 1800000000000, - "webfinger-max-size": 250, - "webfinger-sweep-freq": 900000000000, - "webfinger-ttl": 86400000000000 - }, - "visibility-max-size": 2000, - "visibility-sweep-freq": 60000000000, - "visibility-ttl": 1800000000000 + "account-mem-ratio": 18, + "account-note-mem-ratio": 0.1, + "block-mem-ratio": 3, + "emoji-category-mem-ratio": 0.1, + "emoji-mem-ratio": 3, + "follow-ids-mem-ratio": 4, + "follow-mem-ratio": 4, + "follow-request-ids-mem-ratio": 2, + "follow-request-mem-ratio": 2, + "instance-mem-ratio": 1, + "list-entry-mem-ratio": 3, + "list-mem-ratio": 3, + "marker-mem-ratio": 0.5, + "media-mem-ratio": 4, + "memory-target": 209715200, + "mention-mem-ratio": 5, + "notification-mem-ratio": 5, + "report-mem-ratio": 1, + "status-fave-mem-ratio": 5, + "status-mem-ratio": 18, + "tag-mem-ratio": 3, + "tombstone-mem-ratio": 2, + "user-mem-ratio": 0.1, + "visibility-mem-ratio": 2, + "webfinger-mem-ratio": 0.1 }, "config-path": "internal/config/testdata/test.yaml", "db-address": ":memory:", -- cgit v1.2.3