summaryrefslogtreecommitdiff
path: root/internal/cache
diff options
context:
space:
mode:
authorLibravatar kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>2023-08-14 17:08:19 +0100
committerLibravatar GitHub <noreply@github.com>2023-08-14 17:08:19 +0100
commit815b5291e034864052771cd3561fea479027e00f (patch)
tree18e3d49c441fb15e1f05ea2611e9c5c96b0be86f /internal/cache
parent[fix] Update CSP header for blob images (upload preview) and dev livereload (... (diff)
downloadgotosocial-815b5291e034864052771cd3561fea479027e00f.tar.xz
[bugfix] fix inconsistent calculated cache sizes (#2115)v0.11.0
* use calculated exampleTime instead of `time.Now()` to ensure no locale data, retweak cache ratios * update envparsing test * update default cache memory to 100MiB * fix envparsing with latest cache target default --------- Signed-off-by: kim <grufwub@gmail.com>
Diffstat (limited to 'internal/cache')
-rw-r--r--internal/cache/gts.go58
-rw-r--r--internal/cache/size.go124
2 files changed, 100 insertions, 82 deletions
diff --git a/internal/cache/gts.go b/internal/cache/gts.go
index 8d7ebcd98..12e917919 100644
--- a/internal/cache/gts.go
+++ b/internal/cache/gts.go
@@ -269,7 +269,7 @@ func (c *GTSCaches) initAccount() {
config.GetCacheAccountMemRatio(),
)
- log.Infof(nil, "Account cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.account = result.New([]result.Lookup{
{Name: "ID"},
@@ -296,7 +296,8 @@ func (c *GTSCaches) initAccountNote() {
sizeofAccountNote(), // model in-mem size.
config.GetCacheAccountNoteMemRatio(),
)
- log.Infof(nil, "AccountNote cache size = %d", cap)
+
+ log.Infof(nil, "cache size = %d", cap)
c.accountNote = result.New([]result.Lookup{
{Name: "ID"},
@@ -316,7 +317,8 @@ func (c *GTSCaches) initApplication() {
sizeofApplication(), // model in-mem size.
config.GetCacheApplicationMemRatio(),
)
- log.Infof(nil, "Application cache size = %d", cap)
+
+ log.Infof(nil, "cache size = %d", cap)
c.application = result.New([]result.Lookup{
{Name: "ID"},
@@ -337,7 +339,7 @@ func (c *GTSCaches) initBlock() {
config.GetCacheBlockMemRatio(),
)
- log.Infof(nil, "Block cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.block = result.New([]result.Lookup{
{Name: "ID"},
@@ -360,7 +362,7 @@ func (c *GTSCaches) initBlockIDs() {
config.GetCacheBlockIDsMemRatio(),
)
- log.Infof(nil, "Block IDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.blockIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -374,7 +376,7 @@ func (c *GTSCaches) initBoostOfIDs() {
config.GetCacheBoostOfIDsMemRatio(),
)
- log.Infof(nil, "BoostofIDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.boostOfIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -393,7 +395,7 @@ func (c *GTSCaches) initEmoji() {
config.GetCacheEmojiMemRatio(),
)
- log.Infof(nil, "Emoji cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.emoji = result.New([]result.Lookup{
{Name: "ID"},
@@ -417,7 +419,7 @@ func (c *GTSCaches) initEmojiCategory() {
config.GetCacheEmojiCategoryMemRatio(),
)
- log.Infof(nil, "EmojiCategory cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.emojiCategory = result.New([]result.Lookup{
{Name: "ID"},
@@ -438,7 +440,7 @@ func (c *GTSCaches) initFollow() {
config.GetCacheFollowMemRatio(),
)
- log.Infof(nil, "Follow cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.follow = result.New([]result.Lookup{
{Name: "ID"},
@@ -461,7 +463,7 @@ func (c *GTSCaches) initFollowIDs() {
config.GetCacheFollowIDsMemRatio(),
)
- log.Infof(nil, "Follow IDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.followIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -476,7 +478,7 @@ func (c *GTSCaches) initFollowRequest() {
config.GetCacheFollowRequestMemRatio(),
)
- log.Infof(nil, "FollowRequest cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.followRequest = result.New([]result.Lookup{
{Name: "ID"},
@@ -499,7 +501,7 @@ func (c *GTSCaches) initFollowRequestIDs() {
config.GetCacheFollowRequestIDsMemRatio(),
)
- log.Infof(nil, "Follow Request IDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.followRequestIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -513,7 +515,7 @@ func (c *GTSCaches) initInReplyToIDs() {
config.GetCacheInReplyToIDsMemRatio(),
)
- log.Infof(nil, "InReplyTo IDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.inReplyToIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -528,7 +530,7 @@ func (c *GTSCaches) initInstance() {
config.GetCacheInstanceMemRatio(),
)
- log.Infof(nil, "Instance cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.instance = result.New([]result.Lookup{
{Name: "ID"},
@@ -549,7 +551,7 @@ func (c *GTSCaches) initList() {
config.GetCacheListMemRatio(),
)
- log.Infof(nil, "List cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.list = result.New([]result.Lookup{
{Name: "ID"},
@@ -569,7 +571,7 @@ func (c *GTSCaches) initListEntry() {
config.GetCacheListEntryMemRatio(),
)
- log.Infof(nil, "ListEntry cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.listEntry = result.New([]result.Lookup{
{Name: "ID"},
@@ -591,7 +593,7 @@ func (c *GTSCaches) initMarker() {
config.GetCacheMarkerMemRatio(),
)
- log.Infof(nil, "Marker cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.marker = result.New([]result.Lookup{
{Name: "AccountID.Name"},
@@ -611,7 +613,7 @@ func (c *GTSCaches) initMedia() {
config.GetCacheMediaMemRatio(),
)
- log.Infof(nil, "Media cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.media = result.New([]result.Lookup{
{Name: "ID"},
@@ -631,7 +633,7 @@ func (c *GTSCaches) initMention() {
config.GetCacheMentionMemRatio(),
)
- log.Infof(nil, "Mention cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.mention = result.New([]result.Lookup{
{Name: "ID"},
@@ -651,7 +653,7 @@ func (c *GTSCaches) initNotification() {
config.GetCacheNotificationMemRatio(),
)
- log.Infof(nil, "Notification cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.notification = result.New([]result.Lookup{
{Name: "ID"},
@@ -672,7 +674,7 @@ func (c *GTSCaches) initReport() {
config.GetCacheReportMemRatio(),
)
- log.Infof(nil, "Report cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.report = result.New([]result.Lookup{
{Name: "ID"},
@@ -692,7 +694,7 @@ func (c *GTSCaches) initStatus() {
config.GetCacheStatusMemRatio(),
)
- log.Infof(nil, "Status cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.status = result.New([]result.Lookup{
{Name: "ID"},
@@ -715,7 +717,7 @@ func (c *GTSCaches) initStatusFave() {
config.GetCacheStatusFaveMemRatio(),
)
- log.Infof(nil, "StatusFave cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.statusFave = result.New([]result.Lookup{
{Name: "ID"},
@@ -736,7 +738,7 @@ func (c *GTSCaches) initStatusFaveIDs() {
config.GetCacheStatusFaveIDsMemRatio(),
)
- log.Infof(nil, "StatusFave IDs cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.statusFaveIDs = &SliceCache[string]{Cache: simple.New[string, []string](
0,
@@ -751,7 +753,7 @@ func (c *GTSCaches) initTag() {
config.GetCacheTagMemRatio(),
)
- log.Infof(nil, "Tag cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.tag = result.New([]result.Lookup{
{Name: "ID"},
@@ -772,7 +774,7 @@ func (c *GTSCaches) initTombstone() {
config.GetCacheTombstoneMemRatio(),
)
- log.Infof(nil, "Tombstone cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.tombstone = result.New([]result.Lookup{
{Name: "ID"},
@@ -793,7 +795,7 @@ func (c *GTSCaches) initUser() {
config.GetCacheUserMemRatio(),
)
- log.Infof(nil, "User cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.user = result.New([]result.Lookup{
{Name: "ID"},
@@ -817,7 +819,7 @@ func (c *GTSCaches) initWebfinger() {
config.GetCacheWebfingerMemRatio(),
)
- log.Infof(nil, "Webfinger cache size = %d", cap)
+ log.Infof(nil, "cache size = %d", cap)
c.webfinger = ttl.New[string, string](
0,
diff --git a/internal/cache/size.go b/internal/cache/size.go
index 34586b0b1..f578b9402 100644
--- a/internal/cache/size.go
+++ b/internal/cache/size.go
@@ -66,6 +66,22 @@ you'll make society more equitable for all if you're not careful! :hammer_sickle
sizeofResultKey = 2 * sizeofIDStr
)
+var (
+ // Example time calculated at ~ 14th August, 2023. Because if
+ // we use `time.Now()` in our structs below, it populates
+ // them with locale data which throws-off size calculations.
+ //
+ // This is because the locale data is (relatively) very large
+ // in-memory, but it's global "singletons" ptr'd to by the time
+ // structs, so inconsequential to our calculated cache size.
+ // Unfortunately the size.Of() function is not aware of this!
+ exampleTime = time.Time{}.Add(1692010328 * time.Second)
+
+ // stop trying to collapse this var
+ // block, gofmt, you motherfucker.
+ _ = interface{}(nil)
+)
+
// calculateSliceCacheMax calculates the maximum capacity for a slice cache with given individual ratio.
func calculateSliceCacheMax(ratio float64) int {
return calculateCacheMax(sizeofIDStr, sizeofIDSlice, ratio)
@@ -89,12 +105,12 @@ func calculateResultCacheMax(structSz uintptr, ratio float64) int {
// The result cache wraps each struct result in a wrapping
// struct with further information, and possible error. This
// also needs to be taken into account when calculating value.
- const resultValueOverhead = unsafe.Sizeof(&struct {
+ resultValueOverhead := uintptr(size.Of(&struct {
_ int64
_ []any
_ any
_ error
- }{})
+ }{}))
return calculateCacheMax(
pkeySz+totalLookupKeySz,
@@ -194,9 +210,9 @@ func sizeofAccount() uintptr {
Note: exampleText,
NoteRaw: exampleText,
Memorial: func() *bool { ok := false; return &ok }(),
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- FetchedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
+ FetchedAt: exampleTime,
Bot: func() *bool { ok := true; return &ok }(),
Locked: func() *bool { ok := true; return &ok }(),
Discoverable: func() *bool { ok := false; return &ok }(),
@@ -214,9 +230,9 @@ func sizeofAccount() uintptr {
PrivateKey: &rsa.PrivateKey{},
PublicKey: &rsa.PublicKey{},
PublicKeyURI: exampleURI,
- SensitizedAt: time.Time{},
- SilencedAt: time.Now(),
- SuspendedAt: time.Now(),
+ SensitizedAt: exampleTime,
+ SilencedAt: exampleTime,
+ SuspendedAt: exampleTime,
HideCollections: func() *bool { ok := true; return &ok }(),
SuspensionOrigin: exampleID,
EnableRSS: func() *bool { ok := true; return &ok }(),
@@ -235,8 +251,8 @@ func sizeofAccountNote() uintptr {
func sizeofApplication() uintptr {
return uintptr(size.Of(&gtsmodel.Application{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Name: exampleUsername,
Website: exampleURI,
RedirectURI: exampleURI,
@@ -249,8 +265,8 @@ func sizeofApplication() uintptr {
func sizeofBlock() uintptr {
return uintptr(size.Of(&gtsmodel.Block{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
URI: exampleURI,
AccountID: exampleID,
TargetAccountID: exampleID,
@@ -262,8 +278,8 @@ func sizeofEmoji() uintptr {
ID: exampleID,
Shortcode: exampleTextSmall,
Domain: exampleURI,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
ImageRemoteURL: exampleURI,
ImageStaticRemoteURL: exampleURI,
ImageURL: exampleURI,
@@ -272,7 +288,7 @@ func sizeofEmoji() uintptr {
ImageStaticPath: exampleURI,
ImageContentType: "image/png",
ImageStaticContentType: "image/png",
- ImageUpdatedAt: time.Now(),
+ ImageUpdatedAt: exampleTime,
Disabled: func() *bool { ok := false; return &ok }(),
URI: "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
VisibleInPicker: func() *bool { ok := true; return &ok }(),
@@ -285,16 +301,16 @@ func sizeofEmojiCategory() uintptr {
return uintptr(size.Of(&gtsmodel.EmojiCategory{
ID: exampleID,
Name: exampleUsername,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
}))
}
func sizeofFollow() uintptr {
return uintptr(size.Of(&gtsmodel.Follow{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
AccountID: exampleID,
TargetAccountID: exampleID,
ShowReblogs: func() *bool { ok := true; return &ok }(),
@@ -306,8 +322,8 @@ func sizeofFollow() uintptr {
func sizeofFollowRequest() uintptr {
return uintptr(size.Of(&gtsmodel.FollowRequest{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
AccountID: exampleID,
TargetAccountID: exampleID,
ShowReblogs: func() *bool { ok := true; return &ok }(),
@@ -319,8 +335,8 @@ func sizeofFollowRequest() uintptr {
func sizeofInstance() uintptr {
return uintptr(size.Of(&gtsmodel.Instance{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Domain: exampleURI,
URI: exampleURI,
Title: exampleTextSmall,
@@ -335,8 +351,8 @@ func sizeofInstance() uintptr {
func sizeofList() uintptr {
return uintptr(size.Of(&gtsmodel.List{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Title: exampleTextSmall,
AccountID: exampleID,
RepliesPolicy: gtsmodel.RepliesPolicyFollowed,
@@ -346,8 +362,8 @@ func sizeofList() uintptr {
func sizeofListEntry() uintptr {
return uintptr(size.Of(&gtsmodel.ListEntry{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
ListID: exampleID,
FollowID: exampleID,
}))
@@ -357,7 +373,7 @@ func sizeofMarker() uintptr {
return uintptr(size.Of(&gtsmodel.Marker{
AccountID: exampleID,
Name: gtsmodel.MarkerNameHome,
- UpdatedAt: time.Now(),
+ UpdatedAt: exampleTime,
Version: 0,
LastReadID: exampleID,
}))
@@ -369,8 +385,8 @@ func sizeofMedia() uintptr {
StatusID: exampleID,
URL: exampleURI,
RemoteURL: exampleURI,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Type: gtsmodel.FileTypeImage,
AccountID: exampleID,
Description: exampleText,
@@ -379,12 +395,12 @@ func sizeofMedia() uintptr {
File: gtsmodel.File{
Path: exampleURI,
ContentType: "image/jpeg",
- UpdatedAt: time.Now(),
+ UpdatedAt: exampleTime,
},
Thumbnail: gtsmodel.Thumbnail{
Path: exampleURI,
ContentType: "image/jpeg",
- UpdatedAt: time.Now(),
+ UpdatedAt: exampleTime,
URL: exampleURI,
RemoteURL: exampleURI,
},
@@ -398,8 +414,8 @@ func sizeofMention() uintptr {
return uintptr(size.Of(&gtsmodel.Mention{
ID: exampleURI,
StatusID: exampleURI,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
OriginAccountID: exampleURI,
OriginAccountURI: exampleURI,
TargetAccountID: exampleID,
@@ -413,7 +429,7 @@ func sizeofNotification() uintptr {
return uintptr(size.Of(&gtsmodel.Notification{
ID: exampleID,
NotificationType: gtsmodel.NotificationFave,
- CreatedAt: time.Now(),
+ CreatedAt: exampleTime,
TargetAccountID: exampleID,
OriginAccountID: exampleID,
StatusID: exampleID,
@@ -424,8 +440,8 @@ func sizeofNotification() uintptr {
func sizeofReport() uintptr {
return uintptr(size.Of(&gtsmodel.Report{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
URI: exampleURI,
AccountID: exampleID,
TargetAccountID: exampleID,
@@ -433,7 +449,7 @@ func sizeofReport() uintptr {
StatusIDs: []string{exampleID, exampleID, exampleID},
Forwarded: func() *bool { ok := true; return &ok }(),
ActionTaken: exampleText,
- ActionTakenAt: time.Now(),
+ ActionTakenAt: exampleTime,
ActionTakenByAccountID: exampleID,
}))
}
@@ -449,9 +465,9 @@ func sizeofStatus() uintptr {
TagIDs: []string{exampleID, exampleID, exampleID},
MentionIDs: []string{},
EmojiIDs: []string{exampleID, exampleID, exampleID},
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- FetchedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
+ FetchedAt: exampleTime,
Local: func() *bool { ok := false; return &ok }(),
AccountURI: exampleURI,
AccountID: exampleID,
@@ -476,7 +492,7 @@ func sizeofStatus() uintptr {
func sizeofStatusFave() uintptr {
return uintptr(size.Of(&gtsmodel.StatusFave{
ID: exampleID,
- CreatedAt: time.Now(),
+ CreatedAt: exampleTime,
AccountID: exampleID,
TargetAccountID: exampleID,
StatusID: exampleID,
@@ -488,8 +504,8 @@ func sizeofTag() uintptr {
return uintptr(size.Of(&gtsmodel.Tag{
ID: exampleID,
Name: exampleUsername,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Useable: func() *bool { ok := true; return &ok }(),
Listable: func() *bool { ok := true; return &ok }(),
}))
@@ -498,8 +514,8 @@ func sizeofTag() uintptr {
func sizeofTombstone() uintptr {
return uintptr(size.Of(&gtsmodel.Tombstone{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Domain: exampleUsername,
URI: exampleURI,
}))
@@ -517,29 +533,29 @@ func sizeofVisibility() uintptr {
func sizeofUser() uintptr {
return uintptr(size.Of(&gtsmodel.User{
ID: exampleID,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
+ CreatedAt: exampleTime,
+ UpdatedAt: exampleTime,
Email: exampleURI,
AccountID: exampleID,
EncryptedPassword: exampleTextSmall,
- CurrentSignInAt: time.Now(),
- LastSignInAt: time.Now(),
+ CurrentSignInAt: exampleTime,
+ LastSignInAt: exampleTime,
InviteID: exampleID,
ChosenLanguages: []string{"en", "fr", "jp"},
FilteredLanguages: []string{"en", "fr", "jp"},
Locale: "en",
CreatedByApplicationID: exampleID,
- LastEmailedAt: time.Now(),
+ LastEmailedAt: exampleTime,
ConfirmationToken: exampleTextSmall,
- ConfirmationSentAt: time.Now(),
- ConfirmedAt: time.Now(),
+ ConfirmationSentAt: exampleTime,
+ ConfirmedAt: exampleTime,
UnconfirmedEmail: exampleURI,
Moderator: func() *bool { ok := true; return &ok }(),
Admin: func() *bool { ok := true; return &ok }(),
Disabled: func() *bool { ok := true; return &ok }(),
Approved: func() *bool { ok := true; return &ok }(),
ResetPasswordToken: exampleTextSmall,
- ResetPasswordSentAt: time.Now(),
+ ResetPasswordSentAt: exampleTime,
ExternalID: exampleID,
}))
}