diff options
author | 2024-03-06 02:15:58 -0800 | |
---|---|---|
committer | 2024-03-06 11:15:58 +0100 | |
commit | 61a2b91f454a6eb0dd383fc8614fee154654fa08 (patch) | |
tree | fcf6159f00c3a0833e6647dd00cd03d03774e2b2 /internal/cache | |
parent | [chore]: Bump github.com/stretchr/testify from 1.8.4 to 1.9.0 (#2714) (diff) | |
download | gotosocial-61a2b91f454a6eb0dd383fc8614fee154654fa08.tar.xz |
[feature] Filters v1 (#2594)
* Implement client-side v1 filters
* Exclude linter false positives
* Update test/envparsing.sh
* Fix minor Swagger, style, and Bun usage issues
* Regenerate Swagger
* De-generify filter keywords
* Remove updating filter statuses
This is an operation that the Mastodon v2 filter API doesn't actually have, because filter statuses, unlike keywords, don't have options: the only info they contain is the status ID to be filtered.
* Add a test for filter statuses specifically
* De-generify filter statuses
* Inline FilterEntry
* Use vertical style for Bun operations consistently
* Add comment on Filter DB interface
* Remove GoLand linter control comments
Our existing linters should catch these, or they don't matter very much
* Reduce memory ratio for filters
Diffstat (limited to 'internal/cache')
-rw-r--r-- | internal/cache/cache.go | 6 | ||||
-rw-r--r-- | internal/cache/db.go | 108 | ||||
-rw-r--r-- | internal/cache/size.go | 32 |
3 files changed, 146 insertions, 0 deletions
diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 17fa03323..9b70a565c 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -61,6 +61,9 @@ func (c *Caches) Init() { c.initDomainBlock() c.initEmoji() c.initEmojiCategory() + c.initFilter() + c.initFilterKeyword() + c.initFilterStatus() c.initFollow() c.initFollowIDs() c.initFollowRequest() @@ -119,6 +122,9 @@ func (c *Caches) Sweep(threshold float64) { c.GTS.BlockIDs.Trim(threshold) c.GTS.Emoji.Trim(threshold) c.GTS.EmojiCategory.Trim(threshold) + c.GTS.Filter.Trim(threshold) + c.GTS.FilterKeyword.Trim(threshold) + c.GTS.FilterStatus.Trim(threshold) c.GTS.Follow.Trim(threshold) c.GTS.FollowIDs.Trim(threshold) c.GTS.FollowRequest.Trim(threshold) diff --git a/internal/cache/db.go b/internal/cache/db.go index 275a25451..dc9e385cd 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -67,6 +67,15 @@ type GTSCaches struct { // EmojiCategory provides access to the gtsmodel EmojiCategory database cache. EmojiCategory structr.Cache[*gtsmodel.EmojiCategory] + // Filter provides access to the gtsmodel Filter database cache. + Filter structr.Cache[*gtsmodel.Filter] + + // FilterKeyword provides access to the gtsmodel FilterKeyword database cache. + FilterKeyword structr.Cache[*gtsmodel.FilterKeyword] + + // FilterStatus provides access to the gtsmodel FilterStatus database cache. + FilterStatus structr.Cache[*gtsmodel.FilterStatus] + // Follow provides access to the gtsmodel Follow database cache. Follow structr.Cache[*gtsmodel.Follow] @@ -409,6 +418,105 @@ func (c *Caches) initEmojiCategory() { }) } +func (c *Caches) initFilter() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofFilter(), // model in-mem size. + config.GetCacheFilterMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(filter1 *gtsmodel.Filter) *gtsmodel.Filter { + filter2 := new(gtsmodel.Filter) + *filter2 = *filter1 + + // Don't include ptr fields that + // will be populated separately. + // See internal/db/bundb/filter.go. + filter2.Keywords = nil + filter2.Statuses = nil + + return filter2 + } + + c.GTS.Filter.Init(structr.Config[*gtsmodel.Filter]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "AccountID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + CopyValue: copyF, + }) +} + +func (c *Caches) initFilterKeyword() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofFilterKeyword(), // model in-mem size. + config.GetCacheFilterKeywordMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(filterKeyword1 *gtsmodel.FilterKeyword) *gtsmodel.FilterKeyword { + filterKeyword2 := new(gtsmodel.FilterKeyword) + *filterKeyword2 = *filterKeyword1 + + // Don't include ptr fields that + // will be populated separately. + // See internal/db/bundb/filter.go. + filterKeyword2.Filter = nil + + return filterKeyword2 + } + + c.GTS.FilterKeyword.Init(structr.Config[*gtsmodel.FilterKeyword]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "AccountID", Multiple: true}, + {Fields: "FilterID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + CopyValue: copyF, + }) +} + +func (c *Caches) initFilterStatus() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofFilterStatus(), // model in-mem size. + config.GetCacheFilterStatusMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(filterStatus1 *gtsmodel.FilterStatus) *gtsmodel.FilterStatus { + filterStatus2 := new(gtsmodel.FilterStatus) + *filterStatus2 = *filterStatus1 + + // Don't include ptr fields that + // will be populated separately. + // See internal/db/bundb/filter.go. + filterStatus2.Filter = nil + + return filterStatus2 + } + + c.GTS.FilterStatus.Init(structr.Config[*gtsmodel.FilterStatus]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "AccountID", Multiple: true}, + {Fields: "FilterID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + CopyValue: copyF, + }) +} + func (c *Caches) initFollow() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/size.go b/internal/cache/size.go index 62fb31469..f9d88491d 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -309,6 +309,38 @@ func sizeofEmojiCategory() uintptr { })) } +func sizeofFilter() uintptr { + return uintptr(size.Of(>smodel.Filter{ + ID: exampleID, + CreatedAt: exampleTime, + UpdatedAt: exampleTime, + ExpiresAt: exampleTime, + AccountID: exampleID, + Title: exampleTextSmall, + Action: gtsmodel.FilterActionHide, + })) +} + +func sizeofFilterKeyword() uintptr { + return uintptr(size.Of(>smodel.FilterKeyword{ + ID: exampleID, + CreatedAt: exampleTime, + UpdatedAt: exampleTime, + FilterID: exampleID, + Keyword: exampleTextSmall, + })) +} + +func sizeofFilterStatus() uintptr { + return uintptr(size.Of(>smodel.FilterStatus{ + ID: exampleID, + CreatedAt: exampleTime, + UpdatedAt: exampleTime, + FilterID: exampleID, + StatusID: exampleID, + })) +} + func sizeofFollow() uintptr { return uintptr(size.Of(>smodel.Follow{ ID: exampleID, |