[chore] consolidate caching libraries (#704)

* add miekg/dns dependency

* set/validate accountDomain

* move finger to dereferencer

* totally break GetRemoteAccount

* start reworking finger func a bit

* start reworking getRemoteAccount a bit

* move mention parts to namestring

* rework webfingerget

* use util function to extract webfinger parts

* use accountDomain

* rework finger again, final form

* just a real nasty commit, the worst

* remove refresh from account

* use new ASRepToAccount signature

* fix incorrect debug call

* fix for new getRemoteAccount

* rework GetRemoteAccount

* start updating tests to remove repetition

* break a lot of tests
Move shared test logic into the testrig,
rather than having it scattered all over
the place. This allows us to just mock
the transport controller once, and have
all tests use it (unless they need not to
for some other reason).

* fix up tests to use main mock httpclient

* webfinger only if necessary

* cheeky linting with the lads

* update mentionName regex
recognize instance accounts

* don't finger instance accounts

* test webfinger part extraction

* increase default worker count to 4 per cpu

* don't repeat regex parsing

* final search for discovered accountDomain

* be more permissive in namestring lookup

* add more extraction tests

* simplify GetParseMentionFunc

* skip long search if local account

* fix broken test

* consolidate to all use same caching libraries

Signed-off-by: kim <grufwub@gmail.com>

* perform more caching in the database layer

Signed-off-by: kim <grufwub@gmail.com>

* remove ASNote cache

Signed-off-by: kim <grufwub@gmail.com>

* update cache library, improve db tracing hooks

Signed-off-by: kim <grufwub@gmail.com>

* return ErrNoEntries if no account status IDs found, small formatting changes

Signed-off-by: kim <grufwub@gmail.com>

* fix tests, thanks tobi!

Signed-off-by: kim <grufwub@gmail.com>

Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

6 files changed, 68 insertions, 292 deletions

diff --git a/internal/cache/account.go b/internal/cache/account.go
index 8dbb07848..474afbe44 100644
--- a/internal/cache/account.go
+++ b/internal/cache/account.go
@@ -1,103 +1,62 @@
 package cache
 
 import (
-	"sync"
+	"time"
 
-	"github.com/ReneKroon/ttlcache"
-	"github.com/sirupsen/logrus"
+	"codeberg.org/gruf/go-cache/v2"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// AccountCache is a wrapper around ttlcache.Cache to provide URL and URI lookups for gtsmodel.Account
+// AccountCache is a cache wrapper to provide URL and URI lookups for gtsmodel.Account
 type AccountCache struct {
-	cache *ttlcache.Cache   // map of IDs -> cached accounts
-	urls  map[string]string // map of account URLs -> IDs
-	uris  map[string]string // map of account URIs -> IDs
-	mutex sync.Mutex
+	cache cache.LookupCache[string, string, *gtsmodel.Account]
 }
 
 // NewAccountCache returns a new instantiated AccountCache object
 func NewAccountCache() *AccountCache {
-	c := AccountCache{
-		cache: ttlcache.NewCache(),
-		urls:  make(map[string]string, 100),
-		uris:  make(map[string]string, 100),
-		mutex: sync.Mutex{},
-	}
-
-	// Set callback to purge lookup maps on expiration
-	c.cache.SetExpirationCallback(func(key string, value interface{}) {
-		account, ok := value.(*gtsmodel.Account)
-		if !ok {
-			logrus.Panicf("AccountCache could not assert entry with key %s to *gtsmodel.Account", key)
-		}
-
-		c.mutex.Lock()
-		delete(c.urls, account.URL)
-		delete(c.uris, account.URI)
-		c.mutex.Unlock()
+	c := &AccountCache{}
+	c.cache = cache.NewLookup(cache.LookupCfg[string, string, *gtsmodel.Account]{
+		RegisterLookups: func(lm *cache.LookupMap[string, string]) {
+			lm.RegisterLookup("uri")
+			lm.RegisterLookup("url")
+		},
+
+		AddLookups: func(lm *cache.LookupMap[string, string], acc *gtsmodel.Account) {
+			if uri := acc.URI; uri != "" {
+				lm.Set("uri", uri, acc.ID)
+			}
+			if url := acc.URL; url != "" {
+				lm.Set("url", url, acc.ID)
+			}
+		},
+
+		DeleteLookups: func(lm *cache.LookupMap[string, string], acc *gtsmodel.Account) {
+			if uri := acc.URI; uri != "" {
+				lm.Delete("uri", uri)
+			}
+			if url := acc.URL; url != "" {
+				lm.Delete("url", url)
+			}
+		},
 	})
-
-	return &c
+	c.cache.SetTTL(time.Minute*5, false)
+	c.cache.Start(time.Second * 10)
+	return c
 }
 
 // GetByID attempts to fetch a account from the cache by its ID, you will receive a copy for thread-safety
 func (c *AccountCache) GetByID(id string) (*gtsmodel.Account, bool) {
-	c.mutex.Lock()
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
+	return c.cache.Get(id)
 }
 
 // GetByURL attempts to fetch a account from the cache by its URL, you will receive a copy for thread-safety
 func (c *AccountCache) GetByURL(url string) (*gtsmodel.Account, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.urls[url]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt account lookup
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
+	return c.cache.GetBy("url", url)
 }
 
 // GetByURI attempts to fetch a account from the cache by its URI, you will receive a copy for thread-safety
 func (c *AccountCache) GetByURI(uri string) (*gtsmodel.Account, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.uris[uri]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt account lookup
-	account, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return account, ok
-}
-
-// getByID performs an unsafe (no mutex locks) lookup of account by ID, returning a copy of account in cache
-func (c *AccountCache) getByID(id string) (*gtsmodel.Account, bool) {
-	v, ok := c.cache.Get(id)
-	if !ok {
-		return nil, false
-	}
-
-	a, ok := v.(*gtsmodel.Account)
-	if !ok {
-		panic("account cache entry was not an account")
-	}
-
-	return copyAccount(a), true
+	return c.cache.GetBy("uri", uri)
 }
 
 // Put places a account in the cache, ensuring that the object place is a copy for thread-safety
@@ -105,16 +64,7 @@ func (c *AccountCache) Put(account *gtsmodel.Account) {
 	if account == nil || account.ID == "" {
 		panic("invalid account")
 	}
-
-	c.mutex.Lock()
 	c.cache.Set(account.ID, copyAccount(account))
-	if account.URL != "" {
-		c.urls[account.URL] = account.ID
-	}
-	if account.URI != "" {
-		c.uris[account.URI] = account.ID
-	}
-	c.mutex.Unlock()
 }
 
 // copyAccount performs a surface-level copy of account, only keeping attached IDs intact, not the objects.
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
deleted file mode 100644
index 8de2340f2..000000000
--- a/internal/cache/cache.go
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package cache
-
-import (
-	"time"
-
-	"github.com/ReneKroon/ttlcache"
-)
-
-// Cache defines an in-memory cache that is safe to be wiped when the application is restarted
-type Cache interface {
-	Store(k string, v interface{}) error
-	Fetch(k string) (interface{}, error)
-}
-
-type cache struct {
-	c *ttlcache.Cache
-}
-
-// New returns a new in-memory cache.
-func New() Cache {
-	c := ttlcache.NewCache()
-	c.SetTTL(5 * time.Minute)
-	cache := &cache{
-		c: c,
-	}
-	return cache
-}
diff --git a/internal/cache/error.go b/internal/cache/error.go
deleted file mode 100644
index 3e6fab24c..000000000
--- a/internal/cache/error.go
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package cache
-
-import "errors"
-
-// Error models an error returned by the in-memory cache.
-type Error error
-
-// ErrNotFound means that a value for the requested key was not found in the cache.
-var ErrNotFound = errors.New("value not found in cache")
diff --git a/internal/cache/fetch.go b/internal/cache/fetch.go
deleted file mode 100644
index 2ebaf74da..000000000
--- a/internal/cache/fetch.go
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package cache
-
-func (c *cache) Fetch(k string) (interface{}, error) {
-	i, stored := c.c.Get(k)
-	if !stored {
-		return nil, ErrNotFound
-	}
-
-	return i, nil
-}
diff --git a/internal/cache/status.go b/internal/cache/status.go
index 9a698c608..6c0b5aa9f 100644
--- a/internal/cache/status.go
+++ b/internal/cache/status.go
@@ -1,103 +1,62 @@
 package cache
 
 import (
-	"sync"
+	"time"
 
-	"github.com/ReneKroon/ttlcache"
-	"github.com/sirupsen/logrus"
+	"codeberg.org/gruf/go-cache/v2"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// StatusCache is a wrapper around ttlcache.Cache to provide URL and URI lookups for gtsmodel.Status
+// StatusCache is a cache wrapper to provide URL and URI lookups for gtsmodel.Status
 type StatusCache struct {
-	cache *ttlcache.Cache   // map of IDs -> cached statuses
-	urls  map[string]string // map of status URLs -> IDs
-	uris  map[string]string // map of status URIs -> IDs
-	mutex sync.Mutex
+	cache cache.LookupCache[string, string, *gtsmodel.Status]
 }
 
 // NewStatusCache returns a new instantiated statusCache object
 func NewStatusCache() *StatusCache {
-	c := StatusCache{
-		cache: ttlcache.NewCache(),
-		urls:  make(map[string]string, 100),
-		uris:  make(map[string]string, 100),
-		mutex: sync.Mutex{},
-	}
-
-	// Set callback to purge lookup maps on expiration
-	c.cache.SetExpirationCallback(func(key string, value interface{}) {
-		status, ok := value.(*gtsmodel.Status)
-		if !ok {
-			logrus.Panicf("StatusCache could not assert entry with key %s to *gtsmodel.Status", key)
-		}
-
-		c.mutex.Lock()
-		delete(c.urls, status.URL)
-		delete(c.uris, status.URI)
-		c.mutex.Unlock()
+	c := &StatusCache{}
+	c.cache = cache.NewLookup(cache.LookupCfg[string, string, *gtsmodel.Status]{
+		RegisterLookups: func(lm *cache.LookupMap[string, string]) {
+			lm.RegisterLookup("uri")
+			lm.RegisterLookup("url")
+		},
+
+		AddLookups: func(lm *cache.LookupMap[string, string], status *gtsmodel.Status) {
+			if uri := status.URI; uri != "" {
+				lm.Set("uri", uri, status.ID)
+			}
+			if url := status.URL; url != "" {
+				lm.Set("url", url, status.ID)
+			}
+		},
+
+		DeleteLookups: func(lm *cache.LookupMap[string, string], status *gtsmodel.Status) {
+			if uri := status.URI; uri != "" {
+				lm.Delete("uri", uri)
+			}
+			if url := status.URL; url != "" {
+				lm.Delete("url", url)
+			}
+		},
 	})
-
-	return &c
+	c.cache.SetTTL(time.Minute*5, false)
+	c.cache.Start(time.Second * 10)
+	return c
 }
 
 // GetByID attempts to fetch a status from the cache by its ID, you will receive a copy for thread-safety
 func (c *StatusCache) GetByID(id string) (*gtsmodel.Status, bool) {
-	c.mutex.Lock()
-	status, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return status, ok
+	return c.cache.Get(id)
 }
 
 // GetByURL attempts to fetch a status from the cache by its URL, you will receive a copy for thread-safety
 func (c *StatusCache) GetByURL(url string) (*gtsmodel.Status, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.urls[url]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt status lookup
-	status, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return status, ok
+	return c.cache.GetBy("url", url)
 }
 
 // GetByURI attempts to fetch a status from the cache by its URI, you will receive a copy for thread-safety
 func (c *StatusCache) GetByURI(uri string) (*gtsmodel.Status, bool) {
-	// Perform safe ID lookup
-	c.mutex.Lock()
-	id, ok := c.uris[uri]
-
-	// Not found, unlock early
-	if !ok {
-		c.mutex.Unlock()
-		return nil, false
-	}
-
-	// Attempt status lookup
-	status, ok := c.getByID(id)
-	c.mutex.Unlock()
-	return status, ok
-}
-
-// getByID performs an unsafe (no mutex locks) lookup of status by ID, returning a copy of status in cache
-func (c *StatusCache) getByID(id string) (*gtsmodel.Status, bool) {
-	v, ok := c.cache.Get(id)
-	if !ok {
-		return nil, false
-	}
-
-	s, ok := v.(*gtsmodel.Status)
-	if !ok {
-		panic("status cache entry was not a status")
-	}
-
-	return copyStatus(s), true
+	return c.cache.GetBy("uri", uri)
 }
 
 // Put places a status in the cache, ensuring that the object place is a copy for thread-safety
@@ -105,16 +64,7 @@ func (c *StatusCache) Put(status *gtsmodel.Status) {
 	if status == nil || status.ID == "" {
 		panic("invalid status")
 	}
-
-	c.mutex.Lock()
 	c.cache.Set(status.ID, copyStatus(status))
-	if status.URL != "" {
-		c.urls[status.URL] = status.ID
-	}
-	if status.URI != "" {
-		c.uris[status.URI] = status.ID
-	}
-	c.mutex.Unlock()
 }
 
 // copyStatus performs a surface-level copy of status, only keeping attached IDs intact, not the objects.
diff --git a/internal/cache/store.go b/internal/cache/store.go
deleted file mode 100644
index d3e135fd6..000000000
--- a/internal/cache/store.go
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package cache
-
-func (c *cache) Store(k string, v interface{}) error {
-	c.c.Set(k, v)
-	return nil
-}