[feature] simpler cache size configuration (#2051)

* add automatic cache max size generation based on ratios of a singular fixed memory target

Signed-off-by: kim <grufwub@gmail.com>

* remove now-unused cache max-size config variables

Signed-off-by: kim <grufwub@gmail.com>

* slight ratio tweak

Signed-off-by: kim <grufwub@gmail.com>

* remove unused visibility config var

Signed-off-by: kim <grufwub@gmail.com>

* add secret little ratio config trick

Signed-off-by: kim <grufwub@gmail.com>

* fixed a word

Signed-off-by: kim <grufwub@gmail.com>

* update cache library to remove use of TTL in result caches + slice cache

Signed-off-by: kim <grufwub@gmail.com>

* update other cache usages to use correct interface

Signed-off-by: kim <grufwub@gmail.com>

* update example config to explain the cache memory target

Signed-off-by: kim <grufwub@gmail.com>

* update env parsing test with new config values

Signed-off-by: kim <grufwub@gmail.com>

* do some ratio twiddling

Signed-off-by: kim <grufwub@gmail.com>

* add missing header

* update envparsing with latest defaults

Signed-off-by: kim <grufwub@gmail.com>

* update size calculations to take into account result cache, simple cache and extra map overheads

Signed-off-by: kim <grufwub@gmail.com>

* tweak the ratios some more

Signed-off-by: kim <grufwub@gmail.com>

* more nan rampaging

Signed-off-by: kim <grufwub@gmail.com>

* fix envparsing script

Signed-off-by: kim <grufwub@gmail.com>

* update cache library, add sweep function to keep caches trim

Signed-off-by: kim <grufwub@gmail.com>

* sweep caches once a minute

Signed-off-by: kim <grufwub@gmail.com>

* add a regular job to sweep caches and keep under 80% utilisation

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* add new size library used to libraries section of readme

Signed-off-by: kim <grufwub@gmail.com>

* add better explanations for the mem-ratio numbers

Signed-off-by: kim <grufwub@gmail.com>

* update go-cache

Signed-off-by: kim <grufwub@gmail.com>

* library version bump

Signed-off-by: kim <grufwub@gmail.com>

* update cache.result{} size model estimation

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>

1 files changed, 113 insertions, 113 deletions

diff --git a/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go b/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go
index a86a72c77..f31e6604a 100644
--- a/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go
+++ b/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go
@@ -2,14 +2,38 @@ package result
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"reflect"
-	"time"
 	_ "unsafe"
 
-	"codeberg.org/gruf/go-cache/v3/ttl"
+	"codeberg.org/gruf/go-cache/v3/simple"
 	"codeberg.org/gruf/go-errors/v2"
 )
 
+type result struct {
+	// Result primary key
+	PKey int64
+
+	// keys accessible under
+	Keys cacheKeys
+
+	// cached value
+	Value any
+
+	// cached error
+	Error error
+}
+
+// getResultValue is a safe way of casting and fetching result value.
+func getResultValue[T any](res *result) T {
+	v, ok := res.Value.(T)
+	if !ok {
+		fmt.Fprintf(os.Stderr, "!! BUG: unexpected value type in result: %T\n", res.Value)
+	}
+	return v
+}
+
 // Lookup represents a struct object lookup method in the cache.
 type Lookup struct {
 	// Name is a period ('.') separated string
@@ -23,26 +47,23 @@ type Lookup struct {
 	// Multi allows specifying a key capable of storing
 	// multiple results. Note this only supports invalidate.
 	Multi bool
-
-	// TODO: support toggling case sensitive lookups.
-	// CaseSensitive bool
 }
 
 // Cache provides a means of caching value structures, along with
 // the results of attempting to load them. An example usecase of this
 // cache would be in wrapping a database, allowing caching of sql.ErrNoRows.
-type Cache[Value any] struct {
-	cache   ttl.Cache[int64, result[Value]] // underlying result cache
-	invalid func(Value)                     // store unwrapped invalidate callback.
-	lookups structKeys                      // pre-determined struct lookups
-	ignore  func(error) bool                // determines cacheable errors
-	copy    func(Value) Value               // copies a Value type
-	next    int64                           // update key counter
+type Cache[T any] struct {
+	cache   simple.Cache[int64, *result] // underlying result cache
+	lookups structKeys                   // pre-determined struct lookups
+	invalid func(T)                      // store unwrapped invalidate callback.
+	ignore  func(error) bool             // determines cacheable errors
+	copy    func(T) T                    // copies a Value type
+	next    int64                        // update key counter
 }
 
 // New returns a new initialized Cache, with given lookups, underlying value copy function and provided capacity.
-func New[Value any](lookups []Lookup, copy func(Value) Value, cap int) *Cache[Value] {
-	var z Value
+func New[T any](lookups []Lookup, copy func(T) T, cap int) *Cache[T] {
+	var z T
 
 	// Determine generic type
 	t := reflect.TypeOf(z)
@@ -58,7 +79,7 @@ func New[Value any](lookups []Lookup, copy func(Value) Value, cap int) *Cache[Va
 	}
 
 	// Allocate new cache object
-	c := &Cache[Value]{copy: copy}
+	c := &Cache[T]{copy: copy}
 	c.lookups = make([]structKey, len(lookups))
 
 	for i, lookup := range lookups {
@@ -67,38 +88,20 @@ func New[Value any](lookups []Lookup, copy func(Value) Value, cap int) *Cache[Va
 	}
 
 	// Create and initialize underlying cache
-	c.cache.Init(0, cap, 0)
+	c.cache.Init(0, cap)
 	c.SetEvictionCallback(nil)
 	c.SetInvalidateCallback(nil)
 	c.IgnoreErrors(nil)
 	return c
 }
 
-// Start will start the cache background eviction routine with given sweep frequency. If already
-// running or a freq <= 0 provided, this is a no-op. This will block until eviction routine started.
-func (c *Cache[Value]) Start(freq time.Duration) bool {
-	return c.cache.Start(freq)
-}
-
-// Stop will stop cache background eviction routine. If not running this
-// is a no-op. This will block until the eviction routine has stopped.
-func (c *Cache[Value]) Stop() bool {
-	return c.cache.Stop()
-}
-
-// SetTTL sets the cache item TTL. Update can be specified to force updates of existing items
-// in the cache, this will simply add the change in TTL to their current expiry time.
-func (c *Cache[Value]) SetTTL(ttl time.Duration, update bool) {
-	c.cache.SetTTL(ttl, update)
-}
-
 // SetEvictionCallback sets the eviction callback to the provided hook.
-func (c *Cache[Value]) SetEvictionCallback(hook func(Value)) {
+func (c *Cache[T]) SetEvictionCallback(hook func(T)) {
 	if hook == nil {
 		// Ensure non-nil hook.
-		hook = func(Value) {}
+		hook = func(T) {}
 	}
-	c.cache.SetEvictionCallback(func(pkey int64, res result[Value]) {
+	c.cache.SetEvictionCallback(func(pkey int64, res *result) {
 		c.cache.Lock()
 		for _, key := range res.Keys {
 			// Delete key->pkey lookup
@@ -108,23 +111,25 @@ func (c *Cache[Value]) SetEvictionCallback(hook func(Value)) {
 		c.cache.Unlock()
 
 		if res.Error != nil {
-			// Skip error hooks
+			// Skip value hooks
 			return
 		}
 
-		// Call user hook.
-		hook(res.Value)
+		// Free result and call hook.
+		v := getResultValue[T](res)
+		putResult(res)
+		hook(v)
 	})
 }
 
 // SetInvalidateCallback sets the invalidate callback to the provided hook.
-func (c *Cache[Value]) SetInvalidateCallback(hook func(Value)) {
+func (c *Cache[T]) SetInvalidateCallback(hook func(T)) {
 	if hook == nil {
 		// Ensure non-nil hook.
-		hook = func(Value) {}
+		hook = func(T) {}
 	} // store hook.
 	c.invalid = hook
-	c.cache.SetInvalidateCallback(func(pkey int64, res result[Value]) {
+	c.cache.SetInvalidateCallback(func(pkey int64, res *result) {
 		c.cache.Lock()
 		for _, key := range res.Keys {
 			// Delete key->pkey lookup
@@ -134,17 +139,19 @@ func (c *Cache[Value]) SetInvalidateCallback(hook func(Value)) {
 		c.cache.Unlock()
 
 		if res.Error != nil {
-			// Skip error hooks
+			// Skip value hooks
 			return
 		}
 
-		// Call user hook.
-		hook(res.Value)
+		// Free result and call hook.
+		v := getResultValue[T](res)
+		putResult(res)
+		hook(v)
 	})
 }
 
 // IgnoreErrors allows setting a function hook to determine which error types should / not be cached.
-func (c *Cache[Value]) IgnoreErrors(ignore func(error) bool) {
+func (c *Cache[T]) IgnoreErrors(ignore func(error) bool) {
 	if ignore == nil {
 		ignore = func(err error) bool {
 			return errors.Comparable(
@@ -160,11 +167,10 @@ func (c *Cache[Value]) IgnoreErrors(ignore func(error) bool) {
 }
 
 // Load will attempt to load an existing result from the cacche for the given lookup and key parts, else calling the provided load function and caching the result.
-func (c *Cache[Value]) Load(lookup string, load func() (Value, error), keyParts ...any) (Value, error) {
+func (c *Cache[T]) Load(lookup string, load func() (T, error), keyParts ...any) (T, error) {
 	var (
-		zero Value
-		res  result[Value]
-		ok   bool
+		zero T
+		res  *result
 	)
 
 	// Get lookup key info by name.
@@ -182,24 +188,22 @@ func (c *Cache[Value]) Load(lookup string, load func() (Value, error), keyParts
 	// Look for primary cache key
 	pkeys := keyInfo.pkeys[ckey]
 
-	if ok = (len(pkeys) > 0); ok {
-		var entry *ttl.Entry[int64, result[Value]]
-
+	if len(pkeys) > 0 {
 		// Fetch the result for primary key
-		entry, ok = c.cache.Cache.Get(pkeys[0])
+		entry, ok := c.cache.Cache.Get(pkeys[0])
 		if ok {
 			// Since the invalidation / eviction hooks acquire a mutex
 			// lock separately, and only at this point are the pkeys
 			// updated, there is a chance that a primary key may return
 			// no matching entry. Hence we have to check for it here.
-			res = entry.Value
+			res = entry.Value.(*result)
 		}
 	}
 
 	// Done with lock
 	c.cache.Unlock()
 
-	if !ok {
+	if res == nil {
 		// Generate fresh result.
 		value, err := load()
 
@@ -209,6 +213,9 @@ func (c *Cache[Value]) Load(lookup string, load func() (Value, error), keyParts
 				return zero, err
 			}
 
+			// Alloc result.
+			res = getResult()
+
 			// Store error result.
 			res.Error = err
 
@@ -219,6 +226,9 @@ func (c *Cache[Value]) Load(lookup string, load func() (Value, error), keyParts
 				key:  ckey,
 			}}
 		} else {
+			// Alloc result.
+			res = getResult()
+
 			// Store value result.
 			res.Value = value
 
@@ -251,22 +261,21 @@ func (c *Cache[Value]) Load(lookup string, load func() (Value, error), keyParts
 	}
 
 	// Return a copy of value from cache
-	return c.copy(res.Value), nil
+	return c.copy(getResultValue[T](res)), nil
 }
 
 // Store will call the given store function, and on success store the value in the cache as a positive result.
-func (c *Cache[Value]) Store(value Value, store func() error) error {
+func (c *Cache[T]) Store(value T, store func() error) error {
 	// Attempt to store this value.
 	if err := store(); err != nil {
 		return err
 	}
 
 	// Prepare cached result.
-	result := result[Value]{
-		Keys:  c.lookups.generate(value),
-		Value: c.copy(value),
-		Error: nil,
-	}
+	result := getResult()
+	result.Keys = c.lookups.generate(value)
+	result.Value = c.copy(value)
+	result.Error = nil
 
 	var evict func()
 
@@ -293,9 +302,8 @@ func (c *Cache[Value]) Store(value Value, store func() error) error {
 }
 
 // Has checks the cache for a positive result under the given lookup and key parts.
-func (c *Cache[Value]) Has(lookup string, keyParts ...any) bool {
-	var res result[Value]
-	var ok bool
+func (c *Cache[T]) Has(lookup string, keyParts ...any) bool {
+	var res *result
 
 	// Get lookup key info by name.
 	keyInfo := c.lookups.get(lookup)
@@ -312,29 +320,27 @@ func (c *Cache[Value]) Has(lookup string, keyParts ...any) bool {
 	// Look for primary key for cache key
 	pkeys := keyInfo.pkeys[ckey]
 
-	if ok = (len(pkeys) > 0); ok {
-		var entry *ttl.Entry[int64, result[Value]]
-
+	if len(pkeys) > 0 {
 		// Fetch the result for primary key
-		entry, ok = c.cache.Cache.Get(pkeys[0])
+		entry, ok := c.cache.Cache.Get(pkeys[0])
 		if ok {
 			// Since the invalidation / eviction hooks acquire a mutex
 			// lock separately, and only at this point are the pkeys
 			// updated, there is a chance that a primary key may return
 			// no matching entry. Hence we have to check for it here.
-			res = entry.Value
+			res = entry.Value.(*result)
 		}
 	}
 
 	// Done with lock
 	c.cache.Unlock()
 
-	// Check for non-error result.
-	return ok && (res.Error == nil)
+	// Check for result AND non-error result.
+	return (res != nil && res.Error == nil)
 }
 
 // Invalidate will invalidate any result from the cache found under given lookup and key parts.
-func (c *Cache[Value]) Invalidate(lookup string, keyParts ...any) {
+func (c *Cache[T]) Invalidate(lookup string, keyParts ...any) {
 	// Get lookup key info by name.
 	keyInfo := c.lookups.get(lookup)
 
@@ -351,15 +357,20 @@ func (c *Cache[Value]) Invalidate(lookup string, keyParts ...any) {
 	c.cache.InvalidateAll(pkeys...)
 }
 
-// Clear empties the cache, calling the invalidate callback.
-func (c *Cache[Value]) Clear() { c.cache.Clear() }
+// Clear empties the cache, calling the invalidate callback where necessary.
+func (c *Cache[T]) Clear() { c.Trim(100) }
+
+// Trim ensures the cache stays within percentage of total capacity, truncating where necessary.
+func (c *Cache[T]) Trim(perc float64) { c.cache.Trim(perc) }
 
 // store will cache this result under all of its required cache keys.
-func (c *Cache[Value]) store(res result[Value]) (evict func()) {
+func (c *Cache[T]) store(res *result) (evict func()) {
+	var toEvict []*result
+
 	// Get primary key
-	pnext := c.next
+	res.PKey = c.next
 	c.next++
-	if pnext > c.next {
+	if res.PKey > c.next {
 		panic("cache primary key overflow")
 	}
 
@@ -371,15 +382,19 @@ func (c *Cache[Value]) store(res result[Value]) (evict func()) {
 			for _, conflict := range pkeys {
 				// Get the overlapping result with this key.
 				entry, _ := c.cache.Cache.Get(conflict)
+				confRes := entry.Value.(*result)
 
 				// From conflicting entry, drop this key, this
 				// will prevent eviction cleanup key confusion.
-				entry.Value.Keys.drop(key.info.name)
+				confRes.Keys.drop(key.info.name)
 
-				if len(entry.Value.Keys) == 0 {
+				if len(res.Keys) == 0 {
 					// We just over-wrote the only lookup key for
 					// this value, so we drop its primary key too.
 					c.cache.Cache.Delete(conflict)
+
+					// Add finished result to evict queue.
+					toEvict = append(toEvict, confRes)
 				}
 			}
 
@@ -388,42 +403,27 @@ func (c *Cache[Value]) store(res result[Value]) (evict func()) {
 		}
 
 		// Store primary key lookup.
-		pkeys = append(pkeys, pnext)
+		pkeys = append(pkeys, res.PKey)
 		key.info.pkeys[key.key] = pkeys
 	}
 
-	// Store main entry under primary key, using evict hook if needed
-	c.cache.Cache.SetWithHook(pnext, &ttl.Entry[int64, result[Value]]{
-		Expiry: c.expiry(),
-		Key:    pnext,
-		Value:  res,
-	}, func(_ int64, item *ttl.Entry[int64, result[Value]]) {
-		evict = func() { c.cache.Evict(item.Key, item.Value) }
+	// Store main entry under primary key, catch evicted.
+	c.cache.Cache.SetWithHook(res.PKey, &simple.Entry{
+		Key:   res.PKey,
+		Value: res,
+	}, func(_ int64, item *simple.Entry) {
+		toEvict = append(toEvict, item.Value.(*result))
 	})
 
-	return evict
-}
-
-//go:linkname runtime_nanotime runtime.nanotime
-func runtime_nanotime() uint64
-
-// expiry returns an the next expiry time to use for an entry,
-// which is equivalent to time.Now().Add(ttl), or zero if disabled.
-func (c *Cache[Value]) expiry() uint64 {
-	if ttl := c.cache.TTL; ttl > 0 {
-		return runtime_nanotime() +
-			uint64(c.cache.TTL)
+	if len(toEvict) == 0 {
+		// none evicted.
+		return nil
 	}
-	return 0
-}
-
-type result[Value any] struct {
-	// keys accessible under
-	Keys cacheKeys
-
-	// cached value
-	Value Value
 
-	// cached error
-	Error error
+	return func() {
+		for _, res := range toEvict {
+			// Call evict hook on each entry.
+			c.cache.Evict(res.PKey, res)
+		}
+	}
 }