[feature] Prune timelines once per hour to plug memory leak (#1117)

* export highest/lowest ULIDs as proper const

* add stop + start to timeline manager, other small fixes

* unexport unused interface funcs + tidy up

* add LastGot func

* add timeline Prune function

* test prune

* update lastGot

1 files changed, 82 insertions, 86 deletions

diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go
index c643d1873..a3b0d06ce 100644
--- a/internal/timeline/prepare.go
+++ b/internal/timeline/prepare.go
@@ -26,12 +26,66 @@ import (
 
 	"codeberg.org/gruf/go-kv"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
+func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
+	l := log.WithFields(kv.Fields{{"amount", amount}}...)
+
+	// lazily initialize prepared posts if it hasn't been done already
+	if t.preparedItems.data == nil {
+		t.preparedItems.data = &list.List{}
+		t.preparedItems.data.Init()
+	}
+
+	// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
+	if t.indexedItems.data == nil {
+		l.Debug("postindex.data was nil, indexing behind highest possible ID")
+		if err := t.indexBehind(ctx, id.Highest, amount); err != nil {
+			return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", id.Highest, err)
+		}
+	}
+
+	l.Trace("entering prepareloop")
+	t.Lock()
+	defer t.Unlock()
+	var prepared int
+prepareloop:
+	for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
+		if e == nil {
+			continue
+		}
+
+		entry, ok := e.Value.(*indexedItemsEntry)
+		if !ok {
+			return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
+		}
+
+		if err := t.prepare(ctx, entry.itemID); err != nil {
+			// there's been an error
+			if err != db.ErrNoEntries {
+				// it's a real error
+				return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.itemID, err)
+			}
+			// the status just doesn't exist (anymore) so continue to the next one
+			continue
+		}
+
+		prepared++
+		if prepared == amount {
+			// we're done
+			l.Trace("leaving prepareloop")
+			break prepareloop
+		}
+	}
+
+	l.Trace("leaving function")
+	return nil
+}
+
 func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
 	l := log.WithFields(kv.Fields{
-
 		{"amount", amount},
 		{"maxID", maxID},
 		{"sinceID", sinceID},
@@ -39,39 +93,36 @@ func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID strin
 	}...)
 
 	var err error
-
-	// maxID is defined but sinceID isn't so take from behind
-	if maxID != "" && sinceID == "" {
+	switch {
+	case maxID != "" && sinceID == "":
 		l.Debug("preparing behind maxID")
-		err = t.PrepareBehind(ctx, maxID, amount)
-	}
-
-	// maxID isn't defined, but sinceID || minID are, so take x before
-	if maxID == "" && sinceID != "" {
+		err = t.prepareBehind(ctx, maxID, amount)
+	case maxID == "" && sinceID != "":
 		l.Debug("preparing before sinceID")
-		err = t.PrepareBefore(ctx, sinceID, false, amount)
-	}
-	if maxID == "" && minID != "" {
+		err = t.prepareBefore(ctx, sinceID, false, amount)
+	case maxID == "" && minID != "":
 		l.Debug("preparing before minID")
-		err = t.PrepareBefore(ctx, minID, false, amount)
+		err = t.prepareBefore(ctx, minID, false, amount)
 	}
 
 	return err
 }
 
-func (t *timeline) PrepareBehind(ctx context.Context, itemID string, amount int) error {
+// prepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
+// If include is true, then the given item ID will also be prepared, otherwise only entries behind it will be prepared.
+func (t *timeline) prepareBehind(ctx context.Context, itemID string, amount int) error {
 	// lazily initialize prepared items if it hasn't been done already
 	if t.preparedItems.data == nil {
 		t.preparedItems.data = &list.List{}
 		t.preparedItems.data.Init()
 	}
 
-	if err := t.IndexBehind(ctx, itemID, amount); err != nil {
-		return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", itemID, err)
+	if err := t.indexBehind(ctx, itemID, amount); err != nil {
+		return fmt.Errorf("prepareBehind: error indexing behind id %s: %s", itemID, err)
 	}
 
 	// if the itemindex is nil, nothing has been indexed yet so there's nothing to prepare
-	if t.itemIndex.data == nil {
+	if t.indexedItems.data == nil {
 		return nil
 	}
 
@@ -80,10 +131,10 @@ func (t *timeline) PrepareBehind(ctx context.Context, itemID string, amount int)
 	t.Lock()
 	defer t.Unlock()
 prepareloop:
-	for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*itemIndexEntry)
+	for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*indexedItemsEntry)
 		if !ok {
-			return errors.New("PrepareBehind: could not parse e as itemIndexEntry")
+			return errors.New("prepareBehind: could not parse e as itemIndexEntry")
 		}
 
 		if !preparing {
@@ -98,7 +149,7 @@ prepareloop:
 				// there's been an error
 				if err != db.ErrNoEntries {
 					// it's a real error
-					return fmt.Errorf("PrepareBehind: error preparing item with id %s: %s", entry.itemID, err)
+					return fmt.Errorf("prepareBehind: error preparing item with id %s: %s", entry.itemID, err)
 				}
 				// the status just doesn't exist (anymore) so continue to the next one
 				continue
@@ -114,7 +165,7 @@ prepareloop:
 	return nil
 }
 
-func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include bool, amount int) error {
+func (t *timeline) prepareBefore(ctx context.Context, statusID string, include bool, amount int) error {
 	t.Lock()
 	defer t.Unlock()
 
@@ -125,17 +176,17 @@ func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include b
 	}
 
 	// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
-	if t.itemIndex.data == nil {
+	if t.indexedItems.data == nil {
 		return nil
 	}
 
 	var prepared int
 	var preparing bool
 prepareloop:
-	for e := t.itemIndex.data.Back(); e != nil; e = e.Prev() {
-		entry, ok := e.Value.(*itemIndexEntry)
+	for e := t.indexedItems.data.Back(); e != nil; e = e.Prev() {
+		entry, ok := e.Value.(*indexedItemsEntry)
 		if !ok {
-			return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
+			return errors.New("prepareBefore: could not parse e as a postIndexEntry")
 		}
 
 		if !preparing {
@@ -153,7 +204,7 @@ prepareloop:
 				// there's been an error
 				if err != db.ErrNoEntries {
 					// it's a real error
-					return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.itemID, err)
+					return fmt.Errorf("prepareBefore: error preparing status with id %s: %s", entry.itemID, err)
 				}
 				// the status just doesn't exist (anymore) so continue to the next one
 				continue
@@ -169,63 +220,6 @@ prepareloop:
 	return nil
 }
 
-func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
-	l := log.WithFields(kv.Fields{
-
-		{"amount", amount},
-	}...)
-
-	// lazily initialize prepared posts if it hasn't been done already
-	if t.preparedItems.data == nil {
-		t.preparedItems.data = &list.List{}
-		t.preparedItems.data.Init()
-	}
-
-	// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
-	if t.itemIndex.data == nil {
-		l.Debug("postindex.data was nil, indexing behind highest possible ID")
-		if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", amount); err != nil {
-			return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
-		}
-	}
-
-	l.Trace("entering prepareloop")
-	t.Lock()
-	defer t.Unlock()
-	var prepared int
-prepareloop:
-	for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
-		if e == nil {
-			continue
-		}
-
-		entry, ok := e.Value.(*itemIndexEntry)
-		if !ok {
-			return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
-		}
-
-		if err := t.prepare(ctx, entry.itemID); err != nil {
-			// there's been an error
-			if err != db.ErrNoEntries {
-				// it's a real error
-				return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.itemID, err)
-			}
-			// the status just doesn't exist (anymore) so continue to the next one
-			continue
-		}
-
-		prepared++
-		if prepared == amount {
-			// we're done
-			l.Trace("leaving prepareloop")
-			break prepareloop
-		}
-	}
-
-	l.Trace("leaving function")
-	return nil
-}
-
 func (t *timeline) prepare(ctx context.Context, itemID string) error {
 	// trigger the caller-provided prepare function
 	prepared, err := t.prepareFunction(ctx, t.accountID, itemID)
@@ -245,7 +239,9 @@ func (t *timeline) prepare(ctx context.Context, itemID string) error {
 	return t.preparedItems.insertPrepared(ctx, preparedItemsEntry)
 }
 
-func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
+// oldestPreparedItemID returns the id of the rearmost (ie., the oldest) prepared item, or an error if something goes wrong.
+// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
+func (t *timeline) oldestPreparedItemID(ctx context.Context) (string, error) {
 	var id string
 	if t.preparedItems == nil || t.preparedItems.data == nil {
 		// return an empty string if prepared items hasn't been initialized yet
@@ -260,7 +256,7 @@ func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
 
 	entry, ok := e.Value.(*preparedItemsEntry)
 	if !ok {
-		return id, errors.New("OldestPreparedItemID: could not parse e as a preparedItemsEntry")
+		return id, errors.New("oldestPreparedItemID: could not parse e as a preparedItemsEntry")
 	}
 
 	return entry.itemID, nil