1 files changed, 203 insertions, 98 deletions

diff --git a/internal/timeline/index.go b/internal/timeline/index.go
index 3ab8dbeb9..a45617134 100644
--- a/internal/timeline/index.go
+++ b/internal/timeline/index.go
@@ -24,103 +24,205 @@ import (
 	"fmt"
 
 	"codeberg.org/gruf/go-kv"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
-func (t *timeline) ItemIndexLength(ctx context.Context) int {
-	if t.indexedItems == nil || t.indexedItems.data == nil {
-		return 0
+func (t *timeline) indexXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error {
+	l := log.
+		WithContext(ctx).
+		WithFields(kv.Fields{
+			{"amount", amount},
+			{"behindID", behindID},
+			{"beforeID", beforeID},
+			{"frontToBack", frontToBack},
+		}...)
+	l.Trace("entering indexXBetweenIDs")
+
+	if beforeID >= behindID {
+		// This is an impossible situation, we
+		// can't index anything between these.
+		return nil
 	}
-	return t.indexedItems.data.Len()
-}
 
-func (t *timeline) indexBehind(ctx context.Context, itemID string, amount int) error {
-	l := log.WithContext(ctx).
-		WithFields(kv.Fields{{"amount", amount}}...)
+	t.Lock()
+	defer t.Unlock()
 
-	// lazily initialize index if it hasn't been done already
-	if t.indexedItems.data == nil {
-		t.indexedItems.data = &list.List{}
-		t.indexedItems.data.Init()
+	// Lazily init indexed items.
+	if t.items.data == nil {
+		t.items.data = &list.List{}
+		t.items.data.Init()
 	}
 
-	// If we're already indexedBehind given itemID by the required amount, we can return nil.
-	// First find position of itemID (or as near as possible).
-	var position int
-positionLoop:
-	for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*indexedItemsEntry)
-		if !ok {
-			return errors.New("indexBehind: could not parse e as an itemIndexEntry")
+	// Start by mapping out the list so we know what
+	// we have to do. Depending on the current state
+	// of the list we might not have to do *anything*.
+	var (
+		position         int
+		listLen          = t.items.data.Len()
+		behindIDPosition int
+		beforeIDPosition int
+	)
+
+	for e := t.items.data.Front(); e != nil; e = e.Next() {
+		entry := e.Value.(*indexedItemsEntry) //nolint:forcetypeassert
+
+		position++
+
+		if entry.itemID > behindID {
+			l.Trace("item is too new, continuing")
+			continue
 		}
 
-		if entry.itemID <= itemID {
-			// we've found it
-			break positionLoop
+		if behindIDPosition == 0 {
+			// Gone far enough through the list
+			// and found our behindID mark.
+			// We only need to set this once.
+			l.Tracef("found behindID mark %s at position %d", entry.itemID, position)
+			behindIDPosition = position
+		}
+
+		if entry.itemID >= beforeID {
+			// Push the beforeID mark back
+			// one place every iteration.
+			l.Tracef("setting beforeID mark %s at position %d", entry.itemID, position)
+			beforeIDPosition = position
+		}
+
+		if entry.itemID <= beforeID {
+			// We've gone beyond the bounds of
+			// items we're interested in; stop.
+			l.Trace("reached older items, breaking")
+			break
 		}
-		position++
 	}
 
-	// now check if the length of indexed items exceeds the amount of items required (position of itemID, plus amount of posts requested after that)
-	if t.indexedItems.data.Len() > position+amount {
-		// we have enough indexed behind already to satisfy amount, so don't need to make db calls
-		l.Trace("returning nil since we already have enough items indexed")
-		return nil
+	// We can now figure out if we need to make db calls.
+	var grabMore bool
+	switch {
+	case listLen < amount:
+		// The whole list is shorter than the
+		// amount we're being asked to return,
+		// make up the difference.
+		grabMore = true
+		amount -= listLen
+	case beforeIDPosition-behindIDPosition < amount:
+		// Not enough items between behindID and
+		// beforeID to return amount required,
+		// try to get more.
+		grabMore = true
 	}
 
-	toIndex := []Timelineable{}
-	offsetID := itemID
+	if !grabMore {
+		// We're good!
+		return nil
+	}
 
-	l.Trace("entering grabloop")
-grabloop:
-	for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
-		// first grab items using the caller-provided grab function
-		l.Trace("grabbing...")
-		items, stop, err := t.grabFunction(ctx, t.accountID, offsetID, "", "", amount)
-		if err != nil {
-			return err
-		}
-		if stop {
-			break grabloop
-		}
+	// Fetch additional items.
+	items, err := t.grab(ctx, amount, behindID, beforeID, frontToBack)
+	if err != nil {
+		return err
+	}
 
-		l.Trace("filtering...")
-		// now filter each item using the caller-provided filter function
-		for _, item := range items {
-			shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
-			if err != nil {
-				return err
-			}
-			if shouldIndex {
-				toIndex = append(toIndex, item)
-			}
-			offsetID = item.GetID()
+	// Index all the items we got. We already have
+	// a lock on the timeline, so don't call IndexOne
+	// here, since that will also try to get a lock!
+	for _, item := range items {
+		entry := &indexedItemsEntry{
+			itemID:           item.GetID(),
+			boostOfID:        item.GetBoostOfID(),
+			accountID:        item.GetAccountID(),
+			boostOfAccountID: item.GetBoostOfAccountID(),
 		}
-	}
-	l.Trace("left grabloop")
 
-	// index the items we got
-	for _, s := range toIndex {
-		if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
-			return fmt.Errorf("indexBehind: error indexing item with id %s: %s", s.GetID(), err)
+		if _, err := t.items.insertIndexed(ctx, entry); err != nil {
+			return fmt.Errorf("error inserting entry with itemID %s into index: %w", entry.itemID, err)
 		}
 	}
 
 	return nil
 }
 
-func (t *timeline) IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
-	t.Lock()
-	defer t.Unlock()
+// grab wraps the timeline's grabFunction in paging + filtering logic.
+func (t *timeline) grab(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Timelineable, error) {
+	var (
+		sinceID  string
+		minID    string
+		grabbed  int
+		maxID    = behindID
+		filtered = make([]Timelineable, 0, amount)
+	)
 
-	postIndexEntry := &indexedItemsEntry{
-		itemID:           itemID,
-		boostOfID:        boostOfID,
-		accountID:        accountID,
-		boostOfAccountID: boostOfAccountID,
+	if frontToBack {
+		sinceID = beforeID
+	} else {
+		minID = beforeID
 	}
 
-	return t.indexedItems.insertIndexed(ctx, postIndexEntry)
+	for attempts := 0; attempts < 5; attempts++ {
+		if grabbed >= amount {
+			// We got everything we needed.
+			break
+		}
+
+		items, stop, err := t.grabFunction(
+			ctx,
+			t.accountID,
+			maxID,
+			sinceID,
+			minID,
+			// Don't grab more than we need to.
+			amount-grabbed,
+		)
+
+		if err != nil {
+			// Grab function already checks for
+			// db.ErrNoEntries, so if an error
+			// is returned then it's a real one.
+			return nil, err
+		}
+
+		if stop || len(items) == 0 {
+			// No items left.
+			break
+		}
+
+		// Set next query parameters.
+		if frontToBack {
+			// Page down.
+			maxID = items[len(items)-1].GetID()
+			if maxID <= beforeID {
+				// Can't go any further.
+				break
+			}
+		} else {
+			// Page up.
+			minID = items[0].GetID()
+			if minID >= behindID {
+				// Can't go any further.
+				break
+			}
+		}
+
+		for _, item := range items {
+			ok, err := t.filterFunction(ctx, t.accountID, item)
+			if err != nil {
+				if !errors.Is(err, db.ErrNoEntries) {
+					// Real error here.
+					return nil, err
+				}
+				log.Warnf(ctx, "errNoEntries while filtering item %s: %s", item.GetID(), err)
+				continue
+			}
+
+			if ok {
+				filtered = append(filtered, item)
+				grabbed++ // count this as grabbed
+			}
+		}
+	}
+
+	return filtered, nil
 }
 
 func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
@@ -134,46 +236,49 @@ func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boos
 		boostOfAccountID: boostOfAccountID,
 	}
 
-	inserted, err := t.indexedItems.insertIndexed(ctx, postIndexEntry)
-	if err != nil {
-		return inserted, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
+	if inserted, err := t.items.insertIndexed(ctx, postIndexEntry); err != nil {
+		return false, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %w", err)
+	} else if !inserted {
+		// Entry wasn't inserted, so
+		// don't bother preparing it.
+		return false, nil
 	}
 
-	if inserted {
-		if err := t.prepare(ctx, statusID); err != nil {
-			return inserted, fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
-		}
+	preparable, err := t.prepareFunction(ctx, t.accountID, statusID)
+	if err != nil {
+		return true, fmt.Errorf("IndexAndPrepareOne: error preparing: %w", err)
 	}
+	postIndexEntry.prepared = preparable
 
-	return inserted, nil
+	return true, nil
 }
 
-func (t *timeline) OldestIndexedItemID(ctx context.Context) (string, error) {
-	var id string
-	if t.indexedItems == nil || t.indexedItems.data == nil || t.indexedItems.data.Back() == nil {
-		// return an empty string if postindex hasn't been initialized yet
-		return id, nil
-	}
+func (t *timeline) Len() int {
+	t.Lock()
+	defer t.Unlock()
 
-	e := t.indexedItems.data.Back()
-	entry, ok := e.Value.(*indexedItemsEntry)
-	if !ok {
-		return id, errors.New("OldestIndexedItemID: could not parse e as itemIndexEntry")
+	if t.items == nil || t.items.data == nil {
+		// indexedItems hasnt been initialized yet.
+		return 0
 	}
-	return entry.itemID, nil
+
+	return t.items.data.Len()
 }
 
-func (t *timeline) NewestIndexedItemID(ctx context.Context) (string, error) {
-	var id string
-	if t.indexedItems == nil || t.indexedItems.data == nil || t.indexedItems.data.Front() == nil {
-		// return an empty string if postindex hasn't been initialized yet
-		return id, nil
+func (t *timeline) OldestIndexedItemID() string {
+	t.Lock()
+	defer t.Unlock()
+
+	if t.items == nil || t.items.data == nil {
+		// indexedItems hasnt been initialized yet.
+		return ""
 	}
 
-	e := t.indexedItems.data.Front()
-	entry, ok := e.Value.(*indexedItemsEntry)
-	if !ok {
-		return id, errors.New("NewestIndexedItemID: could not parse e as itemIndexEntry")
+	e := t.items.data.Back()
+	if e == nil {
+		// List was empty.
+		return ""
 	}
-	return entry.itemID, nil
+
+	return e.Value.(*indexedItemsEntry).itemID //nolint:forcetypeassert
 }