summaryrefslogtreecommitdiff
path: root/internal/timeline/get.go
diff options
context:
space:
mode:
Diffstat (limited to 'internal/timeline/get.go')
-rw-r--r--internal/timeline/get.go573
1 files changed, 301 insertions, 272 deletions
diff --git a/internal/timeline/get.go b/internal/timeline/get.go
index f6f885d92..4ca9023f2 100644
--- a/internal/timeline/get.go
+++ b/internal/timeline/get.go
@@ -25,11 +25,11 @@ import (
"time"
"codeberg.org/gruf/go-kv"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
-const retries = 5
-
func (t *timeline) LastGot() time.Time {
t.Lock()
defer t.Unlock()
@@ -47,339 +47,368 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
}...)
l.Trace("entering get and updating t.lastGot")
- // regardless of what happens below, update the
- // last time Get was called for this timeline
+ // Regardless of what happens below, update the
+ // last time Get was called for this timeline.
t.Lock()
t.lastGot = time.Now()
t.Unlock()
- var items []Preparable
- var err error
-
- // no params are defined to just fetch from the top
- // this is equivalent to a user asking for the top x items from their timeline
- if maxID == "" && sinceID == "" && minID == "" {
- items, err = t.getXFromTop(ctx, amount)
- // aysnchronously prepare the next predicted query so it's ready when the user asks for it
- if len(items) != 0 {
+ var (
+ items []Preparable
+ err error
+ )
+
+ switch {
+ case maxID == "" && sinceID == "" && minID == "":
+ // No params are defined so just fetch from the top.
+ // This is equivalent to a user starting to view
+ // their timeline from newest -> older posts.
+ items, err = t.getXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true)
+
+ // Cache expected next query to speed up scrolling.
+ // Assume the user will be scrolling downwards from
+ // the final ID in items.
+ if prepareNext && err == nil && len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
- if prepareNext {
- // already cache the next query to speed up scrolling
- go func() {
- // use context.Background() because we don't want the query to abort when the request finishes
- if err := t.prepareNextQuery(context.Background(), amount, nextMaxID, "", ""); err != nil {
- l.Errorf("error preparing next query: %s", err)
- }
- }()
- }
+ t.prepareNextQuery(amount, nextMaxID, "", "")
}
- }
- // maxID is defined but sinceID isn't so take from behind
- // this is equivalent to a user asking for the next x items from their timeline, starting from maxID
- if maxID != "" && sinceID == "" {
- attempts := 0
- items, err = t.getXBehindID(ctx, amount, maxID, &attempts)
- // aysnchronously prepare the next predicted query so it's ready when the user asks for it
- if len(items) != 0 {
+ case maxID != "" && sinceID == "" && minID == "":
+ // Only maxID is defined, so fetch from maxID onwards.
+ // This is equivalent to a user paging further down
+ // their timeline from newer -> older posts.
+ items, err = t.getXBetweenIDs(ctx, amount, maxID, id.Lowest, true)
+
+ // Cache expected next query to speed up scrolling.
+ // Assume the user will be scrolling downwards from
+ // the final ID in items.
+ if prepareNext && err == nil && len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
- if prepareNext {
- // already cache the next query to speed up scrolling
- go func() {
- // use context.Background() because we don't want the query to abort when the request finishes
- if err := t.prepareNextQuery(context.Background(), amount, nextMaxID, "", ""); err != nil {
- l.Errorf("error preparing next query: %s", err)
- }
- }()
- }
+ t.prepareNextQuery(amount, nextMaxID, "", "")
}
- }
-
- // maxID is defined and sinceID || minID are as well, so take a slice between them
- // this is equivalent to a user asking for items older than x but newer than y
- if maxID != "" && sinceID != "" {
- items, err = t.getXBetweenID(ctx, amount, maxID, minID)
- }
- if maxID != "" && minID != "" {
- items, err = t.getXBetweenID(ctx, amount, maxID, minID)
- }
-
- // maxID isn't defined, but sinceID || minID are, so take x before
- // this is equivalent to a user asking for items newer than x (eg., refreshing the top of their timeline)
- if maxID == "" && sinceID != "" {
- items, err = t.getXBeforeID(ctx, amount, sinceID, true)
- }
- if maxID == "" && minID != "" {
- items, err = t.getXBeforeID(ctx, amount, minID, true)
- }
-
- return items, err
-}
-// getXFromTop returns x amount of items from the top of the timeline, from newest to oldest.
-func (t *timeline) getXFromTop(ctx context.Context, amount int) ([]Preparable, error) {
- // make a slice of preparedItems with the length we need to return
- preparedItems := make([]Preparable, 0, amount)
+ // In the next cases, maxID is defined, and so are
+ // either sinceID or minID. This is equivalent to
+ // a user opening an in-progress timeline and asking
+ // for a slice of posts somewhere in the middle, or
+ // trying to "fill in the blanks" between two points,
+ // paging either up or down.
+ case maxID != "" && sinceID != "":
+ items, err = t.getXBetweenIDs(ctx, amount, maxID, sinceID, true)
+
+ // Cache expected next query to speed up scrolling.
+ // We can assume the caller is scrolling downwards.
+ // Guess id.Lowest as sinceID, since we don't actually
+ // know what the next sinceID would be.
+ if prepareNext && err == nil && len(items) != 0 {
+ nextMaxID := items[len(items)-1].GetID()
+ t.prepareNextQuery(amount, nextMaxID, id.Lowest, "")
+ }
- if t.preparedItems.data == nil {
- t.preparedItems.data = &list.List{}
- }
+ case maxID != "" && minID != "":
+ items, err = t.getXBetweenIDs(ctx, amount, maxID, minID, false)
- // make sure we have enough items prepared to return
- if t.preparedItems.data.Len() < amount {
- if err := t.PrepareFromTop(ctx, amount); err != nil {
- return nil, err
+ // Cache expected next query to speed up scrolling.
+ // We can assume the caller is scrolling upwards.
+ // Guess id.Highest as maxID, since we don't actually
+ // know what the next maxID would be.
+ if prepareNext && err == nil && len(items) != 0 {
+ prevMinID := items[0].GetID()
+ t.prepareNextQuery(amount, id.Highest, "", prevMinID)
}
- }
- // work through the prepared items from the top and return
- var served int
- for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXFromTop: could not parse e as a preparedItemsEntry")
- }
- preparedItems = append(preparedItems, entry.prepared)
- served++
- if served >= amount {
- break
+ // In the final cases, maxID is not defined, but
+ // either sinceID or minID are. This is equivalent to
+ // a user either "pulling up" at the top of their timeline
+ // to refresh it and check if newer posts have come in, or
+ // trying to scroll upwards from an old post to see what
+ // they missed since then.
+ //
+ // In these calls, we use the highest possible ulid as
+ // behindID because we don't have a cap for newest that
+ // we're interested in.
+ case maxID == "" && sinceID != "":
+ items, err = t.getXBetweenIDs(ctx, amount, id.Highest, sinceID, true)
+
+ // We can't cache an expected next query for this one,
+ // since presumably the caller is at the top of their
+ // timeline already.
+
+ case maxID == "" && minID != "":
+ items, err = t.getXBetweenIDs(ctx, amount, id.Highest, minID, false)
+
+ // Cache expected next query to speed up scrolling.
+ // We can assume the caller is scrolling upwards.
+ // Guess id.Highest as maxID, since we don't actually
+ // know what the next maxID would be.
+ if prepareNext && err == nil && len(items) != 0 {
+ prevMinID := items[0].GetID()
+ t.prepareNextQuery(amount, id.Highest, "", prevMinID)
}
+
+ default:
+ err = errors.New("Get: switch statement exhausted with no results")
}
- return preparedItems, nil
+ return items, err
}
-// getXBehindID returns x amount of items from the given id onwards, from newest to oldest.
-// This will NOT include the item with the given ID.
+// getXBetweenIDs returns x amount of items somewhere between (not including) the given IDs.
//
-// This corresponds to an api call to /timelines/home?max_id=WHATEVER
-func (t *timeline) getXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]Preparable, error) {
- l := log.WithContext(ctx).
+// If frontToBack is true, items will be served paging down from behindID.
+// This corresponds to an api call to /timelines/home?max_id=WHATEVER&since_id=WHATEVER
+//
+// If frontToBack is false, items will be served paging up from beforeID.
+// This corresponds to an api call to /timelines/home?max_id=WHATEVER&min_id=WHATEVER
+func (t *timeline) getXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Preparable, error) {
+ l := log.
+ WithContext(ctx).
WithFields(kv.Fields{
{"amount", amount},
{"behindID", behindID},
- {"attempts", attempts},
+ {"beforeID", beforeID},
+ {"frontToBack", frontToBack},
}...)
+ l.Trace("entering getXBetweenID")
- newAttempts := *attempts
- newAttempts++
- attempts = &newAttempts
-
- // make a slice of items with the length we need to return
+ // Assume length we need to return.
items := make([]Preparable, 0, amount)
- if t.preparedItems.data == nil {
- t.preparedItems.data = &list.List{}
- }
-
- // iterate through the modified list until we hit the mark we're looking for
- var position int
- var behindIDMark *list.Element
-
-findMarkLoop:
- for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
- position++
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBehindID: could not parse e as a preparedPostsEntry")
- }
-
- if entry.itemID <= behindID {
- l.Trace("found behindID mark")
- behindIDMark = e
- break findMarkLoop
- }
- }
-
- // we didn't find it, so we need to make sure it's indexed and prepared and then try again
- // this can happen when a user asks for really old items
- if behindIDMark == nil {
- if err := t.prepareBehind(ctx, behindID, amount); err != nil {
- return nil, fmt.Errorf("getXBehindID: error preparing behind and including ID %s", behindID)
- }
- oldestID, err := t.oldestPreparedItemID(ctx)
- if err != nil {
- return nil, err
- }
- if oldestID == "" {
- l.Tracef("oldestID is empty so we can't return behindID %s", behindID)
- return items, nil
- }
- if oldestID == behindID {
- l.Tracef("given behindID %s is the same as oldestID %s so there's nothing to return behind it", behindID, oldestID)
- return items, nil
- }
- if *attempts > retries {
- l.Tracef("exceeded retries looking for behindID %s", behindID)
- return items, nil
- }
- l.Trace("trying getXBehindID again")
- return t.getXBehindID(ctx, amount, behindID, attempts)
- }
-
- // make sure we have enough items prepared behind it to return what we're being asked for
- if t.preparedItems.data.Len() < amount+position {
- if err := t.prepareBehind(ctx, behindID, amount); err != nil {
- return nil, err
- }
- }
-
- // start serving from the entry right after the mark
- var served int
-serveloop:
- for e := behindIDMark.Next(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBehindID: could not parse e as a preparedPostsEntry")
- }
-
- // serve up to the amount requested
- items = append(items, entry.prepared)
- served++
- if served >= amount {
- break serveloop
- }
+ if beforeID >= behindID {
+ // This is an impossible situation, we
+ // can't serve anything between these.
+ return items, nil
}
- return items, nil
-}
-
-// getXBeforeID returns x amount of items up to the given id, from newest to oldest.
-// This will NOT include the item with the given ID.
-//
-// This corresponds to an api call to /timelines/home?since_id=WHATEVER
-func (t *timeline) getXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]Preparable, error) {
- // make a slice of items with the length we need to return
- items := make([]Preparable, 0, amount)
-
- if t.preparedItems.data == nil {
- t.preparedItems.data = &list.List{}
+ // Try to ensure we have enough items prepared.
+ if err := t.prepareXBetweenIDs(ctx, amount, behindID, beforeID, frontToBack); err != nil {
+ // An error here doesn't necessarily mean we
+ // can't serve anything, so log + keep going.
+ l.Debugf("error calling prepareXBetweenIDs: %s", err)
}
- // iterate through the modified list until we hit the mark we're looking for, or as close as possible to it
- var beforeIDMark *list.Element
-findMarkLoop:
- for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
+ var (
+ beforeIDMark *list.Element
+ served int
+ // Our behavior while ranging through the
+ // list changes depending on if we're
+ // going front-to-back or back-to-front.
+ //
+ // To avoid checking which one we're doing
+ // in each loop iteration, define our range
+ // function here outside the loop.
+ //
+ // The bool indicates to the caller whether
+ // iteration should continue (true) or stop
+ // (false).
+ rangeF func(e *list.Element) (bool, error)
+ // If we get certain errors on entries as we're
+ // looking through, we might want to cheekily
+ // remove their elements from the timeline.
+ // Everything added to this slice will be removed.
+ removeElements = []*list.Element{}
+ )
+
+ defer func() {
+ for _, e := range removeElements {
+ t.items.data.Remove(e)
}
-
- if entry.itemID >= beforeID {
- beforeIDMark = e
- } else {
- break findMarkLoop
- }
- }
-
- if beforeIDMark == nil {
- return items, nil
- }
-
- var served int
-
- if startFromTop {
- // start serving from the front/top and keep going until we hit mark or get x amount items
- serveloopFromTop:
- for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
+ }()
+
+ if frontToBack {
+ // We're going front-to-back, which means we
+ // don't need to look for a mark per se, we
+ // just keep serving items until we've reached
+ // a point where the items are out of the range
+ // we're interested in.
+ rangeF = func(e *list.Element) (bool, error) {
+ entry := e.Value.(*indexedItemsEntry) //nolint:forcetypeassert
+
+ if entry.itemID >= behindID {
+ // ID of this item is too high,
+ // just keep iterating.
+ l.Trace("item is too new, continuing")
+ return true, nil
}
- if entry.itemID == beforeID {
- break serveloopFromTop
+ if entry.itemID <= beforeID {
+ // We've gone as far as we can through
+ // the list and reached entries that are
+ // now too old for us, stop here.
+ l.Trace("reached older items, breaking")
+ return false, nil
}
- // serve up to the amount requested
- items = append(items, entry.prepared)
- served++
- if served >= amount {
- break serveloopFromTop
- }
- }
- } else if !startFromTop {
- // start serving from the entry right before the mark
- serveloopFromBottom:
- for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBeforeID: could not parse e as a preparedPostsEntry")
+ l.Trace("entry is just right")
+
+ if entry.prepared == nil {
+ // Whoops, this entry isn't prepared yet; some
+ // race condition? That's OK, we can do it now.
+ prepared, err := t.prepareFunction(ctx, t.accountID, entry.itemID)
+ if err != nil {
+ if errors.Is(err, db.ErrNoEntries) {
+ // ErrNoEntries means something has been deleted,
+ // so we'll likely not be able to ever prepare this.
+ // This means we can remove it and skip past it.
+ l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID)
+ removeElements = append(removeElements, e)
+ return true, nil
+ }
+ // We've got a proper db error.
+ return false, fmt.Errorf("getXBetweenIDs: db error while trying to prepare %s: %w", entry.itemID, err)
+ }
+ entry.prepared = prepared
}
- // serve up to the amount requested
items = append(items, entry.prepared)
+
served++
- if served >= amount {
- break serveloopFromBottom
- }
+ return served < amount, nil
}
- }
-
- return items, nil
-}
+ } else {
+ // Iterate through the list from the top, until
+ // we reach an item with id smaller than beforeID;
+ // ie., an item OLDER than beforeID. At that point,
+ // we can stop looking because we're not interested
+ // in older entries.
+ rangeF = func(e *list.Element) (bool, error) {
+ // Move the mark back one place each loop.
+ beforeIDMark = e
-// getXBetweenID returns x amount of items from the given maxID, up to the given id, from newest to oldest.
-// This will NOT include the item with the given IDs.
-//
-// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
-func (t *timeline) getXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]Preparable, error) {
- // make a slice of items with the length we need to return
- items := make([]Preparable, 0, amount)
+ //nolint:forcetypeassert
+ if entry := e.Value.(*indexedItemsEntry); entry.itemID <= beforeID {
+ // We've gone as far as we can through
+ // the list and reached entries that are
+ // now too old for us, stop here.
+ l.Trace("reached older items, breaking")
+ return false, nil
+ }
- if t.preparedItems.data == nil {
- t.preparedItems.data = &list.List{}
+ return true, nil
+ }
}
- // iterate through the modified list until we hit the mark we're looking for
- var position int
- var behindIDMark *list.Element
-findMarkLoop:
- for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
- position++
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBetweenID: could not parse e as a preparedPostsEntry")
+ // Iterate through the list until the function
+ // we defined above instructs us to stop.
+ for e := t.items.data.Front(); e != nil; e = e.Next() {
+ keepGoing, err := rangeF(e)
+ if err != nil {
+ return nil, err
}
- if entry.itemID == behindID {
- behindIDMark = e
- break findMarkLoop
+ if !keepGoing {
+ break
}
}
- // we didn't find it
- if behindIDMark == nil {
- return nil, fmt.Errorf("getXBetweenID: couldn't find item with ID %s", behindID)
+ if frontToBack || beforeIDMark == nil {
+ // If we're serving front to back, then
+ // items should be populated by now. If
+ // we're serving back to front but didn't
+ // find any items newer than beforeID,
+ // we can just return empty items.
+ return items, nil
}
- // make sure we have enough items prepared behind it to return what we're being asked for
- if t.preparedItems.data.Len() < amount+position {
- if err := t.prepareBehind(ctx, behindID, amount); err != nil {
- return nil, err
+ // We're serving back to front, so iterate upwards
+ // towards the front of the list from the mark we found,
+ // until we either get to the front, serve enough
+ // items, or reach behindID.
+ //
+ // To preserve ordering, we need to reverse the slice
+ // when we're finished.
+ for e := beforeIDMark; e != nil; e = e.Prev() {
+ entry := e.Value.(*indexedItemsEntry) //nolint:forcetypeassert
+
+ if entry.itemID == beforeID {
+ // Don't include the beforeID
+ // entry itself, just continue.
+ l.Trace("entry item ID is equal to beforeID, skipping")
+ continue
}
- }
- // start serving from the entry right after the mark
- var served int
-serveloop:
- for e := behindIDMark.Next(); e != nil; e = e.Next() {
- entry, ok := e.Value.(*preparedItemsEntry)
- if !ok {
- return nil, errors.New("getXBetweenID: could not parse e as a preparedPostsEntry")
+ if entry.itemID >= behindID {
+ // We've reached items that are
+ // newer than what we're looking
+ // for, just stop here.
+ l.Trace("reached newer items, breaking")
+ break
}
- if entry.itemID == beforeID {
- break serveloop
+ if entry.prepared == nil {
+ // Whoops, this entry isn't prepared yet; some
+ // race condition? That's OK, we can do it now.
+ prepared, err := t.prepareFunction(ctx, t.accountID, entry.itemID)
+ if err != nil {
+ if errors.Is(err, db.ErrNoEntries) {
+ // ErrNoEntries means something has been deleted,
+ // so we'll likely not be able to ever prepare this.
+ // This means we can remove it and skip past it.
+ l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID)
+ removeElements = append(removeElements, e)
+ continue
+ }
+ // We've got a proper db error.
+ return nil, fmt.Errorf("getXBetweenIDs: db error while trying to prepare %s: %w", entry.itemID, err)
+ }
+ entry.prepared = prepared
}
- // serve up to the amount requested
items = append(items, entry.prepared)
+
served++
if served >= amount {
- break serveloop
+ break
}
}
+ // Reverse order of items.
+ // https://zchee.github.io/golang-wiki/SliceTricks/#reversing
+ for l, r := 0, len(items)-1; l < r; l, r = l+1, r-1 {
+ items[l], items[r] = items[r], items[l]
+ }
+
return items, nil
}
+
+func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) {
+ var (
+ // We explicitly use context.Background() rather than
+ // accepting a context param because we don't want this
+ // to stop/break when the calling context finishes.
+ ctx = context.Background()
+ err error
+ )
+
+ // Always perform this async so caller doesn't have to wait.
+ go func() {
+ switch {
+ case maxID == "" && sinceID == "" && minID == "":
+ err = t.prepareXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true)
+ case maxID != "" && sinceID == "" && minID == "":
+ err = t.prepareXBetweenIDs(ctx, amount, maxID, id.Lowest, true)
+ case maxID != "" && sinceID != "":
+ err = t.prepareXBetweenIDs(ctx, amount, maxID, sinceID, true)
+ case maxID != "" && minID != "":
+ err = t.prepareXBetweenIDs(ctx, amount, maxID, minID, false)
+ case maxID == "" && sinceID != "":
+ err = t.prepareXBetweenIDs(ctx, amount, id.Highest, sinceID, true)
+ case maxID == "" && minID != "":
+ err = t.prepareXBetweenIDs(ctx, amount, id.Highest, minID, false)
+ default:
+ err = errors.New("Get: switch statement exhausted with no results")
+ }
+
+ if err != nil {
+ log.
+ WithContext(ctx).
+ WithFields(kv.Fields{
+ {"amount", amount},
+ {"maxID", maxID},
+ {"sinceID", sinceID},
+ {"minID", minID},
+ }...).
+ Warnf("error preparing next query: %s", err)
+ }
+ }()
+}
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-25 19:12:14 +0000