summaryrefslogtreecommitdiff
path: root/internal/timeline
diff options
context:
space:
mode:
Diffstat (limited to 'internal/timeline')
-rw-r--r--internal/timeline/get.go309
-rw-r--r--internal/timeline/index.go143
-rw-r--r--internal/timeline/manager.go217
-rw-r--r--internal/timeline/postindex.go57
-rw-r--r--internal/timeline/prepare.go215
-rw-r--r--internal/timeline/preparedposts.go60
-rw-r--r--internal/timeline/remove.go50
-rw-r--r--internal/timeline/timeline.go139
8 files changed, 1190 insertions, 0 deletions
diff --git a/internal/timeline/get.go b/internal/timeline/get.go
new file mode 100644
index 000000000..867e0940b
--- /dev/null
+++ b/internal/timeline/get.go
@@ -0,0 +1,309 @@
+package timeline
+
+import (
+ "container/list"
+ "errors"
+ "fmt"
+
+ "github.com/sirupsen/logrus"
+ apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+)
+
+func (t *timeline) Get(amount int, maxID string, sinceID string, minID string) ([]*apimodel.Status, error) {
+ l := t.log.WithFields(logrus.Fields{
+ "func": "Get",
+ "accountID": t.accountID,
+ })
+
+ var statuses []*apimodel.Status
+ var err error
+
+ // no params are defined to just fetch from the top
+ if maxID == "" && sinceID == "" && minID == "" {
+ statuses, err = t.GetXFromTop(amount)
+ // aysnchronously prepare the next predicted query so it's ready when the user asks for it
+ if len(statuses) != 0 {
+ nextMaxID := statuses[len(statuses)-1].ID
+ go func() {
+ if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
+ l.Errorf("error preparing next query: %s", err)
+ }
+ }()
+ }
+ }
+
+ // maxID is defined but sinceID isn't so take from behind
+ if maxID != "" && sinceID == "" {
+ statuses, err = t.GetXBehindID(amount, maxID)
+ // aysnchronously prepare the next predicted query so it's ready when the user asks for it
+ if len(statuses) != 0 {
+ nextMaxID := statuses[len(statuses)-1].ID
+ go func() {
+ if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
+ l.Errorf("error preparing next query: %s", err)
+ }
+ }()
+ }
+ }
+
+ // maxID is defined and sinceID || minID are as well, so take a slice between them
+ if maxID != "" && sinceID != "" {
+ statuses, err = t.GetXBetweenID(amount, maxID, minID)
+ }
+ if maxID != "" && minID != "" {
+ statuses, err = t.GetXBetweenID(amount, maxID, minID)
+ }
+
+ // maxID isn't defined, but sinceID || minID are, so take x before
+ if maxID == "" && sinceID != "" {
+ statuses, err = t.GetXBeforeID(amount, sinceID, true)
+ }
+ if maxID == "" && minID != "" {
+ statuses, err = t.GetXBeforeID(amount, minID, true)
+ }
+
+ return statuses, err
+}
+
+func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
+ // make a slice of statuses with the length we need to return
+ statuses := make([]*apimodel.Status, 0, amount)
+
+ if t.preparedPosts.data == nil {
+ t.preparedPosts.data = &list.List{}
+ }
+
+ // make sure we have enough posts prepared to return
+ if t.preparedPosts.data.Len() < amount {
+ if err := t.PrepareFromTop(amount); err != nil {
+ return nil, err
+ }
+ }
+
+ // work through the prepared posts from the top and return
+ var served int
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
+ }
+ statuses = append(statuses, entry.prepared)
+ served = served + 1
+ if served >= amount {
+ break
+ }
+ }
+
+ return statuses, nil
+}
+
+func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) {
+ // make a slice of statuses with the length we need to return
+ statuses := make([]*apimodel.Status, 0, amount)
+
+ if t.preparedPosts.data == nil {
+ t.preparedPosts.data = &list.List{}
+ }
+
+ // iterate through the modified list until we hit the mark we're looking for
+ var position int
+ var behindIDMark *list.Element
+
+findMarkLoop:
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ position = position + 1
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
+ }
+
+ if entry.statusID == behindID {
+ behindIDMark = e
+ break findMarkLoop
+ }
+ }
+
+ // we didn't find it, so we need to make sure it's indexed and prepared and then try again
+ if behindIDMark == nil {
+ if err := t.IndexBehind(behindID, amount); err != nil {
+ return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID)
+ }
+ if err := t.PrepareBehind(behindID, amount); err != nil {
+ return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
+ }
+ oldestID, err := t.OldestPreparedPostID()
+ if err != nil {
+ return nil, err
+ }
+ if oldestID == "" || oldestID == behindID {
+ // there is no oldest prepared post, or the oldest prepared post is still the post we're looking for entries after
+ // this means we should just return the empty statuses slice since we don't have any more posts to offer
+ return statuses, nil
+ }
+ return t.GetXBehindID(amount, behindID)
+ }
+
+ // make sure we have enough posts prepared behind it to return what we're being asked for
+ if t.preparedPosts.data.Len() < amount+position {
+ if err := t.PrepareBehind(behindID, amount); err != nil {
+ return nil, err
+ }
+ }
+
+ // start serving from the entry right after the mark
+ var served int
+serveloop:
+ for e := behindIDMark.Next(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
+ }
+
+ // serve up to the amount requested
+ statuses = append(statuses, entry.prepared)
+ served = served + 1
+ if served >= amount {
+ break serveloop
+ }
+ }
+
+ return statuses, nil
+}
+
+func (t *timeline) GetXBeforeID(amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
+ // make a slice of statuses with the length we need to return
+ statuses := make([]*apimodel.Status, 0, amount)
+
+ if t.preparedPosts.data == nil {
+ t.preparedPosts.data = &list.List{}
+ }
+
+ // iterate through the modified list until we hit the mark we're looking for
+ var beforeIDMark *list.Element
+findMarkLoop:
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+ }
+
+ if entry.statusID == beforeID {
+ beforeIDMark = e
+ break findMarkLoop
+ }
+ }
+
+ // we didn't find it, so we need to make sure it's indexed and prepared and then try again
+ if beforeIDMark == nil {
+ if err := t.IndexBefore(beforeID, true, amount); err != nil {
+ return nil, fmt.Errorf("GetXBeforeID: error indexing before and including ID %s", beforeID)
+ }
+ if err := t.PrepareBefore(beforeID, true, amount); err != nil {
+ return nil, fmt.Errorf("GetXBeforeID: error preparing before and including ID %s", beforeID)
+ }
+ return t.GetXBeforeID(amount, beforeID, startFromTop)
+ }
+
+ var served int
+
+ if startFromTop {
+ // start serving from the front/top and keep going until we hit mark or get x amount statuses
+ serveloopFromTop:
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+ }
+
+ if entry.statusID == beforeID {
+ break serveloopFromTop
+ }
+
+ // serve up to the amount requested
+ statuses = append(statuses, entry.prepared)
+ served = served + 1
+ if served >= amount {
+ break serveloopFromTop
+ }
+ }
+ } else if !startFromTop {
+ // start serving from the entry right before the mark
+ serveloopFromBottom:
+ for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+ }
+
+ // serve up to the amount requested
+ statuses = append(statuses, entry.prepared)
+ served = served + 1
+ if served >= amount {
+ break serveloopFromBottom
+ }
+ }
+ }
+
+ return statuses, nil
+}
+
+func (t *timeline) GetXBetweenID(amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
+ // make a slice of statuses with the length we need to return
+ statuses := make([]*apimodel.Status, 0, amount)
+
+ if t.preparedPosts.data == nil {
+ t.preparedPosts.data = &list.List{}
+ }
+
+ // iterate through the modified list until we hit the mark we're looking for
+ var position int
+ var behindIDMark *list.Element
+findMarkLoop:
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ position = position + 1
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
+ }
+
+ if entry.statusID == behindID {
+ behindIDMark = e
+ break findMarkLoop
+ }
+ }
+
+ // we didn't find it
+ if behindIDMark == nil {
+ return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID)
+ }
+
+ // make sure we have enough posts prepared behind it to return what we're being asked for
+ if t.preparedPosts.data.Len() < amount+position {
+ if err := t.PrepareBehind(behindID, amount); err != nil {
+ return nil, err
+ }
+ }
+
+ // start serving from the entry right after the mark
+ var served int
+serveloop:
+ for e := behindIDMark.Next(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
+ }
+
+ if entry.statusID == beforeID {
+ break serveloop
+ }
+
+ // serve up to the amount requested
+ statuses = append(statuses, entry.prepared)
+ served = served + 1
+ if served >= amount {
+ break serveloop
+ }
+ }
+
+ return statuses, nil
+}
diff --git a/internal/timeline/index.go b/internal/timeline/index.go
new file mode 100644
index 000000000..56f5c14df
--- /dev/null
+++ b/internal/timeline/index.go
@@ -0,0 +1,143 @@
+package timeline
+
+import (
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (t *timeline) IndexBefore(statusID string, include bool, amount int) error {
+ // filtered := []*gtsmodel.Status{}
+ // offsetStatus := statusID
+
+ // grabloop:
+ // for len(filtered) < amount {
+ // statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, true)
+ // if err != nil {
+ // if _, ok := err.(db.ErrNoEntries); !ok {
+ // return fmt.Errorf("IndexBeforeAndIncluding: error getting statuses from db: %s", err)
+ // }
+ // break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
+ // }
+
+ // for _, s := range statuses {
+ // relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
+ // if err != nil {
+ // continue
+ // }
+ // visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
+ // if err != nil {
+ // continue
+ // }
+ // if visible {
+ // filtered = append(filtered, s)
+ // }
+ // offsetStatus = s.ID
+ // }
+ // }
+
+ // for _, s := range filtered {
+ // if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
+ // return fmt.Errorf("IndexBeforeAndIncluding: error indexing status with id %s: %s", s.ID, err)
+ // }
+ // }
+
+ return nil
+}
+
+func (t *timeline) IndexBehind(statusID string, amount int) error {
+ filtered := []*gtsmodel.Status{}
+ offsetStatus := statusID
+
+grabloop:
+ for len(filtered) < amount {
+ statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, offsetStatus, "", "", amount, false)
+ if err != nil {
+ if _, ok := err.(db.ErrNoEntries); ok {
+ break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
+ }
+ return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err)
+ }
+
+ for _, s := range statuses {
+ relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
+ if err != nil {
+ continue
+ }
+ visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
+ if err != nil {
+ continue
+ }
+ if visible {
+ filtered = append(filtered, s)
+ }
+ offsetStatus = s.ID
+ }
+ }
+
+ for _, s := range filtered {
+ if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
+ return fmt.Errorf("IndexBehindAndIncluding: error indexing status with id %s: %s", s.ID, err)
+ }
+ }
+
+ return nil
+}
+
+func (t *timeline) IndexOneByID(statusID string) error {
+ return nil
+}
+
+func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error {
+ t.Lock()
+ defer t.Unlock()
+
+ postIndexEntry := &postIndexEntry{
+ statusID: statusID,
+ }
+
+ return t.postIndex.insertIndexed(postIndexEntry)
+}
+
+func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error {
+ t.Lock()
+ defer t.Unlock()
+
+ postIndexEntry := &postIndexEntry{
+ statusID: statusID,
+ }
+
+ if err := t.postIndex.insertIndexed(postIndexEntry); err != nil {
+ return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
+ }
+
+ if err := t.prepare(statusID); err != nil {
+ return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
+ }
+
+ return nil
+}
+
+func (t *timeline) OldestIndexedPostID() (string, error) {
+ var id string
+ if t.postIndex == nil || t.postIndex.data == nil {
+ // return an empty string if postindex hasn't been initialized yet
+ return id, nil
+ }
+
+ e := t.postIndex.data.Back()
+
+ if e == nil {
+ // return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
+ return id, nil
+ }
+
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
+ }
+ return entry.statusID, nil
+}
diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go
new file mode 100644
index 000000000..9d28b5060
--- /dev/null
+++ b/internal/timeline/manager.go
@@ -0,0 +1,217 @@
+/*
+ GoToSocial
+ Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package timeline
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ "github.com/sirupsen/logrus"
+ apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+const (
+ desiredPostIndexLength = 400
+)
+
+// Manager abstracts functions for creating timelines for multiple accounts, and adding, removing, and fetching entries from those timelines.
+//
+// By the time a status hits the manager interface, it should already have been filtered and it should be established that the status indeed
+// belongs in the home timeline of the given account ID.
+//
+// The manager makes a distinction between *indexed* posts and *prepared* posts.
+//
+// Indexed posts consist of just that post's ID (in the database) and the time it was created. An indexed post takes up very little memory, so
+// it's not a huge priority to keep trimming the indexed posts list.
+//
+// Prepared posts consist of the post's database ID, the time it was created, AND the apimodel representation of that post, for quick serialization.
+// Prepared posts of course take up more memory than indexed posts, so they should be regularly pruned if they're not being actively served.
+type Manager interface {
+ // Ingest takes one status and indexes it into the timeline for the given account ID.
+ //
+ // It should already be established before calling this function that the status/post actually belongs in the timeline!
+ Ingest(status *gtsmodel.Status, timelineAccountID string) error
+ // IngestAndPrepare takes one status and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
+ // This is useful in cases where we know the status will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
+ //
+ // It should already be established before calling this function that the status/post actually belongs in the timeline!
+ IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) error
+ // HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
+ // If maxID is provided, it will return entries from that maxID onwards, inclusive.
+ HomeTimeline(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
+ // GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
+ GetIndexedLength(timelineAccountID string) int
+ // GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
+ GetDesiredIndexLength() int
+ // GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
+ GetOldestIndexedID(timelineAccountID string) (string, error)
+ // PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
+ PrepareXFromTop(timelineAccountID string, limit int) error
+ // WipeStatusFromTimeline completely removes a status and from the index and prepared posts of the given account ID
+ //
+ // The returned int indicates how many entries were removed.
+ WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error)
+ // WipeStatusFromAllTimelines removes the status from the index and prepared posts of all timelines
+ WipeStatusFromAllTimelines(statusID string) error
+}
+
+// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
+func NewManager(db db.DB, tc typeutils.TypeConverter, config *config.Config, log *logrus.Logger) Manager {
+ return &manager{
+ accountTimelines: sync.Map{},
+ db: db,
+ tc: tc,
+ config: config,
+ log: log,
+ }
+}
+
+type manager struct {
+ accountTimelines sync.Map
+ db db.DB
+ tc typeutils.TypeConverter
+ config *config.Config
+ log *logrus.Logger
+}
+
+func (m *manager) Ingest(status *gtsmodel.Status, timelineAccountID string) error {
+ l := m.log.WithFields(logrus.Fields{
+ "func": "Ingest",
+ "timelineAccountID": timelineAccountID,
+ "statusID": status.ID,
+ })
+
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ l.Trace("ingesting status")
+ return t.IndexOne(status.CreatedAt, status.ID)
+}
+
+func (m *manager) IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) error {
+ l := m.log.WithFields(logrus.Fields{
+ "func": "IngestAndPrepare",
+ "timelineAccountID": timelineAccountID,
+ "statusID": status.ID,
+ })
+
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ l.Trace("ingesting status")
+ return t.IndexAndPrepareOne(status.CreatedAt, status.ID)
+}
+
+func (m *manager) Remove(statusID string, timelineAccountID string) (int, error) {
+ l := m.log.WithFields(logrus.Fields{
+ "func": "Remove",
+ "timelineAccountID": timelineAccountID,
+ "statusID": statusID,
+ })
+
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ l.Trace("removing status")
+ return t.Remove(statusID)
+}
+
+func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
+ l := m.log.WithFields(logrus.Fields{
+ "func": "HomeTimelineGet",
+ "timelineAccountID": timelineAccountID,
+ })
+
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ statuses, err := t.Get(limit, maxID, sinceID, minID)
+ if err != nil {
+ l.Errorf("error getting statuses: %s", err)
+ }
+ return statuses, nil
+}
+
+func (m *manager) GetIndexedLength(timelineAccountID string) int {
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ return t.PostIndexLength()
+}
+
+func (m *manager) GetDesiredIndexLength() int {
+ return desiredPostIndexLength
+}
+
+func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) {
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ return t.OldestIndexedPostID()
+}
+
+func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error {
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ return t.PrepareFromTop(limit)
+}
+
+func (m *manager) WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error) {
+ t := m.getOrCreateTimeline(timelineAccountID)
+
+ return t.Remove(statusID)
+}
+
+func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
+ errors := []string{}
+ m.accountTimelines.Range(func(k interface{}, i interface{}) bool {
+ t, ok := i.(Timeline)
+ if !ok {
+ panic("couldn't parse entry as Timeline, this should never happen so panic")
+ }
+
+ if _, err := t.Remove(statusID); err != nil {
+ errors = append(errors, err.Error())
+ }
+
+ return false
+ })
+
+ var err error
+ if len(errors) > 0 {
+ err = fmt.Errorf("one or more errors removing status %s from all timelines: %s", statusID, strings.Join(errors, ";"))
+ }
+
+ return err
+}
+
+func (m *manager) getOrCreateTimeline(timelineAccountID string) Timeline {
+ var t Timeline
+ i, ok := m.accountTimelines.Load(timelineAccountID)
+ if !ok {
+ t = NewTimeline(timelineAccountID, m.db, m.tc, m.log)
+ m.accountTimelines.Store(timelineAccountID, t)
+ } else {
+ t, ok = i.(Timeline)
+ if !ok {
+ panic("couldn't parse entry as Timeline, this should never happen so panic")
+ }
+ }
+
+ return t
+}
diff --git a/internal/timeline/postindex.go b/internal/timeline/postindex.go
new file mode 100644
index 000000000..2ab65e087
--- /dev/null
+++ b/internal/timeline/postindex.go
@@ -0,0 +1,57 @@
+package timeline
+
+import (
+ "container/list"
+ "errors"
+)
+
+type postIndex struct {
+ data *list.List
+}
+
+type postIndexEntry struct {
+ statusID string
+}
+
+func (p *postIndex) insertIndexed(i *postIndexEntry) error {
+ if p.data == nil {
+ p.data = &list.List{}
+ }
+
+ // if we have no entries yet, this is both the newest and oldest entry, so just put it in the front
+ if p.data.Len() == 0 {
+ p.data.PushFront(i)
+ return nil
+ }
+
+ var insertMark *list.Element
+ // We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
+ // We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
+ for e := p.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return errors.New("index: could not parse e as a postIndexEntry")
+ }
+
+ // if the post to index is newer than e, insert it before e in the list
+ if insertMark == nil {
+ if i.statusID > entry.statusID {
+ insertMark = e
+ }
+ }
+
+ // make sure we don't insert a duplicate
+ if entry.statusID == i.statusID {
+ return nil
+ }
+ }
+
+ if insertMark != nil {
+ p.data.InsertBefore(i, insertMark)
+ return nil
+ }
+
+ // if we reach this point it's the oldest post we've seen so put it at the back
+ p.data.PushBack(i)
+ return nil
+}
diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go
new file mode 100644
index 000000000..1fb1cd714
--- /dev/null
+++ b/internal/timeline/prepare.go
@@ -0,0 +1,215 @@
+package timeline
+
+import (
+ "errors"
+ "fmt"
+
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) error {
+ var err error
+
+ // maxID is defined but sinceID isn't so take from behind
+ if maxID != "" && sinceID == "" {
+ err = t.PrepareBehind(maxID, amount)
+ }
+
+ // maxID isn't defined, but sinceID || minID are, so take x before
+ if maxID == "" && sinceID != "" {
+ err = t.PrepareBefore(sinceID, false, amount)
+ }
+ if maxID == "" && minID != "" {
+ err = t.PrepareBefore(minID, false, amount)
+ }
+
+ return err
+}
+
+func (t *timeline) PrepareBehind(statusID string, amount int) error {
+ t.Lock()
+ defer t.Unlock()
+
+ var prepared int
+ var preparing bool
+prepareloop:
+ for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return errors.New("PrepareBehind: could not parse e as a postIndexEntry")
+ }
+
+ if !preparing {
+ // we haven't hit the position we need to prepare from yet
+ if entry.statusID == statusID {
+ preparing = true
+ }
+ }
+
+ if preparing {
+ if err := t.prepare(entry.statusID); err != nil {
+ // there's been an error
+ if _, ok := err.(db.ErrNoEntries); !ok {
+ // it's a real error
+ return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err)
+ }
+ // the status just doesn't exist (anymore) so continue to the next one
+ continue
+ }
+ if prepared == amount {
+ // we're done
+ break prepareloop
+ }
+ prepared = prepared + 1
+ }
+ }
+
+ return nil
+}
+
+func (t *timeline) PrepareBefore(statusID string, include bool, amount int) error {
+ t.Lock()
+ defer t.Unlock()
+
+ var prepared int
+ var preparing bool
+prepareloop:
+ for e := t.postIndex.data.Back(); e != nil; e = e.Prev() {
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
+ }
+
+ if !preparing {
+ // we haven't hit the position we need to prepare from yet
+ if entry.statusID == statusID {
+ preparing = true
+ if !include {
+ continue
+ }
+ }
+ }
+
+ if preparing {
+ if err := t.prepare(entry.statusID); err != nil {
+ // there's been an error
+ if _, ok := err.(db.ErrNoEntries); !ok {
+ // it's a real error
+ return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err)
+ }
+ // the status just doesn't exist (anymore) so continue to the next one
+ continue
+ }
+ if prepared == amount {
+ // we're done
+ break prepareloop
+ }
+ prepared = prepared + 1
+ }
+ }
+
+ return nil
+}
+
+func (t *timeline) PrepareFromTop(amount int) error {
+ t.Lock()
+ defer t.Unlock()
+
+ t.preparedPosts.data.Init()
+
+ var prepared int
+prepareloop:
+ for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
+ }
+
+ if err := t.prepare(entry.statusID); err != nil {
+ // there's been an error
+ if _, ok := err.(db.ErrNoEntries); !ok {
+ // it's a real error
+ return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err)
+ }
+ // the status just doesn't exist (anymore) so continue to the next one
+ continue
+ }
+
+ prepared = prepared + 1
+ if prepared == amount {
+ // we're done
+ break prepareloop
+ }
+ }
+
+ return nil
+}
+
+func (t *timeline) prepare(statusID string) error {
+
+ // start by getting the status out of the database according to its indexed ID
+ gtsStatus := &gtsmodel.Status{}
+ if err := t.db.GetByID(statusID, gtsStatus); err != nil {
+ return err
+ }
+
+ // if the account pointer hasn't been set on this timeline already, set it lazily here
+ if t.account == nil {
+ timelineOwnerAccount := &gtsmodel.Account{}
+ if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
+ return err
+ }
+ t.account = timelineOwnerAccount
+ }
+
+ // to convert the status we need relevant accounts from it, so pull them out here
+ relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus)
+ if err != nil {
+ return err
+ }
+
+ // check if this is a boost...
+ var reblogOfStatus *gtsmodel.Status
+ if gtsStatus.BoostOfID != "" {
+ s := &gtsmodel.Status{}
+ if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil {
+ return err
+ }
+ reblogOfStatus = s
+ }
+
+ // serialize the status (or, at least, convert it to a form that's ready to be serialized)
+ apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
+ if err != nil {
+ return err
+ }
+
+ // shove it in prepared posts as a prepared posts entry
+ preparedPostsEntry := &preparedPostsEntry{
+ statusID: statusID,
+ prepared: apiModelStatus,
+ }
+
+ return t.preparedPosts.insertPrepared(preparedPostsEntry)
+}
+
+func (t *timeline) OldestPreparedPostID() (string, error) {
+ var id string
+ if t.preparedPosts == nil || t.preparedPosts.data == nil {
+ // return an empty string if prepared posts hasn't been initialized yet
+ return id, nil
+ }
+
+ e := t.preparedPosts.data.Back()
+ if e == nil {
+ // return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
+ return id, nil
+ }
+
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return id, errors.New("OldestPreparedPostID: could not parse e as a preparedPostsEntry")
+ }
+ return entry.statusID, nil
+}
diff --git a/internal/timeline/preparedposts.go b/internal/timeline/preparedposts.go
new file mode 100644
index 000000000..429ce5415
--- /dev/null
+++ b/internal/timeline/preparedposts.go
@@ -0,0 +1,60 @@
+package timeline
+
+import (
+ "container/list"
+ "errors"
+
+ apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+)
+
+type preparedPosts struct {
+ data *list.List
+}
+
+type preparedPostsEntry struct {
+ statusID string
+ prepared *apimodel.Status
+}
+
+func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
+ if p.data == nil {
+ p.data = &list.List{}
+ }
+
+ // if we have no entries yet, this is both the newest and oldest entry, so just put it in the front
+ if p.data.Len() == 0 {
+ p.data.PushFront(i)
+ return nil
+ }
+
+ var insertMark *list.Element
+ // We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
+ // We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
+ for e := p.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return errors.New("index: could not parse e as a preparedPostsEntry")
+ }
+
+ // if the post to index is newer than e, insert it before e in the list
+ if insertMark == nil {
+ if i.statusID > entry.statusID {
+ insertMark = e
+ }
+ }
+
+ // make sure we don't insert a duplicate
+ if entry.statusID == i.statusID {
+ return nil
+ }
+ }
+
+ if insertMark != nil {
+ p.data.InsertBefore(i, insertMark)
+ return nil
+ }
+
+ // if we reach this point it's the oldest post we've seen so put it at the back
+ p.data.PushBack(i)
+ return nil
+}
diff --git a/internal/timeline/remove.go b/internal/timeline/remove.go
new file mode 100644
index 000000000..2f340d37b
--- /dev/null
+++ b/internal/timeline/remove.go
@@ -0,0 +1,50 @@
+package timeline
+
+import (
+ "container/list"
+ "errors"
+)
+
+func (t *timeline) Remove(statusID string) (int, error) {
+ t.Lock()
+ defer t.Unlock()
+ var removed int
+
+ // remove entr(ies) from the post index
+ removeIndexes := []*list.Element{}
+ if t.postIndex != nil && t.postIndex.data != nil {
+ for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*postIndexEntry)
+ if !ok {
+ return removed, errors.New("Remove: could not parse e as a postIndexEntry")
+ }
+ if entry.statusID == statusID {
+ removeIndexes = append(removeIndexes, e)
+ }
+ }
+ }
+ for _, e := range removeIndexes {
+ t.postIndex.data.Remove(e)
+ removed = removed + 1
+ }
+
+ // remove entr(ies) from prepared posts
+ removePrepared := []*list.Element{}
+ if t.preparedPosts != nil && t.preparedPosts.data != nil {
+ for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+ entry, ok := e.Value.(*preparedPostsEntry)
+ if !ok {
+ return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
+ }
+ if entry.statusID == statusID {
+ removePrepared = append(removePrepared, e)
+ }
+ }
+ }
+ for _, e := range removePrepared {
+ t.preparedPosts.data.Remove(e)
+ removed = removed + 1
+ }
+
+ return removed, nil
+}
diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go
new file mode 100644
index 000000000..7408436dc
--- /dev/null
+++ b/internal/timeline/timeline.go
@@ -0,0 +1,139 @@
+/*
+ GoToSocial
+ Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package timeline
+
+import (
+ "sync"
+ "time"
+
+ "github.com/sirupsen/logrus"
+ apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// Timeline represents a timeline for one account, and contains indexed and prepared posts.
+type Timeline interface {
+ /*
+ RETRIEVAL FUNCTIONS
+ */
+
+ Get(amount int, maxID string, sinceID string, minID string) ([]*apimodel.Status, error)
+ // GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
+ GetXFromTop(amount int) ([]*apimodel.Status, error)
+ // GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest.
+ // This will NOT include the status with the given ID.
+ //
+ // This corresponds to an api call to /timelines/home?max_id=WHATEVER
+ GetXBehindID(amount int, fromID string) ([]*apimodel.Status, error)
+ // GetXBeforeID returns x amount of posts up to the given id, from newest to oldest.
+ // This will NOT include the status with the given ID.
+ //
+ // This corresponds to an api call to /timelines/home?since_id=WHATEVER
+ GetXBeforeID(amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
+ // GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
+ // This will NOT include the status with the given IDs.
+ //
+ // This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
+ GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
+
+ /*
+ INDEXING FUNCTIONS
+ */
+
+ // IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property.
+ IndexOne(statusCreatedAt time.Time, statusID string) error
+
+ // OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
+ // If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
+ OldestIndexedPostID() (string, error)
+
+ /*
+ PREPARATION FUNCTIONS
+ */
+
+ // PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
+ PrepareFromTop(amount int) error
+ // PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
+ // If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
+ PrepareBehind(statusID string, amount int) error
+ // IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
+ // and then immediately prepares it.
+ IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error
+ // OldestPreparedPostID returns the id of the rearmost (ie., the oldest) prepared post, or an error if something goes wrong.
+ // If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
+ OldestPreparedPostID() (string, error)
+
+ /*
+ INFO FUNCTIONS
+ */
+
+ // ActualPostIndexLength returns the actual length of the post index at this point in time.
+ PostIndexLength() int
+
+ /*
+ UTILITY FUNCTIONS
+ */
+
+ // Reset instructs the timeline to reset to its base state -- cache only the minimum amount of posts.
+ Reset() error
+ // Remove removes a status from both the index and prepared posts.
+ //
+ // If a status has multiple entries in a timeline, they will all be removed.
+ //
+ // The returned int indicates the amount of entries that were removed.
+ Remove(statusID string) (int, error)
+}
+
+// timeline fulfils the Timeline interface
+type timeline struct {
+ postIndex *postIndex
+ preparedPosts *preparedPosts
+ accountID string
+ account *gtsmodel.Account
+ db db.DB
+ tc typeutils.TypeConverter
+ log *logrus.Logger
+ sync.Mutex
+}
+
+// NewTimeline returns a new Timeline for the given account ID
+func NewTimeline(accountID string, db db.DB, typeConverter typeutils.TypeConverter, log *logrus.Logger) Timeline {
+ return &timeline{
+ postIndex: &postIndex{},
+ preparedPosts: &preparedPosts{},
+ accountID: accountID,
+ db: db,
+ tc: typeConverter,
+ log: log,
+ }
+}
+
+func (t *timeline) Reset() error {
+ return nil
+}
+
+func (t *timeline) PostIndexLength() int {
+ if t.postIndex == nil || t.postIndex.data == nil {
+ return 0
+ }
+
+ return t.postIndex.data.Len()
+}
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-13 06:20:51 +0000