Improve GetRemoteStatus and db.GetStatus() logic (#174)

* only fetch status parents / children if explicity requested when dereferencing

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Remove recursive DB GetStatus logic, don't fetch parent unless requested

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* StatusCache copies status so there are no thread-safety issues with modified status objects

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* remove sqlite test files

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix bugs introduced by previous commit

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix not continue on error in loop

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* use our own RunInTx implementation (possible fix for nested tx error)

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix cast statement to work with SQLite

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* be less strict about valid status in cache

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add cache=shared ALWAYS for SQLite db instances

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Fix EnrichRemoteAccount when updating account fails

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add nolint tag

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* ensure file: prefixes the filename in sqlite addr

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add an account cache, add status author account from db

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Fix incompatible SQLite query

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* *actually* use the new getAccount() function in accountsDB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update cache tests to use test suite

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add RelationshipTestSuite, add tests for methods with changed SQL

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

1 files changed, 24 insertions, 32 deletions

diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
index 3fa1e4133..7a7f928f1 100644
--- a/internal/federation/dereferencing/status.go
+++ b/internal/federation/dereferencing/status.go
@@ -39,8 +39,8 @@ import (
 //
 // EnrichRemoteStatus is mostly useful for calling after a status has been initially created by
 // the federatingDB's Create function, but additional dereferencing is needed on it.
-func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status) (*gtsmodel.Status, error) {
-	if err := d.populateStatusFields(ctx, status, username); err != nil {
+func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent, includeChilds bool) (*gtsmodel.Status, error) {
+	if err := d.populateStatusFields(ctx, status, username, includeParent, includeChilds); err != nil {
 		return nil, err
 	}
 
@@ -62,7 +62,7 @@ func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status
 // If a dereference was performed, then the function also returns the ap.Statusable representation for further processing.
 //
 // SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored.
-func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
+func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent, includeChilds bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
 	new := true
 
 	// check if we already have the status in our db
@@ -105,7 +105,7 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat
 		}
 		gtsStatus.ID = ulid
 
-		if err := d.populateStatusFields(ctx, gtsStatus, username); err != nil {
+		if err := d.populateStatusFields(ctx, gtsStatus, username, includeParent, includeChilds); err != nil {
 			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
 		}
 
@@ -115,7 +115,7 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat
 	} else {
 		gtsStatus.ID = maybeStatus.ID
 
-		if err := d.populateStatusFields(ctx, gtsStatus, username); err != nil {
+		if err := d.populateStatusFields(ctx, gtsStatus, username, includeParent, includeChilds); err != nil {
 			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
 		}
 
@@ -235,7 +235,7 @@ func (d *deref) dereferenceStatusable(ctx context.Context, username string, remo
 // This function will deference all of the above, insert them in the database as necessary,
 // and attach them to the status. The status itself will not be added to the database yet,
 // that's up the caller to do.
-func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
+func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string, includeParent, includeChilds bool) error {
 	l := d.log.WithFields(logrus.Fields{
 		"func":   "dereferenceStatusFields",
 		"status": fmt.Sprintf("%+v", status),
@@ -275,14 +275,19 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu
 	// 3. Emojis
 	// TODO
 
-	// 4. Mentions
-	if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil {
-		return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err)
+	// 4. Mentions (only if requested)
+	// TODO: do we need to handle removing empty mention objects and just using mention IDs slice?
+	if includeChilds {
+		if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil {
+			return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err)
+		}
 	}
 
-	// 5. Replied-to-status.
-	if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil {
-		return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err)
+	// 5. Replied-to-status (only if requested)
+	if includeParent {
+		if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil {
+			return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err)
+		}
 	}
 
 	return nil
@@ -391,7 +396,6 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
 	attachments := []*gtsmodel.MediaAttachment{}
 
 	for _, a := range status.Attachments {
-
 		aURL, err := url.Parse(a.RemoteURL)
 		if err != nil {
 			l.Errorf("populateStatusAttachments: couldn't parse attachment url %s: %s", a.RemoteURL, err)
@@ -401,6 +405,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
 		attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, aURL, status.AccountID, status.ID, a.File.ContentType)
 		if err != nil {
 			l.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err)
+			continue
 		}
 
 		attachmentIDs = append(attachmentIDs, attachment.ID)
@@ -420,29 +425,16 @@ func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.St
 			return err
 		}
 
-		var replyToStatus *gtsmodel.Status
-		errs := []string{}
-
 		// see if we have the status in our db already
-		if s, err := d.db.GetStatusByURI(ctx, status.InReplyToURI); err != nil {
-			errs = append(errs, err.Error())
-		} else {
-			replyToStatus = s
-		}
-
-		if replyToStatus == nil {
-			// didn't find the status in our db, try to get it remotely
-			if s, _, _, err := d.GetRemoteStatus(ctx, requestingUsername, statusURI, false); err != nil {
-				errs = append(errs, err.Error())
-			} else {
-				replyToStatus = s
+		replyToStatus, err := d.db.GetStatusByURI(ctx, status.InReplyToURI)
+		if err != nil {
+			// Status was not in the DB, try fetch
+			replyToStatus, _, _, err = d.GetRemoteStatus(ctx, requestingUsername, statusURI, false, false, false)
+			if err != nil {
+				return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", status.InReplyToURI, err)
 			}
 		}
 
-		if replyToStatus == nil {
-			return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", statusURI, strings.Join(errs, " : "))
-		}
-
 		// we have the status
 		status.InReplyToID = replyToStatus.ID
 		status.InReplyTo = replyToStatus