Domain block (#76)

* start work on admin domain blocking

* move stuff around + further work on domain blocks

* move + restructure processor

* prep work for deleting account

* tidy

* go fmt

* formatting

* domain blocking more work

* check domain blocks way earlier on

* progress on delete account

* delete more stuff when an account is gone

* and more...

* domain blocky block block

* get individual domain block, delete a block

1 files changed, 25 insertions, 301 deletions

diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go
index cc3ffa153..36568cf13 100644
--- a/internal/processing/fromfederator.go
+++ b/internal/processing/fromfederator.go
@@ -21,7 +21,6 @@ package processing
 import (
 	"errors"
 	"fmt"
-	"net/url"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -49,7 +48,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming status")
-			if err := p.dereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
+			if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
 				return fmt.Errorf("error dereferencing status from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
@@ -72,7 +71,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, "", false); err != nil {
+			if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
@@ -105,7 +104,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 				return errors.New("announce was not parseable as *gtsmodel.Status")
 			}
 
-			if err := p.dereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
+			if err := p.federator.DereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
 				return fmt.Errorf("error dereferencing announce from federator: %s", err)
 			}
 
@@ -140,7 +139,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
+			if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
@@ -160,6 +159,27 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			if !ok {
 				return errors.New("note was not parseable as *gtsmodel.Status")
 			}
+
+			// delete all attachments for this status
+			for _, a := range statusToDelete.Attachments {
+				if err := p.mediaProcessor.Delete(a); err != nil {
+					return err
+				}
+			}
+
+			// delete all mentions for this status
+			for _, m := range statusToDelete.Mentions {
+				if err := p.db.DeleteByID(m, &gtsmodel.Mention{}); err != nil {
+					return err
+				}
+			}
+
+			// delete all notifications for this status
+			if err := p.db.DeleteWhere([]db.Where{{Key: "status_id", Value: statusToDelete.ID}}, &[]*gtsmodel.Notification{}); err != nil {
+				return err
+			}
+
+			// remove this status from any and all timelines
 			return p.deleteStatusFromTimelines(statusToDelete)
 		case gtsmodel.ActivityStreamsProfile:
 			// DELETE A PROFILE/ACCOUNT
@@ -183,299 +203,3 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 
 	return nil
 }
-
-// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
-// federated status, back in the federating db's Create function.
-//
-// When a status comes in from the federation API, there are certain fields that
-// haven't been dereferenced yet, because we needed to provide a snappy synchronous
-// response to the caller. By the time it reaches this function though, it's being
-// processed asynchronously, so we have all the time in the world to fetch the various
-// bits and bobs that are attached to the status, and properly flesh it out, before we
-// send the status to any timelines and notify people.
-//
-// Things to dereference and fetch here:
-//
-// 1. Media attachments.
-// 2. Hashtags.
-// 3. Emojis.
-// 4. Mentions.
-// 5. Posting account.
-// 6. Replied-to-status.
-//
-// SIDE EFFECTS:
-// This function will deference all of the above, insert them in the database as necessary,
-// and attach them to the status. The status itself will not be added to the database yet,
-// that's up the caller to do.
-func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
-	l := p.log.WithFields(logrus.Fields{
-		"func":   "dereferenceStatusFields",
-		"status": fmt.Sprintf("%+v", status),
-	})
-	l.Debug("entering function")
-
-	t, err := p.federator.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error creating transport: %s", err)
-	}
-
-	// the status should have an ID by now, but just in case it doesn't let's generate one here
-	// because we'll need it further down
-	if status.ID == "" {
-		newID, err := id.NewULIDFromTime(status.CreatedAt)
-		if err != nil {
-			return err
-		}
-		status.ID = newID
-	}
-
-	// 1. Media attachments.
-	//
-	// At this point we should know:
-	// * the media type of the file we're looking for (a.File.ContentType)
-	// * the blurhash (a.Blurhash)
-	// * the file type (a.Type)
-	// * the remote URL (a.RemoteURL)
-	// This should be enough to pass along to the media processor.
-	attachmentIDs := []string{}
-	for _, a := range status.GTSMediaAttachments {
-		l.Debugf("dereferencing attachment: %+v", a)
-
-		// it might have been processed elsewhere so check first if it's already in the database or not
-		maybeAttachment := &gtsmodel.MediaAttachment{}
-		err := p.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
-		if err == nil {
-			// we already have it in the db, dereferenced, no need to do it again
-			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
-			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
-			continue
-		}
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			// we have a real error
-			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		// it just doesn't exist yet so carry on
-		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
-		deferencedAttachment, err := p.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
-		if err != nil {
-			p.log.Errorf("error dereferencing status attachment: %s", err)
-			continue
-		}
-		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
-		deferencedAttachment.StatusID = status.ID
-		deferencedAttachment.Description = a.Description
-		if err := p.db.Put(deferencedAttachment); err != nil {
-			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
-	}
-	status.Attachments = attachmentIDs
-
-	// 2. Hashtags
-
-	// 3. Emojis
-
-	// 4. Mentions
-	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
-	//
-	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
-	mentions := []string{}
-	for _, m := range status.GTSMentions {
-		if m.ID == "" {
-			mID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			m.ID = mID
-		}
-
-		uri, err := url.Parse(m.MentionedAccountURI)
-		if err != nil {
-			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
-			continue
-		}
-
-		m.StatusID = status.ID
-		m.OriginAccountID = status.GTSAuthorAccount.ID
-		m.OriginAccountURI = status.GTSAuthorAccount.URI
-
-		targetAccount := &gtsmodel.Account{}
-		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
-			// proper error
-			if _, ok := err.(db.ErrNoEntries); !ok {
-				return fmt.Errorf("db error checking for account with uri %s", uri.String())
-			}
-
-			// we just don't have it yet, so we should go get it....
-			accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, uri)
-			if err != nil {
-				// we can't dereference it so just skip it
-				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
-			if err != nil {
-				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccountID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			targetAccount.ID = targetAccountID
-
-			if err := p.db.Put(targetAccount); err != nil {
-				return fmt.Errorf("db error inserting account with uri %s", uri.String())
-			}
-		}
-
-		// by this point, we know the targetAccount exists in our database with an ID :)
-		m.TargetAccountID = targetAccount.ID
-		if err := p.db.Put(m); err != nil {
-			return fmt.Errorf("error creating mention: %s", err)
-		}
-		mentions = append(mentions, m.ID)
-	}
-	status.Mentions = mentions
-
-	return nil
-}
-
-func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
-	l := p.log.WithFields(logrus.Fields{
-		"func":               "dereferenceAccountFields",
-		"requestingUsername": requestingUsername,
-	})
-
-	t, err := p.federator.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error getting transport for user: %s", err)
-	}
-
-	// fetch the header and avatar
-	if err := p.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
-		// if this doesn't work, just skip it -- we can do it later
-		l.Debugf("error fetching header/avi for account: %s", err)
-	}
-
-	if err := p.db.UpdateByID(account.ID, account); err != nil {
-		return fmt.Errorf("error updating account in database: %s", err)
-	}
-
-	return nil
-}
-
-func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
-	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
-		// we can't do anything unfortunately
-		return errors.New("dereferenceAnnounce: no URI to dereference")
-	}
-
-	// check if we already have the boosted status in the database
-	boostedStatus := &gtsmodel.Status{}
-	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
-	if err == nil {
-		// nice, we already have it so we don't actually need to dereference it from remote
-		announce.Content = boostedStatus.Content
-		announce.ContentWarning = boostedStatus.ContentWarning
-		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-		announce.Sensitive = boostedStatus.Sensitive
-		announce.Language = boostedStatus.Language
-		announce.Text = boostedStatus.Text
-		announce.BoostOfID = boostedStatus.ID
-		announce.Visibility = boostedStatus.Visibility
-		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-		announce.GTSBoostedStatus = boostedStatus
-		return nil
-	}
-
-	// we don't have it so we need to dereference it
-	remoteStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error parsing url %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	statusable, err := p.federator.DereferenceRemoteStatus(requestingUsername, remoteStatusURI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// make sure we have the author account in the db
-	attributedToProp := statusable.GetActivityStreamsAttributedTo()
-	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
-		accountURI := iter.GetIRI()
-		if accountURI == nil {
-			continue
-		}
-
-		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
-			// we already have it, fine
-			continue
-		}
-
-		// we don't have the boosted status author account yet so dereference it
-		accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, accountURI)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
-		}
-		account, err := p.tc.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
-		}
-
-		accountID, err := id.NewRandomULID()
-		if err != nil {
-			return err
-		}
-		account.ID = accountID
-
-		if err := p.db.Put(account); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
-		}
-
-		if err := p.dereferenceAccountFields(account, requestingUsername, false); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
-		}
-	}
-
-	// now convert the statusable into something we can understand
-	boostedStatus, err = p.tc.ASStatusToStatus(statusable)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
-	if err != nil {
-		return nil
-	}
-	boostedStatus.ID = boostedStatusID
-
-	if err := p.db.Put(boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// now dereference additional fields straight away (we're already async here so we have time)
-	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// update with the newly dereferenced fields
-	if err := p.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
-	}
-
-	// we have everything we need!
-	announce.Content = boostedStatus.Content
-	announce.ContentWarning = boostedStatus.ContentWarning
-	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-	announce.Sensitive = boostedStatus.Sensitive
-	announce.Language = boostedStatus.Language
-	announce.Text = boostedStatus.Text
-	announce.BoostOfID = boostedStatus.ID
-	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-	announce.GTSBoostedStatus = boostedStatus
-	return nil
-}