[feature] Implement backfilling statuses thru scheduled_at (#3685)

* Implement backfilling statuses thru scheduled_at

* Forbid mentioning others in backfills

* Update error messages & codes

* Add new tests for backfilled statuses

* Test that backfilling doesn't timeline or notify

* Fix check for absence of notification

* Test that backfills do not cause federation

* Fix type of apimodel.StatusCreateRequest.ScheduledAt in tests

* Add config file switch and min date check

4 files changed, 294 insertions, 19 deletions

diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go
index b77d0af9c..46052d0aa 100644
--- a/internal/processing/status/create.go
+++ b/internal/processing/status/create.go
@@ -19,10 +19,14 @@ package status
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
@@ -92,11 +96,54 @@ func (p *Processor) Create(
 	// Get current time.
 	now := time.Now()
 
+	// Default to current time as creation time.
+	createdAt := now
+
+	// Handle backfilled/scheduled statuses.
+	backfill := false
+	if form.ScheduledAt != nil {
+		scheduledAt := *form.ScheduledAt
+
+		// Statuses may only be scheduled a minimum time into the future.
+		if now.Before(scheduledAt) {
+			const errText = "scheduled statuses are not yet supported"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorNotImplemented(err, errText)
+		}
+
+		// If not scheduled into the future, this status is being backfilled.
+		if !config.GetInstanceAllowBackdatingStatuses() {
+			const errText = "backdating statuses has been disabled on this instance"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorForbidden(err)
+		}
+
+		// Statuses can't be backdated to or before the UNIX epoch
+		// since this would prevent generating a ULID.
+		// If backdated even further to the Go epoch,
+		// this would also cause issues with time.Time.IsZero() checks
+		// that normally signify an absent optional time,
+		// but this check covers both cases.
+		if scheduledAt.Compare(time.UnixMilli(0)) <= 0 {
+			const errText = "statuses can't be backdated to or before the UNIX epoch"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorNotAcceptable(err, errText)
+		}
+
+		// Allow the backfill and generate an appropriate ID for the creation time.
+		backfill = true
+		createdAt = scheduledAt
+		var err error
+		if statusID, err = p.backfilledStatusID(ctx, createdAt); err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+	}
+
 	status := &gtsmodel.Status{
 		ID:                       statusID,
 		URI:                      accountURIs.StatusesURI + "/" + statusID,
 		URL:                      accountURIs.StatusesURL + "/" + statusID,
-		CreatedAt:                now,
+		CreatedAt:                createdAt,
 		Local:                    util.Ptr(true),
 		Account:                  requester,
 		AccountID:                requester.ID,
@@ -134,11 +181,24 @@ func (p *Processor) Create(
 		PendingApproval: util.Ptr(false),
 	}
 
+	if backfill {
+		log.Infof(ctx, "%d mentions", len(status.Mentions))
+		for _, mention := range status.Mentions {
+			log.Infof(ctx, "mention: target account ID = %s, requester ID = %s", mention.TargetAccountID, requester.ID)
+			if mention.TargetAccountID != requester.ID {
+				const errText = "statuses mentioning others can't be backfilled"
+				err := gtserror.New(errText)
+				return nil, gtserror.NewErrorForbidden(err, errText)
+			}
+		}
+	}
+
 	// Check + attach in-reply-to status.
 	if errWithCode := p.processInReplyTo(ctx,
 		requester,
 		status,
 		form.InReplyToID,
+		backfill,
 	); errWithCode != nil {
 		return nil, errWithCode
 	}
@@ -165,11 +225,17 @@ func (p *Processor) Create(
 	}
 
 	if form.Poll != nil {
+		if backfill {
+			const errText = "statuses with polls can't be backfilled"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorForbidden(err, errText)
+		}
+
 		// Process poll, inserting into database.
 		poll, errWithCode := p.processPoll(ctx,
 			statusID,
 			form.Poll,
-			now,
+			createdAt,
 		)
 		if errWithCode != nil {
 			return nil, errWithCode
@@ -200,10 +266,14 @@ func (p *Processor) Create(
 	}
 
 	// Send it to the client API worker for async side-effects.
+	var model any = status
+	if backfill {
+		model = &gtsmodel.BackfillStatus{Status: status}
+	}
 	p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{
 		APObjectType:   ap.ObjectNote,
 		APActivityType: ap.ActivityCreate,
-		GTSModel:       status,
+		GTSModel:       model,
 		Origin:         requester,
 	})
 
@@ -227,7 +297,40 @@ func (p *Processor) Create(
 	return p.c.GetAPIStatus(ctx, requester, status)
 }
 
-func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status, inReplyToID string) gtserror.WithCode {
+// backfilledStatusID tries to find an unused ULID for a backfilled status.
+func (p *Processor) backfilledStatusID(ctx context.Context, createdAt time.Time) (string, error) {
+	// backfilledStatusIDRetries should be more than enough attempts.
+	const backfilledStatusIDRetries = 100
+
+	for try := 0; try < backfilledStatusIDRetries; try++ {
+		var err error
+
+		// Generate a ULID based on the backfilled status's original creation time.
+		statusID := id.NewULIDFromTime(createdAt)
+
+		// Check for an existing status with that ID.
+		_, err = p.state.DB.GetStatusByID(gtscontext.SetBarebones(ctx), statusID)
+		if errors.Is(err, db.ErrNoEntries) {
+			// We found an unused one.
+			return statusID, nil
+		} else if err != nil {
+			err := gtserror.Newf("DB error checking if a status ID was in use: %w", err)
+			return "", err
+		}
+		// That status ID is in use. Try again.
+	}
+
+	err := gtserror.Newf("failed to find an unused ID after %d tries", backfilledStatusIDRetries)
+	return "", err
+}
+
+func (p *Processor) processInReplyTo(
+	ctx context.Context,
+	requester *gtsmodel.Account,
+	status *gtsmodel.Status,
+	inReplyToID string,
+	backfill bool,
+) gtserror.WithCode {
 	if inReplyToID == "" {
 		// Not a reply.
 		// Nothing to do.
@@ -269,6 +372,13 @@ func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Ac
 		return gtserror.NewErrorForbidden(err, errText)
 	}
 
+	// When backfilling, only self-replies are allowed.
+	if backfill && requester.ID != inReplyTo.AccountID {
+		const errText = "replies to others can't be backfilled"
+		err := gtserror.New(errText)
+		return gtserror.NewErrorForbidden(err, errText)
+	}
+
 	// Derive pendingApproval status.
 	var pendingApproval bool
 	switch {
diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go
index d0a5c7f92..16cefcebf 100644
--- a/internal/processing/status/create_test.go
+++ b/internal/processing/status/create_test.go
@@ -48,7 +48,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithQuotationMarks(
 		SpoilerText: "\"test\"", // these should not be html-escaped when the final text is rendered
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypePlain,
 	}
@@ -75,7 +75,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuot
 		SpoilerText: "&#34test&#34", // the html-escaped quotation marks should appear as normal quotation marks in the finished text
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypePlain,
 	}
@@ -106,7 +106,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji
 		Sensitive:   false,
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypeMarkdown,
 	}
@@ -133,7 +133,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithSpoilerTextEmoj
 		Sensitive:   false,
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypeMarkdown,
 	}
@@ -164,7 +164,7 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() {
 		SpoilerText: "",
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypePlain,
 	}
@@ -189,7 +189,7 @@ func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() {
 		SpoilerText: "",
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "zh-Hans",
 		ContentType: apimodel.StatusContentTypePlain,
 	}
@@ -219,7 +219,7 @@ func (suite *StatusCreateTestSuite) TestProcessReplyToUnthreadedRemoteStatus() {
 		SpoilerText: "this is a reply",
 		Visibility:  apimodel.VisibilityPublic,
 		LocalOnly:   util.Ptr(false),
-		ScheduledAt: "",
+		ScheduledAt: nil,
 		Language:    "en",
 		ContentType: apimodel.StatusContentTypePlain,
 	}
diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go
index c5dfc157d..a208d97b0 100644
--- a/internal/processing/workers/fromclientapi.go
+++ b/internal/processing/workers/fromclientapi.go
@@ -260,9 +260,16 @@ func (p *clientAPI) CreateUser(ctx context.Context, cMsg *messages.FromClientAPI
 }
 
 func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientAPI) error {
-	status, ok := cMsg.GTSModel.(*gtsmodel.Status)
-	if !ok {
-		return gtserror.Newf("%T not parseable as *gtsmodel.Status", cMsg.GTSModel)
+	var status *gtsmodel.Status
+	backfill := false
+	if backfillStatus, ok := cMsg.GTSModel.(*gtsmodel.BackfillStatus); ok {
+		status = backfillStatus.Status
+		backfill = true
+	} else {
+		status, ok = cMsg.GTSModel.(*gtsmodel.Status)
+		if !ok {
+			return gtserror.Newf("%T not parseable as *gtsmodel.Status or *gtsmodel.BackfillStatus", cMsg.GTSModel)
+		}
 	}
 
 	// If pending approval is true then status must
@@ -344,12 +351,14 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA
 		log.Errorf(ctx, "error updating account stats: %v", err)
 	}
 
-	if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
-		log.Errorf(ctx, "error timelining and notifying status: %v", err)
-	}
+	if !backfill {
+		if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
+			log.Errorf(ctx, "error timelining and notifying status: %v", err)
+		}
 
-	if err := p.federate.CreateStatus(ctx, status); err != nil {
-		log.Errorf(ctx, "error federating status: %v", err)
+		if err := p.federate.CreateStatus(ctx, status); err != nil {
+			log.Errorf(ctx, "error federating status: %v", err)
+		}
 	}
 
 	if status.InReplyToID != "" {
diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go
index acb25673d..1d70eb96c 100644
--- a/internal/processing/workers/fromclientapi_test.go
+++ b/internal/processing/workers/fromclientapi_test.go
@@ -368,6 +368,162 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() {
 	suite.checkWebPushed(testStructs.WebPushSender, receivingAccount.ID, gtsmodel.NotificationStatus)
 }
 
+// Even with notifications on for a user, backfilling a status should not notify or timeline it.
+func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithNotification() {
+	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	var (
+		ctx              = context.Background()
+		postingAccount   = suite.testAccounts["admin_account"]
+		receivingAccount = suite.testAccounts["local_account_1"]
+		testList         = suite.testLists["local_account_1_list_1"]
+		streams          = suite.openStreams(ctx,
+			testStructs.Processor,
+			receivingAccount,
+			[]string{testList.ID},
+		)
+		homeStream  = streams[stream.TimelineHome]
+		listStream  = streams[stream.TimelineList+":"+testList.ID]
+		notifStream = streams[stream.TimelineNotifications]
+
+		// Admin account posts a new top-level status.
+		status = suite.newStatus(
+			ctx,
+			testStructs.State,
+			postingAccount,
+			gtsmodel.VisibilityPublic,
+			nil,
+			nil,
+			nil,
+			false,
+			nil,
+		)
+	)
+
+	// Update the follow from receiving account -> posting account so
+	// that receiving account wants notifs when posting account posts.
+	follow := new(gtsmodel.Follow)
+	*follow = *suite.testFollows["local_account_1_admin_account"]
+
+	follow.Notify = util.Ptr(true)
+	if err := testStructs.State.DB.UpdateFollow(ctx, follow); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process the new status as a backfill.
+	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+		ctx,
+		&messages.FromClientAPI{
+			APObjectType:   ap.ObjectNote,
+			APActivityType: ap.ActivityCreate,
+			GTSModel:       &gtsmodel.BackfillStatus{Status: status},
+			Origin:         postingAccount,
+		},
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// There should be no message in the home stream.
+	suite.checkStreamed(
+		homeStream,
+		false,
+		"",
+		"",
+	)
+
+	// There should be no message in the list stream.
+	suite.checkStreamed(
+		listStream,
+		false,
+		"",
+		"",
+	)
+
+	// No notification should appear for the status.
+	if testrig.WaitFor(func() bool {
+		var err error
+		_, err = testStructs.State.DB.GetNotification(
+			ctx,
+			gtsmodel.NotificationStatus,
+			receivingAccount.ID,
+			postingAccount.ID,
+			status.ID,
+		)
+		return err == nil
+	}) {
+		suite.FailNow("a status notification was created, but should not have been")
+	}
+
+	// There should be no message in the notification stream.
+	suite.checkStreamed(
+		notifStream,
+		false,
+		"",
+		"",
+	)
+
+	// There should be no Web Push status notification.
+	suite.checkNotWebPushed(testStructs.WebPushSender, receivingAccount.ID)
+}
+
+// Backfilled statuses should not federate when created.
+func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithRemoteFollower() {
+	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	var (
+		ctx              = context.Background()
+		postingAccount   = suite.testAccounts["local_account_1"]
+		receivingAccount = suite.testAccounts["remote_account_1"]
+
+		// Local account posts a new top-level status.
+		status = suite.newStatus(
+			ctx,
+			testStructs.State,
+			postingAccount,
+			gtsmodel.VisibilityPublic,
+			nil,
+			nil,
+			nil,
+			false,
+			nil,
+		)
+	)
+
+	// Follow the local account from the remote account.
+	follow := &gtsmodel.Follow{
+		ID:              "01JJHW9RW28SC1NEPZ0WBJQ4ZK",
+		CreatedAt:       testrig.TimeMustParse("2022-05-14T13:21:09+02:00"),
+		UpdatedAt:       testrig.TimeMustParse("2022-05-14T13:21:09+02:00"),
+		AccountID:       receivingAccount.ID,
+		TargetAccountID: postingAccount.ID,
+		ShowReblogs:     util.Ptr(true),
+		URI:             "http://fossbros-anonymous.io/users/foss_satan/follow/01JJHWEVC7F8W2JDW1136K431K",
+		Notify:          util.Ptr(false),
+	}
+
+	if err := testStructs.State.DB.PutFollow(ctx, follow); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process the new status as a backfill.
+	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+		ctx,
+		&messages.FromClientAPI{
+			APObjectType:   ap.ObjectNote,
+			APActivityType: ap.ActivityCreate,
+			GTSModel:       &gtsmodel.BackfillStatus{Status: status},
+			Origin:         postingAccount,
+		},
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// No deliveries should be queued.
+	suite.Zero(testStructs.State.Workers.Delivery.Queue.Len())
+}
+
 func (suite *FromClientAPITestSuite) TestProcessCreateStatusReply() {
 	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
 	defer testrig.TearDownTestStructs(testStructs)