summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLibravatar Vyr Cossont <VyrCossont@users.noreply.github.com>2025-02-12 09:49:33 -0800
committerLibravatar GitHub <noreply@github.com>2025-02-12 09:49:33 -0800
commitfccb0bc102f2a54a21eed343cda64f9a5221b677 (patch)
treeb7c1858f4a92841dfaf59e7102189a635d7136f8
parent[performance] improved enum migrations (#3782) (diff)
downloadgotosocial-fccb0bc102f2a54a21eed343cda64f9a5221b677.tar.xz
[feature] Implement backfilling statuses thru scheduled_at (#3685)
* Implement backfilling statuses thru scheduled_at * Forbid mentioning others in backfills * Update error messages & codes * Add new tests for backfilled statuses * Test that backfilling doesn't timeline or notify * Fix check for absence of notification * Test that backfills do not cause federation * Fix type of apimodel.StatusCreateRequest.ScheduledAt in tests * Add config file switch and min date check
-rw-r--r--docs/api/swagger.yaml8
-rw-r--r--docs/configuration/instance.md13
-rw-r--r--example/config.yaml13
-rw-r--r--internal/api/client/statuses/statuscreate.go14
-rw-r--r--internal/api/client/statuses/statuscreate_test.go132
-rw-r--r--internal/api/model/status.go15
-rw-r--r--internal/config/config.go1
-rw-r--r--internal/config/defaults.go1
-rw-r--r--internal/config/flags.go1
-rw-r--r--internal/config/helpers.gen.go25
-rw-r--r--internal/federation/dereferencing/instance_test.go6
-rw-r--r--internal/gtsmodel/status.go9
-rw-r--r--internal/processing/status/create.go118
-rw-r--r--internal/processing/status/create_test.go14
-rw-r--r--internal/processing/workers/fromclientapi.go25
-rw-r--r--internal/processing/workers/fromclientapi_test.go156
-rwxr-xr-xtest/envparsing.sh1
-rw-r--r--testrig/config.go1
18 files changed, 513 insertions, 40 deletions
diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml
index df8f09321..836df83e8 100644
--- a/docs/api/swagger.yaml
+++ b/docs/api/swagger.yaml
@@ -10397,10 +10397,14 @@ paths:
x-go-name: Federated
- description: |-
ISO 8601 Datetime at which to schedule a status.
- Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+
+ Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
Must be at least 5 minutes in the future.
+ This feature isn't implemented yet.
- This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+ Providing this parameter with a *past* time will cause the status to be backdated,
+ and will not push it to the user's followers. This is intended for importing old statuses.
+ format: date-time
in: formData
name: scheduled_at
type: string
diff --git a/docs/configuration/instance.md b/docs/configuration/instance.md
index bffec8f70..2a945eed2 100644
--- a/docs/configuration/instance.md
+++ b/docs/configuration/instance.md
@@ -171,4 +171,17 @@ instance-subscriptions-process-every: "24h"
# Options: ["", "zero", "serve", "baffle"]
# Default: ""
instance-stats-mode: ""
+
+# Bool. This flag controls whether local accounts may backdate statuses
+# using past dates with the scheduled_at param to /api/v1/statuses.
+# This flag does not affect scheduling posts in the future
+# (which is currently not implemented anyway),
+# nor can it prevent remote accounts from backdating their own statuses.
+#
+# If true, all local accounts may backdate statuses.
+# If false, status backdating will be disabled and an error will be returned if it's used.
+#
+# Options: [true, false]
+# Default: true
+instance-allow-backdating-statuses: true
```
diff --git a/example/config.yaml b/example/config.yaml
index b618ad7ba..2b3a873fb 100644
--- a/example/config.yaml
+++ b/example/config.yaml
@@ -458,6 +458,19 @@ instance-subscriptions-process-every: "24h"
# Default: ""
instance-stats-mode: ""
+# Bool. This flag controls whether local accounts may backdate statuses
+# using past dates with the scheduled_at param to /api/v1/statuses.
+# This flag does not affect scheduling posts in the future
+# (which is currently not implemented anyway),
+# nor can it prevent remote accounts from backdating their own statuses.
+#
+# If true, all local accounts may backdate statuses.
+# If false, status backdating will be disabled and an error will be returned if it's used.
+#
+# Options: [true, false]
+# Default: true
+instance-allow-backdating-statuses: true
+
###########################
##### ACCOUNTS CONFIG #####
###########################
diff --git a/internal/api/client/statuses/statuscreate.go b/internal/api/client/statuses/statuscreate.go
index d187e823f..bfb1c486d 100644
--- a/internal/api/client/statuses/statuscreate.go
+++ b/internal/api/client/statuses/statuscreate.go
@@ -179,11 +179,15 @@ import (
// x-go-name: ScheduledAt
// description: |-
// ISO 8601 Datetime at which to schedule a status.
-// Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+//
+// Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
// Must be at least 5 minutes in the future.
+// This feature isn't implemented yet.
//
-// This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+// Providing this parameter with a *past* time will cause the status to be backdated,
+// and will not push it to the user's followers. This is intended for importing old statuses.
// type: string
+// format: date-time
// in: formData
// -
// name: language
@@ -384,12 +388,6 @@ func parseStatusCreateForm(c *gin.Context) (*apimodel.StatusCreateRequest, gtser
return nil, gtserror.NewErrorNotAcceptable(errors.New(text), text)
}
- // Check not scheduled status.
- if form.ScheduledAt != "" {
- const text = "scheduled_at is not yet implemented"
- return nil, gtserror.NewErrorNotImplemented(errors.New(text), text)
- }
-
// Check if the deprecated "federated" field was
// set in lieu of "local_only", and use it if so.
if form.LocalOnly == nil && form.Federated != nil { // nolint:staticcheck
diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go
index 227e7d83e..53e517a6e 100644
--- a/internal/api/client/statuses/statuscreate_test.go
+++ b/internal/api/client/statuses/statuscreate_test.go
@@ -20,14 +20,18 @@ package statuses_test
import (
"bytes"
"context"
+ "encoding/json"
"fmt"
+ "io"
"net/http"
"net/http/httptest"
"testing"
+ "time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -41,10 +45,11 @@ const (
statusMarkdown = "# Title\n\n## Smaller title\n\nThis is a post written in [markdown](https://www.markdownguide.org/)\n\n<img src=\"https://d33wubrfki0l68.cloudfront.net/f1f475a6fda1c2c4be4cac04033db5c3293032b4/513a4/assets/images/markdown-mark-white.svg\"/>"
)
-func (suite *StatusCreateTestSuite) postStatus(
+// Post a status.
+func (suite *StatusCreateTestSuite) postStatusCore(
formData map[string][]string,
jsonData string,
-) (string, *httptest.ResponseRecorder) {
+) *httptest.ResponseRecorder {
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
@@ -77,9 +82,42 @@ func (suite *StatusCreateTestSuite) postStatus(
// Trigger handler.
suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+ return recorder
+}
+
+// Post a status and return the result as deterministic JSON.
+func (suite *StatusCreateTestSuite) postStatus(
+ formData map[string][]string,
+ jsonData string,
+) (string, *httptest.ResponseRecorder) {
+ recorder := suite.postStatusCore(formData, jsonData)
return suite.parseStatusResponse(recorder)
}
+// Post a status and return the result as a non-deterministic API structure.
+func (suite *StatusCreateTestSuite) postStatusStruct(
+ formData map[string][]string,
+ jsonData string,
+) (*apimodel.Status, *httptest.ResponseRecorder) {
+ recorder := suite.postStatusCore(formData, jsonData)
+
+ result := recorder.Result()
+ defer result.Body.Close()
+
+ data, err := io.ReadAll(result.Body)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ apiStatus := apimodel.Status{}
+ if err := json.Unmarshal(data, &apiStatus); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ return &apiStatus, recorder
+}
+
// Post a new status with some custom visibility settings
func (suite *StatusCreateTestSuite) TestPostNewStatus() {
out, recorder := suite.postStatus(map[string][]string{
@@ -383,10 +421,98 @@ func (suite *StatusCreateTestSuite) TestPostNewScheduledStatus() {
// We should have a helpful error message.
suite.Equal(`{
- "error": "Not Implemented: scheduled_at is not yet implemented"
+ "error": "Not Implemented: scheduled statuses are not yet supported"
}`, out)
}
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatus() {
+ // A time in the past.
+ scheduledAtStr := "2020-10-04T15:32:02.018Z"
+ scheduledAt, err := time.Parse(time.RFC3339Nano, scheduledAtStr)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ status, recorder := suite.postStatusStruct(map[string][]string{
+ "status": {"this is a recycled status from the past!"},
+ "scheduled_at": {scheduledAtStr},
+ }, "")
+
+ // Creating a status in the past should succeed.
+ suite.Equal(http.StatusOK, recorder.Code)
+
+ // The status should be backdated.
+ createdAt, err := time.Parse(time.RFC3339Nano, status.CreatedAt)
+ if err != nil {
+ suite.FailNow(err.Error())
+ return
+ }
+ suite.Equal(scheduledAt, createdAt.UTC())
+
+ // The status's ULID should be backdated.
+ timeFromULID, err := id.TimeFromULID(status.ID)
+ if err != nil {
+ suite.FailNow(err.Error())
+ return
+ }
+ suite.Equal(scheduledAt, timeFromULID.UTC())
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfMention() {
+ _, recorder := suite.postStatus(map[string][]string{
+ "status": {"@the_mighty_zork this is a recycled mention from the past!"},
+ "scheduled_at": {"2020-10-04T15:32:02.018Z"},
+ }, "")
+
+ // Mentioning yourself is allowed in backfilled statuses.
+ suite.Equal(http.StatusOK, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithMention() {
+ _, recorder := suite.postStatus(map[string][]string{
+ "status": {"@admin this is a recycled mention from the past!"},
+ "scheduled_at": {"2020-10-04T15:32:02.018Z"},
+ }, "")
+
+ // Mentioning others is forbidden in backfilled statuses.
+ suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfReply() {
+ _, recorder := suite.postStatus(map[string][]string{
+ "status": {"this is a recycled reply from the past!"},
+ "scheduled_at": {"2020-10-04T15:32:02.018Z"},
+ "in_reply_to_id": {suite.testStatuses["local_account_1_status_1"].ID},
+ }, "")
+
+ // Replying to yourself is allowed in backfilled statuses.
+ suite.Equal(http.StatusOK, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithReply() {
+ _, recorder := suite.postStatus(map[string][]string{
+ "status": {"this is a recycled reply from the past!"},
+ "scheduled_at": {"2020-10-04T15:32:02.018Z"},
+ "in_reply_to_id": {suite.testStatuses["admin_account_status_1"].ID},
+ }, "")
+
+ // Replying to others is forbidden in backfilled statuses.
+ suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
+func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithPoll() {
+ _, recorder := suite.postStatus(map[string][]string{
+ "status": {"this is a recycled poll from the past!"},
+ "scheduled_at": {"2020-10-04T15:32:02.018Z"},
+ "poll[options][]": {"first option", "second option"},
+ "poll[expires_in]": {"3600"},
+ "poll[multiple]": {"true"},
+ }, "")
+
+ // Polls are forbidden in backfilled statuses.
+ suite.Equal(http.StatusForbidden, recorder.Code)
+}
+
func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() {
out, recorder := suite.postStatus(map[string][]string{
"status": {statusMarkdown},
diff --git a/internal/api/model/status.go b/internal/api/model/status.go
index ea9fbaa35..2ee3123e6 100644
--- a/internal/api/model/status.go
+++ b/internal/api/model/status.go
@@ -17,7 +17,11 @@
package model
-import "github.com/superseriousbusiness/gotosocial/internal/language"
+import (
+ "time"
+
+ "github.com/superseriousbusiness/gotosocial/internal/language"
+)
// Status models a status or post.
//
@@ -231,9 +235,14 @@ type StatusCreateRequest struct {
Federated *bool `form:"federated" json:"federated"`
// ISO 8601 Datetime at which to schedule a status.
- // Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+ //
+ // Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
// Must be at least 5 minutes in the future.
- ScheduledAt string `form:"scheduled_at" json:"scheduled_at"`
+ // This feature isn't implemented yet.
+ //
+ // Providing this parameter with a *past* time will cause the status to be backdated,
+ // and will not push it to the user's followers. This is intended for importing old statuses.
+ ScheduledAt *time.Time `form:"scheduled_at" json:"scheduled_at"`
// ISO 639 language code for this status.
Language string `form:"language" json:"language"`
diff --git a/internal/config/config.go b/internal/config/config.go
index 33003d0f9..8ce2105b4 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -91,6 +91,7 @@ type Configuration struct {
InstanceSubscriptionsProcessFrom string `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
InstanceSubscriptionsProcessEvery time.Duration `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
InstanceStatsMode string `name:"instance-stats-mode" usage:"Allows you to customize the way stats are served to crawlers: one of '', 'serve', 'zero', 'baffle'. Home page stats remain unchanged."`
+ InstanceAllowBackdatingStatuses bool `name:"instance-allow-backdating-statuses" usage:"Allow local accounts to backdate statuses using the scheduled_at param to /api/v1/statuses"`
AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 7f66e4209..78a8230d5 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -67,6 +67,7 @@ var Defaults = Configuration{
InstanceLanguages: make(language.Languages, 0),
InstanceSubscriptionsProcessFrom: "23:00", // 11pm,
InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
+ InstanceAllowBackdatingStatuses: true,
AccountsRegistrationOpen: false,
AccountsReasonRequired: true,
diff --git a/internal/config/flags.go b/internal/config/flags.go
index d67085d6d..3a2564c94 100644
--- a/internal/config/flags.go
+++ b/internal/config/flags.go
@@ -93,6 +93,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage"))
cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage"))
cmd.Flags().String(InstanceStatsModeFlag(), cfg.InstanceStatsMode, fieldtag("InstanceStatsMode", "usage"))
+ cmd.Flags().Bool(InstanceAllowBackdatingStatusesFlag(), cfg.InstanceAllowBackdatingStatuses, fieldtag("InstanceAllowBackdatingStatuses", "usage"))
// Accounts
cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage"))
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index d3ccf16ea..156c19fd5 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -1082,6 +1082,31 @@ func GetInstanceStatsMode() string { return global.GetInstanceStatsMode() }
// SetInstanceStatsMode safely sets the value for global configuration 'InstanceStatsMode' field
func SetInstanceStatsMode(v string) { global.SetInstanceStatsMode(v) }
+// GetInstanceAllowBackdatingStatuses safely fetches the Configuration value for state's 'InstanceAllowBackdatingStatuses' field
+func (st *ConfigState) GetInstanceAllowBackdatingStatuses() (v bool) {
+ st.mutex.RLock()
+ v = st.config.InstanceAllowBackdatingStatuses
+ st.mutex.RUnlock()
+ return
+}
+
+// SetInstanceAllowBackdatingStatuses safely sets the Configuration value for state's 'InstanceAllowBackdatingStatuses' field
+func (st *ConfigState) SetInstanceAllowBackdatingStatuses(v bool) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.InstanceAllowBackdatingStatuses = v
+ st.reloadToViper()
+}
+
+// InstanceAllowBackdatingStatusesFlag returns the flag name for the 'InstanceAllowBackdatingStatuses' field
+func InstanceAllowBackdatingStatusesFlag() string { return "instance-allow-backdating-statuses" }
+
+// GetInstanceAllowBackdatingStatuses safely fetches the value for global configuration 'InstanceAllowBackdatingStatuses' field
+func GetInstanceAllowBackdatingStatuses() bool { return global.GetInstanceAllowBackdatingStatuses() }
+
+// SetInstanceAllowBackdatingStatuses safely sets the value for global configuration 'InstanceAllowBackdatingStatuses' field
+func SetInstanceAllowBackdatingStatuses(v bool) { global.SetInstanceAllowBackdatingStatuses(v) }
+
// GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
st.mutex.RLock()
diff --git a/internal/federation/dereferencing/instance_test.go b/internal/federation/dereferencing/instance_test.go
index 15f075479..c07490d4b 100644
--- a/internal/federation/dereferencing/instance_test.go
+++ b/internal/federation/dereferencing/instance_test.go
@@ -50,7 +50,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
//
// Debug-level logs should show something like:
//
- // - "can't fetch /nodeinfo/2.1: robots.txt disallows it"
+ // - "can't fetch /nodeinfo/2.1: robots.txt disallows it"
instanceIRI: testrig.URLMustParse("https://furtive-nerds.example.org"),
expectedSoftware: "",
},
@@ -60,7 +60,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
//
// Debug-level logs should show something like:
//
- // - "can't fetch api/v1/instance: robots.txt disallows it"
+ // - "can't fetch api/v1/instance: robots.txt disallows it"
// - "can't fetch .well-known/nodeinfo: robots.txt disallows it"
instanceIRI: testrig.URLMustParse("https://robotic.furtive-nerds.example.org"),
expectedSoftware: "",
@@ -71,7 +71,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() {
//
// Debug-level logs should show something like:
//
- // - "can't use fetched .well-known/nodeinfo: robots tags disallows it"
+ // - "can't use fetched .well-known/nodeinfo: robots tags disallows it"
instanceIRI: testrig.URLMustParse("https://really.furtive-nerds.example.org"),
expectedSoftware: "",
},
diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go
index d28898ed1..e170e7464 100644
--- a/internal/gtsmodel/status.go
+++ b/internal/gtsmodel/status.go
@@ -86,7 +86,7 @@ func (s *Status) GetAccountID() string {
return s.AccountID
}
-// GetBoostID implements timeline.Timelineable{}.
+// GetBoostOfID implements timeline.Timelineable{}.
func (s *Status) GetBoostOfID() string {
return s.BoostOfID
}
@@ -171,7 +171,7 @@ func (s *Status) EditsPopulated() bool {
return true
}
-// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date
+// EmojisUpToDate returns whether status emoji attachments of receiving status are up-to-date
// according to emoji attachments of the passed status, by comparing their emoji URIs. We don't
// use IDs as this is used to determine whether there are new emojis to fetch.
func (s *Status) EmojisUpToDate(other *Status) bool {
@@ -386,3 +386,8 @@ type Content struct {
Content string
ContentMap map[string]string
}
+
+// BackfillStatus is a wrapper for creating a status without pushing notifications to followers.
+type BackfillStatus struct {
+ *Status
+}
diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go
index b77d0af9c..46052d0aa 100644
--- a/internal/processing/status/create.go
+++ b/internal/processing/status/create.go
@@ -19,10 +19,14 @@ package status
import (
"context"
+ "errors"
"time"
"github.com/superseriousbusiness/gotosocial/internal/ap"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
@@ -92,11 +96,54 @@ func (p *Processor) Create(
// Get current time.
now := time.Now()
+ // Default to current time as creation time.
+ createdAt := now
+
+ // Handle backfilled/scheduled statuses.
+ backfill := false
+ if form.ScheduledAt != nil {
+ scheduledAt := *form.ScheduledAt
+
+ // Statuses may only be scheduled a minimum time into the future.
+ if now.Before(scheduledAt) {
+ const errText = "scheduled statuses are not yet supported"
+ err := gtserror.New(errText)
+ return nil, gtserror.NewErrorNotImplemented(err, errText)
+ }
+
+ // If not scheduled into the future, this status is being backfilled.
+ if !config.GetInstanceAllowBackdatingStatuses() {
+ const errText = "backdating statuses has been disabled on this instance"
+ err := gtserror.New(errText)
+ return nil, gtserror.NewErrorForbidden(err)
+ }
+
+ // Statuses can't be backdated to or before the UNIX epoch
+ // since this would prevent generating a ULID.
+ // If backdated even further to the Go epoch,
+ // this would also cause issues with time.Time.IsZero() checks
+ // that normally signify an absent optional time,
+ // but this check covers both cases.
+ if scheduledAt.Compare(time.UnixMilli(0)) <= 0 {
+ const errText = "statuses can't be backdated to or before the UNIX epoch"
+ err := gtserror.New(errText)
+ return nil, gtserror.NewErrorNotAcceptable(err, errText)
+ }
+
+ // Allow the backfill and generate an appropriate ID for the creation time.
+ backfill = true
+ createdAt = scheduledAt
+ var err error
+ if statusID, err = p.backfilledStatusID(ctx, createdAt); err != nil {
+ return nil, gtserror.NewErrorInternalError(err)
+ }
+ }
+
status := &gtsmodel.Status{
ID: statusID,
URI: accountURIs.StatusesURI + "/" + statusID,
URL: accountURIs.StatusesURL + "/" + statusID,
- CreatedAt: now,
+ CreatedAt: createdAt,
Local: util.Ptr(true),
Account: requester,
AccountID: requester.ID,
@@ -134,11 +181,24 @@ func (p *Processor) Create(
PendingApproval: util.Ptr(false),
}
+ if backfill {
+ log.Infof(ctx, "%d mentions", len(status.Mentions))
+ for _, mention := range status.Mentions {
+ log.Infof(ctx, "mention: target account ID = %s, requester ID = %s", mention.TargetAccountID, requester.ID)
+ if mention.TargetAccountID != requester.ID {
+ const errText = "statuses mentioning others can't be backfilled"
+ err := gtserror.New(errText)
+ return nil, gtserror.NewErrorForbidden(err, errText)
+ }
+ }
+ }
+
// Check + attach in-reply-to status.
if errWithCode := p.processInReplyTo(ctx,
requester,
status,
form.InReplyToID,
+ backfill,
); errWithCode != nil {
return nil, errWithCode
}
@@ -165,11 +225,17 @@ func (p *Processor) Create(
}
if form.Poll != nil {
+ if backfill {
+ const errText = "statuses with polls can't be backfilled"
+ err := gtserror.New(errText)
+ return nil, gtserror.NewErrorForbidden(err, errText)
+ }
+
// Process poll, inserting into database.
poll, errWithCode := p.processPoll(ctx,
statusID,
form.Poll,
- now,
+ createdAt,
)
if errWithCode != nil {
return nil, errWithCode
@@ -200,10 +266,14 @@ func (p *Processor) Create(
}
// Send it to the client API worker for async side-effects.
+ var model any = status
+ if backfill {
+ model = &gtsmodel.BackfillStatus{Status: status}
+ }
p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{
APObjectType: ap.ObjectNote,
APActivityType: ap.ActivityCreate,
- GTSModel: status,
+ GTSModel: model,
Origin: requester,
})
@@ -227,7 +297,40 @@ func (p *Processor) Create(
return p.c.GetAPIStatus(ctx, requester, status)
}
-func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status, inReplyToID string) gtserror.WithCode {
+// backfilledStatusID tries to find an unused ULID for a backfilled status.
+func (p *Processor) backfilledStatusID(ctx context.Context, createdAt time.Time) (string, error) {
+ // backfilledStatusIDRetries should be more than enough attempts.
+ const backfilledStatusIDRetries = 100
+
+ for try := 0; try < backfilledStatusIDRetries; try++ {
+ var err error
+
+ // Generate a ULID based on the backfilled status's original creation time.
+ statusID := id.NewULIDFromTime(createdAt)
+
+ // Check for an existing status with that ID.
+ _, err = p.state.DB.GetStatusByID(gtscontext.SetBarebones(ctx), statusID)
+ if errors.Is(err, db.ErrNoEntries) {
+ // We found an unused one.
+ return statusID, nil
+ } else if err != nil {
+ err := gtserror.Newf("DB error checking if a status ID was in use: %w", err)
+ return "", err
+ }
+ // That status ID is in use. Try again.
+ }
+
+ err := gtserror.Newf("failed to find an unused ID after %d tries", backfilledStatusIDRetries)
+ return "", err
+}
+
+func (p *Processor) processInReplyTo(
+ ctx context.Context,
+ requester *gtsmodel.Account,
+ status *gtsmodel.Status,
+ inReplyToID string,
+ backfill bool,
+) gtserror.WithCode {
if inReplyToID == "" {
// Not a reply.
// Nothing to do.
@@ -269,6 +372,13 @@ func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Ac
return gtserror.NewErrorForbidden(err, errText)
}
+ // When backfilling, only self-replies are allowed.
+ if backfill && requester.ID != inReplyTo.AccountID {
+ const errText = "replies to others can't be backfilled"
+ err := gtserror.New(errText)
+ return gtserror.NewErrorForbidden(err, errText)
+ }
+
// Derive pendingApproval status.
var pendingApproval bool
switch {
diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go
index d0a5c7f92..16cefcebf 100644
--- a/internal/processing/status/create_test.go
+++ b/internal/processing/status/create_test.go
@@ -48,7 +48,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithQuotationMarks(
SpoilerText: "\"test\"", // these should not be html-escaped when the final text is rendered
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypePlain,
}
@@ -75,7 +75,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuot
SpoilerText: "&#34test&#34", // the html-escaped quotation marks should appear as normal quotation marks in the finished text
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypePlain,
}
@@ -106,7 +106,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji
Sensitive: false,
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypeMarkdown,
}
@@ -133,7 +133,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithSpoilerTextEmoj
Sensitive: false,
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypeMarkdown,
}
@@ -164,7 +164,7 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() {
SpoilerText: "",
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypePlain,
}
@@ -189,7 +189,7 @@ func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() {
SpoilerText: "",
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "zh-Hans",
ContentType: apimodel.StatusContentTypePlain,
}
@@ -219,7 +219,7 @@ func (suite *StatusCreateTestSuite) TestProcessReplyToUnthreadedRemoteStatus() {
SpoilerText: "this is a reply",
Visibility: apimodel.VisibilityPublic,
LocalOnly: util.Ptr(false),
- ScheduledAt: "",
+ ScheduledAt: nil,
Language: "en",
ContentType: apimodel.StatusContentTypePlain,
}
diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go
index c5dfc157d..a208d97b0 100644
--- a/internal/processing/workers/fromclientapi.go
+++ b/internal/processing/workers/fromclientapi.go
@@ -260,9 +260,16 @@ func (p *clientAPI) CreateUser(ctx context.Context, cMsg *messages.FromClientAPI
}
func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientAPI) error {
- status, ok := cMsg.GTSModel.(*gtsmodel.Status)
- if !ok {
- return gtserror.Newf("%T not parseable as *gtsmodel.Status", cMsg.GTSModel)
+ var status *gtsmodel.Status
+ backfill := false
+ if backfillStatus, ok := cMsg.GTSModel.(*gtsmodel.BackfillStatus); ok {
+ status = backfillStatus.Status
+ backfill = true
+ } else {
+ status, ok = cMsg.GTSModel.(*gtsmodel.Status)
+ if !ok {
+ return gtserror.Newf("%T not parseable as *gtsmodel.Status or *gtsmodel.BackfillStatus", cMsg.GTSModel)
+ }
}
// If pending approval is true then status must
@@ -344,12 +351,14 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA
log.Errorf(ctx, "error updating account stats: %v", err)
}
- if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
- log.Errorf(ctx, "error timelining and notifying status: %v", err)
- }
+ if !backfill {
+ if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
+ log.Errorf(ctx, "error timelining and notifying status: %v", err)
+ }
- if err := p.federate.CreateStatus(ctx, status); err != nil {
- log.Errorf(ctx, "error federating status: %v", err)
+ if err := p.federate.CreateStatus(ctx, status); err != nil {
+ log.Errorf(ctx, "error federating status: %v", err)
+ }
}
if status.InReplyToID != "" {
diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go
index acb25673d..1d70eb96c 100644
--- a/internal/processing/workers/fromclientapi_test.go
+++ b/internal/processing/workers/fromclientapi_test.go
@@ -368,6 +368,162 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() {
suite.checkWebPushed(testStructs.WebPushSender, receivingAccount.ID, gtsmodel.NotificationStatus)
}
+// Even with notifications on for a user, backfilling a status should not notify or timeline it.
+func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithNotification() {
+ testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+ defer testrig.TearDownTestStructs(testStructs)
+
+ var (
+ ctx = context.Background()
+ postingAccount = suite.testAccounts["admin_account"]
+ receivingAccount = suite.testAccounts["local_account_1"]
+ testList = suite.testLists["local_account_1_list_1"]
+ streams = suite.openStreams(ctx,
+ testStructs.Processor,
+ receivingAccount,
+ []string{testList.ID},
+ )
+ homeStream = streams[stream.TimelineHome]
+ listStream = streams[stream.TimelineList+":"+testList.ID]
+ notifStream = streams[stream.TimelineNotifications]
+
+ // Admin account posts a new top-level status.
+ status = suite.newStatus(
+ ctx,
+ testStructs.State,
+ postingAccount,
+ gtsmodel.VisibilityPublic,
+ nil,
+ nil,
+ nil,
+ false,
+ nil,
+ )
+ )
+
+ // Update the follow from receiving account -> posting account so
+ // that receiving account wants notifs when posting account posts.
+ follow := new(gtsmodel.Follow)
+ *follow = *suite.testFollows["local_account_1_admin_account"]
+
+ follow.Notify = util.Ptr(true)
+ if err := testStructs.State.DB.UpdateFollow(ctx, follow); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // Process the new status as a backfill.
+ if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+ ctx,
+ &messages.FromClientAPI{
+ APObjectType: ap.ObjectNote,
+ APActivityType: ap.ActivityCreate,
+ GTSModel: &gtsmodel.BackfillStatus{Status: status},
+ Origin: postingAccount,
+ },
+ ); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // There should be no message in the home stream.
+ suite.checkStreamed(
+ homeStream,
+ false,
+ "",
+ "",
+ )
+
+ // There should be no message in the list stream.
+ suite.checkStreamed(
+ listStream,
+ false,
+ "",
+ "",
+ )
+
+ // No notification should appear for the status.
+ if testrig.WaitFor(func() bool {
+ var err error
+ _, err = testStructs.State.DB.GetNotification(
+ ctx,
+ gtsmodel.NotificationStatus,
+ receivingAccount.ID,
+ postingAccount.ID,
+ status.ID,
+ )
+ return err == nil
+ }) {
+ suite.FailNow("a status notification was created, but should not have been")
+ }
+
+ // There should be no message in the notification stream.
+ suite.checkStreamed(
+ notifStream,
+ false,
+ "",
+ "",
+ )
+
+ // There should be no Web Push status notification.
+ suite.checkNotWebPushed(testStructs.WebPushSender, receivingAccount.ID)
+}
+
+// Backfilled statuses should not federate when created.
+func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithRemoteFollower() {
+ testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+ defer testrig.TearDownTestStructs(testStructs)
+
+ var (
+ ctx = context.Background()
+ postingAccount = suite.testAccounts["local_account_1"]
+ receivingAccount = suite.testAccounts["remote_account_1"]
+
+ // Local account posts a new top-level status.
+ status = suite.newStatus(
+ ctx,
+ testStructs.State,
+ postingAccount,
+ gtsmodel.VisibilityPublic,
+ nil,
+ nil,
+ nil,
+ false,
+ nil,
+ )
+ )
+
+ // Follow the local account from the remote account.
+ follow := &gtsmodel.Follow{
+ ID: "01JJHW9RW28SC1NEPZ0WBJQ4ZK",
+ CreatedAt: testrig.TimeMustParse("2022-05-14T13:21:09+02:00"),
+ UpdatedAt: testrig.TimeMustParse("2022-05-14T13:21:09+02:00"),
+ AccountID: receivingAccount.ID,
+ TargetAccountID: postingAccount.ID,
+ ShowReblogs: util.Ptr(true),
+ URI: "http://fossbros-anonymous.io/users/foss_satan/follow/01JJHWEVC7F8W2JDW1136K431K",
+ Notify: util.Ptr(false),
+ }
+
+ if err := testStructs.State.DB.PutFollow(ctx, follow); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // Process the new status as a backfill.
+ if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+ ctx,
+ &messages.FromClientAPI{
+ APObjectType: ap.ObjectNote,
+ APActivityType: ap.ActivityCreate,
+ GTSModel: &gtsmodel.BackfillStatus{Status: status},
+ Origin: postingAccount,
+ },
+ ); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // No deliveries should be queued.
+ suite.Zero(testStructs.State.Workers.Delivery.Queue.Len())
+}
+
func (suite *FromClientAPITestSuite) TestProcessCreateStatusReply() {
testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
defer testrig.TearDownTestStructs(testStructs)
diff --git a/test/envparsing.sh b/test/envparsing.sh
index 0e7c0db20..fc6afbcc2 100755
--- a/test/envparsing.sh
+++ b/test/envparsing.sh
@@ -108,6 +108,7 @@ EXPECT=$(cat << "EOF"
"timeout": 30000000000,
"tls-insecure-skip-verify": false
},
+ "instance-allow-backdating-statuses": true,
"instance-deliver-to-shared-inboxes": false,
"instance-expose-peers": true,
"instance-expose-public-timeline": true,
diff --git a/testrig/config.go b/testrig/config.go
index f68a8ffb7..9f17530c4 100644
--- a/testrig/config.go
+++ b/testrig/config.go
@@ -101,6 +101,7 @@ func testDefaults() config.Configuration {
},
InstanceSubscriptionsProcessFrom: "23:00", // 11pm,
InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
+ InstanceAllowBackdatingStatuses: true,
AccountsRegistrationOpen: true,
AccountsReasonRequired: true,