From 23fc70f4e68730b7eec91d58dac54ec00099ed8d Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:35:07 +0000 Subject: [feature] add support for receiving federated status edits (#3597) * add support for extracting Updated field from Statusable implementers * add support for status edits in the database, and update status dereferencer to handle them * remove unused AdditionalInfo{}.CreatedAt * remove unused AdditionalEmojiInfo{}.CreatedAt * update new mention creation to use status.UpdatedAt * remove mention.UpdatedAt, fixes related to NewULIDFromTime() change * add migration to remove Mention{}.UpdatedAt field * add migration to add the StatusEdit{} table * start adding tests, add delete function for status edits * add more of status edit migrations, fill in more of the necessary edit delete functionality * remove unused function * allow generating gotosocial compatible ulid via CLI with `go run ./cmd/gen-ulid` * add StatusEdit{} test models * fix new statusedits sql * use model instead of table name * actually remove the Mention.UpdatedAt field... * fix tests now new models are added, add more status edit DB tests * fix panic wording * add test for deleting status edits * don't automatically set `updated_at` field on updated statuses * flesh out more of the dereferencer status edit tests, ensure updated at field set on outgoing AS statuses * remove media_attachments.updated_at column * fix up more tests, further complete the dereferencer status edit tests * update more status serialization tests not expecting 'updated' AS property * gah!! json serialization tests!! * undo some gtscontext wrapping changes * more serialization test fixing :smiling_face_with_tear: * more test fixing, ensure the edit.status_id field is actually set :facepalm: * fix status edit test * grrr linter * add edited_at field to apimodel status * remove the choice of paging on the timeline public filtered test (otherwise it needs updating every time you add statuses ...) * ensure that status.updated_at always fits chronologically * fix more serialization tests ... * add more code comments * fix envparsing * update swagger file * properly handle media description changes during status edits * slight formatting tweak * code comment --- internal/processing/account/rss_test.go | 16 ++++++-- internal/processing/media/getfile.go | 4 +- internal/processing/media/unattach_test.go | 3 -- internal/processing/status/get.go | 2 +- internal/processing/stream/notification_test.go | 4 +- internal/processing/stream/statusupdate_test.go | 5 ++- internal/processing/timeline/public_test.go | 4 +- internal/processing/workers/util.go | 51 +++++++++++++++---------- 8 files changed, 53 insertions(+), 36 deletions(-) (limited to 'internal/processing') diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index e4706d3b7..5606151c2 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -70,7 +70,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(context.Background(), "the_mighty_zork") suite.NoError(err) - suite.EqualValues(1704878640, lastModified.Unix()) + suite.EqualValues(1730451600, lastModified.Unix()) feed, err := getFeed() suite.NoError(err) @@ -79,13 +79,23 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { Posts from @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork Posts from @the_mighty_zork@localhost:8080 - Wed, 10 Jan 2024 09:24:00 +0000 - Wed, 10 Jan 2024 09:24:00 +0000 + Fri, 01 Nov 2024 09:00:00 +0000 + Fri, 01 Nov 2024 09:00:00 +0000 http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp Avatar for @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork + + edited status + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + @the_mighty_zork@localhost:8080 made a new post: "this is the latest revision of the status, with a content-warning" + this is the latest revision of the status, with a content-warning

]]> + @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + Fri, 01 Nov 2024 09:00:00 +0000 + http://localhost:8080/@the_mighty_zork/feed.rss + HTML in post http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40 diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 6962601f2..11d8f7eb5 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -177,9 +177,7 @@ func (p *Processor) getAttachmentContent( } // Start preparing API content model. - apiContent := &apimodel.Content{ - ContentUpdated: attach.UpdatedAt, - } + apiContent := &apimodel.Content{} // Retrieve appropriate // size file from storage. diff --git a/internal/processing/media/unattach_test.go b/internal/processing/media/unattach_test.go index 051caa4d3..02d2c7077 100644 --- a/internal/processing/media/unattach_test.go +++ b/internal/processing/media/unattach_test.go @@ -20,7 +20,6 @@ package media_test import ( "context" "testing" - "time" "github.com/stretchr/testify/suite" ) @@ -42,8 +41,6 @@ func (suite *UnattachTestSuite) TestUnattachMedia() { dbAttachment, errWithCode := suite.db.GetAttachmentByID(ctx, a.ID) suite.NoError(errWithCode) - - suite.WithinDuration(dbAttachment.UpdatedAt, time.Now(), 1*time.Minute) suite.Empty(dbAttachment.StatusID) } diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 75a687db2..470b93a8f 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -67,7 +67,6 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account if errWithCode != nil { return nil, errWithCode } - return p.c.GetAPIStatus(ctx, requestingAccount, targetStatus) } @@ -106,5 +105,6 @@ func (p *Processor) SourceGet(ctx context.Context, requestingAccount *gtsmodel.A err = gtserror.Newf("error converting status: %w", err) return nil, gtserror.NewErrorInternalError(err) } + return statusSource, nil } diff --git a/internal/processing/stream/notification_test.go b/internal/processing/stream/notification_test.go index 169e4f5ce..5c89e1f40 100644 --- a/internal/processing/stream/notification_test.go +++ b/internal/processing/stream/notification_test.go @@ -79,8 +79,8 @@ func (suite *NotificationTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index b61a9c623..6bf5e436c 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -54,6 +54,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.Equal(`{ "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -90,8 +91,8 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, diff --git a/internal/processing/timeline/public_test.go b/internal/processing/timeline/public_test.go index 6b01c9849..ab8e33429 100644 --- a/internal/processing/timeline/public_test.go +++ b/internal/processing/timeline/public_test.go @@ -102,8 +102,8 @@ func (suite *PublicTestSuite) TestPublicTimelineGetHideFiltered() { requester = suite.testAccounts["local_account_1"] maxID = "" sinceID = "" - minID = "01F8MHAAY43M6RJ473VQFCVH36" // 1 before filteredStatus - limit = 10 + minID = "" + limit = 100 local = false filteredStatus = suite.testStatuses["admin_account_status_2"] filteredStatusFound = false diff --git a/internal/processing/workers/util.go b/internal/processing/workers/util.go index 62ea6c95c..b358dc951 100644 --- a/internal/processing/workers/util.go +++ b/internal/processing/workers/util.go @@ -75,6 +75,21 @@ func (u *utils) wipeStatus( } } + // Before handling media, ensure + // historic edits are populated. + if !status.EditsPopulated() { + var err error + + // Fetch all historical edits of status from database. + status.Edits, err = u.state.DB.GetStatusEditsByIDs( + gtscontext.SetBarebones(ctx), + status.EditIDs, + ) + if err != nil { + errs.Appendf("error getting status edits from database: %w", err) + } + } + // Either delete all attachments for this status, // or simply detach + clean them separately later. // @@ -83,20 +98,27 @@ func (u *utils) wipeStatus( // status immediately (in case of delete + redraft). if deleteAttachments { // todo:u.state.DB.DeleteAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if err := u.media.Delete(ctx, id); err != nil { errs.Appendf("error deleting media: %w", err) } } } else { // todo:u.state.DB.UnattachAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if _, err := u.media.Unattach(ctx, status.Account, id); err != nil { errs.Appendf("error unattaching media: %w", err) } } } + // Delete all historical edits of status. + if ids := status.EditIDs; len(ids) > 0 { + if err := u.state.DB.DeleteStatusEdits(ctx, ids); err != nil { + errs.Appendf("error deleting status edits: %w", err) + } + } + // Delete all mentions generated by this status. // todo:u.state.DB.DeleteMentionsForStatus for _, id := range status.MentionIDs { @@ -120,19 +142,20 @@ func (u *utils) wipeStatus( errs.Appendf("error deleting status faves: %w", err) } - if pollID := status.PollID; pollID != "" { + if id := status.PollID; id != "" { // Delete this poll by ID from the database. - if err := u.state.DB.DeletePollByID(ctx, pollID); err != nil { + if err := u.state.DB.DeletePollByID(ctx, id); err != nil { errs.Appendf("error deleting status poll: %w", err) } // Cancel any scheduled expiry task for poll. - _ = u.state.Workers.Scheduler.Cancel(pollID) + _ = u.state.Workers.Scheduler.Cancel(id) } // Get all boost of this status so that we can // delete those boosts + remove them from timelines. boosts, err := u.state.DB.GetStatusBoosts( + // We MUST set a barebones context here, // as depending on where it came from the // original BoostOf may already be gone. @@ -537,11 +560,7 @@ func (u *utils) requestFave( } // Create + store new interaction request. - req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusFaveToInteractionRequest(fave) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -584,11 +603,7 @@ func (u *utils) requestReply( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(reply) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -631,11 +646,7 @@ func (u *utils) requestAnnounce( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(boost) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } -- cgit v1.2.3