diff options
| author |  kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> | 2024-12-05 13:35:07 +0000 |
|---|---|---|
| committer |  GitHub <noreply@github.com> | 2024-12-05 13:35:07 +0000 |
| commit | 23fc70f4e68730b7eec91d58dac54ec00099ed8d (patch) | |
| tree | 9ab22b85d1d2c9c7ff2db9371ab2dc752c085f07 /internal | |
| parent | [feature] unending polls (#3592) (diff) | |
| download | gotosocial-23fc70f4e68730b7eec91d58dac54ec00099ed8d.tar.xz | |
[feature] add support for receiving federated status edits (#3597)
* add support for extracting Updated field from Statusable implementers
* add support for status edits in the database, and update status dereferencer to handle them
* remove unused AdditionalInfo{}.CreatedAt
* remove unused AdditionalEmojiInfo{}.CreatedAt
* update new mention creation to use status.UpdatedAt
* remove mention.UpdatedAt, fixes related to NewULIDFromTime() change
* add migration to remove Mention{}.UpdatedAt field
* add migration to add the StatusEdit{} table
* start adding tests, add delete function for status edits
* add more of status edit migrations, fill in more of the necessary edit delete functionality
* remove unused function
* allow generating gotosocial compatible ulid via CLI with `go run ./cmd/gen-ulid`
* add StatusEdit{} test models
* fix new statusedits sql
* use model instead of table name
* actually remove the Mention.UpdatedAt field...
* fix tests now new models are added, add more status edit DB tests
* fix panic wording
* add test for deleting status edits
* don't automatically set `updated_at` field on updated statuses
* flesh out more of the dereferencer status edit tests, ensure updated at field set on outgoing AS statuses
* remove media_attachments.updated_at column
* fix up more tests, further complete the dereferencer status edit tests
* update more status serialization tests not expecting 'updated' AS property
* gah!! json serialization tests!!
* undo some gtscontext wrapping changes
* more serialization test fixing :smiling_face_with_tear:
* more test fixing, ensure the edit.status_id field is actually set :facepalm:
* fix status edit test
* grrr linter
* add edited_at field to apimodel status
* remove the choice of paging on the timeline public filtered test (otherwise it needs updating every time you add statuses ...)
* ensure that status.updated_at always fits chronologically
* fix more serialization tests ...
* add more code comments
* fix envparsing
* update swagger file
* properly handle media description changes during status edits
* slight formatting tweak
* code comment
Diffstat (limited to 'internal')
81 files changed, 2250 insertions, 590 deletions
diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index a721fa997..1f08fde37 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -25,8 +25,11 @@ import ( // IsActivityable returns whether AS vocab type name is acceptable as Activityable. func IsActivityable(typeName string) bool { - return isActivity(typeName) || - isIntransitiveActivity(typeName) + return isActivity(typeName) + // See interfaces_test.go comment + // about intransitive activities: + // + // || isIntransitiveActivity(typeName) } // ToActivityable safely tries to cast vocab.Type as Activityable, also checking for expected AS type names. @@ -184,6 +187,7 @@ type Accountable interface { WithEndpoints WithTag WithPublished + WithUpdated } // Statusable represents the minimum activitypub interface for representing a 'status'. @@ -196,6 +200,7 @@ type Statusable interface { WithName WithInReplyTo WithPublished + WithUpdated WithURL WithAttributedTo WithTo diff --git a/internal/ap/interfaces_test.go b/internal/ap/interfaces_test.go new file mode 100644 index 000000000..d3248cb1d --- /dev/null +++ b/internal/ap/interfaces_test.go @@ -0,0 +1,93 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package ap_test + +import ( + "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" +) + +var ( + // NOTE: the below aren't actually tests that are run, + // we just move them into an _test.go file to declutter + // the main interfaces.go file, which is already long. + + // Compile-time checks for Activityable interface methods. + _ ap.Activityable = (vocab.ActivityStreamsAccept)(nil) + _ ap.Activityable = (vocab.ActivityStreamsTentativeAccept)(nil) + _ ap.Activityable = (vocab.ActivityStreamsAdd)(nil) + _ ap.Activityable = (vocab.ActivityStreamsCreate)(nil) + _ ap.Activityable = (vocab.ActivityStreamsDelete)(nil) + _ ap.Activityable = (vocab.ActivityStreamsFollow)(nil) + _ ap.Activityable = (vocab.ActivityStreamsIgnore)(nil) + _ ap.Activityable = (vocab.ActivityStreamsJoin)(nil) + _ ap.Activityable = (vocab.ActivityStreamsLeave)(nil) + _ ap.Activityable = (vocab.ActivityStreamsLike)(nil) + _ ap.Activityable = (vocab.ActivityStreamsOffer)(nil) + _ ap.Activityable = (vocab.ActivityStreamsInvite)(nil) + _ ap.Activityable = (vocab.ActivityStreamsReject)(nil) + _ ap.Activityable = (vocab.ActivityStreamsTentativeReject)(nil) + _ ap.Activityable = (vocab.ActivityStreamsRemove)(nil) + _ ap.Activityable = (vocab.ActivityStreamsUndo)(nil) + _ ap.Activityable = (vocab.ActivityStreamsUpdate)(nil) + _ ap.Activityable = (vocab.ActivityStreamsView)(nil) + _ ap.Activityable = (vocab.ActivityStreamsListen)(nil) + _ ap.Activityable = (vocab.ActivityStreamsRead)(nil) + _ ap.Activityable = (vocab.ActivityStreamsMove)(nil) + _ ap.Activityable = (vocab.ActivityStreamsAnnounce)(nil) + _ ap.Activityable = (vocab.ActivityStreamsBlock)(nil) + _ ap.Activityable = (vocab.ActivityStreamsFlag)(nil) + _ ap.Activityable = (vocab.ActivityStreamsDislike)(nil) + + // the below intransitive activities don't fit the interface definition because they're + // missing an attached object (as the activity itself contains the details), but we don't + // actually end up using them so it's simpler to just comment them out and not have to do + // a WithObject{} interface check on every single incoming activity: + // + // _ Activityable = (vocab.ActivityStreamsArrive)(nil) + // _ Activityable = (vocab.ActivityStreamsTravel)(nil) + // _ Activityable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for Accountable interface methods. + _ ap.Accountable = (vocab.ActivityStreamsPerson)(nil) + _ ap.Accountable = (vocab.ActivityStreamsApplication)(nil) + _ ap.Accountable = (vocab.ActivityStreamsOrganization)(nil) + _ ap.Accountable = (vocab.ActivityStreamsService)(nil) + _ ap.Accountable = (vocab.ActivityStreamsGroup)(nil) + + // Compile-time checks for Statusable interface methods. + _ ap.Statusable = (vocab.ActivityStreamsArticle)(nil) + _ ap.Statusable = (vocab.ActivityStreamsDocument)(nil) + _ ap.Statusable = (vocab.ActivityStreamsImage)(nil) + _ ap.Statusable = (vocab.ActivityStreamsVideo)(nil) + _ ap.Statusable = (vocab.ActivityStreamsNote)(nil) + _ ap.Statusable = (vocab.ActivityStreamsPage)(nil) + _ ap.Statusable = (vocab.ActivityStreamsEvent)(nil) + _ ap.Statusable = (vocab.ActivityStreamsPlace)(nil) + _ ap.Statusable = (vocab.ActivityStreamsProfile)(nil) + _ ap.Statusable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for Pollable interface methods. + _ ap.Pollable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for PollOptionable interface methods. + _ ap.PollOptionable = (vocab.ActivityStreamsNote)(nil) + + // Compile-time checks for Acceptable interface methods. + _ ap.Acceptable = (vocab.ActivityStreamsAccept)(nil) +) diff --git a/internal/ap/properties.go b/internal/ap/properties.go index 38e58ebc0..0a2564168 100644 --- a/internal/ap/properties.go +++ b/internal/ap/properties.go @@ -408,6 +408,25 @@ func SetPublished(with WithPublished, published time.Time) { publishProp.Set(published) } +// GetUpdated returns the time contained in the Updated property of 'with'. +func GetUpdated(with WithUpdated) time.Time { + updateProp := with.GetActivityStreamsUpdated() + if updateProp == nil || !updateProp.IsXMLSchemaDateTime() { + return time.Time{} + } + return updateProp.Get() +} + +// SetUpdated sets the given time on the Updated property of 'with'. +func SetUpdated(with WithUpdated, updated time.Time) { + updateProp := with.GetActivityStreamsUpdated() + if updateProp == nil { + updateProp = streams.NewActivityStreamsUpdatedProperty() + with.SetActivityStreamsUpdated(updateProp) + } + updateProp.Set(updated) +} + // GetEndTime returns the time contained in the EndTime property of 'with'. func GetEndTime(with WithEndTime) time.Time { endTimeProp := with.GetActivityStreamsEndTime() diff --git a/internal/api/activitypub/users/outboxget_test.go b/internal/api/activitypub/users/outboxget_test.go index cba1ef31d..2de3b0456 100644 --- a/internal/api/activitypub/users/outboxget_test.go +++ b/internal/api/activitypub/users/outboxget_test.go @@ -82,7 +82,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { "@context": "https://www.w3.org/ns/activitystreams", "first": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40", "id": "http://localhost:8080/users/the_mighty_zork/outbox", - "totalItems": 8, + "totalItems": 9, "type": "OrderedCollection" }`, dst.String()) @@ -145,6 +145,14 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { { "actor": "http://localhost:8080/users/the_mighty_zork", "cc": "http://localhost:8080/users/the_mighty_zork/followers", + "id": "http://localhost:8080/users/the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR/activity#Create", + "object": "http://localhost:8080/users/the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Create" + }, + { + "actor": "http://localhost:8080/users/the_mighty_zork", + "cc": "http://localhost:8080/users/the_mighty_zork/followers", "id": "http://localhost:8080/users/the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40/activity#Create", "object": "http://localhost:8080/users/the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40", "to": "https://www.w3.org/ns/activitystreams#Public", @@ -160,8 +168,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { } ], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", - "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01HH9KYNQPA416TNJ53NSATP40", - "totalItems": 8, + "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01JDPZC707CKDN8N4QVWM4Z1NR", + "totalItems": 9, "type": "OrderedCollectionPage" }`, dst.String()) @@ -224,7 +232,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", "orderedItems": [], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", - "totalItems": 8, + "totalItems": 9, "type": "OrderedCollectionPage" }`, dst.String()) diff --git a/internal/api/client/accounts/accountverify_test.go b/internal/api/client/accounts/accountverify_test.go index 3f67cdefb..df5c21389 100644 --- a/internal/api/client/accounts/accountverify_test.go +++ b/internal/api/client/accounts/accountverify_test.go @@ -97,7 +97,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.webp", apimodelAccount.HeaderStatic) suite.Equal(2, apimodelAccount.FollowersCount) suite.Equal(2, apimodelAccount.FollowingCount) - suite.Equal(8, apimodelAccount.StatusesCount) + suite.Equal(9, apimodelAccount.StatusesCount) suite.EqualValues(apimodel.VisibilityPublic, apimodelAccount.Source.Privacy) suite.Equal(testAccount.Settings.Language, apimodelAccount.Source.Language) suite.Equal(testAccount.NoteRaw, apimodelAccount.Source.Note) diff --git a/internal/api/client/admin/accountsgetv2_test.go b/internal/api/client/admin/accountsgetv2_test.go index 77ca135eb..489a245d0 100644 --- a/internal/api/client/admin/accountsgetv2_test.go +++ b/internal/api/client/admin/accountsgetv2_test.go @@ -99,8 +99,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -262,8 +262,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -403,8 +403,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/admin/reportsget_test.go b/internal/api/client/admin/reportsget_test.go index 12a307836..255e32c3b 100644 --- a/internal/api/client/admin/reportsget_test.go +++ b/internal/api/client/admin/reportsget_test.go @@ -186,8 +186,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -232,8 +232,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -414,8 +414,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -473,8 +473,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -485,6 +485,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -521,8 +522,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -667,8 +668,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -726,8 +727,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -738,6 +739,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -774,8 +776,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -920,8 +922,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -979,8 +981,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -991,6 +993,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1027,8 +1030,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, diff --git a/internal/api/client/exports/exports_test.go b/internal/api/client/exports/exports_test.go index 62997af5d..13f7bea05 100644 --- a/internal/api/client/exports/exports_test.go +++ b/internal/api/client/exports/exports_test.go @@ -229,7 +229,7 @@ Cool Ass Posters From This Instance,admin@localhost:8080 "media_storage": "", "followers_count": 2, "following_count": 2, - "statuses_count": 8, + "statuses_count": 9, "lists_count": 1, "blocks_count": 0, "mutes_count": 0 diff --git a/internal/api/client/instance/instancepatch_test.go b/internal/api/client/instance/instancepatch_test.go index efcb3762f..f126ee6ae 100644 --- a/internal/api/client/instance/instancepatch_test.go +++ b/internal/api/client/instance/instancepatch_test.go @@ -155,7 +155,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -296,7 +296,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -437,7 +437,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -629,7 +629,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -792,7 +792,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/attachment/original/`+instanceAccount.AvatarMediaAttachment.ID+`.gif",`+` @@ -974,7 +974,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", diff --git a/internal/api/client/mutes/mutesget_test.go b/internal/api/client/mutes/mutesget_test.go index fa52c9aa9..13d826398 100644 --- a/internal/api/client/mutes/mutesget_test.go +++ b/internal/api/client/mutes/mutesget_test.go @@ -148,7 +148,7 @@ func (suite *MutesTestSuite) TestIndefinitelyMutedAccountSerializesMuteExpiratio // Fetch all muted accounts for the logged-in account. // The expected body contains `"mute_expires_at":null`. - _, err = suite.getMutedAccounts(http.StatusOK, `[{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"discoverable":true,"bot":false,"created_at":"2021-09-26T10:52:36.000Z","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.webp","header_static":"http://localhost:8080/assets/default_header.webp","header_description":"Flat gray background (default header).","followers_count":0,"following_count":0,"statuses_count":3,"last_status_at":"2021-09-11","emojis":[],"fields":[],"mute_expires_at":null}]`) + _, err = suite.getMutedAccounts(http.StatusOK, `[{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"discoverable":true,"bot":false,"created_at":"2021-09-26T10:52:36.000Z","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.webp","header_static":"http://localhost:8080/assets/default_header.webp","header_description":"Flat gray background (default header).","followers_count":0,"following_count":0,"statuses_count":4,"last_status_at":"2024-11-01","emojis":[],"fields":[],"mute_expires_at":null}]`) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/api/client/reports/reportget_test.go b/internal/api/client/reports/reportget_test.go index 8c9dfa1e5..afbcb2e28 100644 --- a/internal/api/client/reports/reportget_test.go +++ b/internal/api/client/reports/reportget_test.go @@ -130,8 +130,8 @@ func (suite *ReportGetTestSuite) TestGetReport1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/reports/reportsget_test.go b/internal/api/client/reports/reportsget_test.go index 0eb66e778..b5988e331 100644 --- a/internal/api/client/reports/reportsget_test.go +++ b/internal/api/client/reports/reportsget_test.go @@ -156,8 +156,8 @@ func (suite *ReportsGetTestSuite) TestGetReports() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -247,8 +247,8 @@ func (suite *ReportsGetTestSuite) TestGetReports4() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -322,8 +322,8 @@ func (suite *ReportsGetTestSuite) TestGetReports6() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -381,8 +381,8 @@ func (suite *ReportsGetTestSuite) TestGetReports7() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/search/searchget_test.go b/internal/api/client/search/searchget_test.go index ab4f46689..2c4efd19c 100644 --- a/internal/api/client/search/searchget_test.go +++ b/internal/api/client/search/searchget_test.go @@ -916,7 +916,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() { } suite.Len(searchResult.Accounts, 5) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } @@ -959,7 +959,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() { } suite.Len(searchResult.Accounts, 2) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } @@ -1002,7 +1002,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() { } suite.Len(searchResult.Accounts, 0) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } diff --git a/internal/api/client/statuses/statusboost_test.go b/internal/api/client/statuses/statusboost_test.go index 1f92d8b3f..51b7d7652 100644 --- a/internal/api/client/statuses/statusboost_test.go +++ b/internal/api/client/statuses/statusboost_test.go @@ -100,6 +100,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 0, @@ -145,6 +146,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() { "card": null, "content": "hello world! #welcome ! first post on the instance :rainbow: !", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [ { "category": "reactions", @@ -280,6 +282,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -329,6 +332,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() { "card": null, "content": "hi!", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -494,6 +498,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -539,6 +544,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() { "card": null, "content": "<p>Hi <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span>, can I reply?</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go index 5f5386dd5..227e7d83e 100644 --- a/internal/api/client/statuses/statuscreate_test.go +++ b/internal/api/client/statuses/statuscreate_test.go @@ -102,6 +102,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() { "card": null, "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -187,6 +188,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicy() { "card": null, "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -282,6 +284,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicyJSON() { "card": null, "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -407,6 +410,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() { "card": null, "content": "<h1>Title</h1><h2>Smaller title</h2><p>This is a post written in <a href=\"https://www.markdownguide.org/\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">markdown</a></p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -490,6 +494,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() { "card": null, "content": "<p>hello <span class=\"h-card\"><a href=\"https://unknown-instance.com/@brand_new_person\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>brand_new_person</span></a></span></p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -567,6 +572,7 @@ func (suite *StatusCreateTestSuite) TestPostStatusWithLinksAndTags() { "card": null, "content": "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -650,6 +656,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() { "card": null, "content": "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [ { "category": "reactions", @@ -747,6 +754,7 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() { "card": null, "content": "<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> this reply should work!</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -829,6 +837,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() { "card": null, "content": "<p>here's an image attachment</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -933,6 +942,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithNoncanonicalLanguageTag "card": null, "content": "<p>English? what's English? i speak American</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -1007,6 +1017,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollForm() { "card": null, "content": "<p>this is a status with a poll!</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -1103,6 +1114,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollJSON() { "card": null, "content": "<p>this is a status with a poll!</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, diff --git a/internal/api/client/statuses/statusfave_test.go b/internal/api/client/statuses/statusfave_test.go index bd81c0cf9..8851b4d58 100644 --- a/internal/api/client/statuses/statusfave_test.go +++ b/internal/api/client/statuses/statusfave_test.go @@ -105,6 +105,7 @@ func (suite *StatusFaveTestSuite) TestPostFave() { "card": null, "content": "🐕🐕🐕🐕🐕", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 1, @@ -228,6 +229,7 @@ func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() { "card": null, "content": "<p>Hi <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span>, can I reply?</p>", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 1, diff --git a/internal/api/client/statuses/statushistory_test.go b/internal/api/client/statuses/statushistory_test.go index aea666dbb..3878f54e4 100644 --- a/internal/api/client/statuses/statushistory_test.go +++ b/internal/api/client/statuses/statushistory_test.go @@ -116,8 +116,8 @@ func (suite *StatusHistoryTestSuite) TestGetHistory() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/api/client/statuses/statusmute_test.go b/internal/api/client/statuses/statusmute_test.go index 384761fc6..66bd4a420 100644 --- a/internal/api/client/statuses/statusmute_test.go +++ b/internal/api/client/statuses/statusmute_test.go @@ -91,6 +91,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { suite.Equal(`{ "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -134,8 +135,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -178,6 +179,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { suite.Equal(`{ "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -221,8 +223,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/api/model/content.go b/internal/api/model/content.go index 7da389ed1..5af81b11b 100644 --- a/internal/api/model/content.go +++ b/internal/api/model/content.go @@ -19,7 +19,6 @@ package model import ( "io" - "time" "github.com/superseriousbusiness/gotosocial/internal/storage" ) @@ -30,8 +29,6 @@ type Content struct { ContentType string // ContentLength in bytes ContentLength int64 - // Time when the content was last updated. - ContentUpdated time.Time // Actual content Content io.ReadCloser // Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL) diff --git a/internal/api/model/status.go b/internal/api/model/status.go index c29ab3e82..724134b77 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -29,6 +29,10 @@ type Status struct { // The date when this status was created (ISO 8601 Datetime). // example: 2021-07-30T09:20:25+00:00 CreatedAt string `json:"created_at"` + // Timestamp of when the status was last edited (ISO 8601 Datetime). + // example: 2021-07-30T09:20:25+00:00 + // nullable: true + EditedAt *string `json:"edited_at"` // ID of the status being replied to. // example: 01FBVD42CQ3ZEEVMW180SBX03B // nullable: true diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a4f9f2044..1a66fcd6b 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -105,6 +105,7 @@ func (c *Caches) Init() { c.initStatus() c.initStatusBookmark() c.initStatusBookmarkIDs() + c.initStatusEdit() c.initStatusFave() c.initStatusFaveIDs() c.initTag() diff --git a/internal/cache/db.go b/internal/cache/db.go index aac11236a..dc47bc31c 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -226,6 +226,9 @@ type DBCaches struct { // StatusBookmarkIDs provides access to the status bookmark IDs list database cache. StatusBookmarkIDs SliceCache[string] + // StatusEdit provides access to the gtsmodel StatusEdit database cache. + StatusEdit StructCache[*gtsmodel.StatusEdit] + // StatusFave provides access to the gtsmodel StatusFave database cache. StatusFave StructCache[*gtsmodel.StatusFave] @@ -1385,6 +1388,38 @@ func (c *Caches) initStatusBookmarkIDs() { c.DB.StatusBookmarkIDs.Init(0, cap) } +func (c *Caches) initStatusEdit() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofStatusEdit(), // model in-mem size. + config.GetCacheStatusEditMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(s1 *gtsmodel.StatusEdit) *gtsmodel.StatusEdit { + s2 := new(gtsmodel.StatusEdit) + *s2 = *s1 + + // Don't include ptr fields that + // will be populated separately. + s2.Attachments = nil + + return s2 + } + + c.DB.StatusEdit.Init(structr.CacheConfig[*gtsmodel.StatusEdit]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "StatusID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + Copy: copyF, + Invalidate: c.OnInvalidateStatusEdit, + }) +} + func (c *Caches) initStatusFave() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 9b42e88f6..42d7b7399 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -273,6 +273,11 @@ func (c *Caches) OnInvalidateStatusBookmark(bookmark *gtsmodel.StatusBookmark) { c.DB.StatusBookmarkIDs.Invalidate(bookmark.StatusID) } +func (c *Caches) OnInvalidateStatusEdit(edit *gtsmodel.StatusEdit) { + // Invalidate cache of related status model. + c.DB.Status.Invalidate("ID", edit.StatusID) +} + func (c *Caches) OnInvalidateStatusFave(fave *gtsmodel.StatusFave) { // Invalidate status fave ID list for this status. c.DB.StatusFaveIDs.Invalidate(fave.StatusID) diff --git a/internal/cache/size.go b/internal/cache/size.go index 26f4096ed..988755099 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -505,7 +505,6 @@ func sizeofMedia() uintptr { URL: exampleURI, RemoteURL: exampleURI, CreatedAt: exampleTime, - UpdatedAt: exampleTime, Type: gtsmodel.FileTypeImage, AccountID: exampleID, Description: exampleText, @@ -532,7 +531,6 @@ func sizeofMention() uintptr { ID: exampleURI, StatusID: exampleURI, CreatedAt: exampleTime, - UpdatedAt: exampleTime, OriginAccountID: exampleURI, OriginAccountURI: exampleURI, TargetAccountID: exampleID, @@ -674,6 +672,23 @@ func sizeofStatusBookmark() uintptr { })) } +func sizeofStatusEdit() uintptr { + return uintptr(size.Of(>smodel.StatusEdit{ + ID: exampleID, + Content: exampleText, + ContentWarning: exampleUsername, // similar length + Text: exampleText, + Language: "en", + Sensitive: func() *bool { ok := false; return &ok }(), + AttachmentIDs: []string{exampleID, exampleID, exampleID}, + Attachments: nil, + PollOptions: []string{exampleTextSmall, exampleTextSmall, exampleTextSmall, exampleTextSmall}, + PollVotes: []int{69, 420, 1337, 1969}, + StatusID: exampleID, + CreatedAt: exampleTime, + })) +} + func sizeofStatusFave() uintptr { return uintptr(size.Of(>smodel.StatusFave{ ID: exampleID, diff --git a/internal/config/config.go b/internal/config/config.go index 2e3ad8ec1..413743409 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -238,6 +238,7 @@ type CacheConfiguration struct { StatusMemRatio float64 `name:"status-mem-ratio"` StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"` StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"` + StatusEditMemRatio float64 `name:"status-edit-mem-ratio"` StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"` TagMemRatio float64 `name:"tag-mem-ratio"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 9b45002d0..f77c5c456 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -199,6 +199,7 @@ var Defaults = Configuration{ StatusMemRatio: 5, StatusBookmarkMemRatio: 0.5, StatusBookmarkIDsMemRatio: 2, + StatusEditMemRatio: 2, StatusFaveMemRatio: 2, StatusFaveIDsMemRatio: 3, TagMemRatio: 2, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a35622f8e..543292ebe 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3912,6 +3912,31 @@ func GetCacheStatusBookmarkIDsMemRatio() float64 { return global.GetCacheStatusB // SetCacheStatusBookmarkIDsMemRatio safely sets the value for global configuration 'Cache.StatusBookmarkIDsMemRatio' field func SetCacheStatusBookmarkIDsMemRatio(v float64) { global.SetCacheStatusBookmarkIDsMemRatio(v) } +// GetCacheStatusEditMemRatio safely fetches the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) GetCacheStatusEditMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.StatusEditMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheStatusEditMemRatio safely sets the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) SetCacheStatusEditMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.StatusEditMemRatio = v + st.reloadToViper() +} + +// CacheStatusEditMemRatioFlag returns the flag name for the 'Cache.StatusEditMemRatio' field +func CacheStatusEditMemRatioFlag() string { return "cache-status-edit-mem-ratio" } + +// GetCacheStatusEditMemRatio safely fetches the value for global configuration 'Cache.StatusEditMemRatio' field +func GetCacheStatusEditMemRatio() float64 { return global.GetCacheStatusEditMemRatio() } + +// SetCacheStatusEditMemRatio safely sets the value for global configuration 'Cache.StatusEditMemRatio' field +func SetCacheStatusEditMemRatio(v float64) { global.SetCacheStatusEditMemRatio(v) } + // GetCacheStatusFaveMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveMemRatio' field func (st *ConfigState) GetCacheStatusFaveMemRatio() (v float64) { st.mutex.RLock() diff --git a/internal/db/bundb/account_test.go b/internal/db/bundb/account_test.go index 7dcc0f9e7..879250408 100644 --- a/internal/db/bundb/account_test.go +++ b/internal/db/bundb/account_test.go @@ -46,7 +46,7 @@ type AccountTestSuite struct { func (suite *AccountTestSuite) TestGetAccountStatuses() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, false, false, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 8) + suite.Len(statuses, 9) } func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { @@ -69,7 +69,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { if err != nil { suite.FailNow(err.Error()) } - suite.Len(statuses, 2) + suite.Len(statuses, 3) // try to get the last page (should be empty) statuses, err = suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 3, false, false, statuses[len(statuses)-1].ID, "", false, false) @@ -80,13 +80,13 @@ func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesAndReblogs() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, true, true, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 7) + suite.Len(statuses, 8) } func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesAndReblogsPublicOnly() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, true, true, "", "", false, true) suite.NoError(err) - suite.Len(statuses, 3) + suite.Len(statuses, 4) } // populateTestStatus adds mandatory fields to a partially populated status. @@ -173,7 +173,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesExcludesSelfR testAccount := suite.testAccounts["local_account_1"] statuses, err := suite.db.GetAccountStatuses(context.Background(), testAccount.ID, 20, true, true, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 8) + suite.Len(statuses, 9) for _, status := range statuses { if status.InReplyToID != "" && status.InReplyToAccountID != testAccount.ID { suite.FailNowf("", "Status with ID %s is a non-self reply and should have been excluded", status.ID) diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index 56159dc25..e20aab765 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -114,7 +114,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 25) + suite.Len(s, 28) } func (suite *BasicTestSuite) TestGetAllNotNull() { diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 70132fe58..cf612fd2e 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -81,6 +81,7 @@ type DBService struct { db.SinBinStatus db.Status db.StatusBookmark + db.StatusEdit db.StatusFave db.Tag db.Thread @@ -272,6 +273,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { db: db, state: state, }, + StatusEdit: &statusEditDB{ + db: db, + state: state, + }, StatusFave: &statusFaveDB{ db: db, state: state, diff --git a/internal/db/bundb/bundb_test.go b/internal/db/bundb/bundb_test.go index e976199e4..2fcf61aed 100644 --- a/internal/db/bundb/bundb_test.go +++ b/internal/db/bundb/bundb_test.go @@ -57,6 +57,7 @@ type BunDBStandardTestSuite struct { testPolls map[string]*gtsmodel.Poll testPollVotes map[string]*gtsmodel.PollVote testInteractionRequests map[string]*gtsmodel.InteractionRequest + testStatusEdits map[string]*gtsmodel.StatusEdit } func (suite *BunDBStandardTestSuite) SetupSuite() { @@ -83,6 +84,7 @@ func (suite *BunDBStandardTestSuite) SetupSuite() { suite.testPolls = testrig.NewTestPolls() suite.testPollVotes = testrig.NewTestPollVotes() suite.testInteractionRequests = testrig.NewTestInteractionRequests() + suite.testStatusEdits = testrig.NewTestStatusEdits() } func (suite *BunDBStandardTestSuite) SetupTest() { diff --git a/internal/db/bundb/instance_test.go b/internal/db/bundb/instance_test.go index 4b8ec9962..1364bacc2 100644 --- a/internal/db/bundb/instance_test.go +++ b/internal/db/bundb/instance_test.go @@ -47,13 +47,13 @@ func (suite *InstanceTestSuite) TestCountInstanceUsersRemote() { func (suite *InstanceTestSuite) TestCountInstanceStatuses() { count, err := suite.db.CountInstanceStatuses(context.Background(), config.GetHost()) suite.NoError(err) - suite.Equal(19, count) + suite.Equal(21, count) } func (suite *InstanceTestSuite) TestCountInstanceStatusesRemote() { count, err := suite.db.CountInstanceStatuses(context.Background(), "fossbros-anonymous.io") suite.NoError(err) - suite.Equal(3, count) + suite.Equal(4, count) } func (suite *InstanceTestSuite) TestCountInstanceDomains() { diff --git a/internal/db/bundb/interaction_test.go b/internal/db/bundb/interaction_test.go index 37684f18c..1eb8154c1 100644 --- a/internal/db/bundb/interaction_test.go +++ b/internal/db/bundb/interaction_test.go @@ -59,11 +59,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this reply. - req, err := typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusToInteractionRequest(reply) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } @@ -90,11 +86,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this boost. - req, err := typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusToInteractionRequest(boost) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } @@ -121,11 +113,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this fave. - req, err := typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusFaveToInteractionRequest(fave) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index 453ad856a..09c8188f0 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -104,12 +104,6 @@ func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttach } func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error { - media.UpdatedAt = time.Now() - if len(columns) > 0 { - // If we're updating by column, ensure "updated_at" is included. - columns = append(columns, "updated_at") - } - return m.state.Caches.DB.Media.Store(media, func() error { _, err := m.db.NewUpdate(). Model(media). diff --git a/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go b/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go index 82c2b4016..a3fb8675e 100644 --- a/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go +++ b/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go @@ -93,11 +93,7 @@ func init() { // For each currently pending status, check whether it's a reply or // a boost, and insert a corresponding interaction request into the db. for _, pendingStatus := range pendingStatuses { - req, err := typeutils.StatusToInteractionRequest(ctx, pendingStatus) - if err != nil { - return err - } - + req := typeutils.StatusToInteractionRequest(pendingStatus) if _, err := tx. NewInsert(). Model(req). @@ -125,10 +121,7 @@ func init() { } for _, pendingFave := range pendingFaves { - req, err := typeutils.StatusFaveToInteractionRequest(ctx, pendingFave) - if err != nil { - return err - } + req := typeutils.StatusFaveToInteractionRequest(pendingFave) if _, err := tx. NewInsert(). diff --git a/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go b/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go new file mode 100644 index 000000000..bd72dc109 --- /dev/null +++ b/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go @@ -0,0 +1,57 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package migrations + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + // Check for 'updated_at' column on mentions table, else return. + exists, err := doesColumnExist(ctx, tx, "mentions", "updated_at") + if err != nil { + return err + } else if !exists { + return nil + } + + // Remove 'updated_at' column. + _, err = tx.NewDropColumn(). + Model((*gtsmodel.Mention)(nil)). + Column("updated_at"). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits.go b/internal/db/bundb/migrations/20241113152126_add_status_edits.go new file mode 100644 index 000000000..aa0b0d4b9 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits.go @@ -0,0 +1,67 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package migrations + +import ( + "context" + "reflect" + + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20241113152126_add_status_edits" + + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + statusType := reflect.TypeOf((*gtsmodel.Status)(nil)) + + // Generate new Status.EditIDs column definition from bun. + colDef, err := getBunColumnDef(tx, statusType, "EditIDs") + if err != nil { + return err + } + + // Add EditIDs column to Status table. + _, err = tx.NewAddColumn(). + Model((*gtsmodel.Status)(nil)). + ColumnExpr(colDef). + Exec(ctx) + if err != nil { + return err + } + + // Create the main StatusEdits table. + _, err = tx.NewCreateTable(). + IfNotExists(). + Model((*gtsmodel.StatusEdit)(nil)). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go b/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go new file mode 100644 index 000000000..1b7d93f70 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go @@ -0,0 +1,97 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import ( + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// Status represents a user-created 'post' or 'status' in the database, either remote or local +type Status struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` // when was item (remote) last fetched. + PinnedAt time.Time `bun:"type:timestamptz,nullzero"` // Status was pinned by owning account at this time. + URI string `bun:",unique,nullzero,notnull"` // activitypub URI of this status + URL string `bun:",nullzero"` // web url for viewing this status + Content string `bun:""` // content of this status; likely html-formatted but not guaranteed + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of any media attachments associated with this status + Attachments []*gtsmodel.MediaAttachment `bun:"attached_media,rel:has-many"` // Attachments corresponding to attachmentIDs + TagIDs []string `bun:"tags,array"` // Database IDs of any tags used in this status + Tags []*gtsmodel.Tag `bun:"attached_tags,m2m:status_to_tags"` // Tags corresponding to tagIDs. https://bun.uptrace.dev/guide/relations.html#many-to-many-relation + MentionIDs []string `bun:"mentions,array"` // Database IDs of any mentions in this status + Mentions []*gtsmodel.Mention `bun:"attached_mentions,rel:has-many"` // Mentions corresponding to mentionIDs + EmojiIDs []string `bun:"emojis,array"` // Database IDs of any emojis used in this status + Emojis []*gtsmodel.Emoji `bun:"attached_emojis,m2m:status_to_emojis"` // Emojis corresponding to emojiIDs. https://bun.uptrace.dev/guide/relations.html#many-to-many-relation + Local *bool `bun:",nullzero,notnull,default:false"` // is this status from a local account? + AccountID string `bun:"type:CHAR(26),nullzero,notnull"` // which account posted this status? + Account *gtsmodel.Account `bun:"rel:belongs-to"` // account corresponding to accountID + AccountURI string `bun:",nullzero,notnull"` // activitypub uri of the owner of this status + InReplyToID string `bun:"type:CHAR(26),nullzero"` // id of the status this status replies to + InReplyToURI string `bun:",nullzero"` // activitypub uri of the status this status is a reply to + InReplyToAccountID string `bun:"type:CHAR(26),nullzero"` // id of the account that this status replies to + InReplyTo *Status `bun:"-"` // status corresponding to inReplyToID + InReplyToAccount *gtsmodel.Account `bun:"rel:belongs-to"` // account corresponding to inReplyToAccountID + BoostOfID string `bun:"type:CHAR(26),nullzero"` // id of the status this status is a boost of + BoostOfURI string `bun:"-"` // URI of the status this status is a boost of; field not inserted in the db, just for dereferencing purposes. + BoostOfAccountID string `bun:"type:CHAR(26),nullzero"` // id of the account that owns the boosted status + BoostOf *Status `bun:"-"` // status that corresponds to boostOfID + BoostOfAccount *gtsmodel.Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID + ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null + EditIDs []string `bun:"edits,array"` // + Edits []*StatusEdit `bun:"-"` // + PollID string `bun:"type:CHAR(26),nullzero"` // + Poll *gtsmodel.Poll `bun:"-"` // + ContentWarning string `bun:",nullzero"` // cw string for this status + Visibility Visibility `bun:",nullzero,notnull"` // visibility entry for this status + Sensitive *bool `bun:",nullzero,notnull,default:false"` // mark the status as sensitive? + Language string `bun:",nullzero"` // what language is this status written in? + CreatedWithApplicationID string `bun:"type:CHAR(26),nullzero"` // Which application was used to create this status? + CreatedWithApplication *gtsmodel.Application `bun:"rel:belongs-to"` // application corresponding to createdWithApplicationID + ActivityStreamsType string `bun:",nullzero,notnull"` // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!. + Text string `bun:""` // Original text of the status without formatting + Federated *bool `bun:",notnull"` // This status will be federated beyond the local timeline(s) + InteractionPolicy *gtsmodel.InteractionPolicy `bun:""` // InteractionPolicy for this status. If null then the default InteractionPolicy should be assumed for this status's Visibility. Always null for boost wrappers. + PendingApproval *bool `bun:",nullzero,notnull,default:false"` // If true then status is a reply or boost wrapper that must be Approved by the reply-ee or boost-ee before being fully distributed. + PreApproved bool `bun:"-"` // If true, then status is a reply to or boost wrapper of a status on our instance, has permission to do the interaction, and an Accept should be sent out for it immediately. Field not stored in the DB. + ApprovedByURI string `bun:",nullzero"` // URI of an Accept Activity that approves the Announce or Create Activity that this status was/will be attached to. +} + +// Visibility represents the visibility granularity of a status. +type Visibility string + +const ( + // VisibilityNone means nobody can see this. + // It's only used for web status visibility. + VisibilityNone Visibility = "none" + // VisibilityPublic means this status will be visible to everyone on all timelines. + VisibilityPublic Visibility = "public" + // VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists. + VisibilityUnlocked Visibility = "unlocked" + // VisibilityFollowersOnly means this status is viewable to followers only. + VisibilityFollowersOnly Visibility = "followers_only" + // VisibilityMutualsOnly means this status is visible to mutual followers only. + VisibilityMutualsOnly Visibility = "mutuals_only" + // VisibilityDirect means this status is visible only to mentioned recipients. + VisibilityDirect Visibility = "direct" + // VisibilityDefault is used when no other setting can be found. + VisibilityDefault Visibility = VisibilityUnlocked +) diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go b/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go new file mode 100644 index 000000000..b27c3b343 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go @@ -0,0 +1,48 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import ( + "time" +) + +// StatusEdit represents a **historical** view of a Status +// after a received edit. The Status itself will always +// contain the latest up-to-date information. +// +// Note that stored status edits may not exactly match that +// of the origin server, they are a best-effort by receiver +// to store version history. There is no AP history endpoint. +type StatusEdit struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. + Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed. + ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit. + Text string `bun:""` // Original status text, without formatting, at time of edit. + Language string `bun:",nullzero"` // Status language at time of edit. + Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit. + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of media attachments associated with status at time of edit. + AttachmentDescriptions []string `bun:",array"` // Previous media descriptions of media attachments associated with status at time of edit. + PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll. + PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset. + StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of. + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server). + + // We don't bother having a *gtsmodel.Status model here + // as the StatusEdit is always just attached to a Status, + // so it doesn't need a self-reference back to it. +} diff --git a/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go b/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go index 10ae95c17..7621ddc6c 100644 --- a/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go +++ b/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go @@ -19,12 +19,9 @@ package migrations import ( "context" - "errors" old_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" new_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/uptrace/bun" @@ -128,97 +125,6 @@ func init() { } } -// convertEnums performs a transaction that converts -// a table's column of our old-style enums (strings) to -// more performant and space-saving integer types. -func convertEnums[OldType ~string, NewType ~int16]( - ctx context.Context, - tx bun.Tx, - table string, - column string, - mapping map[OldType]NewType, - defaultValue *NewType, -) error { - if len(mapping) == 0 { - return errors.New("empty mapping") - } - - // Generate new column name. - newColumn := column + "_new" - - log.Infof(ctx, "converting %s.%s enums; "+ - "this may take a while, please don't interrupt!", - table, column, - ) - - // Ensure a default value. - if defaultValue == nil { - var zero NewType - defaultValue = &zero - } - - // Add new column to database. - if _, err := tx.NewAddColumn(). - Table(table). - ColumnExpr("? SMALLINT NOT NULL DEFAULT ?", - bun.Ident(newColumn), - *defaultValue). - Exec(ctx); err != nil { - return gtserror.Newf("error adding new column: %w", err) - } - - // Get a count of all in table. - total, err := tx.NewSelect(). - Table(table). - Count(ctx) - if err != nil { - return gtserror.Newf("error selecting total count: %w", err) - } - - var updated int - for old, new := range mapping { - - // Update old to new values. - res, err := tx.NewUpdate(). - Table(table). - Where("? = ?", bun.Ident(column), old). - Set("? = ?", bun.Ident(newColumn), new). - Exec(ctx) - if err != nil { - return gtserror.Newf("error updating old column values: %w", err) - } - - // Count number items updated. - n, _ := res.RowsAffected() - updated += int(n) - } - - // Check total updated. - if total != updated { - log.Warnf(ctx, "total=%d does not match updated=%d", total, updated) - } - - // Drop the old column from table. - if _, err := tx.NewDropColumn(). - Table(table). - ColumnExpr("?", bun.Ident(column)). - Exec(ctx); err != nil { - return gtserror.Newf("error dropping old column: %w", err) - } - - // Rename new to old name. - if _, err := tx.NewRaw( - "ALTER TABLE ? RENAME COLUMN ? TO ?", - bun.Ident(table), - bun.Ident(newColumn), - bun.Ident(column), - ).Exec(ctx); err != nil { - return gtserror.Newf("error renaming new column: %w", err) - } - - return nil -} - // visibilityEnumMapping maps old Visibility enum values to their newer integer type. func visibilityEnumMapping[T ~string]() map[T]new_gtsmodel.Visibility { return map[T]new_gtsmodel.Visibility{ diff --git a/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go b/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go new file mode 100644 index 000000000..344168b38 --- /dev/null +++ b/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go @@ -0,0 +1,57 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package migrations + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + // Check for 'updated_at' column on media attachments table, else return. + exists, err := doesColumnExist(ctx, tx, "media_attachments", "updated_at") + if err != nil { + return err + } else if !exists { + return nil + } + + // Remove 'updated_at' column. + _, err = tx.NewDropColumn(). + Model((*gtsmodel.MediaAttachment)(nil)). + Column("updated_at"). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/util.go b/internal/db/bundb/migrations/util.go index 47de09e23..edf7c1d05 100644 --- a/internal/db/bundb/migrations/util.go +++ b/internal/db/bundb/migrations/util.go @@ -19,11 +19,209 @@ package migrations import ( "context" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "codeberg.org/gruf/go-byteutil" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" + "github.com/uptrace/bun/dialect/feature" + "github.com/uptrace/bun/dialect/sqltype" + "github.com/uptrace/bun/schema" ) +// convertEnums performs a transaction that converts +// a table's column of our old-style enums (strings) to +// more performant and space-saving integer types. +func convertEnums[OldType ~string, NewType ~int16]( + ctx context.Context, + tx bun.Tx, + table string, + column string, + mapping map[OldType]NewType, + defaultValue *NewType, +) error { + if len(mapping) == 0 { + return errors.New("empty mapping") + } + + // Generate new column name. + newColumn := column + "_new" + + log.Infof(ctx, "converting %s.%s enums; "+ + "this may take a while, please don't interrupt!", + table, column, + ) + + // Ensure a default value. + if defaultValue == nil { + var zero NewType + defaultValue = &zero + } + + // Add new column to database. + if _, err := tx.NewAddColumn(). + Table(table). + ColumnExpr("? SMALLINT NOT NULL DEFAULT ?", + bun.Ident(newColumn), + *defaultValue). + Exec(ctx); err != nil { + return gtserror.Newf("error adding new column: %w", err) + } + + // Get a count of all in table. + total, err := tx.NewSelect(). + Table(table). + Count(ctx) + if err != nil { + return gtserror.Newf("error selecting total count: %w", err) + } + + var updated int + for old, new := range mapping { + + // Update old to new values. + res, err := tx.NewUpdate(). + Table(table). + Where("? = ?", bun.Ident(column), old). + Set("? = ?", bun.Ident(newColumn), new). + Exec(ctx) + if err != nil { + return gtserror.Newf("error updating old column values: %w", err) + } + + // Count number items updated. + n, _ := res.RowsAffected() + updated += int(n) + } + + // Check total updated. + if total != updated { + log.Warnf(ctx, "total=%d does not match updated=%d", total, updated) + } + + // Drop the old column from table. + if _, err := tx.NewDropColumn(). + Table(table). + ColumnExpr("?", bun.Ident(column)). + Exec(ctx); err != nil { + return gtserror.Newf("error dropping old column: %w", err) + } + + // Rename new to old name. + if _, err := tx.NewRaw( + "ALTER TABLE ? RENAME COLUMN ? TO ?", + bun.Ident(table), + bun.Ident(newColumn), + bun.Ident(column), + ).Exec(ctx); err != nil { + return gtserror.Newf("error renaming new column: %w", err) + } + + return nil +} + +// getBunColumnDef generates a column definition string for the SQL table represented by +// Go type, with the SQL column represented by the given Go field name. This ensures when +// adding a new column for table by migration that it will end up as bun would create it. +// +// NOTE: this function must stay in sync with (*bun.CreateTableQuery{}).AppendQuery(), +// specifically where it loops over table fields appending each column definition. +func getBunColumnDef(db bun.IDB, rtype reflect.Type, fieldName string) (string, error) { + d := db.Dialect() + f := d.Features() + + // Get bun schema definitions for Go type and its field. + field, table, err := getModelField(db, rtype, fieldName) + if err != nil { + return "", err + } + + // Start with reasonable buf. + buf := make([]byte, 0, 64) + + // Start with the SQL column name. + buf = append(buf, field.SQLName...) + buf = append(buf, " "...) + + // Append the SQL + // type information. + switch { + + // Most of the time these two will match, but for the cases where DiscoveredSQLType is dialect-specific, + // e.g. pgdialect would change sqltype.SmallInt to pgTypeSmallSerial for columns that have `bun:",autoincrement"` + case !strings.EqualFold(field.CreateTableSQLType, field.DiscoveredSQLType): + buf = append(buf, field.CreateTableSQLType...) + + // For all common SQL types except VARCHAR, both UserDefinedSQLType and DiscoveredSQLType specify the correct type, + // and we needn't modify it. For VARCHAR columns, we will stop to check if a valid length has been set in .Varchar(int). + case !strings.EqualFold(field.CreateTableSQLType, sqltype.VarChar): + buf = append(buf, field.CreateTableSQLType...) + + // All else falls back + // to a default varchar. + default: + if d.Name() == dialect.Oracle { + buf = append(buf, "VARCHAR2"...) + } else { + buf = append(buf, sqltype.VarChar...) + } + buf = append(buf, "("...) + buf = strconv.AppendInt(buf, int64(d.DefaultVarcharLen()), 10) + buf = append(buf, ")"...) + } + + // Append not null definition if field requires. + if field.NotNull && d.Name() != dialect.Oracle { + buf = append(buf, " NOT NULL"...) + } + + // Append autoincrement definition if field requires. + if field.Identity && f.Has(feature.GeneratedIdentity) || + (field.AutoIncrement && (f.Has(feature.AutoIncrement) || f.Has(feature.Identity))) { + buf = d.AppendSequence(buf, table, field) + } + + // Append any default value. + if field.SQLDefault != "" { + buf = append(buf, " DEFAULT "...) + buf = append(buf, field.SQLDefault...) + } + + return byteutil.B2S(buf), nil +} + +// getModelField returns the uptrace/bun schema details for given Go type and field name. +func getModelField(db bun.IDB, rtype reflect.Type, fieldName string) (*schema.Field, *schema.Table, error) { + + // Get the associated table for Go type. + table := db.Dialect().Tables().Get(rtype) + if table == nil { + return nil, nil, fmt.Errorf("no table found for type: %s", rtype) + } + + var field *schema.Field + + // Look for field matching Go name. + for i := range table.Fields { + if table.Fields[i].GoName == fieldName { + field = table.Fields[i] + break + } + } + + if field == nil { + return nil, nil, fmt.Errorf("no bun field found on %s with name: %s", rtype, fieldName) + } + + return field, table, nil +} + // doesColumnExist safely checks whether given column exists on table, handling both SQLite and PostgreSQL appropriately. func doesColumnExist(ctx context.Context, tx bun.Tx, table, col string) (bool, error) { var n int diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 45e9864a3..fa31f3459 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -21,7 +21,6 @@ import ( "context" "errors" "slices" - "time" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" @@ -181,7 +180,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) error { var ( err error - errs = gtserror.NewMultiError(9) + errs gtserror.MultiError ) if status.Account == nil { @@ -257,7 +256,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.AttachmentsPopulated() { // Status attachments are out-of-date with IDs, repopulate. status.Attachments, err = s.state.DB.GetAttachmentsByIDs( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.AttachmentIDs, ) if err != nil { @@ -268,7 +267,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.TagsPopulated() { // Status tags are out-of-date with IDs, repopulate. status.Tags, err = s.state.DB.GetTags( - ctx, + gtscontext.SetBarebones(ctx), status.TagIDs, ) if err != nil { @@ -279,7 +278,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.MentionsPopulated() { // Status mentions are out-of-date with IDs, repopulate. status.Mentions, err = s.state.DB.GetMentions( - ctx, // leave fully populated for now + ctx, // TODO: manually populate mentions for places expecting these populated status.MentionIDs, ) if err != nil { @@ -290,7 +289,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.EmojisPopulated() { // Status emojis are out-of-date with IDs, repopulate. status.Emojis, err = s.state.DB.GetEmojisByIDs( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.EmojiIDs, ) if err != nil { @@ -298,10 +297,21 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) } } + if !status.EditsPopulated() { + // Status edits are out-of-date with IDs, repopulate. + status.Edits, err = s.state.DB.GetStatusEditsByIDs( + gtscontext.SetBarebones(ctx), + status.EditIDs, + ) + if err != nil { + errs.Appendf("error populating status edits: %w", err) + } + } + if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil { // Populate the status' expected CreatedWithApplication (not always set). status.CreatedWithApplication, err = s.state.DB.GetApplicationByID( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.CreatedWithApplicationID, ) if err != nil { @@ -350,14 +360,14 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error } } - // change the status ID of the media attachments to the new status + // change the status ID of the media + // attachments to the current status for _, a := range status.Attachments { a.StatusID = status.ID - a.UpdatedAt = time.Now() if _, err := tx. NewUpdate(). Model(a). - Column("status_id", "updated_at"). + Column("status_id"). Where("? = ?", bun.Ident("media_attachment.id"), a.ID). Exec(ctx); err != nil { if !errors.Is(err, db.ErrAlreadyExists) { @@ -384,19 +394,15 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error } // Finally, insert the status - _, err := tx.NewInsert().Model(status).Exec(ctx) + _, err := tx.NewInsert(). + Model(status). + Exec(ctx) return err }) }) } func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) error { - status.UpdatedAt = time.Now() - if len(columns) > 0 { - // If we're updating by column, ensure "updated_at" is included. - columns = append(columns, "updated_at") - } - return s.state.Caches.DB.Status.Store(status, func() error { // It is safe to run this database transaction within cache.Store // as the cache does not attempt a mutex lock until AFTER hook. @@ -434,13 +440,14 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co } } - // change the status ID of the media attachments to the new status + // change the status ID of the media + // attachments to the current status. for _, a := range status.Attachments { a.StatusID = status.ID - a.UpdatedAt = time.Now() if _, err := tx. NewUpdate(). Model(a). + Column("status_id"). Where("? = ?", bun.Ident("media_attachment.id"), a.ID). Exec(ctx); err != nil { if !errors.Is(err, db.ErrAlreadyExists) { @@ -467,8 +474,7 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co } // Finally, update the status - _, err := tx. - NewUpdate(). + _, err := tx.NewUpdate(). Model(status). Column(columns...). Where("? = ?", bun.Ident("status.id"), status.ID). diff --git a/internal/db/bundb/statusedit.go b/internal/db/bundb/statusedit.go new file mode 100644 index 000000000..c932968fd --- /dev/null +++ b/internal/db/bundb/statusedit.go @@ -0,0 +1,198 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package bundb + +import ( + "context" + "errors" + "slices" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/util/xslices" + "github.com/uptrace/bun" +) + +type statusEditDB struct { + db *bun.DB + state *state.State +} + +func (s *statusEditDB) GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) { + // Fetch edit from database cache with loader callback. + edit, err := s.state.Caches.DB.StatusEdit.LoadOne("ID", + func() (*gtsmodel.StatusEdit, error) { + var edit gtsmodel.StatusEdit + + // Not cached, load edit + // from database by its ID. + if err := s.db.NewSelect(). + Model(&edit). + Where("? = ?", bun.Ident("id"), id). + Scan(ctx); err != nil { + return nil, err + } + + return &edit, nil + }, id, + ) + if err != nil { + return nil, err + } + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edit, nil + } + + // Further populate the edit fields where applicable. + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + return nil, err + } + + return edit, nil +} + +func (s *statusEditDB) GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) { + // Load status edits for IDs via cache loader callbacks. + edits, err := s.state.Caches.DB.StatusEdit.LoadIDs("ID", + ids, + func(uncached []string) ([]*gtsmodel.StatusEdit, error) { + // Preallocate expected length of uncached edits. + edits := make([]*gtsmodel.StatusEdit, 0, len(uncached)) + + // Perform database query scanning + // the remaining (uncached) edit IDs. + if err := s.db.NewSelect(). + Model(&edits). + Where("? IN (?)", bun.Ident("id"), bun.In(uncached)). + Scan(ctx); err != nil { + return nil, err + } + + return edits, nil + }, + ) + if err != nil { + return nil, err + } + + // Reorder the edits by their + // IDs to ensure in correct order. + getID := func(e *gtsmodel.StatusEdit) string { return e.ID } + xslices.OrderBy(edits, ids, getID) + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edits, nil + } + + // Populate all loaded edits, removing those we fail to + // populate (removes needing so many nil checks everywhere). + edits = slices.DeleteFunc(edits, func(edit *gtsmodel.StatusEdit) bool { + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + log.Errorf(ctx, "error populating edit %s: %v", edit.ID, err) + return true + } + return false + }) + + return edits, nil +} + +func (s *statusEditDB) PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + var err error + var errs gtserror.MultiError + + // For sub-models we only want + // barebones versions of them. + ctx = gtscontext.SetBarebones(ctx) + + if !edit.AttachmentsPopulated() { + // Fetch all attachments for status edit's IDs. + edit.Attachments, err = s.state.DB.GetAttachmentsByIDs( + ctx, + edit.AttachmentIDs, + ) + if err != nil { + errs.Appendf("error populating edit attachments: %w", err) + } + } + + return errs.Combine() +} + +func (s *statusEditDB) PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + return s.state.Caches.DB.StatusEdit.Store(edit, func() error { + _, err := s.db.NewInsert().Model(edit).Exec(ctx) + return err + }) +} + +func (s *statusEditDB) DeleteStatusEdits(ctx context.Context, ids []string) error { + // Gather necessary fields from + // deleted for cache invalidation. + deleted := make([]*gtsmodel.StatusEdit, 0, len(ids)) + + // Delete all edits with IDs pertaining + // to given slice, returning status IDs. + if _, err := s.db.NewDelete(). + Model(&deleted). + Where("? IN (?)", bun.Ident("id"), bun.In(ids)). + Returning("?", bun.Ident("status_id")). + Exec(ctx); err != nil && + !errors.Is(err, db.ErrNoEntries) { + return err + } + + // Check for no deletes. + if len(deleted) == 0 { + return nil + } + + // Invalidate all the cached status edits with IDs. + s.state.Caches.DB.StatusEdit.InvalidateIDs("ID", ids) + + // With each invalidate hook mark status ID of + // edit we just called for. We only want to call + // invalidate hooks of edits from unique statuses. + invalidated := make(map[string]struct{}, 1) + + // Invalidate the first delete manually, this + // opt negates need for initial hashmap lookup. + s.state.Caches.OnInvalidateStatusEdit(deleted[0]) + invalidated[deleted[0].StatusID] = struct{}{} + + for _, edit := range deleted { + // Check not already called for status. + _, ok := invalidated[edit.StatusID] + if ok { + continue + } + + // Manually call status edit invalidate hook. + s.state.Caches.OnInvalidateStatusEdit(edit) + invalidated[edit.StatusID] = struct{}{} + } + + return nil +} diff --git a/internal/db/bundb/statusedit_test.go b/internal/db/bundb/statusedit_test.go new file mode 100644 index 000000000..b6a15e825 --- /dev/null +++ b/internal/db/bundb/statusedit_test.go @@ -0,0 +1,168 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package bundb_test + +import ( + "context" + "errors" + "reflect" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type StatusEditTestSuite struct { + BunDBStandardTestSuite +} + +func (suite *StatusEditTestSuite) TestGetStatusEditBy() { + t := suite.T() + + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // Sentinel error to mark avoiding a test case. + sentinelErr := errors.New("sentinel") + + for _, edit := range suite.testStatusEdits { + for lookup, dbfunc := range map[string]func() (*gtsmodel.StatusEdit, error){ + "id": func() (*gtsmodel.StatusEdit, error) { + return suite.db.GetStatusEditByID(ctx, edit.ID) + }, + } { + // Clear database caches. + suite.state.Caches.Init() + + t.Logf("checking database lookup %q", lookup) + + // Perform database function. + checkEdit, err := dbfunc() + if err != nil { + if err == sentinelErr { + continue + } + + t.Errorf("error encountered for database lookup %q: %v", lookup, err) + continue + } + + // Check received account data. + if !areEditsEqual(edit, checkEdit) { + t.Errorf("edit does not contain expected data: %+v", checkEdit) + continue + } + } + } +} + +func (suite *StatusEditTestSuite) TestGetStatusEditsByIDs() { + t := suite.T() + + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // editsByStatus returns all test edits by the given status with ID. + editsByStatus := func(status *gtsmodel.Status) []*gtsmodel.StatusEdit { + var edits []*gtsmodel.StatusEdit + for _, edit := range suite.testStatusEdits { + if edit.StatusID == status.ID { + edits = append(edits, edit) + } + } + return edits + } + + for _, status := range suite.testStatuses { + // Get test status edit models + // that should be found for status. + check := editsByStatus(status) + + // Fetch edits for the slice of IDs attached to status from database. + edits, err := suite.state.DB.GetStatusEditsByIDs(ctx, status.EditIDs) + suite.NoError(err) + + // Ensure both slices + // sorted the same. + sortEdits(check) + sortEdits(edits) + + // Check whether slices of status edits match. + if !slices.EqualFunc(check, edits, areEditsEqual) { + t.Error("status edit slices do not match") + } + } +} + +func (suite *StatusEditTestSuite) TestDeleteStatusEdits() { + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + for _, status := range suite.testStatuses { + // Delete all edits for status with given IDs from database. + err := suite.state.DB.DeleteStatusEdits(ctx, status.EditIDs) + suite.NoError(err) + + // Now attempt to fetch these edits from database, should be empty. + edits, err := suite.state.DB.GetStatusEditsByIDs(ctx, status.EditIDs) + suite.NoError(err) + suite.Empty(edits) + } +} + +func TestStatusEditTestSuite(t *testing.T) { + suite.Run(t, new(StatusEditTestSuite)) +} + +func areEditsEqual(e1, e2 *gtsmodel.StatusEdit) bool { + // Clone the 1st status edit. + e1Copy := new(gtsmodel.StatusEdit) + *e1Copy = *e1 + e1 = e1Copy + + // Clone the 2nd status edit. + e2Copy := new(gtsmodel.StatusEdit) + *e2Copy = *e2 + e2 = e2Copy + + // Clear populated sub-models. + e1.Attachments = nil + e2.Attachments = nil + + // Clear database-set fields. + e1.CreatedAt = time.Time{} + e2.CreatedAt = time.Time{} + + return reflect.DeepEqual(*e1, *e2) +} + +func sortEdits(edits []*gtsmodel.StatusEdit) { + slices.SortFunc(edits, func(a, b *gtsmodel.StatusEdit) int { + if a.CreatedAt.Before(b.CreatedAt) { + return +1 + } else if b.CreatedAt.Before(a.CreatedAt) { + return -1 + } + return 0 + }) +} diff --git a/internal/db/bundb/timeline.go b/internal/db/bundb/timeline.go index bcb7953d4..fcea0178a 100644 --- a/internal/db/bundb/timeline.go +++ b/internal/db/bundb/timeline.go @@ -123,13 +123,8 @@ func (t *timelineDB) GetHomeTimeline(ctx context.Context, accountID string, maxI if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -223,13 +218,8 @@ func (t *timelineDB) GetPublicTimeline(ctx context.Context, maxID string, sinceI if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -409,13 +399,8 @@ func (t *timelineDB) GetListTimeline( if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -508,13 +493,8 @@ func (t *timelineDB) GetTagTimeline( if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID diff --git a/internal/db/bundb/timeline_test.go b/internal/db/bundb/timeline_test.go index 00df2b3a6..75a335512 100644 --- a/internal/db/bundb/timeline_test.go +++ b/internal/db/bundb/timeline_test.go @@ -37,10 +37,7 @@ type TimelineTestSuite struct { func getFutureStatus() *gtsmodel.Status { theDistantFuture := time.Now().Add(876600 * time.Hour) - id, err := id.NewULIDFromTime(theDistantFuture) - if err != nil { - panic(err) - } + id := id.NewULIDFromTime(theDistantFuture) return >smodel.Status{ ID: id, @@ -182,7 +179,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 8) + suite.checkStatuses(s, id.Highest, id.Lowest, 9) // Remove admin account from the exclusive list. listEntry := suite.testListEntries["local_account_1_list_1_entry_2"] @@ -196,7 +193,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 12) + suite.checkStatuses(s, id.Highest, id.Lowest, 13) } func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { @@ -228,7 +225,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 8) + suite.checkStatuses(s, id.Highest, id.Lowest, 9) } func (suite *TimelineTestSuite) TestGetHomeTimelineWithFutureStatus() { @@ -281,8 +278,8 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineFromHighest() { } suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01J2M1HPFSS54S60Y0KYV23KJE", s[0].ID) - suite.Equal("01G36SF3V6Y6V5BF9P4R7PQG7G", s[len(s)-1].ID) + suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) + suite.Equal("01HEN2RZ8BG29Y5Z9VJC73HZW7", s[len(s)-1].ID) } func (suite *TimelineTestSuite) TestGetListTimelineNoParams() { @@ -296,7 +293,7 @@ func (suite *TimelineTestSuite) TestGetListTimelineNoParams() { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 12) + suite.checkStatuses(s, id.Highest, id.Lowest, 13) } func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { @@ -311,8 +308,8 @@ func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { } suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01HEN2PRXT0TF4YDRA64FZZRN7", s[0].ID) - suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", s[len(s)-1].ID) + suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) + suite.Equal("01FN3VJGFH10KR7S2PB0GFJZYG", s[len(s)-1].ID) } func (suite *TimelineTestSuite) TestGetListTimelineMinID() { diff --git a/internal/db/db.go b/internal/db/db.go index c42985912..11dd2e507 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -51,6 +51,7 @@ type DB interface { SinBinStatus Status StatusBookmark + StatusEdit StatusFave Tag Thread diff --git a/internal/db/statusedit.go b/internal/db/statusedit.go new file mode 100644 index 000000000..32e770fb9 --- /dev/null +++ b/internal/db/statusedit.go @@ -0,0 +1,43 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type StatusEdit interface { + + // GetStatusEditByID fetches the StatusEdit with given ID from the database. + GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) + + // GetStatusEditsByIDs fetches all StatusEdits with given IDs from database, + // this is optimized and faster than multiple calls to GetStatusEditByID. + GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) + + // PopulateStatusEdit ensures the given StatusEdit's sub-models are populated. + PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error + + // PutStatusEdit inserts the given new StatusEdit into the database. + PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error + + // DeleteStatusEdits deletes the StatusEdits with given IDs from the database. + DeleteStatusEdits(ctx context.Context, ids []string) error +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go index a3eaf199d..eb949f159 100644 --- a/internal/federation/dereferencing/announce.go +++ b/internal/federation/dereferencing/announce.go @@ -87,7 +87,7 @@ func (d *Dereferencer) EnrichAnnounce( boost.Federated = target.Federated // Ensure this Announce is permitted by the Announcee. - permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost) + permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost, true) if err != nil { return nil, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err) } @@ -99,10 +99,7 @@ func (d *Dereferencer) EnrichAnnounce( } // Generate an ID for the boost wrapper status. - boost.ID, err = id.NewULIDFromTime(boost.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating id: %w", err) - } + boost.ID = id.NewULIDFromTime(boost.CreatedAt) // Store the boost wrapper status in database. switch err = d.state.DB.PutStatus(ctx, boost); { diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 3bed4b198..d22eeb237 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -128,6 +128,7 @@ func (d *Dereferencer) RefreshMedia( // Check emoji is up-to-date // with provided extra info. switch { + case force: case info.Blurhash != nil && *info.Blurhash != attach.Blurhash: attach.Blurhash = *info.Blurhash diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index c90730826..d19669891 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -302,6 +302,7 @@ func (d *Dereferencer) enrichStatusSafely( uri, status, statusable, + isNew, ) // Check for a returned HTTP code via error. @@ -374,6 +375,7 @@ func (d *Dereferencer) enrichStatus( uri *url.URL, status *gtsmodel.Status, statusable ap.Statusable, + isNew bool, ) ( *gtsmodel.Status, ap.Statusable, @@ -476,8 +478,7 @@ func (d *Dereferencer) enrichStatus( // Ensure the final parsed status URI or URL matches // the input URI we fetched (or received) it as. - matches, err := util.URIMatches( - uri, + matches, err := util.URIMatches(uri, append( ap.GetURL(statusable), // status URL(s) ap.GetJSONLDId(statusable), // status URI @@ -497,19 +498,10 @@ func (d *Dereferencer) enrichStatus( ) } - var isNew bool - - // Based on the original provided - // status model, determine whether - // this is a new insert / update. - if isNew = (status.ID == ""); isNew { + if isNew { // Generate new status ID from the provided creation date. - latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - latestStatus.ID = id.NewULID() // just use "now" - } + latestStatus.ID = id.NewULIDFromTime(latestStatus.CreatedAt) } else { // Reuse existing status ID. @@ -519,7 +511,6 @@ func (d *Dereferencer) enrichStatus( // Set latest fetch time and carry- // over some values from "old" status. latestStatus.FetchedAt = time.Now() - latestStatus.UpdatedAt = status.UpdatedAt latestStatus.Local = status.Local latestStatus.PinnedAt = status.PinnedAt @@ -538,8 +529,9 @@ func (d *Dereferencer) enrichStatus( } // Check if this is a permitted status we should accept. - // Function also sets "PendingApproval" bool as necessary. - permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus) + // Function also sets "PendingApproval" bool as necessary, + // and handles removal of existing statuses no longer permitted. + permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus, isNew) if err != nil { return nil, nil, gtserror.Newf("error checking permissibility for status %s: %w", uri, err) } @@ -550,59 +542,113 @@ func (d *Dereferencer) enrichStatus( return nil, nil, gtserror.SetNotPermitted(err) } - // Ensure the status' mentions are populated, and pass in existing to check for changes. - if err := d.fetchStatusMentions(ctx, requestUser, status, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) + // Insert / update any attached status poll. + pollChanged, err := d.handleStatusPoll(ctx, + status, + latestStatus, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling poll for status %s: %w", uri, err) } - // Ensure the status' poll remains consistent, else reset the poll. - if err := d.fetchStatusPoll(ctx, status, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error populating poll for status %s: %w", uri, err) + // Populate mentions associated with status, passing + // in existing status to reuse old where possible. + // (especially important here to reduce need to dereference). + mentionsChanged, err := d.fetchStatusMentions(ctx, + requestUser, + status, + latestStatus, + ) + if err != nil { + return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) } - // Now that we know who this status replies to (handled by ASStatusToStatus) - // and who it mentions, we can add a ThreadID to it if necessary. - if err := d.threadStatus(ctx, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error checking / creating threadID for status %s: %w", uri, err) + // Ensure status in a thread is connected. + threadChanged, err := d.threadStatus(ctx, + status, + latestStatus, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling threading for status %s: %w", uri, err) } - // Ensure the status' tags are populated, (changes are expected / okay). - if err := d.fetchStatusTags(ctx, status, latestStatus); err != nil { + // Populate tags associated with status, passing + // in existing status to reuse old where possible. + tagsChanged, err := d.fetchStatusTags(ctx, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err) } - // Ensure the status' media attachments are populated, passing in existing to check for changes. - if err := d.fetchStatusAttachments(ctx, requestUser, status, latestStatus); err != nil { + // Populate media attachments associated with status, + // passing in existing status to reuse old where possible + // (especially important here to reduce need to dereference). + mediaChanged, err := d.fetchStatusAttachments(ctx, + requestUser, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating attachments for status %s: %w", uri, err) } - // Ensure the status' emoji attachments are populated, passing in existing to check for changes. - if err := d.fetchStatusEmojis(ctx, status, latestStatus); err != nil { + // Populate emoji associated with status, passing + // in existing status to reuse old where possible + // (especially important here to reduce need to dereference). + emojiChanged, err := d.fetchStatusEmojis(ctx, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating emojis for status %s: %w", uri, err) } if isNew { - // This is new, put the status in the database. - err := d.state.DB.PutStatus(ctx, latestStatus) - if err != nil { - return nil, nil, gtserror.Newf("error putting in database: %w", err) + // Simplest case, insert this new status into the database. + if err := d.state.DB.PutStatus(ctx, latestStatus); err != nil { + return nil, nil, gtserror.Newf("error inserting new status %s: %w", uri, err) } } else { - // This is an existing status, update the model in the database. - if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error updating database: %w", err) + // Check for and handle any edits to status, inserting + // historical edit if necessary. Also determines status + // columns that need updating in below query. + cols, err := d.handleStatusEdit(ctx, + status, + latestStatus, + pollChanged, + mentionsChanged, + threadChanged, + tagsChanged, + mediaChanged, + emojiChanged, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling edit for status %s: %w", uri, err) + } + + // With returned changed columns, now update the existing status entry. + if err := d.state.DB.UpdateStatus(ctx, latestStatus, cols...); err != nil { + return nil, nil, gtserror.Newf("error updating existing status %s: %w", uri, err) } } return latestStatus, statusable, nil } +// fetchStatusMentions populates the mentions on 'status', creating +// new where needed, or using unchanged mentions from 'existing' status. func (d *Dereferencer) fetchStatusMentions( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be created mention IDs. status.MentionIDs = make([]string, len(status.Mentions)) @@ -610,7 +656,6 @@ func (d *Dereferencer) fetchStatusMentions( var ( mention = status.Mentions[i] alreadyExists bool - err error ) // Search existing status for a mention already stored, @@ -633,19 +678,16 @@ func (d *Dereferencer) fetchStatusMentions( continue } + // Mark status as + // having changed. + changed = true + // This mention didn't exist yet. - // Generate new ID according to status creation. - // TODO: update this to use "edited_at" when we add - // support for edited status revision history. - mention.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - mention.ID = id.NewULID() // just use "now" - } + // Generate new ID according to latest update. + mention.ID = id.NewULIDFromTime(status.UpdatedAt) // Set known further mention details. - mention.CreatedAt = status.CreatedAt - mention.UpdatedAt = status.UpdatedAt + mention.CreatedAt = status.UpdatedAt mention.OriginAccount = status.Account mention.OriginAccountID = status.AccountID mention.OriginAccountURI = status.AccountURI @@ -657,7 +699,7 @@ func (d *Dereferencer) fetchStatusMentions( // Place the new mention into the database. if err := d.state.DB.PutMention(ctx, mention); err != nil { - return gtserror.Newf("error putting mention in database: %w", err) + return changed, gtserror.Newf("error putting mention in database: %w", err) } // Set the *new* mention and ID. @@ -678,17 +720,42 @@ func (d *Dereferencer) fetchStatusMentions( i++ } - return nil + return changed, nil } -func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status) error { - if status.InReplyTo != nil { - if parentThreadID := status.InReplyTo.ThreadID; parentThreadID != "" { - // Simplest case: parent status - // is threaded, so inherit threadID. - status.ThreadID = parentThreadID - return nil +// threadStatus ensures that given status is threaded correctly +// where necessary. that is it will inherit a thread ID from the +// existing copy if it is threaded correctly, else it will inherit +// a thread ID from a parent with existing thread, else it will +// generate a new thread ID if status mentions a local account. +func (d *Dereferencer) threadStatus( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) ( + changed bool, + err error, +) { + + // Check for existing status + // that is already threaded. + if existing.ThreadID != "" { + + // Existing is threaded correctly. + if existing.InReplyTo == nil || + existing.InReplyTo.ThreadID == existing.ThreadID { + status.ThreadID = existing.ThreadID + return false, nil } + + // TODO: delete incorrect thread + } + + // Check for existing parent to inherit threading from. + if inReplyTo := status.InReplyTo; inReplyTo != nil && + inReplyTo.ThreadID != "" { + status.ThreadID = inReplyTo.ThreadID + return true, nil } // Parent wasn't threaded. If this @@ -711,7 +778,7 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status // Status doesn't mention a // local account, so we don't // need to thread it. - return nil + return false, nil } // Status mentions a local account. @@ -719,24 +786,30 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status // it to the status. threadID := id.NewULID() - if err := d.state.DB.PutThread( - ctx, - >smodel.Thread{ - ID: threadID, - }, + // Insert new thread model into db. + if err := d.state.DB.PutThread(ctx, + >smodel.Thread{ID: threadID}, ); err != nil { - return gtserror.Newf("error inserting new thread in db: %w", err) + return false, gtserror.Newf("error inserting new thread in db: %w", err) } + // Set thread on latest status. status.ThreadID = threadID - return nil + return true, nil } +// fetchStatusTags populates the tags on 'status', fetching existing +// from the database and creating new where needed. 'existing' is used +// to fetch tags that have not changed since previous stored status. func (d *Dereferencer) fetchStatusTags( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be determined tag IDs. status.TagIDs = make([]string, len(status.Tags)) @@ -751,10 +824,14 @@ func (d *Dereferencer) fetchStatusTags( continue } + // Mark status as + // having changed. + changed = true + // Look for existing tag with name in the database. existing, err := d.state.DB.GetTagByName(ctx, tag.Name) if err != nil && !errors.Is(err, db.ErrNoEntries) { - return gtserror.Newf("db error getting tag %s: %w", tag.Name, err) + return changed, gtserror.Newf("db error getting tag %s: %w", tag.Name, err) } else if existing != nil { status.Tags[i] = existing status.TagIDs[i] = existing.ID @@ -788,106 +865,21 @@ func (d *Dereferencer) fetchStatusTags( i++ } - return nil -} - -func (d *Dereferencer) fetchStatusPoll( - ctx context.Context, - existing *gtsmodel.Status, - status *gtsmodel.Status, -) error { - var ( - // insertStatusPoll generates ID and inserts the poll attached to status into the database. - insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error { - var err error - - // Generate new ID for poll from the status CreatedAt. - // TODO: update this to use "edited_at" when we add - // support for edited status revision history. - status.Poll.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - status.Poll.ID = id.NewULID() // just use "now" - } - - // Update the status<->poll links. - status.PollID = status.Poll.ID - status.Poll.StatusID = status.ID - status.Poll.Status = status - - // Insert this latest poll into the database. - err = d.state.DB.PutPoll(ctx, status.Poll) - if err != nil { - return gtserror.Newf("error putting in database: %w", err) - } - - return nil - } - - // deleteStatusPoll deletes the poll with ID, and all attached votes, from the database. - deleteStatusPoll = func(ctx context.Context, pollID string) error { - if err := d.state.DB.DeletePollByID(ctx, pollID); err != nil { - return gtserror.Newf("error deleting existing poll from database: %w", err) - } - return nil - } - ) - - switch { - case existing.Poll == nil && status.Poll == nil: - // no poll before or after, nothing to do. - return nil - - case existing.Poll == nil && status.Poll != nil: - // no previous poll, insert new poll! - return insertStatusPoll(ctx, status) - - case status.Poll == nil: - // existing poll has been deleted, remove this. - return deleteStatusPoll(ctx, existing.PollID) - - case pollChanged(existing.Poll, status.Poll): - // poll has changed since original, delete and reinsert new. - if err := deleteStatusPoll(ctx, existing.PollID); err != nil { - return err - } - return insertStatusPoll(ctx, status) - - case pollUpdated(existing.Poll, status.Poll): - // Since we last saw it, the poll has updated! - // Whether that be stats, or close time. - poll := existing.Poll - poll.Closing = pollJustClosed(existing.Poll, status.Poll) - poll.ClosedAt = status.Poll.ClosedAt - poll.Voters = status.Poll.Voters - poll.Votes = status.Poll.Votes - - // Update poll model in the database (specifically only the possible changed columns). - if err := d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { - return gtserror.Newf("error updating poll: %w", err) - } - - // Update poll on status. - status.PollID = poll.ID - status.Poll = poll - return nil - - default: - // latest and existing - // polls are up to date. - poll := existing.Poll - status.PollID = poll.ID - status.Poll = poll - return nil - } + return changed, nil } +// fetchStatusAttachments populates the attachments on 'status', creating new database +// entries where needed and dereferencing it, or using unchanged from 'existing' status. func (d *Dereferencer) fetchStatusAttachments( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) @@ -897,9 +889,26 @@ func (d *Dereferencer) fetchStatusAttachments( // Look for existing media attachment with remote URL first. existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL) if ok && existing.ID != "" { + var info media.AdditionalMediaInfo - // Ensure the existing media attachment is up-to-date and cached. - existing, err := d.updateAttachment(ctx, requestUser, existing, placeholder) + // Look for any difference in stored media description. + diff := (existing.Description != placeholder.Description) + if diff { + info.Description = &placeholder.Description + } + + // If description changed, + // we mark media as changed. + changed = changed || diff + + // Store any attachment updates and + // ensure media is locally cached. + existing, err := d.RefreshMedia(ctx, + requestUser, + existing, + info, + diff, + ) if err != nil { log.Errorf(ctx, "error updating existing attachment: %v", err) @@ -915,9 +924,12 @@ func (d *Dereferencer) fetchStatusAttachments( continue } + // Mark status as + // having changed. + changed = true + // Load this new media attachment. - attachment, err := d.GetMedia( - ctx, + attachment, err := d.GetMedia(ctx, requestUser, status.AccountID, placeholder.RemoteURL, @@ -955,28 +967,34 @@ func (d *Dereferencer) fetchStatusAttachments( i++ } - return nil + return changed, nil } +// fetchStatusEmojis populates the emojis on 'status', creating new database entries +// where needed and dereferencing it, or using unchanged from 'existing' status. func (d *Dereferencer) fetchStatusEmojis( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Fetch the updated emojis for our status. emojis, changed, err := d.fetchEmojis(ctx, existing.Emojis, status.Emojis, ) if err != nil { - return gtserror.Newf("error fetching emojis: %w", err) + return changed, gtserror.Newf("error fetching emojis: %w", err) } if !changed { // Use existing status emoji objects. status.EmojiIDs = existing.EmojiIDs status.Emojis = existing.Emojis - return nil + return false, nil } // Set latest emojis. @@ -988,9 +1006,254 @@ func (d *Dereferencer) fetchStatusEmojis( status.EmojiIDs[i] = emoji.ID } + return true, nil +} + +// handleStatusPoll handles both inserting of new status poll or the +// update of an existing poll. this handles the case of simple vote +// count updates (without being classified as a change of the poll +// itself), as well as full poll changes that delete existing instance. +func (d *Dereferencer) handleStatusPoll( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) ( + changed bool, + err error, +) { + switch { + case existing.Poll == nil && status.Poll == nil: + // no poll before or after, nothing to do. + return false, nil + + case existing.Poll == nil && status.Poll != nil: + // no previous poll, insert new status poll! + return true, d.insertStatusPoll(ctx, status) + + case status.Poll == nil: + // existing status poll has been deleted, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + err = gtserror.Newf("error deleting poll from database: %w", err) + } + return true, err + + case pollChanged(existing.Poll, status.Poll): + // existing status poll has been changed, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + return true, gtserror.Newf("error deleting poll from database: %w", err) + } + + // insert latest poll version into database. + return true, d.insertStatusPoll(ctx, status) + + case pollStateUpdated(existing.Poll, status.Poll): + // Since we last saw it, the poll has updated! + // Whether that be stats, or close time. + poll := existing.Poll + poll.Closing = pollJustClosed(existing.Poll, status.Poll) + poll.ClosedAt = status.Poll.ClosedAt + poll.Voters = status.Poll.Voters + poll.Votes = status.Poll.Votes + + // Update poll model in the database (specifically only the possible changed columns). + if err = d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { + return false, gtserror.Newf("error updating poll: %w", err) + } + + // Update poll on status. + status.PollID = poll.ID + status.Poll = poll + return false, nil + + default: + // latest and existing + // polls are up to date. + poll := existing.Poll + status.PollID = poll.ID + status.Poll = poll + return false, nil + } +} + +// insertStatusPoll inserts an assumed new poll attached to status into the database, this +// also handles generating new ID for the poll and setting necessary fields on the status. +func (d *Dereferencer) insertStatusPoll(ctx context.Context, status *gtsmodel.Status) error { + var err error + + // Generate new ID for poll from latest updated time. + status.Poll.ID = id.NewULIDFromTime(status.UpdatedAt) + + // Update the status<->poll links. + status.PollID = status.Poll.ID + status.Poll.StatusID = status.ID + status.Poll.Status = status + + // Insert this latest poll into the database. + err = d.state.DB.PutPoll(ctx, status.Poll) + if err != nil { + return gtserror.Newf("error putting poll in database: %w", err) + } + return nil } +// handleStatusEdit compiles a list of changed status table columns between +// existing and latest status model, and where necessary inserts a historic +// edit of the status into the database to store its previous state. the +// returned slice is a list of columns requiring updating in the database. +func (d *Dereferencer) handleStatusEdit( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, + pollChanged bool, + mentionsChanged bool, + threadChanged bool, + tagsChanged bool, + mediaChanged bool, + emojiChanged bool, +) ( + cols []string, + err error, +) { + var edited bool + + // Preallocate max slice length. + cols = make([]string, 0, 13) + + // Always update `fetched_at`. + cols = append(cols, "fetched_at") + + // Check for edited status content. + if existing.Content != status.Content { + cols = append(cols, "content") + edited = true + } + + // Check for edited status content warning. + if existing.ContentWarning != status.ContentWarning { + cols = append(cols, "content_warning") + edited = true + } + + // Check for edited status sensitive flag. + if *existing.Sensitive != *status.Sensitive { + cols = append(cols, "sensitive") + edited = true + } + + // Check for edited status language tag. + if existing.Language != status.Language { + cols = append(cols, "language") + edited = true + } + + if pollChanged { + // Attached poll was changed. + cols = append(cols, "poll_id") + edited = true + } + + if mentionsChanged { + cols = append(cols, "mentions") // i.e. MentionIDs + + // Mentions changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if threadChanged { + cols = append(cols, "thread_id") + + // Thread changed doesn't necessarily + // indicate an edit, it may just now + // actually be included in a thread. + } + + if tagsChanged { + cols = append(cols, "tags") // i.e. TagIDs + + // Tags changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if mediaChanged { + // Attached media was changed. + cols = append(cols, "attachments") // i.e. AttachmentIDs + edited = true + } + + if emojiChanged { + // Attached emojis changed. + cols = append(cols, "emojis") // i.e. EmojiIDs + + // Emojis changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if edited { + // We prefer to use provided 'upated_at', but ensure + // it fits chronologically with creation / last update. + if !status.UpdatedAt.After(status.CreatedAt) || + !status.UpdatedAt.After(existing.UpdatedAt) { + + // Else fallback to now as update time. + status.UpdatedAt = status.FetchedAt + } + + // Status has been editted since last + // we saw it, take snapshot of existing. + var edit gtsmodel.StatusEdit + edit.ID = id.NewULIDFromTime(status.UpdatedAt) + edit.Content = existing.Content + edit.ContentWarning = existing.ContentWarning + edit.Text = existing.Text + edit.Language = existing.Language + edit.Sensitive = existing.Sensitive + edit.StatusID = status.ID + + // Copy existing attachments and descriptions. + edit.AttachmentIDs = existing.AttachmentIDs + edit.Attachments = existing.Attachments + if l := len(existing.Attachments); l > 0 { + edit.AttachmentDescriptions = make([]string, l) + for i, attach := range existing.Attachments { + edit.AttachmentDescriptions[i] = attach.Description + } + } + + // Edit creation is last update time. + edit.CreatedAt = existing.UpdatedAt + + if existing.Poll != nil { + // Poll only set if existing contained them. + edit.PollOptions = existing.Poll.Options + + if !*existing.Poll.HideCounts || pollChanged { + // If the counts are allowed to be + // shown, or poll has changed, then + // include poll vote counts in edit. + edit.PollVotes = existing.Poll.Votes + } + } + + // Insert this new edit of existing status into database. + if err := d.state.DB.PutStatusEdit(ctx, &edit); err != nil { + return nil, gtserror.Newf("error putting edit in database: %w", err) + } + + // Add edit to list of edits on the status. + status.EditIDs = append(status.EditIDs, edit.ID) + status.Edits = append(status.Edits, &edit) + + // Add updated_at and edits to list of cols. + cols = append(cols, "updated_at", "edits") + } + + return cols, nil +} + // getPopulatedMention tries to populate the given // mention with the correct TargetAccount and (if not // yet set) TargetAccountURI, returning the populated diff --git a/internal/federation/dereferencing/status_permitted.go b/internal/federation/dereferencing/status_permitted.go index 9ad425c2f..5d05c5de4 100644 --- a/internal/federation/dereferencing/status_permitted.go +++ b/internal/federation/dereferencing/status_permitted.go @@ -62,6 +62,7 @@ func (d *Dereferencer) isPermittedStatus( requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, + isNew bool, ) ( permitted bool, // is permitted? err error, @@ -98,7 +99,7 @@ func (d *Dereferencer) isPermittedStatus( permitted = true } - if !permitted && existing != nil { + if !permitted && !isNew { log.Infof(ctx, "deleting unpermitted: %s", existing.URI) // Delete existing status from database as it's no longer permitted. @@ -110,11 +111,13 @@ func (d *Dereferencer) isPermittedStatus( return } +// isPermittedReply ... func (d *Dereferencer) isPermittedReply( ctx context.Context, requestUser string, reply *gtsmodel.Status, ) (bool, error) { + var ( replyURI = reply.URI // Definitely set. inReplyToURI = reply.InReplyToURI // Definitely set. @@ -149,8 +152,7 @@ func (d *Dereferencer) isPermittedReply( // If this status's parent was rejected, // implicitly this reply should be too; // there's nothing more to check here. - return false, d.unpermittedByParent( - ctx, + return false, d.unpermittedByParent(ctx, reply, thisReq, parentReq, @@ -164,6 +166,7 @@ func (d *Dereferencer) isPermittedReply( // be approved, then we should just reject it // again, as nothing's changed since last time. if thisRejected && acceptIRI == "" { + // Nothing changed, // still rejected. return false, nil @@ -174,6 +177,7 @@ func (d *Dereferencer) isPermittedReply( // to be approved. Continue permission checks. if inReplyTo == nil { + // If we didn't have the replied-to status // in our database (yet), we can't check // right now if this reply is permitted. diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go index 3b2c2bff2..4b3bd6d67 100644 --- a/internal/federation/dereferencing/status_test.go +++ b/internal/federation/dereferencing/status_test.go @@ -21,14 +21,21 @@ import ( "context" "fmt" "testing" + "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/superseriousbusiness/gotosocial/testrig" ) +// instantFreshness is the shortest possible freshness window. +var instantFreshness = util.Ptr(dereferencing.FreshnessWindow(0)) + type StatusTestSuite struct { DereferencerStandardTestSuite } @@ -229,6 +236,219 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithNonMatchingURI() { suite.Nil(fetchedStatus) } +func (suite *StatusTestSuite) TestDereferencerRefreshStatusUpdated() { + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // The local account we will be fetching statuses as. + fetchingAccount := suite.testAccounts["local_account_1"] + + // The test status in question that we will be dereferencing from "remote". + testURIStr := "https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839" + testURI := testrig.URLMustParse(testURIStr) + testStatusable := suite.client.TestRemoteStatuses[testURIStr] + + // Fetch the remote status first to load it into instance. + testStatus, statusable, err := suite.dereferencer.GetStatusByURI(ctx, + fetchingAccount.Username, + testURI, + ) + suite.NotNil(statusable) + suite.NoError(err) + + // Run through multiple possible edits. + for _, testCase := range []struct { + editedContent string + editedContentWarning string + editedLanguage string + editedSensitive bool + editedAttachmentIDs []string + editedPollOptions []string + editedPollVotes []int + editedAt time.Time + }{ + { + editedContent: "updated status content!", + editedContentWarning: "CW: edited status content", + editedLanguage: testStatus.Language, // no change + editedSensitive: *testStatus.Sensitive, // no change + editedAttachmentIDs: testStatus.AttachmentIDs, // no change + editedPollOptions: getPollOptions(testStatus), // no change + editedPollVotes: getPollVotes(testStatus), // no change + editedAt: time.Now(), + }, + } { + // Take a snapshot of current + // state of the test status. + testStatus = copyStatus(testStatus) + + // Edit the "remote" statusable obj. + suite.editStatusable(testStatusable, + testCase.editedContent, + testCase.editedContentWarning, + testCase.editedLanguage, + testCase.editedSensitive, + testCase.editedAttachmentIDs, + testCase.editedPollOptions, + testCase.editedPollVotes, + testCase.editedAt, + ) + + // Refresh with a given statusable to updated to edited copy. + latest, statusable, err := suite.dereferencer.RefreshStatus(ctx, + fetchingAccount.Username, + testStatus, + nil, // NOTE: can provide testStatusable here to test as being received (not deref'd) + instantFreshness, + ) + suite.NotNil(statusable) + suite.NoError(err) + + // verify updated status details. + suite.verifyEditedStatusUpdate( + + // the original status + // before any changes. + testStatus, + + // latest status + // being tested. + latest, + + // expected current state. + >smodel.StatusEdit{ + Content: testCase.editedContent, + ContentWarning: testCase.editedContentWarning, + Language: testCase.editedLanguage, + Sensitive: &testCase.editedSensitive, + AttachmentIDs: testCase.editedAttachmentIDs, + PollOptions: testCase.editedPollOptions, + PollVotes: testCase.editedPollVotes, + // createdAt never changes + }, + + // expected historic edit. + >smodel.StatusEdit{ + Content: testStatus.Content, + ContentWarning: testStatus.ContentWarning, + Language: testStatus.Language, + Sensitive: testStatus.Sensitive, + AttachmentIDs: testStatus.AttachmentIDs, + PollOptions: getPollOptions(testStatus), + PollVotes: getPollVotes(testStatus), + CreatedAt: testStatus.UpdatedAt, + }, + ) + } +} + +// editStatusable updates the given statusable attributes. +// note that this acts on the original object, no copying. +func (suite *StatusTestSuite) editStatusable( + statusable ap.Statusable, + content string, + contentWarning string, + language string, + sensitive bool, + attachmentIDs []string, // TODO: this will require some thinking as to how ... + pollOptions []string, // TODO: this will require changing statusable type to question + pollVotes []int, // TODO: this will require changing statusable type to question + editedAt time.Time, +) { + // simply reset all mentions / emojis / tags + statusable.SetActivityStreamsTag(nil) + + // Update the statusable content property + language (if set). + contentProp := streams.NewActivityStreamsContentProperty() + statusable.SetActivityStreamsContent(contentProp) + contentProp.AppendXMLSchemaString(content) + if language != "" { + contentProp.AppendRDFLangString(map[string]string{ + language: content, + }) + } + + // Update the statusable content-warning property. + summaryProp := streams.NewActivityStreamsSummaryProperty() + statusable.SetActivityStreamsSummary(summaryProp) + summaryProp.AppendXMLSchemaString(contentWarning) + + // Update the statusable sensitive property. + sensitiveProp := streams.NewActivityStreamsSensitiveProperty() + statusable.SetActivityStreamsSensitive(sensitiveProp) + sensitiveProp.AppendXMLSchemaBoolean(sensitive) + + // Update the statusable updated property. + ap.SetUpdated(statusable, editedAt) +} + +// verifyEditedStatusUpdate verifies that a given status has +// the expected number of historic edits, the 'current' status +// attributes (encapsulated as an edit for minimized no. args), +// and the last given 'historic' status edit attributes. +func (suite *StatusTestSuite) verifyEditedStatusUpdate( + testStatus *gtsmodel.Status, // the original model + status *gtsmodel.Status, // the status to check + current *gtsmodel.StatusEdit, // expected current state + historic *gtsmodel.StatusEdit, // historic edit we expect to have +) { + // don't use this func + // name in error msgs. + suite.T().Helper() + + // Check we have expected number of edits. + previousEdits := len(testStatus.Edits) + suite.Len(status.Edits, previousEdits+1) + suite.Len(status.EditIDs, previousEdits+1) + + // Check current state of status. + suite.Equal(current.Content, status.Content) + suite.Equal(current.ContentWarning, status.ContentWarning) + suite.Equal(current.Language, status.Language) + suite.Equal(*current.Sensitive, *status.Sensitive) + suite.Equal(current.AttachmentIDs, status.AttachmentIDs) + suite.Equal(current.PollOptions, getPollOptions(status)) + suite.Equal(current.PollVotes, getPollVotes(status)) + + // Check the latest historic edit matches expected. + latestEdit := status.Edits[len(status.Edits)-1] + suite.Equal(historic.Content, latestEdit.Content) + suite.Equal(historic.ContentWarning, latestEdit.ContentWarning) + suite.Equal(historic.Language, latestEdit.Language) + suite.Equal(*historic.Sensitive, *latestEdit.Sensitive) + suite.Equal(historic.AttachmentIDs, latestEdit.AttachmentIDs) + suite.Equal(historic.PollOptions, latestEdit.PollOptions) + suite.Equal(historic.PollVotes, latestEdit.PollVotes) + suite.Equal(historic.CreatedAt, latestEdit.CreatedAt) + + // The status creation date should never change. + suite.Equal(testStatus.CreatedAt, status.CreatedAt) +} + func TestStatusTestSuite(t *testing.T) { suite.Run(t, new(StatusTestSuite)) } + +// copyStatus returns a copy of the given status model (not including sub-structs). +func copyStatus(status *gtsmodel.Status) *gtsmodel.Status { + copy := new(gtsmodel.Status) + *copy = *status + return copy +} + +// getPollOptions extracts poll option strings from status (if poll is set). +func getPollOptions(status *gtsmodel.Status) []string { + if status.Poll != nil { + return status.Poll.Options + } + return nil +} + +// getPollVotes extracts poll vote counts from status (if poll is set). +func getPollVotes(status *gtsmodel.Status) []int { + if status.Poll != nil { + return status.Poll.Votes + } + return nil +} diff --git a/internal/federation/dereferencing/util.go b/internal/federation/dereferencing/util.go index 297e90adc..208117660 100644 --- a/internal/federation/dereferencing/util.go +++ b/internal/federation/dereferencing/util.go @@ -52,15 +52,15 @@ func emojiChanged(existing, latest *gtsmodel.Emoji) bool { // pollChanged returns whether a poll has changed in way that // indicates that this should be an entirely new poll. i.e. if -// the available options have changed, or the expiry has increased. +// the available options have changed, or the expiry has changed. func pollChanged(existing, latest *gtsmodel.Poll) bool { return !slices.Equal(existing.Options, latest.Options) || !existing.ExpiresAt.Equal(latest.ExpiresAt) } -// pollUpdated returns whether a poll has updated, i.e. if the +// pollStateUpdated returns whether a poll has updated, i.e. if // vote counts have changed, or if it has expired / been closed. -func pollUpdated(existing, latest *gtsmodel.Poll) bool { +func pollStateUpdated(existing, latest *gtsmodel.Poll) bool { return *existing.Voters != *latest.Voters || !slices.Equal(existing.Votes, latest.Votes) || !existing.ClosedAt.Equal(latest.ClosedAt) diff --git a/internal/federation/federatingdb/announce_test.go b/internal/federation/federatingdb/announce_test.go index 264279253..5bb2fc877 100644 --- a/internal/federation/federatingdb/announce_test.go +++ b/internal/federation/federatingdb/announce_test.go @@ -79,7 +79,7 @@ func (suite *AnnounceTestSuite) TestAnnounceTwice() { // Insert the boost-of status into the // DB cache to emulate processor handling - boost.ID, _ = id.NewULIDFromTime(boost.CreatedAt) + boost.ID = id.NewULIDFromTime(boost.CreatedAt) suite.state.Caches.DB.Status.Put(boost) // only the URI will be set for the boosted status diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go index f4bfb5929..5cf6f60a6 100644 --- a/internal/gtsmodel/mediaattachment.go +++ b/internal/gtsmodel/mediaattachment.go @@ -26,7 +26,6 @@ import ( type MediaAttachment struct { ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated StatusID string `bun:"type:CHAR(26),nullzero"` // ID of the status to which this is attached URL string `bun:",nullzero"` // Where can the attachment be retrieved on *this* server RemoteURL string `bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media) diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go index 24e83f904..180193f0f 100644 --- a/internal/gtsmodel/mention.go +++ b/internal/gtsmodel/mention.go @@ -26,7 +26,6 @@ import ( type Mention struct { ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // ID of the status this mention originates from Status *Status `bun:"rel:belongs-to"` // status referred to by statusID OriginAccountID string `bun:"type:CHAR(26),nullzero,notnull"` // ID of the mention creator account diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index f8bd068ab..4c65d8a88 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -20,6 +20,8 @@ package gtsmodel import ( "slices" "time" + + "github.com/superseriousbusiness/gotosocial/internal/util/xslices" ) // Status represents a user-created 'post' or 'status' in the database, either remote or local @@ -55,6 +57,8 @@ type Status struct { BoostOf *Status `bun:"-"` // status that corresponds to boostOfID BoostOfAccount *Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null + EditIDs []string `bun:"edits,array"` // + Edits []*StatusEdit `bun:"-"` // PollID string `bun:"type:CHAR(26),nullzero"` // Poll *Poll `bun:"-"` // ContentWarning string `bun:",nullzero"` // cw string for this status @@ -92,7 +96,8 @@ func (s *Status) GetBoostOfAccountID() string { return s.BoostOfAccountID } -// AttachmentsPopulated returns whether media attachments are populated according to current AttachmentIDs. +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. func (s *Status) AttachmentsPopulated() bool { if len(s.AttachmentIDs) != len(s.Attachments) { // this is the quickest indicator. @@ -106,7 +111,8 @@ func (s *Status) AttachmentsPopulated() bool { return true } -// TagsPopulated returns whether tags are populated according to current TagIDs. +// TagsPopulated returns whether tags are +// populated according to current TagIDs. func (s *Status) TagsPopulated() bool { if len(s.TagIDs) != len(s.Tags) { // this is the quickest indicator. @@ -120,7 +126,8 @@ func (s *Status) TagsPopulated() bool { return true } -// MentionsPopulated returns whether mentions are populated according to current MentionIDs. +// MentionsPopulated returns whether mentions are +// populated according to current MentionIDs. func (s *Status) MentionsPopulated() bool { if len(s.MentionIDs) != len(s.Mentions) { // this is the quickest indicator. @@ -134,7 +141,8 @@ func (s *Status) MentionsPopulated() bool { return true } -// EmojisPopulated returns whether emojis are populated according to current EmojiIDs. +// EmojisPopulated returns whether emojis are +// populated according to current EmojiIDs. func (s *Status) EmojisPopulated() bool { if len(s.EmojiIDs) != len(s.Emojis) { // this is the quickest indicator. @@ -148,6 +156,21 @@ func (s *Status) EmojisPopulated() bool { return true } +// EditsPopulated returns whether edits are +// populated according to current EditIDs. +func (s *Status) EditsPopulated() bool { + if len(s.EditIDs) != len(s.Edits) { + // this is quickest indicator. + return false + } + for i, id := range s.EditIDs { + if s.Edits[i].ID != id { + return false + } + } + return true +} + // EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. @@ -247,6 +270,35 @@ func (s *Status) IsLocalOnly() bool { return s.Federated == nil || !*s.Federated } +// AllAttachmentIDs gathers ALL media attachment IDs from both the +// receiving Status{}, and any historical Status{}.Edits. Note that +// this function will panic if Status{}.Edits is not populated. +func (s *Status) AllAttachmentIDs() []string { + var total int + + if len(s.EditIDs) != len(s.Edits) { + panic("status edits not populated") + } + + // Get count of attachment IDs. + total += len(s.Attachments) + for _, edit := range s.Edits { + total += len(edit.AttachmentIDs) + } + + // Start gathering of all IDs with *current* attachment IDs. + attachmentIDs := make([]string, len(s.AttachmentIDs), total) + copy(attachmentIDs, s.AttachmentIDs) + + // Append IDs of historical edits. + for _, edit := range s.Edits { + attachmentIDs = append(attachmentIDs, edit.AttachmentIDs...) + } + + // Deduplicate these IDs in case of shared media. + return xslices.Deduplicate(attachmentIDs) +} + // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags. type StatusToTag struct { StatusID string `bun:"type:CHAR(26),unique:statustag,nullzero,notnull"` diff --git a/internal/gtsmodel/statusedit.go b/internal/gtsmodel/statusedit.go new file mode 100644 index 000000000..199d47736 --- /dev/null +++ b/internal/gtsmodel/statusedit.go @@ -0,0 +1,62 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import "time" + +// StatusEdit represents a **historical** view of a Status +// after a received edit. The Status itself will always +// contain the latest up-to-date information. +// +// Note that stored status edits may not exactly match that +// of the origin server, they are a best-effort by receiver +// to store version history. There is no AP history endpoint. +type StatusEdit struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. + Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed. + ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit. + Text string `bun:""` // Original status text, without formatting, at time of edit. + Language string `bun:",nullzero"` // Status language at time of edit. + Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit. + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of media attachments associated with status at time of edit. + AttachmentDescriptions []string `bun:",array"` // Previous media descriptions of media attachments associated with status at time of edit. + Attachments []*MediaAttachment `bun:"-"` // Media attachments relating to .AttachmentIDs field (not always populated). + PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll. + PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset. + StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of. + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server). + + // We don't bother having a *gtsmodel.Status model here + // as the StatusEdit is always just attached to a Status, + // so it doesn't need a self-reference back to it. +} + +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. +func (e *StatusEdit) AttachmentsPopulated() bool { + if len(e.AttachmentIDs) != len(e.Attachments) { + // this is the quickest indicator. + return false + } + for i, id := range e.AttachmentIDs { + if e.Attachments[i].ID != id { + return false + } + } + return true +} diff --git a/internal/id/ulid.go b/internal/id/ulid.go index 8de4cc4cc..8c0b1e94c 100644 --- a/internal/id/ulid.go +++ b/internal/id/ulid.go @@ -22,7 +22,9 @@ import ( "math/big" "time" + "codeberg.org/gruf/go-kv" "github.com/oklog/ulid" + "github.com/superseriousbusiness/gotosocial/internal/log" ) const ( @@ -45,13 +47,19 @@ func NewULID() string { return ulid.String() } -// NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong. -func NewULIDFromTime(t time.Time) (string, error) { - newUlid, err := ulid.New(ulid.Timestamp(t), rand.Reader) - if err != nil { - return "", err +// NewULIDFromTime returns a new ULID string using +// given time, or from current time on any error. +func NewULIDFromTime(t time.Time) string { + ts := ulid.Timestamp(t) + if ts > ulid.MaxTime() { + log.WarnKVs(nil, kv.Fields{ + {K: "caller", V: log.Caller(2)}, + {K: "value", V: t}, + {K: "msg", V: "invalid ulid time"}, + }...) + ts = ulid.Now() } - return newUlid.String(), nil + return ulid.MustNew(ts, rand.Reader).String() } // NewRandomULID returns a new ULID string using a random time in an ~80 year range around the current datetime, or an error if something goes wrong. diff --git a/internal/media/manager.go b/internal/media/manager.go index 2807848bd..6aa13c17b 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -118,15 +118,11 @@ func (m *Manager) CreateMedia( Header: util.Ptr(false), Cached: util.Ptr(false), CreatedAt: now, - UpdatedAt: now, } // Check if we were provided additional info // to add to the attachment, and overwrite // some of the attachment fields if so. - if info.CreatedAt != nil { - attachment.CreatedAt = *info.CreatedAt - } if info.StatusID != nil { attachment.StatusID = *info.StatusID } @@ -372,9 +368,6 @@ func (m *Manager) createOrUpdateEmoji( if info.URI != nil { emoji.URI = *info.URI } - if info.CreatedAt != nil { - emoji.CreatedAt = *info.CreatedAt - } if info.Domain != nil { emoji.Domain = *info.Domain } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index e175369f5..5b6882100 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -109,7 +109,6 @@ func (suite *ManagerTestSuite) TestEmojiProcessRefresh() { emojiToUpdate, data, media.AdditionalEmojiInfo{ - CreatedAt: &emojiToUpdate.CreatedAt, Domain: &emojiToUpdate.Domain, ImageRemoteURL: &newImageRemoteURL, }, diff --git a/internal/media/types.go b/internal/media/types.go index 9631a15bd..827752941 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,7 +20,6 @@ package media import ( "context" "io" - "time" ) type Size string @@ -44,10 +43,6 @@ const ( // should be added to attachment when processing a piece of media. type AdditionalMediaInfo struct { - // Time that this media was - // created; defaults to time.Now(). - CreatedAt *time.Time - // ID of the status to which this // media is attached; defaults to "". StatusID *string @@ -93,10 +88,6 @@ type AdditionalEmojiInfo struct { // this remote emoji. URI *string - // Time that this emoji was - // created; defaults to time.Now(). - CreatedAt *time.Time - // Domain the emoji originated from. Blank // for this instance's domain. Defaults to "". Domain *string diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index e4706d3b7..5606151c2 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -70,7 +70,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(context.Background(), "the_mighty_zork") suite.NoError(err) - suite.EqualValues(1704878640, lastModified.Unix()) + suite.EqualValues(1730451600, lastModified.Unix()) feed, err := getFeed() suite.NoError(err) @@ -79,14 +79,24 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { <title>Posts from @the_mighty_zork@localhost:8080</title> <link>http://localhost:8080/@the_mighty_zork</link> <description>Posts from @the_mighty_zork@localhost:8080</description> - <pubDate>Wed, 10 Jan 2024 09:24:00 +0000</pubDate> - <lastBuildDate>Wed, 10 Jan 2024 09:24:00 +0000</lastBuildDate> + <pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate> + <lastBuildDate>Fri, 01 Nov 2024 09:00:00 +0000</lastBuildDate> <image> <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp</url> <title>Avatar for @the_mighty_zork@localhost:8080</title> <link>http://localhost:8080/@the_mighty_zork</link> </image> <item> + <title>edited status</title> + <link>http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</link> + <description>@the_mighty_zork@localhost:8080 made a new post: "this is the latest revision of the status, with a content-warning"</description> + <content:encoded><![CDATA[<p>this is the latest revision of the status, with a content-warning</p>]]></content:encoded> + <author>@the_mighty_zork@localhost:8080</author> + <guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</guid> + <pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate> + <source>http://localhost:8080/@the_mighty_zork/feed.rss</source> + </item> + <item> <title>HTML in post</title> <link>http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</link> <description>@the_mighty_zork@localhost:8080 made a new post: "Here's a bunch of HTML, read it and weep, weep then!

`+"```"+`html
<section class="about-user">
 <div class="col-header">
 <h2>About</h2>
 </div> 
 <div class="fields">
 <h3 class="sr-only">Fields</h3>
 <dl>
...</description> diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 6962601f2..11d8f7eb5 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -177,9 +177,7 @@ func (p *Processor) getAttachmentContent( } // Start preparing API content model. - apiContent := &apimodel.Content{ - ContentUpdated: attach.UpdatedAt, - } + apiContent := &apimodel.Content{} // Retrieve appropriate // size file from storage. diff --git a/internal/processing/media/unattach_test.go b/internal/processing/media/unattach_test.go index 051caa4d3..02d2c7077 100644 --- a/internal/processing/media/unattach_test.go +++ b/internal/processing/media/unattach_test.go @@ -20,7 +20,6 @@ package media_test import ( "context" "testing" - "time" "github.com/stretchr/testify/suite" ) @@ -42,8 +41,6 @@ func (suite *UnattachTestSuite) TestUnattachMedia() { dbAttachment, errWithCode := suite.db.GetAttachmentByID(ctx, a.ID) suite.NoError(errWithCode) - - suite.WithinDuration(dbAttachment.UpdatedAt, time.Now(), 1*time.Minute) suite.Empty(dbAttachment.StatusID) } diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 75a687db2..470b93a8f 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -67,7 +67,6 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account if errWithCode != nil { return nil, errWithCode } - return p.c.GetAPIStatus(ctx, requestingAccount, targetStatus) } @@ -106,5 +105,6 @@ func (p *Processor) SourceGet(ctx context.Context, requestingAccount *gtsmodel.A err = gtserror.Newf("error converting status: %w", err) return nil, gtserror.NewErrorInternalError(err) } + return statusSource, nil } diff --git a/internal/processing/stream/notification_test.go b/internal/processing/stream/notification_test.go index 169e4f5ce..5c89e1f40 100644 --- a/internal/processing/stream/notification_test.go +++ b/internal/processing/stream/notification_test.go @@ -79,8 +79,8 @@ func (suite *NotificationTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index b61a9c623..6bf5e436c 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -54,6 +54,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.Equal(`{ "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -90,8 +91,8 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, diff --git a/internal/processing/timeline/public_test.go b/internal/processing/timeline/public_test.go index 6b01c9849..ab8e33429 100644 --- a/internal/processing/timeline/public_test.go +++ b/internal/processing/timeline/public_test.go @@ -102,8 +102,8 @@ func (suite *PublicTestSuite) TestPublicTimelineGetHideFiltered() { requester = suite.testAccounts["local_account_1"] maxID = "" sinceID = "" - minID = "01F8MHAAY43M6RJ473VQFCVH36" // 1 before filteredStatus - limit = 10 + minID = "" + limit = 100 local = false filteredStatus = suite.testStatuses["admin_account_status_2"] filteredStatusFound = false diff --git a/internal/processing/workers/util.go b/internal/processing/workers/util.go index 62ea6c95c..b358dc951 100644 --- a/internal/processing/workers/util.go +++ b/internal/processing/workers/util.go @@ -75,6 +75,21 @@ func (u *utils) wipeStatus( } } + // Before handling media, ensure + // historic edits are populated. + if !status.EditsPopulated() { + var err error + + // Fetch all historical edits of status from database. + status.Edits, err = u.state.DB.GetStatusEditsByIDs( + gtscontext.SetBarebones(ctx), + status.EditIDs, + ) + if err != nil { + errs.Appendf("error getting status edits from database: %w", err) + } + } + // Either delete all attachments for this status, // or simply detach + clean them separately later. // @@ -83,20 +98,27 @@ func (u *utils) wipeStatus( // status immediately (in case of delete + redraft). if deleteAttachments { // todo:u.state.DB.DeleteAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if err := u.media.Delete(ctx, id); err != nil { errs.Appendf("error deleting media: %w", err) } } } else { // todo:u.state.DB.UnattachAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if _, err := u.media.Unattach(ctx, status.Account, id); err != nil { errs.Appendf("error unattaching media: %w", err) } } } + // Delete all historical edits of status. + if ids := status.EditIDs; len(ids) > 0 { + if err := u.state.DB.DeleteStatusEdits(ctx, ids); err != nil { + errs.Appendf("error deleting status edits: %w", err) + } + } + // Delete all mentions generated by this status. // todo:u.state.DB.DeleteMentionsForStatus for _, id := range status.MentionIDs { @@ -120,19 +142,20 @@ func (u *utils) wipeStatus( errs.Appendf("error deleting status faves: %w", err) } - if pollID := status.PollID; pollID != "" { + if id := status.PollID; id != "" { // Delete this poll by ID from the database. - if err := u.state.DB.DeletePollByID(ctx, pollID); err != nil { + if err := u.state.DB.DeletePollByID(ctx, id); err != nil { errs.Appendf("error deleting status poll: %w", err) } // Cancel any scheduled expiry task for poll. - _ = u.state.Workers.Scheduler.Cancel(pollID) + _ = u.state.Workers.Scheduler.Cancel(id) } // Get all boost of this status so that we can // delete those boosts + remove them from timelines. boosts, err := u.state.DB.GetStatusBoosts( + // We MUST set a barebones context here, // as depending on where it came from the // original BoostOf may already be gone. @@ -537,11 +560,7 @@ func (u *utils) requestFave( } // Create + store new interaction request. - req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusFaveToInteractionRequest(fave) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -584,11 +603,7 @@ func (u *utils) requestReply( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(reply) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -631,11 +646,7 @@ func (u *utils) requestAnnounce( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(boost) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index 6b01ca812..91a456560 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -228,7 +228,7 @@ func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossible() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 20) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) } func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() { @@ -255,7 +255,7 @@ func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 20) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) } func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() { @@ -284,7 +284,7 @@ func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 8) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 9) for _, s := range statuses { if s.GetAccountID() != testAccount.ID { diff --git a/internal/timeline/prune_test.go b/internal/timeline/prune_test.go index e78db64e8..4b909540c 100644 --- a/internal/timeline/prune_test.go +++ b/internal/timeline/prune_test.go @@ -40,7 +40,7 @@ func (suite *PruneTestSuite) TestPrune() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(20, pruned) + suite.Equal(23, pruned) suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } @@ -56,7 +56,7 @@ func (suite *PruneTestSuite) TestPruneTwice() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(20, pruned) + suite.Equal(23, pruned) suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) // Prune same again, nothing should be pruned this time. @@ -78,7 +78,7 @@ func (suite *PruneTestSuite) TestPruneTo0() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(25, pruned) + suite.Equal(28, pruned) suite.Equal(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } @@ -95,7 +95,7 @@ func (suite *PruneTestSuite) TestPruneToInfinityAndBeyond() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) suite.Equal(0, pruned) - suite.Equal(25, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) + suite.Equal(28, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } func TestPruneTestSuite(t *testing.T) { diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index cf0c0719a..1a7098673 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -111,6 +111,13 @@ func (c *Converter) ASRepresentationToAccount( acct.UpdatedAt = pub } + // Extract updated time if possible, i.e. last edited. + if upd := ap.GetUpdated(accountable); !upd.IsZero() { + acct.UpdatedAt = upd + } else { + acct.UpdatedAt = acct.CreatedAt + } + // Extract a preferred name (display name), fallback to username. if displayName := ap.ExtractName(accountable); displayName != "" { acct.DisplayName = displayName @@ -348,18 +355,25 @@ func (c *Converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab // zero-time will fall back to db defaults. if pub := ap.GetPublished(statusable); !pub.IsZero() { status.CreatedAt = pub - status.UpdatedAt = pub } else { log.Warnf(ctx, "unusable published property on %s", uri) } + // status.Updated + // + // Extract updated time for status, defaults to Published. + if upd := ap.GetUpdated(statusable); !upd.IsZero() { + status.UpdatedAt = upd + } else { + status.UpdatedAt = status.CreatedAt + } + // status.AccountURI // status.AccountID // status.Account // - // Account that created the status. Assume we have - // this in the db by the time this function is called, - // error if we don't. + // Account that created the status. Assume we have this + // in the db by the time this function is called, else error. status.Account, err = c.getASAttributedToAccount(ctx, status.URI, statusable, diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index ccde6a38f..573495e0a 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -104,14 +104,8 @@ func (c *Converter) StatusToBoost( return boost, nil } -func StatusToInteractionRequest( - ctx context.Context, - status *gtsmodel.Status, -) (*gtsmodel.InteractionRequest, error) { - reqID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating ID: %w", err) - } +func StatusToInteractionRequest(status *gtsmodel.Status) *gtsmodel.InteractionRequest { + reqID := id.NewULIDFromTime(status.CreatedAt) var ( targetID string @@ -154,17 +148,11 @@ func StatusToInteractionRequest( InteractionType: interactionType, Reply: reply, Announce: announce, - }, nil + } } -func StatusFaveToInteractionRequest( - ctx context.Context, - fave *gtsmodel.StatusFave, -) (*gtsmodel.InteractionRequest, error) { - reqID, err := id.NewULIDFromTime(fave.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating ID: %w", err) - } +func StatusFaveToInteractionRequest(fave *gtsmodel.StatusFave) *gtsmodel.InteractionRequest { + reqID := id.NewULIDFromTime(fave.CreatedAt) return >smodel.InteractionRequest{ ID: reqID, @@ -178,7 +166,7 @@ func StatusFaveToInteractionRequest( InteractionURI: fave.URI, InteractionType: gtsmodel.InteractionLike, Like: fave, - }, nil + } } func (c *Converter) StatusToSinBinStatus( diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index d46ce64e2..7d0c483dd 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -484,10 +484,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat status.SetActivityStreamsInReplyTo(inReplyToProp) } - // published - publishedProp := streams.NewActivityStreamsPublishedProperty() - publishedProp.Set(s.CreatedAt) - status.SetActivityStreamsPublished(publishedProp) + // Set created / updated at properties. + ap.SetPublished(status, s.CreatedAt) + ap.SetUpdated(status, s.UpdatedAt) // url if s.URL != "" { diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index c847cfc93..9870c760a 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -499,6 +499,7 @@ func (suite *InternalToASTestSuite) TestStatusToAS() { "tag": [], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T12:40:37+02:00", "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY" }`, string(bytes)) } @@ -598,6 +599,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() { ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T11:36:45Z", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" }`, string(bytes)) } @@ -698,6 +700,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T11:36:45Z", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" }`, string(bytes)) } @@ -778,6 +781,7 @@ func (suite *InternalToASTestSuite) TestStatusToASWithMentions() { }, "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-11-20T13:32:16Z", "url": "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0" }`, string(bytes)) } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index fda59610b..e0276a53b 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1399,17 +1399,13 @@ func (c *Converter) baseStatusToFrontend( } // Nullable fields. - if s.InReplyToID != "" { - apiStatus.InReplyToID = util.Ptr(s.InReplyToID) - } - - if s.InReplyToAccountID != "" { - apiStatus.InReplyToAccountID = util.Ptr(s.InReplyToAccountID) - } - - if s.Language != "" { - apiStatus.Language = util.Ptr(s.Language) + if !s.UpdatedAt.Equal(s.CreatedAt) { + timestamp := util.FormatISO8601(s.UpdatedAt) + apiStatus.EditedAt = util.Ptr(timestamp) } + apiStatus.InReplyToID = util.PtrIf(s.InReplyToID) + apiStatus.InReplyToAccountID = util.PtrIf(s.InReplyToAccountID) + apiStatus.Language = util.PtrIf(s.Language) if app := s.CreatedWithApplication; app != nil { apiStatus.Application, err = c.AppToAPIAppPublic(ctx, app) diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index db37d3abd..0ec9ea05f 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -67,8 +67,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontend() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -119,8 +119,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendAliasedAndMoved() "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "source": { @@ -162,8 +162,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendAliasedAndMoved() "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -217,8 +217,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct() "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [ { "shortcode": "rainbow", @@ -266,8 +266,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [ { "shortcode": "rainbow", @@ -311,8 +311,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "source": { @@ -463,6 +463,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -641,6 +642,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -807,6 +809,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { suite.Equal(`{ "id": "01G36SF3V6Y6V5BF9P4R7PQG7G", "created_at": "2021-10-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -827,6 +830,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { "reblog": { "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -870,8 +874,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -1218,6 +1222,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments suite.Equal(`{ "id": "01HE7XJ1CG84TBKH5V9XKBVGF5", "created_at": "2023-11-02T10:44:25.000Z", + "edited_at": null, "in_reply_to_id": "01F8MH75CBF9JFX4ZAD54N0W0R", "in_reply_to_account_id": "01F8MH17FWEB39HZJ76B6VXSKF", "sensitive": true, @@ -1350,6 +1355,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToWebStatus() { suite.Equal(`{ "id": "01HE7XJ1CG84TBKH5V9XKBVGF5", "created_at": "2023-11-02T10:44:25.000Z", + "edited_at": null, "in_reply_to_id": "01F8MH75CBF9JFX4ZAD54N0W0R", "in_reply_to_account_id": "01F8MH17FWEB39HZJ76B6VXSKF", "sensitive": true, @@ -1511,6 +1517,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1654,6 +1661,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction suite.Equal(`{ "id": "01F8MHBBN8120SYH7D5S050MGK", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1697,8 +1705,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -1764,6 +1772,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval() suite.Equal(`{ "id": "01J5QVB9VC76NPPRQ207GG4DRZ", "created_at": "2024-02-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": "01F8MHC8VWDRBQR0N1BATDDEM5", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "sensitive": false, @@ -1993,7 +2002,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -2277,8 +2286,8 @@ func (suite *InternalToFrontendTestSuite) TestReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2321,8 +2330,8 @@ func (suite *InternalToFrontendTestSuite) TestReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2398,8 +2407,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2444,8 +2453,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2636,8 +2645,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2695,8 +2704,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2707,6 +2716,7 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -2743,8 +2753,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -2902,8 +2912,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontendSuspendedLoca "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -3214,6 +3224,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "status": { "id": "01F8MHC8VWDRBQR0N1BATDDEM5", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3254,8 +3265,8 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -3307,6 +3318,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "reply": { "id": "01J5QVB9VC76NPPRQ207GG4DRZ", "created_at": "2024-02-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": "01F8MHC8VWDRBQR0N1BATDDEM5", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "sensitive": false, @@ -3464,8 +3476,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -3474,6 +3486,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "last_status": { "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3517,8 +3530,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -3619,8 +3632,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -3640,6 +3653,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "last_status": { "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3683,8 +3697,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/typeutils/wrap_test.go b/internal/typeutils/wrap_test.go index 1085c8c66..c2c9c9464 100644 --- a/internal/typeutils/wrap_test.go +++ b/internal/typeutils/wrap_test.go @@ -131,6 +131,7 @@ func (suite *WrapTestSuite) TestWrapNoteInCreate() { "tag": [], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T12:40:37+02:00", "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY" }, "published": "2021-10-20T12:40:37+02:00", |