From d8113c11e4d84a6d04d56b58d337c235154a535b Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:04:34 +0100 Subject: [feature] Parse content warning to HTML, serialize via client API as plaintext (#3876) * [feature] Parse content warning as HTML, serialize via API to plaintext * tidy up some cruft * whoops * oops * i'm da joker baybee * clemency muy lorde * rename some of the text functions for clarity * jiggle the opts * fiddle de deee * hopefully the last test fix i ever have to do in my beautiful life --- internal/processing/account/rss_test.go | 8 +++---- internal/processing/account/update.go | 18 +++++++------- internal/processing/admin/domainallow.go | 4 ++-- internal/processing/admin/domainblock.go | 4 ++-- internal/processing/instance.go | 4 ++-- internal/processing/media/update.go | 2 +- internal/processing/status/common.go | 31 ++++++++++++++----------- internal/processing/status/create.go | 7 ++++++ internal/processing/status/create_test.go | 27 --------------------- internal/processing/status/delete.go | 7 ++++++ internal/processing/status/edit.go | 9 ++++++- internal/processing/status/get.go | 13 ++++++++++- internal/processing/stream/statusupdate_test.go | 2 +- internal/processing/user/create.go | 2 +- 14 files changed, 73 insertions(+), 65 deletions(-) (limited to 'internal/processing') diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index 5606151c2..9b6b1ee75 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -43,10 +43,10 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { Wed, 20 Oct 2021 10:41:37 +0000 Wed, 20 Oct 2021 10:41:37 +0000 - open to see some puppies + open to see some <strong>puppies</strong> http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37 @admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕" - + 🐕🐕🐕🐕🐕

]]> @admin@localhost:8080 http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37 Wed, 20 Oct 2021 12:36:45 +0000 @@ -56,7 +56,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { hello world! #welcome ! first post on the instance :rainbow: ! http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R @admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !" - !]]> + hello world! #welcome ! first post on the instance :rainbow: !

]]> @admin@localhost:8080 http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R @@ -145,7 +145,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { introduction post http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY @the_mighty_zork@localhost:8080 made a new post: "hello everyone!" - + hello everyone!

]]> @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY Wed, 20 Oct 2021 10:40:37 +0000 diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 2bdbf96f4..3a59dbdf3 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -97,8 +97,8 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form return nil, gtserror.NewErrorBadRequest(err, err.Error()) } - // Parse new display name (always from plaintext). - account.DisplayName = text.SanitizeToPlaintext(displayName) + // HTML tags not allowed in display name. + account.DisplayName = text.StripHTMLFromText(displayName) acctColumns = append(acctColumns, "display_name") } @@ -145,7 +145,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form } if form.AvatarDescription != nil { - desc := text.SanitizeToPlaintext(*form.AvatarDescription) + desc := text.StripHTMLFromText(*form.AvatarDescription) form.AvatarDescription = &desc } @@ -175,7 +175,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form } if form.HeaderDescription != nil { - desc := text.SanitizeToPlaintext(*form.HeaderDescription) + desc := text.StripHTMLFromText(*form.HeaderDescription) form.HeaderDescription = util.Ptr(desc) } @@ -265,7 +265,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form return nil, gtserror.NewErrorBadRequest(err, err.Error()) } - account.Settings.CustomCSS = text.SanitizeToPlaintext(customCSS) + account.Settings.CustomCSS = text.StripHTMLFromText(customCSS) settingsColumns = append(settingsColumns, "custom_css") } @@ -356,8 +356,8 @@ func (p *Processor) updateFields( // Sanitize raw field values. fieldRaw := >smodel.Field{ - Name: text.SanitizeToPlaintext(name), - Value: text.SanitizeToPlaintext(value), + Name: text.StripHTMLFromText(name), + Value: text.StripHTMLFromText(value), } fieldsRaw = append(fieldsRaw, fieldRaw) } @@ -385,7 +385,7 @@ func (p *Processor) processAccountText( emojis := make(map[string]*gtsmodel.Emoji) // Retrieve display name emojis. - for _, emoji := range p.formatter.FromPlainEmojiOnly( + for _, emoji := range p.formatter.FromPlainBasic( ctx, p.parseMention, account.ID, @@ -413,7 +413,7 @@ func (p *Processor) processAccountText( // Name stays plain, but we still need to // see if there are any emojis set in it. field.Name = fieldRaw.Name - for _, emoji := range p.formatter.FromPlainEmojiOnly( + for _, emoji := range p.formatter.FromPlainBasic( ctx, p.parseMention, account.ID, diff --git a/internal/processing/admin/domainallow.go b/internal/processing/admin/domainallow.go index 13f0307f2..02101ccff 100644 --- a/internal/processing/admin/domainallow.go +++ b/internal/processing/admin/domainallow.go @@ -53,8 +53,8 @@ func (p *Processor) createDomainAllow( ID: id.NewULID(), Domain: domain, CreatedByAccountID: adminAcct.ID, - PrivateComment: text.SanitizeToPlaintext(privateComment), - PublicComment: text.SanitizeToPlaintext(publicComment), + PrivateComment: text.StripHTMLFromText(privateComment), + PublicComment: text.StripHTMLFromText(publicComment), Obfuscate: &obfuscate, SubscriptionID: subscriptionID, } diff --git a/internal/processing/admin/domainblock.go b/internal/processing/admin/domainblock.go index f8c1a6708..249df744c 100644 --- a/internal/processing/admin/domainblock.go +++ b/internal/processing/admin/domainblock.go @@ -53,8 +53,8 @@ func (p *Processor) createDomainBlock( ID: id.NewULID(), Domain: domain, CreatedByAccountID: adminAcct.ID, - PrivateComment: text.SanitizeToPlaintext(privateComment), - PublicComment: text.SanitizeToPlaintext(publicComment), + PrivateComment: text.StripHTMLFromText(privateComment), + PublicComment: text.StripHTMLFromText(publicComment), Obfuscate: &obfuscate, SubscriptionID: subscriptionID, } diff --git a/internal/processing/instance.go b/internal/processing/instance.go index 2f4c40416..4cbbb742a 100644 --- a/internal/processing/instance.go +++ b/internal/processing/instance.go @@ -165,7 +165,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe } // Don't allow html in site title. - instance.Title = text.SanitizeToPlaintext(title) + instance.Title = text.StripHTMLFromText(title) columns = append(columns, "title") } @@ -235,7 +235,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe return nil, gtserror.NewErrorBadRequest(err, err.Error()) } - instance.CustomCSS = text.SanitizeToPlaintext(customCSS) + instance.CustomCSS = text.StripHTMLFromText(customCSS) columns = append(columns, []string{"custom_css"}...) } diff --git a/internal/processing/media/update.go b/internal/processing/media/update.go index c8592395f..5afa5d63c 100644 --- a/internal/processing/media/update.go +++ b/internal/processing/media/update.go @@ -87,7 +87,7 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, media // processDescription will sanitize and valid description against server configuration. func processDescription(description string) (string, gtserror.WithCode) { - description = text.SanitizeToPlaintext(description) + description = text.StripHTMLFromText(description) chars := len([]rune(description)) if min := config.GetMediaDescriptionMinChars(); chars < min { diff --git a/internal/processing/status/common.go b/internal/processing/status/common.go index 6a4851f51..77058ed10 100644 --- a/internal/processing/status/common.go +++ b/internal/processing/status/common.go @@ -171,19 +171,25 @@ func (p *Processor) processContent( ) } - // format is the currently set text formatting - // function, according to the provided content-type. - var format text.FormatFunc + var ( + // format is the currently set text formatting + // function, according to the provided content-type. + format text.FormatFunc + // formatCW is like format, but for content warning. + formatCW text.FormatFunc + ) switch contentType { // Format status according to text/plain. case gtsmodel.StatusContentTypePlain: format = p.formatter.FromPlain + formatCW = p.formatter.FromPlainBasic // Format status according to text/markdown. case gtsmodel.StatusContentTypeMarkdown: format = p.formatter.FromMarkdown + formatCW = p.formatter.FromMarkdownBasic // Unknown. default: @@ -215,26 +221,23 @@ func (p *Processor) processContent( status.Emojis = contentRes.Emojis status.Tags = contentRes.Tags - // From here-on-out just use emoji-only - // plain-text formatting as the FormatFunc. - format = p.formatter.FromPlainEmojiOnly - // Sanitize content warning and format. - warning := text.SanitizeToPlaintext(contentWarning) - warningRes := formatInput(format, warning) + cwRes := formatInput(formatCW, contentWarning) // Gather results of the formatted. - status.ContentWarning = warningRes.HTML - status.Emojis = append(status.Emojis, warningRes.Emojis...) + status.ContentWarning = cwRes.HTML + status.Emojis = append(status.Emojis, cwRes.Emojis...) if poll != nil { // Pre-allocate slice of poll options of expected length. status.PollOptions = make([]string, len(poll.Options)) for i, option := range poll.Options { - // Sanitize each poll option and format. - option = text.SanitizeToPlaintext(option) - optionRes := formatInput(format, option) + // Strip each poll option and format. + // + // For polls just use basic formatting. + option = text.StripHTMLFromText(option) + optionRes := formatInput(p.formatter.FromPlainBasic, option) // Gather results of the formatted. status.PollOptions[i] = optionRes.HTML diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index 73ac8d677..10a5560b6 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -189,6 +189,13 @@ func (p *Processor) Create( PendingApproval: util.Ptr(false), } + // Only store ContentWarningText if the parsed + // result is different from the given SpoilerText, + // otherwise skip to avoid duplicating db columns. + if content.ContentWarning != form.SpoilerText { + status.ContentWarningText = form.SpoilerText + } + if backfill { // Ensure backfilled status contains no // mentions to anyone other than author. diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go index ec1ded3b0..1199642b6 100644 --- a/internal/processing/status/create_test.go +++ b/internal/processing/status/create_test.go @@ -60,33 +60,6 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithQuotationMarks( suite.Equal("\"test\"", apiStatus.SpoilerText) } -func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuotationMarks() { - ctx := context.Background() - - creatingAccount := suite.testAccounts["local_account_1"] - creatingApplication := suite.testApplications["application_1"] - - statusCreateForm := &apimodel.StatusCreateRequest{ - Status: "poopoo peepee", - MediaIDs: []string{}, - Poll: nil, - InReplyToID: "", - Sensitive: false, - SpoilerText: ""test"", // the html-escaped quotation marks should appear as normal quotation marks in the finished text - Visibility: apimodel.VisibilityPublic, - LocalOnly: util.Ptr(false), - ScheduledAt: nil, - Language: "en", - ContentType: apimodel.StatusContentTypePlain, - } - - apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm) - suite.NoError(err) - suite.NotNil(apiStatus) - - suite.Equal("\"test\"", apiStatus.SpoilerText) -} - func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji() { ctx := context.Background() diff --git a/internal/processing/status/delete.go b/internal/processing/status/delete.go index 700909f44..8fec8fc5e 100644 --- a/internal/processing/status/delete.go +++ b/internal/processing/status/delete.go @@ -50,6 +50,13 @@ func (p *Processor) Delete(ctx context.Context, requestingAccount *gtsmodel.Acco return nil, errWithCode } + // Replace content warning with raw + // version if it's available, to make + // delete + redraft work nicer. + if targetStatus.ContentWarningText != "" { + apiStatus.SpoilerText = targetStatus.ContentWarningText + } + // Process delete side effects. p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{ APObjectType: ap.ObjectNote, diff --git a/internal/processing/status/edit.go b/internal/processing/status/edit.go index 590fe565a..a9323a72c 100644 --- a/internal/processing/status/edit.go +++ b/internal/processing/status/edit.go @@ -301,7 +301,7 @@ func (p *Processor) Edit( // update the other necessary status fields. status.Content = content.Content status.ContentWarning = content.ContentWarning - status.Text = form.Status + status.Text = form.Status // raw status.ContentType = contentType status.Language = content.Language status.Sensitive = &form.Sensitive @@ -309,6 +309,13 @@ func (p *Processor) Edit( status.Attachments = media status.EditedAt = now + // Only store ContentWarningText if the parsed + // result is different from the given SpoilerText, + // otherwise skip to avoid duplicating db columns. + if content.ContentWarning != form.SpoilerText { + status.ContentWarningText = form.SpoilerText + } + if poll != nil { // Set relevent fields for latest with poll. status.ActivityStreamsType = ap.ActivityQuestion diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 6065c7cfe..e97d9e1b2 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -53,10 +53,21 @@ func (p *Processor) SourceGet(ctx context.Context, requester *gtsmodel.Account, "target status not found", ) } + + // Try to use unparsed content + // warning text if available, + // fall back to parsed cw html. + var spoilerText string + if status.ContentWarningText != "" { + spoilerText = status.ContentWarningText + } else { + spoilerText = status.ContentWarning + } + return &apimodel.StatusSource{ ID: status.ID, Text: status.Text, - SpoilerText: status.ContentWarning, + SpoilerText: spoilerText, ContentType: typeutils.ContentTypeToAPIContentType(status.ContentType), }, nil } diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index 180538c60..1f0bcd142 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -71,7 +71,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { "muted": false, "bookmarked": false, "pinned": false, - "content": "dark souls status bot: \"thoughts of dog\"", + "content": "\u003cp\u003edark souls status bot: \"thoughts of dog\"\u003c/p\u003e", "reblog": null, "account": { "id": "01F8MH5ZK5VRH73AKHQM6Y9VNX", diff --git a/internal/processing/user/create.go b/internal/processing/user/create.go index d2891ef0e..dde69a6ef 100644 --- a/internal/processing/user/create.go +++ b/internal/processing/user/create.go @@ -122,7 +122,7 @@ func (p *Processor) Create( Username: form.Username, Email: form.Email, Password: form.Password, - Reason: text.SanitizeToPlaintext(reason), + Reason: text.StripHTMLFromText(reason), SignUpIP: form.IP, Locale: form.Locale, AppID: app.ID, -- cgit v1.2.3