From c84384e6608368a13a774d6d33a8cc32da7cf209 Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Tue, 19 Jul 2022 15:21:17 +0200
Subject: [bugfix] html escape special characters in text instead of totally
removing them (#719)
* remove minify dependency
* tidy up some tests
* remove pre + postformat funcs
* rework sanitization + formatting
* update tests
* add some more markdown tests
---
internal/api/client/status/statuscreate_test.go | 6 +-
internal/processing/account/update_test.go | 13 ++-
internal/processing/status/util_test.go | 120 ++++++++++--------------
internal/text/common.go | 33 -------
internal/text/common_test.go | 46 ++-------
internal/text/link_test.go | 20 ++--
internal/text/markdown.go | 6 +-
internal/text/markdown_test.go | 32 +++++--
internal/text/minify.go | 39 --------
internal/text/plain.go | 10 +-
internal/text/plain_test.go | 36 ++++---
internal/text/sanitize.go | 7 +-
12 files changed, 131 insertions(+), 237 deletions(-)
delete mode 100644 internal/text/minify.go
(limited to 'internal')
diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go
index c6c9b4dab..f83ac8e8f 100644
--- a/internal/api/client/status/statuscreate_test.go
+++ b/internal/api/client/status/statuscreate_test.go
@@ -178,7 +178,7 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
err = json.Unmarshal(b, statusReply)
suite.NoError(err)
- suite.Equal("#test alright, should be able to post #links with fragments in them now, let's see........
docs.gotosocial.org/en/latest/user_guide/posts/#links
#gotosocial
(tobi remember to pull the docker image challenge)
", statusReply.Content)
+ suite.Equal("#test alright, should be able to post #links with fragments in them now, let's see........
docs.gotosocial.org/en/latest/user_guide/posts/#links
#gotosocial
(tobi remember to pull the docker image challenge)
", statusReply.Content)
}
func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
@@ -211,7 +211,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
suite.NoError(err)
suite.Equal("", statusReply.SpoilerText)
- suite.Equal("here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
", statusReply.Content)
+ suite.Equal("here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
", statusReply.Content)
suite.Len(statusReply.Emojis, 1)
apiEmoji := statusReply.Emojis[0]
@@ -330,7 +330,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
suite.NoError(err)
suite.Equal("", statusResponse.SpoilerText)
- suite.Equal("here's an image attachment
", statusResponse.Content)
+ suite.Equal("here's an image attachment
", statusResponse.Content)
suite.False(statusResponse.Sensitive)
suite.Equal(model.VisibilityPublic, statusResponse.Visibility)
diff --git a/internal/processing/account/update_test.go b/internal/processing/account/update_test.go
index 582dc82e9..7e4ca818e 100644
--- a/internal/processing/account/update_test.go
+++ b/internal/processing/account/update_test.go
@@ -73,13 +73,12 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateSimple() {
func (suite *AccountUpdateTestSuite) TestAccountUpdateWithMention() {
testAccount := suite.testAccounts["local_account_1"]
- locked := true
- displayName := "new display name"
- note := `#hello here i am!
-
-go check out @1happyturtle, they have a cool account!
-`
- noteExpected := `#hello here i am!
go check out @1happyturtle, they have a cool account!
`
+ var (
+ locked = true
+ displayName = "new display name"
+ note = "#hello here i am!\n\ngo check out @1happyturtle, they have a cool account!\n"
+ noteExpected = "#hello here i am!
go check out @1happyturtle, they have a cool account!
"
+ )
form := &apimodel.UpdateCredentialsRequest{
DisplayName: &displayName,
diff --git a/internal/processing/status/util_test.go b/internal/processing/status/util_test.go
index f1b826bd0..6f551d63d 100644
--- a/internal/processing/status/util_test.go
+++ b/internal/processing/status/util_test.go
@@ -23,31 +23,20 @@ import (
"fmt"
"testing"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
-const statusText1 = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-Text`
-
const (
- statusText1ExpectedFull = "Another test @foss_satan
#Hashtag
Text
"
- statusText1ExpectedPartial = "Another test @foss_satan
#Hashtag
Text
"
+ statusText1 = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText"
+ statusText1ExpectedFull = "Another test @foss_satan
#Hashtag
Text
"
+ statusText1ExpectedPartial = "Another test @foss_satan
#Hashtag
Text
"
+ statusText2 = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\n#hashTAG"
+ status2TextExpectedFull = "Another test @foss_satan
#Hashtag
#hashTAG
"
+ status2TextExpectedPartial = "Another test @foss_satan
#Hashtag
#hashTAG
"
)
-const statusText2 = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-#hashTAG`
-
-const status2TextExpectedFull = "Another test @foss_satan
#Hashtag
#hashTAG
"
-
type UtilTestSuite struct {
StatusStandardTestSuite
}
@@ -82,21 +71,21 @@ func (suite *UtilTestSuite) TestProcessMentions1() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Len(suite.T(), status.Mentions, 1)
+ suite.Len(status.Mentions, 1)
newMention := status.Mentions[0]
- assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
- assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
- assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
- assert.Equal(suite.T(), status.ID, newMention.StatusID)
- assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
- assert.Equal(suite.T(), mentionedAccount.URI, newMention.TargetAccountURI)
- assert.Equal(suite.T(), mentionedAccount.URL, newMention.TargetAccountURL)
- assert.NotNil(suite.T(), newMention.OriginAccount)
-
- assert.Len(suite.T(), status.MentionIDs, 1)
- assert.Equal(suite.T(), newMention.ID, status.MentionIDs[0])
+ suite.Equal(mentionedAccount.ID, newMention.TargetAccountID)
+ suite.Equal(creatingAccount.ID, newMention.OriginAccountID)
+ suite.Equal(creatingAccount.URI, newMention.OriginAccountURI)
+ suite.Equal(status.ID, newMention.StatusID)
+ suite.Equal(fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+ suite.Equal(mentionedAccount.URI, newMention.TargetAccountURI)
+ suite.Equal(mentionedAccount.URL, newMention.TargetAccountURL)
+ suite.NotNil(newMention.OriginAccount)
+
+ suite.Len(status.MentionIDs, 1)
+ suite.Equal(newMention.ID, status.MentionIDs[0])
}
func (suite *UtilTestSuite) TestProcessContentFull1() {
@@ -131,20 +120,20 @@ func (suite *UtilTestSuite) TestProcessContentFull1() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+ suite.NoError(err)
+ suite.Empty(status.Content) // shouldn't be set yet
err = suite.status.ProcessTags(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+ suite.NoError(err)
+ suite.Empty(status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Equal(suite.T(), statusText1ExpectedFull, status.Content)
+ suite.NoError(err)
+ suite.Equal(statusText1ExpectedFull, status.Content)
}
func (suite *UtilTestSuite) TestProcessContentPartial1() {
@@ -179,16 +168,16 @@ func (suite *UtilTestSuite) TestProcessContentPartial1() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+ suite.NoError(err)
+ suite.Empty(status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Equal(suite.T(), statusText1ExpectedPartial, status.Content)
+ suite.NoError(err)
+ suite.Equal(statusText1ExpectedPartial, status.Content)
}
func (suite *UtilTestSuite) TestProcessMentions2() {
@@ -221,21 +210,21 @@ func (suite *UtilTestSuite) TestProcessMentions2() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Len(suite.T(), status.Mentions, 1)
+ suite.Len(status.Mentions, 1)
newMention := status.Mentions[0]
- assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
- assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
- assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
- assert.Equal(suite.T(), status.ID, newMention.StatusID)
- assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
- assert.Equal(suite.T(), mentionedAccount.URI, newMention.TargetAccountURI)
- assert.Equal(suite.T(), mentionedAccount.URL, newMention.TargetAccountURL)
- assert.NotNil(suite.T(), newMention.OriginAccount)
-
- assert.Len(suite.T(), status.MentionIDs, 1)
- assert.Equal(suite.T(), newMention.ID, status.MentionIDs[0])
+ suite.Equal(mentionedAccount.ID, newMention.TargetAccountID)
+ suite.Equal(creatingAccount.ID, newMention.OriginAccountID)
+ suite.Equal(creatingAccount.URI, newMention.OriginAccountURI)
+ suite.Equal(status.ID, newMention.StatusID)
+ suite.Equal(fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+ suite.Equal(mentionedAccount.URI, newMention.TargetAccountURI)
+ suite.Equal(mentionedAccount.URL, newMention.TargetAccountURL)
+ suite.NotNil(newMention.OriginAccount)
+
+ suite.Len(status.MentionIDs, 1)
+ suite.Equal(newMention.ID, status.MentionIDs[0])
}
func (suite *UtilTestSuite) TestProcessContentFull2() {
@@ -270,21 +259,21 @@ func (suite *UtilTestSuite) TestProcessContentFull2() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+ suite.NoError(err)
+ suite.Empty(status.Content) // shouldn't be set yet
err = suite.status.ProcessTags(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+ suite.NoError(err)
+ suite.Empty(status.Content) // shouldn't be set yet
/*
ACTUAL TEST
*/
err = suite.status.ProcessContent(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Equal(suite.T(), status2TextExpectedFull, status.Content)
+ suite.Equal(status2TextExpectedFull, status.Content)
}
func (suite *UtilTestSuite) TestProcessContentPartial2() {
@@ -319,18 +308,13 @@ func (suite *UtilTestSuite) TestProcessContentPartial2() {
}
err := suite.status.ProcessMentions(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
- assert.Empty(suite.T(), status.Content) // shouldn't be set yet
-
- /*
- ACTUAL TEST
- */
+ suite.NoError(err)
+ suite.Empty(status.Content)
err = suite.status.ProcessContent(context.Background(), form, creatingAccount.ID, status)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- fmt.Println(status.Content)
- // assert.Equal(suite.T(), statusText2ExpectedPartial, status.Content)
+ suite.Equal(status2TextExpectedPartial, status.Content)
}
func TestUtilTestSuite(t *testing.T) {
diff --git a/internal/text/common.go b/internal/text/common.go
index 9ed3fb06f..005f9dfe1 100644
--- a/internal/text/common.go
+++ b/internal/text/common.go
@@ -21,7 +21,6 @@ package text
import (
"bytes"
"context"
- "html"
"strings"
"unicode"
@@ -30,38 +29,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/regexes"
)
-// preformat contains some common logic for making a string ready for formatting, which should be used for all user-input text.
-func preformat(in string) string {
- // do some preformatting of the text
-
- // 1. unescape everything that might be html escaped
- s := html.UnescapeString(in)
-
- // 2. trim leading or trailing whitespace
- s = strings.TrimSpace(s)
- return s
-}
-
-// postformat contains some common logic for html sanitization of text, wrapping elements, and trimming newlines and whitespace
-func postformat(in string) string {
- // do some postformatting of the text
-
- // 1. sanitize html to remove potentially dangerous elements
- s := SanitizeHTML(in)
-
- // 2. the sanitize step tends to escape characters inside codeblocks, which is behavior we don't want, so unescape everything again
- s = html.UnescapeString(s)
-
- // 3. minify html to remove any trailing newlines, spaces, unnecessary elements, etc etc
- mini, err := MinifyHTML(s)
- if err != nil {
- // if the minify failed, just return what we have
- return s
- }
- // return minified version of the html
- return mini
-}
-
func (f *formatter) ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string {
return regexes.ReplaceAllStringFunc(regexes.HashtagFinder, in, func(match string, buf *bytes.Buffer) string {
// we have a match
diff --git a/internal/text/common_test.go b/internal/text/common_test.go
index 5e8f05b30..48f5240d2 100644
--- a/internal/text/common_test.go
+++ b/internal/text/common_test.go
@@ -28,44 +28,14 @@ import (
)
const (
- replaceMentionsString = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-Text`
- replaceMentionsExpected = `Another test @foss_satan
-
-#Hashtag
-
-Text`
-
- replaceHashtagsExpected = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-Text`
-
- replaceHashtagsAfterMentionsExpected = `Another test @foss_satan
-
-#Hashtag
-
-Text`
-
- replaceMentionsWithLinkString = `Another test @foss_satan@fossbros-anonymous.io
-
-http://fossbros-anonymous.io/@foss_satan/statuses/6675ee73-fccc-4562-a46a-3e8cd9798060`
-
- replaceMentionsWithLinkStringExpected = `Another test @foss_satan
-
-http://fossbros-anonymous.io/@foss_satan/statuses/6675ee73-fccc-4562-a46a-3e8cd9798060`
-
- replaceMentionsWithLinkSelfString = `Mentioning myself: @the_mighty_zork
-
-and linking to my own status: https://localhost:8080/@the_mighty_zork/statuses/01FGXKJRX2PMERJQ9EQF8Y6HCR`
-
- replaceMemtionsWithLinkSelfExpected = `Mentioning myself: @the_mighty_zork
-
-and linking to my own status: https://localhost:8080/@the_mighty_zork/statuses/01FGXKJRX2PMERJQ9EQF8Y6HCR`
+ replaceMentionsString = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText"
+ replaceMentionsExpected = "Another test @foss_satan\n\n#Hashtag\n\nText"
+ replaceHashtagsExpected = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText"
+ replaceHashtagsAfterMentionsExpected = "Another test @foss_satan\n\n#Hashtag\n\nText"
+ replaceMentionsWithLinkString = "Another test @foss_satan@fossbros-anonymous.io\n\nhttp://fossbros-anonymous.io/@foss_satan/statuses/6675ee73-fccc-4562-a46a-3e8cd9798060"
+ replaceMentionsWithLinkStringExpected = "Another test @foss_satan\n\nhttp://fossbros-anonymous.io/@foss_satan/statuses/6675ee73-fccc-4562-a46a-3e8cd9798060"
+ replaceMentionsWithLinkSelfString = "Mentioning myself: @the_mighty_zork\n\nand linking to my own status: https://localhost:8080/@the_mighty_zork/statuses/01FGXKJRX2PMERJQ9EQF8Y6HCR"
+ replaceMemtionsWithLinkSelfExpected = "Mentioning myself: @the_mighty_zork\n\nand linking to my own status: https://localhost:8080/@the_mighty_zork/statuses/01FGXKJRX2PMERJQ9EQF8Y6HCR"
)
type CommonTestSuite struct {
diff --git a/internal/text/link_test.go b/internal/text/link_test.go
index 24484e02d..e50a8dd69 100644
--- a/internal/text/link_test.go
+++ b/internal/text/link_test.go
@@ -71,16 +71,16 @@ type LinkTestSuite struct {
func (suite *LinkTestSuite) TestParseSimple() {
f := suite.formatter.FromPlain(context.Background(), simple, nil, nil)
- assert.Equal(suite.T(), simpleExpected, f)
+ suite.Equal(simpleExpected, f)
}
func (suite *LinkTestSuite) TestParseURLsFromText1() {
urls := text.FindLinks(text1)
- assert.Equal(suite.T(), "https://example.org/link/to/something#fragment", urls[0].String())
- assert.Equal(suite.T(), "http://test.example.org?q=bahhhhhhhhhhhh", urls[1].String())
- assert.Equal(suite.T(), "https://another.link.example.org/with/a/pretty/long/path/at/the/end/of/it", urls[2].String())
- assert.Equal(suite.T(), "https://example.orghttps://google.com", urls[3].String())
+ suite.Equal("https://example.org/link/to/something#fragment", urls[0].String())
+ suite.Equal("http://test.example.org?q=bahhhhhhhhhhhh", urls[1].String())
+ suite.Equal("https://another.link.example.org/with/a/pretty/long/path/at/the/end/of/it", urls[2].String())
+ suite.Equal("https://example.orghttps://google.com", urls[3].String())
}
func (suite *LinkTestSuite) TestParseURLsFromText2() {
@@ -99,7 +99,7 @@ func (suite *LinkTestSuite) TestParseURLsFromText3() {
func (suite *LinkTestSuite) TestReplaceLinksFromText1() {
replaced := suite.formatter.ReplaceLinks(context.Background(), text1)
- assert.Equal(suite.T(), `
+ suite.Equal(`
This is a text with some links in it. Here's link number one: example.org/link/to/something#fragment
Here's link number two: test.example.org?q=bahhhhhhhhhhhh
@@ -114,7 +114,7 @@ really.cool.website <-- this one shouldn't be parsed as a link because it doesn'
func (suite *LinkTestSuite) TestReplaceLinksFromText2() {
replaced := suite.formatter.ReplaceLinks(context.Background(), text2)
- assert.Equal(suite.T(), `
+ suite.Equal(`
this is one link: example.org
this is the same link again: example.org
@@ -126,14 +126,14 @@ these should be deduplicated
func (suite *LinkTestSuite) TestReplaceLinksFromText3() {
// we know mailto links won't be replaced with hrefs -- we only accept https and http
replaced := suite.formatter.ReplaceLinks(context.Background(), text3)
- assert.Equal(suite.T(), `
+ suite.Equal(`
here's a mailto link: mailto:whatever@test.org
`, replaced)
}
func (suite *LinkTestSuite) TestReplaceLinksFromText4() {
replaced := suite.formatter.ReplaceLinks(context.Background(), text4)
- assert.Equal(suite.T(), `
+ suite.Equal(`
two similar links:
example.org
@@ -145,7 +145,7 @@ two similar links:
func (suite *LinkTestSuite) TestReplaceLinksFromText5() {
// we know this one doesn't work properly, which is why html should always be sanitized before being passed into the ReplaceLinks function
replaced := suite.formatter.ReplaceLinks(context.Background(), text5)
- assert.Equal(suite.T(), `
+ suite.Equal(`
what happens when we already have a link within an href?
example.org">example.org
diff --git a/internal/text/markdown.go b/internal/text/markdown.go
index 01238954f..a5c62f23f 100644
--- a/internal/text/markdown.go
+++ b/internal/text/markdown.go
@@ -26,13 +26,11 @@ import (
)
func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
- content := preformat(md)
-
// do the markdown parsing *first*
- contentBytes := blackfriday.Run([]byte(content))
+ contentBytes := blackfriday.Run([]byte(md))
// format tags nicely
- content = f.ReplaceTags(ctx, string(contentBytes), tags)
+ content := f.ReplaceTags(ctx, string(contentBytes), tags)
// format mentions nicely
content = f.ReplaceMentions(ctx, content, mentions)
diff --git a/internal/text/markdown_test.go b/internal/text/markdown_test.go
index 111cfe473..74a18a685 100644
--- a/internal/text/markdown_test.go
+++ b/internal/text/markdown_test.go
@@ -44,15 +44,19 @@ that was some JSON :)
`
const (
- simpleMarkdown = "# Title\n\nHere's a simple text in markdown.\n\nHere's a [link](https://example.org)."
- simpleMarkdownExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a link.
\n"
- withCodeBlockExpected = "Title
\n\nBelow is some JSON.
\n\n{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\n
\n\nthat was some JSON :)
\n"
- withInlineCode = "`Nobody tells you about the SECRET CODENobody tells you about the <code><del>SECRET CODE</del></code>, do they?
\n"
- withInlineCode2 = "`Nobody tells you about the SECRET CODE, do they?`"
- withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
\n"
- withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
- withHashtagExpected = "Title
\n\nhere’s a simple status that uses hashtag #Hashtag!
\n"
+ simpleMarkdown = "# Title\n\nHere's a simple text in markdown.\n\nHere's a [link](https://example.org)."
+ simpleMarkdownExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a link.
\n"
+ withCodeBlockExpected = "Title
\n\nBelow is some JSON.
\n\n{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\n
\n\nthat was some JSON :)
\n"
+ withInlineCode = "`Nobody tells you about the SECRET CODENobody tells you about the <code><del>SECRET CODE</del></code>, do they?
\n"
+ withInlineCode2 = "`Nobody tells you about the SECRET CODE, do they?`"
+ withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
\n"
+ withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
+ withHashtagExpected = "Title
\n\nhere’s a simple status that uses hashtag #Hashtag!
\n"
+ mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a link.\n\nHere's an image: ![\"The](\"https://gts.superseriousbusiness.org/assets/logo.png\")
"
+ mdWithHTMLExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a link.
\n\nHere’s an image:
\n"
+ mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: "
+ mdWithCheekyHTMLExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a cheeky little script:
\n"
)
type MarkdownTestSuite struct {
@@ -88,6 +92,16 @@ func (suite *MarkdownTestSuite) TestParseWithHashtag() {
suite.Equal(withHashtagExpected, s)
}
+func (suite *MarkdownTestSuite) TestParseWithHTML() {
+ s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil)
+ suite.Equal(mdWithHTMLExpected, s)
+}
+
+func (suite *MarkdownTestSuite) TestParseWithCheekyHTML() {
+ s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil)
+ suite.Equal(mdWithCheekyHTMLExpected, s)
+}
+
func TestMarkdownTestSuite(t *testing.T) {
suite.Run(t, new(MarkdownTestSuite))
}
diff --git a/internal/text/minify.go b/internal/text/minify.go
deleted file mode 100644
index e2515b9a4..000000000
--- a/internal/text/minify.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- GoToSocial
- Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see .
-*/
-
-package text
-
-import (
- "github.com/tdewolff/minify/v2"
- "github.com/tdewolff/minify/v2/html"
-)
-
-var m *minify.M
-
-// MinifyHTML runs html through a minifier, reducing it in size.
-func MinifyHTML(in string) (string, error) {
- if m == nil {
- m = minify.New()
- m.Add("text/html", &html.Minifier{
- KeepQuotes: true,
- KeepEndTags: true,
- KeepDocumentTags: true,
- })
- }
- return m.String("text/html", in)
-}
diff --git a/internal/text/plain.go b/internal/text/plain.go
index bc10d1b67..3daea5686 100644
--- a/internal/text/plain.go
+++ b/internal/text/plain.go
@@ -20,6 +20,7 @@ package text
import (
"context"
+ "html"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -32,10 +33,11 @@ var breakReplacer = strings.NewReplacer(
)
func (f *formatter) FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
- content := preformat(plain)
+ // trim any crap
+ content := strings.TrimSpace(plain)
- // sanitize any html elements
- content = removeHTML(content)
+ // clean 'er up
+ content = html.EscapeString(content)
// format links nicely
content = f.ReplaceLinks(ctx, content)
@@ -52,5 +54,5 @@ func (f *formatter) FromPlain(ctx context.Context, plain string, mentions []*gts
// wrap the whole thing in a pee
content = `` + content + `
`
- return postformat(content)
+ return SanitizeHTML(content)
}
diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go
index 2b7b50d5e..cd82e0d1b 100644
--- a/internal/text/plain_test.go
+++ b/internal/text/plain_test.go
@@ -20,27 +20,21 @@ package text_test
import (
"context"
- "fmt"
"testing"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
const (
- simple = "this is a plain and simple status"
- simpleExpected = "this is a plain and simple status
"
-
- withTag = "here's a simple status that uses hashtag #welcome!"
- withTagExpected = "here's a simple status that uses hashtag #welcome!
"
-
- moreComplex = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-Text`
- moreComplexFull = "Another test @foss_satan
#Hashtag
Text
"
+ simple = "this is a plain and simple status"
+ simpleExpected = "this is a plain and simple status
"
+ withTag = "here's a simple status that uses hashtag #welcome!"
+ withTagExpected = "here's a simple status that uses hashtag #welcome!
"
+ withHTML = "blah this should just be html escaped blah
"
+ withHTMLExpected = "<div>blah this should just be html escaped blah</div>
"
+ moreComplex = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText"
+ moreComplexFull = "Another test @foss_satan
#Hashtag
Text
"
)
type PlainTestSuite struct {
@@ -49,7 +43,7 @@ type PlainTestSuite struct {
func (suite *PlainTestSuite) TestParseSimple() {
f := suite.formatter.FromPlain(context.Background(), simple, nil, nil)
- assert.Equal(suite.T(), simpleExpected, f)
+ suite.Equal(simpleExpected, f)
}
func (suite *PlainTestSuite) TestParseWithTag() {
@@ -58,7 +52,12 @@ func (suite *PlainTestSuite) TestParseWithTag() {
}
f := suite.formatter.FromPlain(context.Background(), withTag, nil, foundTags)
- assert.Equal(suite.T(), withTagExpected, f)
+ suite.Equal(withTagExpected, f)
+}
+
+func (suite *PlainTestSuite) TestParseWithHTML() {
+ f := suite.formatter.FromPlain(context.Background(), withHTML, nil, nil)
+ suite.Equal(withHTMLExpected, f)
}
func (suite *PlainTestSuite) TestParseMoreComplex() {
@@ -71,10 +70,7 @@ func (suite *PlainTestSuite) TestParseMoreComplex() {
}
f := suite.formatter.FromPlain(context.Background(), moreComplex, foundMentions, foundTags)
-
- fmt.Println(f)
-
- assert.Equal(suite.T(), moreComplexFull, f)
+ suite.Equal(moreComplexFull, f)
}
func TestPlainTestSuite(t *testing.T) {
diff --git a/internal/text/sanitize.go b/internal/text/sanitize.go
index d4faabbb1..96b7ef994 100644
--- a/internal/text/sanitize.go
+++ b/internal/text/sanitize.go
@@ -19,7 +19,9 @@
package text
import (
+ "html"
"regexp"
+ "strings"
"github.com/microcosm-cc/bluemonday"
)
@@ -59,7 +61,8 @@ func SanitizeHTML(in string) string {
// SanitizePlaintext runs text through basic sanitization. This removes
// any html elements that were in the string, and returns clean plaintext.
func SanitizePlaintext(in string) string {
- content := preformat(in)
+ content := html.UnescapeString(in)
content = removeHTML(content)
- return postformat(content)
+ content = html.UnescapeString(content)
+ return strings.TrimSpace(content)
}
--
cgit v1.2.3