[bugfix] Store and expose status content type (#3870)

* Add ContentType to internal models

* Add ContentType to API models StatusSource and StatusEdit

* Add helpers to convert between API/internal StatusContentType

* Write status content type on create/edit

* Add migration

* Update API docs

go run github.com/go-swagger/go-swagger/cmd/swagger generate spec --scan-models --exclude-deps --output docs/api/swagger.yaml

* ensure ContentType is updated anywhere Text is

* Update docs, take care of TODOs

* Set ContentType in more places where Text is set

* We don't actually use ContentType on the API status model

* Update StatusSource test

* Remove unused helper function I copied

* Revert change to StatusContentType swagger annotation

I'm going to include this in a follow-on PR instead.

* Add test for updating content type in edits

* Return a value from processContentType instead of modifying the existing status

Fixes an issue that was caught by the test I just added - the recorded edit would be marked with the *new* content type instead of the old one, which is obviously bad

* Add test for handling of statuses with no stored content type

* repurpose an existing test status instead of adding a new one to avoid breaking other tests

* Add test to ensure newly created statuses always have content type saved

* Do include content type on status API model actually

This is mostly important when deleting and redrafting.

The comment on `apimodel.Status.Text` implies that it's not sent except in response to status deletion, but actually this doesn't seem to be the case; it also appears to be present in responses to creations and normal fetches and stuff. So I'm treating `ContentType` the same here.

* Update new tests to check content type on API statuses

* Check content type of API statuses in all tests where text is checked

* update other api tests with status content type field

* Add test ensuring text and content type are returned when deleting a status

* Convert processContentType to free function and remove unused parameter

* check for the correct value in the deletion test

* Be explicit about this test status having an empty content type

* Use omitempty consistently on API models

* clean up the final diff a bit

* one more swagger regen for the road

* Handle nil statuses in processContentType

* Don't pass processContentType the entire edit form, it doesn't need it

* Move processContentType to common.go and use for creation as well

* Remove unused parameters to ContentTypeToAPIContentType

7 files changed, 32 insertions, 1 deletions

diff --git a/internal/api/client/statuses/statusboost_test.go b/internal/api/client/statuses/statusboost_test.go
index 51b7d7652..fc750ca38 100644
--- a/internal/api/client/statuses/statusboost_test.go
+++ b/internal/api/client/statuses/statusboost_test.go
@@ -145,6 +145,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() {
     "bookmarked": true,
     "card": null,
     "content": "hello world! #welcome ! first post on the instance :rainbow: !",
+    "content_type": "text/plain",
     "created_at": "right the hell just now babyee",
     "edited_at": null,
     "emojis": [
@@ -331,6 +332,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() {
     "bookmarked": false,
     "card": null,
     "content": "hi!",
+    "content_type": "text/plain",
     "created_at": "right the hell just now babyee",
     "edited_at": null,
     "emojis": [],
@@ -543,6 +545,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() {
     "bookmarked": false,
     "card": null,
     "content": "<p>Hi <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span>, can I reply?</p>",
+    "content_type": "text/markdown",
     "created_at": "right the hell just now babyee",
     "edited_at": null,
     "emojis": [],
diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go
index 53e517a6e..aba39ba23 100644
--- a/internal/api/client/statuses/statuscreate_test.go
+++ b/internal/api/client/statuses/statuscreate_test.go
@@ -139,6 +139,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
   "bookmarked": false,
   "card": null,
   "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -225,6 +226,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicy() {
   "bookmarked": false,
   "card": null,
   "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -321,6 +323,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicyJSON() {
   "bookmarked": false,
   "card": null,
   "content": "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -535,6 +538,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() {
   "bookmarked": false,
   "card": null,
   "content": "<h1>Title</h1><h2>Smaller title</h2><p>This is a post written in <a href=\"https://www.markdownguide.org/\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">markdown</a></p>",
+  "content_type": "text/markdown",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -619,6 +623,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
   "bookmarked": false,
   "card": null,
   "content": "<p>hello <span class=\"h-card\"><a href=\"https://unknown-instance.com/@brand_new_person\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>brand_new_person</span></a></span></p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -697,6 +702,7 @@ func (suite *StatusCreateTestSuite) TestPostStatusWithLinksAndTags() {
   "bookmarked": false,
   "card": null,
   "content": "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -781,6 +787,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
   "bookmarked": false,
   "card": null,
   "content": "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [
@@ -879,6 +886,7 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() {
   "bookmarked": false,
   "card": null,
   "content": "<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> this reply should work!</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -962,6 +970,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
   "bookmarked": false,
   "card": null,
   "content": "<p>here's an image attachment</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -1067,6 +1076,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithNoncanonicalLanguageTag
   "bookmarked": false,
   "card": null,
   "content": "<p>English? what's English? i speak American</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -1142,6 +1152,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollForm() {
   "bookmarked": false,
   "card": null,
   "content": "<p>this is a status with a poll!</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -1239,6 +1250,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollJSON() {
   "bookmarked": false,
   "card": null,
   "content": "<p>this is a status with a poll!</p>",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
diff --git a/internal/api/client/statuses/statusdelete_test.go b/internal/api/client/statuses/statusdelete_test.go
index 6f62ef45f..d7efd5725 100644
--- a/internal/api/client/statuses/statusdelete_test.go
+++ b/internal/api/client/statuses/statusdelete_test.go
@@ -77,6 +77,10 @@ func (suite *StatusDeleteTestSuite) TestPostDelete() {
 	suite.NoError(err)
 	suite.NotNil(statusReply)
 
+	// Check that text and content type are returned for delete and redraft
+	suite.Equal("hello everyone!", statusReply.Text)
+	suite.Equal(apimodel.StatusContentTypePlain, statusReply.ContentType)
+
 	if !testrig.WaitFor(func() bool {
 		_, err := suite.db.GetStatusByID(ctx, targetStatus.ID)
 		return errors.Is(err, db.ErrNoEntries)
diff --git a/internal/api/client/statuses/statusfave_test.go b/internal/api/client/statuses/statusfave_test.go
index 8851b4d58..42b812fdf 100644
--- a/internal/api/client/statuses/statusfave_test.go
+++ b/internal/api/client/statuses/statusfave_test.go
@@ -104,6 +104,7 @@ func (suite *StatusFaveTestSuite) TestPostFave() {
   "bookmarked": false,
   "card": null,
   "content": "🐕🐕🐕🐕🐕",
+  "content_type": "text/plain",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
@@ -228,6 +229,7 @@ func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() {
   "bookmarked": false,
   "card": null,
   "content": "<p>Hi <span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span>, can I reply?</p>",
+  "content_type": "text/markdown",
   "created_at": "right the hell just now babyee",
   "edited_at": null,
   "emojis": [],
diff --git a/internal/api/client/statuses/statusmute_test.go b/internal/api/client/statuses/statusmute_test.go
index 20ee590f8..174ac14dc 100644
--- a/internal/api/client/statuses/statusmute_test.go
+++ b/internal/api/client/statuses/statusmute_test.go
@@ -149,6 +149,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() {
   "card": null,
   "poll": null,
   "text": "hello everyone!",
+  "content_type": "text/plain",
   "interaction_policy": {
     "can_favourite": {
       "always": [
@@ -238,6 +239,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() {
   "card": null,
   "poll": null,
   "text": "hello everyone!",
+  "content_type": "text/plain",
   "interaction_policy": {
     "can_favourite": {
       "always": [
diff --git a/internal/api/client/statuses/statussource_test.go b/internal/api/client/statuses/statussource_test.go
index 797a462ed..e36a939ef 100644
--- a/internal/api/client/statuses/statussource_test.go
+++ b/internal/api/client/statuses/statussource_test.go
@@ -92,7 +92,8 @@ func (suite *StatusSourceTestSuite) TestGetSource() {
 	suite.Equal(`{
   "id": "01F8MHAMCHF6Y650WCRSCP4WMY",
   "text": "hello everyone!",
-  "spoiler_text": "introduction post"
+  "spoiler_text": "introduction post",
+  "content_type": "text/plain"
 }`, dst.String())
 }
 
diff --git a/internal/api/model/status.go b/internal/api/model/status.go
index 2ee3123e6..fe82f09e3 100644
--- a/internal/api/model/status.go
+++ b/internal/api/model/status.go
@@ -110,6 +110,10 @@ type Status struct {
 	// so the user may redraft from the source text without the client having to reverse-engineer
 	// the original text from the HTML content.
 	Text string `json:"text,omitempty"`
+	// Content type that was used to parse the status's text. Returned when
+	// status is deleted, so if the user is redrafting the message the client
+	// can default to the same content type.
+	ContentType StatusContentType `json:"content_type,omitempty"`
 	// A list of filters that matched this status and why they matched, if there are any such filters.
 	Filtered []FilterResult `json:"filtered,omitempty"`
 	// The interaction policy for this status, as set by the status author.
@@ -320,6 +324,9 @@ type StatusSource struct {
 
 	// Plain-text version of spoiler text.
 	SpoilerText string `json:"spoiler_text"`
+
+	// Content type that was used to parse the text.
+	ContentType StatusContentType `json:"content_type,omitempty"`
 }
 
 // StatusEdit represents one historical revision of a status, containing