[bugfix] Normalize status content (don't parse status content as IRI) (#1665)

* start fannying about

* finish up Normalize

* tidy up

* pin to tag

* move errors about just a little bit

5 files changed, 385 insertions, 0 deletions

diff --git a/internal/ap/error.go b/internal/ap/error.go
new file mode 100644
index 000000000..ef27d5ac7
--- /dev/null
+++ b/internal/ap/error.go
@@ -0,0 +1,35 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ap
+
+import "fmt"
+
+// ErrWrongType indicates that we tried to resolve a type into
+// an interface that it's not compatible with, eg a Person into
+// a Statusable.
+type ErrWrongType struct {
+	wrapped error
+}
+
+func (err *ErrWrongType) Error() string {
+	return fmt.Sprintf("wrong received type: %v", err.wrapped)
+}
+
+func newErrWrongType(err error) error {
+	return &ErrWrongType{wrapped: err}
+}
diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go
index f8c18ffc8..33b2eb9ca 100644
--- a/internal/ap/interfaces.go
+++ b/internal/ap/interfaces.go
@@ -60,6 +60,7 @@ type Statusable interface {
 	WithSensitive
 	WithConversation
 	WithContent
+	WithSetContent
 	WithAttachment
 	WithTag
 	WithReplies
@@ -281,6 +282,11 @@ type WithContent interface {
 	GetActivityStreamsContent() vocab.ActivityStreamsContentProperty
 }
 
+// WithSetContent represents an activity that can have content set on it.
+type WithSetContent interface {
+	SetActivityStreamsContent(vocab.ActivityStreamsContentProperty)
+}
+
 // WithPublished represents an activity with ActivityStreamsPublishedProperty
 type WithPublished interface {
 	GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty
diff --git a/internal/ap/normalize.go b/internal/ap/normalize.go
new file mode 100644
index 000000000..2425b35a0
--- /dev/null
+++ b/internal/ap/normalize.go
@@ -0,0 +1,116 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ap
+
+import (
+	"github.com/superseriousbusiness/activity/pub"
+	"github.com/superseriousbusiness/activity/streams"
+)
+
+// NormalizeActivityObject normalizes the 'object'.'content' field of the given Activity.
+//
+// The rawActivity map should the freshly deserialized json representation of the Activity.
+//
+// This function is a noop if the type passed in is anything except a Create with a Statusable as its Object.
+func NormalizeActivityObject(activity pub.Activity, rawActivity map[string]interface{}) {
+	if activity.GetTypeName() != ActivityCreate {
+		// Only interested in Create right now.
+		return
+	}
+
+	withObject, ok := activity.(WithObject)
+	if !ok {
+		// Create was not a WithObject.
+		return
+	}
+
+	createObject := withObject.GetActivityStreamsObject()
+	if createObject == nil {
+		// No object set.
+		return
+	}
+
+	if createObject.Len() != 1 {
+		// Not interested in Object arrays.
+		return
+	}
+
+	// We now know length is 1 so get the first
+	// item from the iter.  We need this to be
+	// a Statusable if we're to continue.
+	i := createObject.At(0)
+	if i == nil {
+		// This is awkward.
+		return
+	}
+
+	t := i.GetType()
+	if t == nil {
+		// This is also awkward.
+		return
+	}
+
+	statusable, ok := t.(Statusable)
+	if !ok {
+		// Object is not Statusable;
+		// we're not interested.
+		return
+	}
+
+	object, ok := rawActivity["object"]
+	if !ok {
+		// No object in raw map.
+		return
+	}
+
+	rawStatusable, ok := object.(map[string]interface{})
+	if !ok {
+		// Object wasn't a json object.
+		return
+	}
+
+	// Pass in the statusable and its raw JSON representation.
+	NormalizeStatusableContent(statusable, rawStatusable)
+}
+
+// NormalizeStatusableContent replaces the Content of the given statusable
+// with the raw 'content' value from the given json object map.
+//
+// noop if there was no content in the json object map or the content was
+// not a plain string.
+func NormalizeStatusableContent(statusable Statusable, rawStatusable map[string]interface{}) {
+	content, ok := rawStatusable["content"]
+	if !ok {
+		// No content in rawStatusable.
+		// TODO: In future we might also
+		// look for "contentMap" property.
+		return
+	}
+
+	rawContent, ok := content.(string)
+	if !ok {
+		// Not interested in content arrays.
+		return
+	}
+
+	// Set normalized content property from the raw string; this
+	// will replace any existing content property on the statusable.
+	contentProp := streams.NewActivityStreamsContentProperty()
+	contentProp.AppendXMLSchemaString(rawContent)
+	statusable.SetActivityStreamsContent(contentProp)
+}
diff --git a/internal/ap/normalize_test.go b/internal/ap/normalize_test.go
new file mode 100644
index 000000000..d2a74a19e
--- /dev/null
+++ b/internal/ap/normalize_test.go
@@ -0,0 +1,110 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ap_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type NormalizeTestSuite struct {
+	suite.Suite
+}
+
+func (suite *NormalizeTestSuite) GetStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) {
+	rawJson := `{
+		"@context": [
+		  "https://www.w3.org/ns/activitystreams",
+		  "https://example.org/schemas/litepub-0.1.jsonld",
+		  {
+			"@language": "und"
+		  }
+		],
+		"actor": "https://example.org/users/someone",
+		"attachment": [],
+		"attributedTo": "https://example.org/users/someone",
+		"cc": [
+		  "https://example.org/users/someone/followers"
+		],
+		"content": "UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the <a class=\"hashtag\" data-tag=\"twittermigration\" href=\"https://example.org/tag/twittermigration\" rel=\"tag ugc\">#TwitterMigration</a>.<br><br>In fact, 100,000 new accounts have been created since last night.<br><br>Since last night&#39;s spike 8,000-12,000 new accounts are being created every hour.<br><br>Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.",
+		"context": "https://example.org/contexts/01GX0MSHPER1E0FT022Q209EJZ",
+		"conversation": "https://example.org/contexts/01GX0MSHPER1E0FT022Q209EJZ",
+		"id": "https://example.org/objects/01GX0MT2PA58JNSMK11MCS65YD",
+		"published": "2022-11-18T17:43:58.489995Z",
+		"replies": {
+		  "items": [
+			"https://example.org/objects/01GX0MV12MGEG3WF9SWB5K3KRJ"
+		  ],
+		  "type": "Collection"
+		},
+		"repliesCount": 0,
+		"sensitive": null,
+		"source": "UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the #TwitterMigration.\r\n\r\nIn fact, 100,000 new accounts have been created since last night.\r\n\r\nSince last night's spike 8,000-12,000 new accounts are being created every hour.\r\n\r\nYesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.",
+		"summary": "",
+		"tag": [
+		  {
+			"href": "https://example.org/tags/twittermigration",
+			"name": "#twittermigration",
+			"type": "Hashtag"
+		  }
+		],
+		"to": [
+		  "https://www.w3.org/ns/activitystreams#Public"
+		],
+		"type": "Note"
+	  }`
+
+	var rawNote map[string]interface{}
+	err := json.Unmarshal([]byte(rawJson), &rawNote)
+	if err != nil {
+		panic(err)
+	}
+
+	t, err := streams.ToType(context.Background(), rawNote)
+	if err != nil {
+		panic(err)
+	}
+
+	return t.(vocab.ActivityStreamsNote), rawNote
+}
+
+func (suite *NormalizeTestSuite) TestNormalizeActivityObject() {
+	note, rawNote := suite.GetStatusable()
+	suite.Equal(`update: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration%3C/a%3E.%3Cbr%3E%3Cbr%3EIn%20fact,%20100,000%20new%20accounts%20have%20been%20created%20since%20last%20night.%3Cbr%3E%3Cbr%3ESince%20last%20night&%2339;s%20spike%208,000-12,000%20new%20accounts%20are%20being%20created%20every%20hour.%3Cbr%3E%3Cbr%3EYesterday,%20I%20estimated%20that%20Mastodon%20would%20have%208%20million%20users%20by%20the%20end%20of%20the%20week.%20That%20might%20happen%20a%20lot%20sooner%20if%20this%20trend%20continues.`, ap.ExtractContent(note))
+
+	create := testrig.WrapAPNoteInCreate(
+		testrig.URLMustParse("https://example.org/create_something"),
+		testrig.URLMustParse("https://example.org/users/someone"),
+		testrig.TimeMustParse("2022-11-18T17:43:58.489995Z"),
+		note,
+	)
+
+	ap.NormalizeActivityObject(create, map[string]interface{}{"object": rawNote})
+	suite.Equal(`UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration</a>.<br><br>In fact, 100,000 new accounts have been created since last night.<br><br>Since last night&#39;s spike 8,000-12,000 new accounts are being created every hour.<br><br>Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.`, ap.ExtractContent(note))
+}
+
+func TestNormalizeTestSuite(t *testing.T) {
+	suite.Run(t, new(NormalizeTestSuite))
+}
diff --git a/internal/ap/resolve.go b/internal/ap/resolve.go
new file mode 100644
index 000000000..c5c9efd65
--- /dev/null
+++ b/internal/ap/resolve.go
@@ -0,0 +1,118 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ap
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/activity/streams/vocab"
+)
+
+// ResolveStatusable tries to resolve the given bytes into an ActivityPub Statusable representation.
+// It will then perform normalization on the Statusable by calling NormalizeStatusable, so that
+// callers don't need to bother doing extra steps.
+//
+// Works for: Article, Document, Image, Video, Note, Page, Event, Place, Profile
+func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) {
+	rawStatusable := make(map[string]interface{})
+	if err := json.Unmarshal(b, &rawStatusable); err != nil {
+		return nil, fmt.Errorf("ResolveStatusable: error unmarshalling bytes into json: %w", err)
+	}
+
+	t, err := streams.ToType(ctx, rawStatusable)
+	if err != nil {
+		return nil, fmt.Errorf("ResolveStatusable: error resolving json into ap vocab type: %w", err)
+	}
+
+	var (
+		statusable Statusable
+		ok         bool
+	)
+
+	switch t.GetTypeName() {
+	case ObjectArticle:
+		statusable, ok = t.(vocab.ActivityStreamsArticle)
+	case ObjectDocument:
+		statusable, ok = t.(vocab.ActivityStreamsDocument)
+	case ObjectImage:
+		statusable, ok = t.(vocab.ActivityStreamsImage)
+	case ObjectVideo:
+		statusable, ok = t.(vocab.ActivityStreamsVideo)
+	case ObjectNote:
+		statusable, ok = t.(vocab.ActivityStreamsNote)
+	case ObjectPage:
+		statusable, ok = t.(vocab.ActivityStreamsPage)
+	case ObjectEvent:
+		statusable, ok = t.(vocab.ActivityStreamsEvent)
+	case ObjectPlace:
+		statusable, ok = t.(vocab.ActivityStreamsPlace)
+	case ObjectProfile:
+		statusable, ok = t.(vocab.ActivityStreamsProfile)
+	}
+
+	if !ok {
+		err = fmt.Errorf("ResolveStatusable: could not resolve %T to Statusable", t)
+		return nil, newErrWrongType(err)
+	}
+
+	NormalizeStatusableContent(statusable, rawStatusable)
+	return statusable, nil
+}
+
+// ResolveStatusable tries to resolve the given bytes into an ActivityPub Accountable representation.
+//
+// Works for: Application, Group, Organization, Person, Service
+func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) {
+	rawAccountable := make(map[string]interface{})
+	if err := json.Unmarshal(b, &rawAccountable); err != nil {
+		return nil, fmt.Errorf("ResolveAccountable: error unmarshalling bytes into json: %w", err)
+	}
+
+	t, err := streams.ToType(ctx, rawAccountable)
+	if err != nil {
+		return nil, fmt.Errorf("ResolveAccountable: error resolving json into ap vocab type: %w", err)
+	}
+
+	var (
+		accountable Accountable
+		ok          bool
+	)
+
+	switch t.GetTypeName() {
+	case ActorApplication:
+		accountable, ok = t.(vocab.ActivityStreamsApplication)
+	case ActorGroup:
+		accountable, ok = t.(vocab.ActivityStreamsGroup)
+	case ActorOrganization:
+		accountable, ok = t.(vocab.ActivityStreamsOrganization)
+	case ActorPerson:
+		accountable, ok = t.(vocab.ActivityStreamsPerson)
+	case ActorService:
+		accountable, ok = t.(vocab.ActivityStreamsService)
+	}
+
+	if !ok {
+		err = fmt.Errorf("ResolveAccountable: could not resolve %T to Accountable", t)
+		return nil, newErrWrongType(err)
+	}
+
+	return accountable, nil
+}