Serve `outbox` for Actor (#289)

* add statusesvisible convenience function

* add minID + onlyPublic to account statuses get

* move swagger collection stuff to common

* start working on Outbox GETting

* move functions into federationProcessor

* outboxToASCollection

* add statusesvisible convenience function

* add minID + onlyPublic to account statuses get

* move swagger collection stuff to common

* start working on Outbox GETting

* move functions into federationProcessor

* outboxToASCollection

* bit more work on outbox paging

* wrapNoteInCreate function

* test + hook up the processor functions

* don't do prev + next links on empty reply

* test get outbox through api

* don't fail on no status entries

* add outbox implementation doc

* typo

5 files changed, 384 insertions, 3 deletions

diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go
index 812c37935..eeb5bead1 100644
--- a/internal/typeutils/converter.go
+++ b/internal/typeutils/converter.go
@@ -155,6 +155,19 @@ type TypeConverter interface {
 	StatusToASRepliesCollection(ctx context.Context, status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error)
 	// StatusURIsToASRepliesPage returns a collection page with appropriate next/part of pagination.
 	StatusURIsToASRepliesPage(ctx context.Context, status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error)
+	// OutboxToASCollection returns an ordered collection with appropriate id, next, and last fields.
+	// The returned collection won't have any actual entries; just links to where entries can be obtained.
+	OutboxToASCollection(ctx context.Context, outboxID string) (vocab.ActivityStreamsOrderedCollection, error)
+	// StatusesToASOutboxPage returns an ordered collection page using the given statuses and parameters as contents.
+	//
+	// The maxID and minID should be the parameters that were passed to the database to obtain the given statuses.
+	// These will be used to create the 'id' field of the collection.
+	//
+	// OutboxID is used to create the 'partOf' field in the collection.
+	//
+	// Appropriate 'next' and 'prev' fields will be created based on the highest and lowest IDs present in the statuses slice.
+	StatusesToASOutboxPage(ctx context.Context, outboxID string, maxID string, minID string, statuses []*gtsmodel.Status) (vocab.ActivityStreamsOrderedCollectionPage, error)
+
 	/*
 		INTERNAL (gts) MODEL TO INTERNAL MODEL
 	*/
@@ -170,6 +183,12 @@ type TypeConverter interface {
 
 	// WrapPersonInUpdate
 	WrapPersonInUpdate(person vocab.ActivityStreamsPerson, originAccount *gtsmodel.Account) (vocab.ActivityStreamsUpdate, error)
+	// WrapNoteInCreate wraps a Note with a Create activity.
+	//
+	// If objectIRIOnly is set to true, then the function won't put the *entire* note in the Object field of the Create,
+	// but just the AP URI of the note. This is useful in cases where you want to give a remote server something to dereference,
+	// and still have control over whether or not they're allowed to actually see the contents.
+	WrapNoteInCreate(note vocab.ActivityStreamsNote, objectIRIOnly bool) (vocab.ActivityStreamsCreate, error)
 }
 
 type converter struct {
diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go
index 73d274908..047d61b80 100644
--- a/internal/typeutils/internaltoas.go
+++ b/internal/typeutils/internaltoas.go
@@ -32,6 +32,13 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
+// const (
+// 	// highestID is the highest possible ULID
+// 	highestID = "ZZZZZZZZZZZZZZZZZZZZZZZZZZ"
+// 	// lowestID is the lowest possible ULID
+// 	lowestID = "00000000000000000000000000"
+// )
+
 // Converts a gts model account into an Activity Streams person type.
 func (c *converter) AccountToAS(ctx context.Context, a *gtsmodel.Account) (vocab.ActivityStreamsPerson, error) {
 	person := streams.NewActivityStreamsPerson()
@@ -1013,3 +1020,140 @@ func (c *converter) StatusURIsToASRepliesPage(ctx context.Context, status *gtsmo
 
 	return page, nil
 }
+
+/*
+	the goal is to end up with something like this:
+	{
+		"id": "https://example.org/users/whatever/outbox?page=true",
+		"type": "OrderedCollectionPage",
+		"next": "https://example.org/users/whatever/outbox?max_id=01FJC1Q0E3SSQR59TD2M1KP4V8&page=true",
+		"prev": "https://example.org/users/whatever/outbox?min_id=01FJC1Q0E3SSQR59TD2M1KP4V8&page=true",
+		"partOf": "https://example.org/users/whatever/outbox",
+		"orderedItems": [
+			"id": "https://example.org/users/whatever/statuses/01FJC1MKPVX2VMWP2ST93Q90K7/activity",
+			"type": "Create",
+			"actor": "https://example.org/users/whatever",
+			"published": "2021-10-18T20:06:18Z",
+			"to": [
+				"https://www.w3.org/ns/activitystreams#Public"
+			],
+			"cc": [
+				"https://example.org/users/whatever/followers"
+			],
+			"object": "https://example.org/users/whatever/statuses/01FJC1MKPVX2VMWP2ST93Q90K7"
+		]
+	}
+*/
+func (c *converter) StatusesToASOutboxPage(ctx context.Context, outboxID string, maxID string, minID string, statuses []*gtsmodel.Status) (vocab.ActivityStreamsOrderedCollectionPage, error) {
+	page := streams.NewActivityStreamsOrderedCollectionPage()
+
+	// .id
+	pageIDProp := streams.NewJSONLDIdProperty()
+	pageID := fmt.Sprintf("%s?page=true", outboxID)
+	if minID != "" {
+		pageID = fmt.Sprintf("%s&minID=%s", pageID, minID)
+	}
+	if maxID != "" {
+		pageID = fmt.Sprintf("%s&maxID=%s", pageID, maxID)
+	}
+	pageIDURI, err := url.Parse(pageID)
+	if err != nil {
+		return nil, err
+	}
+	pageIDProp.SetIRI(pageIDURI)
+	page.SetJSONLDId(pageIDProp)
+
+	// .partOf
+	collectionIDURI, err := url.Parse(outboxID)
+	if err != nil {
+		return nil, err
+	}
+	partOfProp := streams.NewActivityStreamsPartOfProperty()
+	partOfProp.SetIRI(collectionIDURI)
+	page.SetActivityStreamsPartOf(partOfProp)
+
+	// .orderedItems
+	itemsProp := streams.NewActivityStreamsOrderedItemsProperty()
+	var highest string
+	var lowest string
+	for _, s := range statuses {
+		note, err := c.StatusToAS(ctx, s)
+		if err != nil {
+			return nil, err
+		}
+
+		create, err := c.WrapNoteInCreate(note, true)
+		if err != nil {
+			return nil, err
+		}
+
+		itemsProp.AppendActivityStreamsCreate(create)
+
+		if highest == "" || s.ID > highest {
+			highest = s.ID
+		}
+		if lowest == "" || s.ID < lowest {
+			lowest = s.ID
+		}
+	}
+	page.SetActivityStreamsOrderedItems(itemsProp)
+
+	// .next
+	if lowest != "" {
+		nextProp := streams.NewActivityStreamsNextProperty()
+		nextPropIDString := fmt.Sprintf("%s?page=true&max_id=%s", outboxID, lowest)
+		nextPropIDURI, err := url.Parse(nextPropIDString)
+		if err != nil {
+			return nil, err
+		}
+		nextProp.SetIRI(nextPropIDURI)
+		page.SetActivityStreamsNext(nextProp)
+	}
+
+	// .prev
+	if highest != "" {
+		prevProp := streams.NewActivityStreamsPrevProperty()
+		prevPropIDString := fmt.Sprintf("%s?page=true&min_id=%s", outboxID, highest)
+		prevPropIDURI, err := url.Parse(prevPropIDString)
+		if err != nil {
+			return nil, err
+		}
+		prevProp.SetIRI(prevPropIDURI)
+		page.SetActivityStreamsPrev(prevProp)
+	}
+
+	return page, nil
+}
+
+/*
+	we want something that looks like this:
+
+	{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/whatever/outbox",
+		"type": "OrderedCollection",
+		"first": "https://example.org/users/whatever/outbox?page=true"
+	}
+*/
+func (c *converter) OutboxToASCollection(ctx context.Context, outboxID string) (vocab.ActivityStreamsOrderedCollection, error) {
+	collection := streams.NewActivityStreamsOrderedCollection()
+
+	collectionIDProp := streams.NewJSONLDIdProperty()
+	outboxIDURI, err := url.Parse(outboxID)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing url %s", outboxID)
+	}
+	collectionIDProp.SetIRI(outboxIDURI)
+	collection.SetJSONLDId(collectionIDProp)
+
+	collectionFirstProp := streams.NewActivityStreamsFirstProperty()
+	collectionFirstPropID := fmt.Sprintf("%s?page=true", outboxID)
+	collectionFirstPropIDURI, err := url.Parse(collectionFirstPropID)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing url %s", collectionFirstPropID)
+	}
+	collectionFirstProp.SetIRI(collectionFirstPropIDURI)
+	collection.SetActivityStreamsFirst(collectionFirstProp)
+
+	return collection, nil
+}
diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go
index 50f92e043..d8098098f 100644
--- a/internal/typeutils/internaltoas_test.go
+++ b/internal/typeutils/internaltoas_test.go
@@ -37,18 +37,98 @@ func (suite *InternalToASTestSuite) TestAccountToAS() {
 	testAccount := suite.testAccounts["local_account_1"] // take zork for this test
 
 	asPerson, err := suite.typeconverter.AccountToAS(context.Background(), testAccount)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	ser, err := streams.Serialize(asPerson)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	bytes, err := json.Marshal(ser)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	fmt.Println(string(bytes))
 	// TODO: write assertions here, rn we're just eyeballing the output
 }
 
+func (suite *InternalToASTestSuite) TestOutboxToASCollection() {
+	testAccount := suite.testAccounts["admin_account"]
+	ctx := context.Background()
+
+	collection, err := suite.typeconverter.OutboxToASCollection(ctx, testAccount.OutboxURI)
+	suite.NoError(err)
+
+	ser, err := streams.Serialize(collection)
+	assert.NoError(suite.T(), err)
+
+	bytes, err := json.Marshal(ser)
+	suite.NoError(err)
+
+	/*
+		we want this:
+		{
+			"@context": "https://www.w3.org/ns/activitystreams",
+			"first": "http://localhost:8080/users/admin/outbox?page=true",
+			"id": "http://localhost:8080/users/admin/outbox",
+			"type": "OrderedCollection"
+		}
+	*/
+
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","first":"http://localhost:8080/users/admin/outbox?page=true","id":"http://localhost:8080/users/admin/outbox","type":"OrderedCollection"}`, string(bytes))
+}
+
+func (suite *InternalToASTestSuite) TestStatusesToASOutboxPage() {
+	testAccount := suite.testAccounts["admin_account"]
+	ctx := context.Background()
+
+	// get public statuses from testaccount
+	statuses, err := suite.db.GetAccountStatuses(ctx, testAccount.ID, 30, true, "", "", false, false, true)
+	suite.NoError(err)
+
+	page, err := suite.typeconverter.StatusesToASOutboxPage(ctx, testAccount.OutboxURI, "", "", statuses)
+	suite.NoError(err)
+
+	ser, err := streams.Serialize(page)
+	assert.NoError(suite.T(), err)
+
+	bytes, err := json.Marshal(ser)
+	suite.NoError(err)
+
+	/*
+
+		we want this:
+
+		{
+			"@context": "https://www.w3.org/ns/activitystreams",
+			"id": "http://localhost:8080/users/admin/outbox?page=true",
+			"next": "http://localhost:8080/users/admin/outbox?page=true&max_id=01F8MH75CBF9JFX4ZAD54N0W0R",
+			"orderedItems": [
+				{
+					"actor": "http://localhost:8080/users/admin",
+					"cc": "http://localhost:8080/users/admin/followers",
+					"id": "http://localhost:8080/users/admin/statuses/01F8MHAAY43M6RJ473VQFCVH37/activity",
+					"object": "http://localhost:8080/users/admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
+					"published": "2021-10-20T12:36:45Z",
+					"to": "https://www.w3.org/ns/activitystreams#Public",
+					"type": "Create"
+				},
+				{
+					"actor": "http://localhost:8080/users/admin",
+					"cc": "http://localhost:8080/users/admin/followers",
+					"id": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/activity",
+					"object": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
+					"published": "2021-10-20T11:36:45Z",
+					"to": "https://www.w3.org/ns/activitystreams#Public",
+					"type": "Create"
+				}
+			],
+			"partOf": "http://localhost:8080/users/admin/outbox",
+			"prev": "http://localhost:8080/users/admin/outbox?page=true&min_id=01F8MHAAY43M6RJ473VQFCVH37",
+			"type": "OrderedCollectionPage"
+		}
+	*/
+
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/admin/outbox?page=true","next":"http://localhost:8080/users/admin/outbox?page=true\u0026max_id=01F8MH75CBF9JFX4ZAD54N0W0R","orderedItems":[{"actor":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","id":"http://localhost:8080/users/admin/statuses/01F8MHAAY43M6RJ473VQFCVH37/activity","object":"http://localhost:8080/users/admin/statuses/01F8MHAAY43M6RJ473VQFCVH37","published":"2021-10-20T12:36:45Z","to":"https://www.w3.org/ns/activitystreams#Public","type":"Create"},{"actor":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/activity","object":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","to":"https://www.w3.org/ns/activitystreams#Public","type":"Create"}],"partOf":"http://localhost:8080/users/admin/outbox","prev":"http://localhost:8080/users/admin/outbox?page=true\u0026min_id=01F8MHAAY43M6RJ473VQFCVH37","type":"OrderedCollectionPage"}`, string(bytes))
+}
+
 func TestInternalToASTestSuite(t *testing.T) {
 	suite.Run(t, new(InternalToASTestSuite))
 }
diff --git a/internal/typeutils/wrap.go b/internal/typeutils/wrap.go
index d6aa11cb4..e7c422eba 100644
--- a/internal/typeutils/wrap.go
+++ b/internal/typeutils/wrap.go
@@ -7,6 +7,7 @@ import (
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -66,3 +67,66 @@ func (c *converter) WrapPersonInUpdate(person vocab.ActivityStreamsPerson, origi
 
 	return update, nil
 }
+
+func (c *converter) WrapNoteInCreate(note vocab.ActivityStreamsNote, objectIRIOnly bool) (vocab.ActivityStreamsCreate, error) {
+	create := streams.NewActivityStreamsCreate()
+
+	// Object property
+	objectProp := streams.NewActivityStreamsObjectProperty()
+	if objectIRIOnly {
+		objectProp.AppendIRI(note.GetJSONLDId().GetIRI())
+	} else {
+		objectProp.AppendActivityStreamsNote(note)
+	}
+	create.SetActivityStreamsObject(objectProp)
+
+	// ID property
+	idProp := streams.NewJSONLDIdProperty()
+	createID := fmt.Sprintf("%s/activity", note.GetJSONLDId().GetIRI().String())
+	createIDIRI, err := url.Parse(createID)
+	if err != nil {
+		return nil, err
+	}
+	idProp.SetIRI(createIDIRI)
+	create.SetJSONLDId(idProp)
+
+	// Actor Property
+	actorProp := streams.NewActivityStreamsActorProperty()
+	actorIRI, err := ap.ExtractAttributedTo(note)
+	if err != nil {
+		return nil, fmt.Errorf("WrapNoteInCreate: couldn't extract AttributedTo: %s", err)
+	}
+	actorProp.AppendIRI(actorIRI)
+	create.SetActivityStreamsActor(actorProp)
+
+	// Published Property
+	publishedProp := streams.NewActivityStreamsPublishedProperty()
+	published, err := ap.ExtractPublished(note)
+	if err != nil {
+		return nil, fmt.Errorf("WrapNoteInCreate: couldn't extract Published: %s", err)
+	}
+	publishedProp.Set(published)
+	create.SetActivityStreamsPublished(publishedProp)
+
+	// To Property
+	toProp := streams.NewActivityStreamsToProperty()
+	tos, err := ap.ExtractTos(note)
+	if err == nil {
+		for _, to := range tos {
+			toProp.AppendIRI(to)
+		}
+		create.SetActivityStreamsTo(toProp)
+	}
+
+	// Cc Property
+	ccProp := streams.NewActivityStreamsCcProperty()
+	ccs, err := ap.ExtractCCs(note)
+	if err == nil {
+		for _, cc := range ccs {
+			ccProp.AppendIRI(cc)
+		}
+		create.SetActivityStreamsCc(ccProp)
+	}
+
+	return create, nil
+}
diff --git a/internal/typeutils/wrap_test.go b/internal/typeutils/wrap_test.go
new file mode 100644
index 000000000..14309c00c
--- /dev/null
+++ b/internal/typeutils/wrap_test.go
@@ -0,0 +1,74 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package typeutils_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/stretchr/testify/suite"
+)
+
+type WrapTestSuite struct {
+	TypeUtilsTestSuite
+}
+
+func (suite *WrapTestSuite) TestWrapNoteInCreateIRIOnly() {
+	testStatus := suite.testStatuses["local_account_1_status_1"]
+
+	note, err := suite.typeconverter.StatusToAS(context.Background(), testStatus)
+	suite.NoError(err)
+
+	create, err := suite.typeconverter.WrapNoteInCreate(note, true)
+	suite.NoError(err)
+	suite.NotNil(create)
+
+	createI, err := streams.Serialize(create)
+	suite.NoError(err)
+
+	bytes, err := json.Marshal(createI)
+	suite.NoError(err)
+
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","cc":"http://localhost:8080/users/the_mighty_zork/followers","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/activity","object":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY","published":"2021-10-20T12:40:37+02:00","to":"https://www.w3.org/ns/activitystreams#Public","type":"Create"}`, string(bytes))
+}
+
+func (suite *WrapTestSuite) TestWrapNoteInCreate() {
+	testStatus := suite.testStatuses["local_account_1_status_1"]
+
+	note, err := suite.typeconverter.StatusToAS(context.Background(), testStatus)
+	suite.NoError(err)
+
+	create, err := suite.typeconverter.WrapNoteInCreate(note, false)
+	suite.NoError(err)
+	suite.NotNil(create)
+
+	createI, err := streams.Serialize(create)
+	suite.NoError(err)
+
+	bytes, err := json.Marshal(createI)
+	suite.NoError(err)
+
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","cc":"http://localhost:8080/users/the_mighty_zork/followers","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/activity","object":{"attachment":[],"attributedTo":"http://localhost:8080/users/the_mighty_zork","cc":"http://localhost:8080/users/the_mighty_zork/followers","content":"hello everyone!","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY","published":"2021-10-20T12:40:37+02:00","replies":{"first":{"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"Collection"},"summary":"introduction post","tag":[],"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY"},"published":"2021-10-20T12:40:37+02:00","to":"https://www.w3.org/ns/activitystreams#Public","type":"Create"}`, string(bytes))
+}
+
+func TestWrapTestSuite(t *testing.T) {
+	suite.Run(t, new(WrapTestSuite))
+}