Serve `outbox` for Actor (#289)

* add statusesvisible convenience function

* add minID + onlyPublic to account statuses get

* move swagger collection stuff to common

* start working on Outbox GETting

* move functions into federationProcessor

* outboxToASCollection

* add statusesvisible convenience function

* add minID + onlyPublic to account statuses get

* move swagger collection stuff to common

* start working on Outbox GETting

* move functions into federationProcessor

* outboxToASCollection

* bit more work on outbox paging

* wrapNoteInCreate function

* test + hook up the processor functions

* don't do prev + next links on empty reply

* test get outbox through api

* don't fail on no status entries

* add outbox implementation doc

* typo

6 files changed, 393 insertions, 38 deletions

diff --git a/internal/api/s2s/user/common.go b/internal/api/s2s/user/common.go
index e96082c3e..9f426274d 100644
--- a/internal/api/s2s/user/common.go
+++ b/internal/api/s2s/user/common.go
@@ -57,3 +57,41 @@ func negotiateFormat(c *gin.Context) (string, error) {
 	}
 	return format, nil
 }
+
+// SwaggerCollection represents an activitypub collection.
+// swagger:model swaggerCollection
+type SwaggerCollection struct {
+	// ActivityStreams context.
+	// example: https://www.w3.org/ns/activitystreams
+	Context string `json:"@context"`
+	// ActivityStreams ID.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
+	ID string `json:"id"`
+	// ActivityStreams type.
+	// example: Collection
+	Type string `json:"type"`
+	// ActivityStreams first property.
+	First SwaggerCollectionPage `json:"first"`
+	// ActivityStreams last property.
+	Last SwaggerCollectionPage `json:"last,omitempty"`
+}
+
+// SwaggerCollectionPage represents one page of a collection.
+// swagger:model swaggerCollectionPage
+type SwaggerCollectionPage struct {
+	// ActivityStreams ID.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true
+	ID string `json:"id"`
+	// ActivityStreams type.
+	// example: CollectionPage
+	Type string `json:"type"`
+	// Link to the next page.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true
+	Next string `json:"next"`
+	// Collection this page belongs to.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
+	PartOf string `json:"partOf"`
+	// Items on this page.
+	// example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"]
+	Items []string `json:"items"`
+}
diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go
index 79c44116d..554f3d729 100644
--- a/internal/api/s2s/user/inboxpost_test.go
+++ b/internal/api/s2s/user/inboxpost_test.go
@@ -436,7 +436,7 @@ func (suite *InboxPostTestSuite) TestPostDelete() {
 	suite.ErrorIs(err, db.ErrNoEntries)
 
 	// no statuses from foss satan should be left in the database
-	dbStatuses, err := suite.db.GetAccountStatuses(ctx, deletedAccount.ID, 0, false, "", false, false)
+	dbStatuses, err := suite.db.GetAccountStatuses(ctx, deletedAccount.ID, 0, false, "", "", false, false, false)
 	suite.ErrorIs(err, db.ErrNoEntries)
 	suite.Empty(dbStatuses)
 
diff --git a/internal/api/s2s/user/outboxget.go b/internal/api/s2s/user/outboxget.go
new file mode 100644
index 000000000..46f9d2ded
--- /dev/null
+++ b/internal/api/s2s/user/outboxget.go
@@ -0,0 +1,142 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package user
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+// OutboxGETHandler swagger:operation GET /users/{username}/outbox s2sOutboxGet
+//
+// Get the public outbox collection for an actor.
+//
+// Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`.
+//
+// If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`.
+//
+// HTTP signature is required on the request.
+//
+// ---
+// tags:
+// - s2s/federation
+//
+// produces:
+// - application/activity+json
+//
+// parameters:
+// - name: username
+//   type: string
+//   description: Username of the account.
+//   in: path
+//   required: true
+// - name: page
+//   type: boolean
+//   description: Return response as a CollectionPage.
+//   in: query
+//   default: false
+// - name: min_id
+//   type: string
+//   description: Minimum ID of the next status, used for paging.
+//   in: query
+// - name: max_id
+//   type: string
+//   description: Maximum ID of the next status, used for paging.
+//   in: query
+//
+// responses:
+//   '200':
+//      in: body
+//      schema:
+//        "$ref": "#/definitions/swaggerCollection"
+//   '400':
+//      description: bad request
+//   '401':
+//      description: unauthorized
+//   '403':
+//      description: forbidden
+//   '404':
+//      description: not found
+func (m *Module) OutboxGETHandler(c *gin.Context) {
+	l := logrus.WithFields(logrus.Fields{
+		"func": "OutboxGETHandler",
+		"url":  c.Request.RequestURI,
+	})
+
+	requestedUsername := c.Param(UsernameKey)
+	if requestedUsername == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"})
+		return
+	}
+
+	page := false
+	pageString := c.Query(PageKey)
+	if pageString != "" {
+		i, err := strconv.ParseBool(pageString)
+		if err != nil {
+			l.Debugf("error parsing page string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse page query param"})
+			return
+		}
+		page = i
+	}
+
+	minID := ""
+	minIDString := c.Query(MinIDKey)
+	if minIDString != "" {
+		minID = minIDString
+	}
+
+	maxID := ""
+	maxIDString := c.Query(MaxIDKey)
+	if maxIDString != "" {
+		maxID = maxIDString
+	}
+
+	format, err := negotiateFormat(c)
+	if err != nil {
+		c.JSON(http.StatusNotAcceptable, gin.H{"error": fmt.Sprintf("could not negotiate format with given Accept header(s): %s", err)})
+		return
+	}
+	l.Tracef("negotiated format: %s", format)
+
+	ctx := transferContext(c)
+
+	outbox, errWithCode := m.processor.GetFediOutbox(ctx, requestedUsername, page, maxID, minID, c.Request.URL)
+	if errWithCode != nil {
+		l.Info(errWithCode.Error())
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+		return
+	}
+
+	b, mErr := json.Marshal(outbox)
+	if mErr != nil {
+		err := fmt.Errorf("could not marshal json: %s", mErr)
+		l.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.Data(http.StatusOK, format, b)
+}
diff --git a/internal/api/s2s/user/outboxget_test.go b/internal/api/s2s/user/outboxget_test.go
new file mode 100644
index 000000000..f1818683e
--- /dev/null
+++ b/internal/api/s2s/user/outboxget_test.go
@@ -0,0 +1,206 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package user_test
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type OutboxGetTestSuite struct {
+	UserStandardTestSuite
+}
+
+func (suite *OutboxGetTestSuite) TestGetOutbox() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_zork_outbox"]
+	targetAccount := suite.testAccounts["local_account_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI, nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+	}
+
+	// trigger the function being tested
+	userModule.OutboxGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","first":"http://localhost:8080/users/the_mighty_zork/outbox?page=true","id":"http://localhost:8080/users/the_mighty_zork/outbox","type":"OrderedCollection"}`, string(b))
+
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	suite.NoError(err)
+
+	t, err := streams.ToType(context.Background(), m)
+	suite.NoError(err)
+
+	_, ok := t.(vocab.ActivityStreamsOrderedCollection)
+	suite.True(ok)
+}
+
+func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_zork_outbox_first"]
+	targetAccount := suite.testAccounts["local_account_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true", nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+	}
+
+	// trigger the function being tested
+	userModule.OutboxGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/outbox?page=true","next":"http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY","orderedItems":{"actor":"http://localhost:8080/users/the_mighty_zork","cc":"http://localhost:8080/users/the_mighty_zork/followers","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/activity","object":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY","published":"2021-10-20T10:40:37Z","to":"https://www.w3.org/ns/activitystreams#Public","type":"Create"},"partOf":"http://localhost:8080/users/the_mighty_zork/outbox","prev":"http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026min_id=01F8MHAMCHF6Y650WCRSCP4WMY","type":"OrderedCollectionPage"}`, string(b))
+
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	suite.NoError(err)
+
+	t, err := streams.ToType(context.Background(), m)
+	suite.NoError(err)
+
+	_, ok := t.(vocab.ActivityStreamsOrderedCollectionPage)
+	suite.True(ok)
+}
+
+func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_zork_outbox_next"]
+	targetAccount := suite.testAccounts["local_account_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+		gin.Param{
+			Key:   user.MaxIDKey,
+			Value: "01F8MHAMCHF6Y650WCRSCP4WMY",
+		},
+	}
+
+	// trigger the function being tested
+	userModule.OutboxGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026maxID=01F8MHAMCHF6Y650WCRSCP4WMY","orderedItems":[],"partOf":"http://localhost:8080/users/the_mighty_zork/outbox","type":"OrderedCollectionPage"}`, string(b))
+
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	suite.NoError(err)
+
+	t, err := streams.ToType(context.Background(), m)
+	suite.NoError(err)
+
+	_, ok := t.(vocab.ActivityStreamsOrderedCollectionPage)
+	suite.True(ok)
+}
+
+func TestOutboxGetTestSuite(t *testing.T) {
+	suite.Run(t, new(OutboxGetTestSuite))
+}
diff --git a/internal/api/s2s/user/repliesget.go b/internal/api/s2s/user/repliesget.go
index b24dead31..e6328a26b 100644
--- a/internal/api/s2s/user/repliesget.go
+++ b/internal/api/s2s/user/repliesget.go
@@ -75,7 +75,7 @@ import (
 //   '200':
 //      in: body
 //      schema:
-//        "$ref": "#/definitions/swaggerStatusRepliesCollection"
+//        "$ref": "#/definitions/swaggerCollection"
 //   '400':
 //      description: bad request
 //   '401':
@@ -158,39 +158,3 @@ func (m *Module) StatusRepliesGETHandler(c *gin.Context) {
 
 	c.Data(http.StatusOK, format, b)
 }
-
-// SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies.
-// swagger:model swaggerStatusRepliesCollection
-type SwaggerStatusRepliesCollection struct {
-	// ActivityStreams context.
-	// example: https://www.w3.org/ns/activitystreams
-	Context string `json:"@context"`
-	// ActivityStreams ID.
-	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
-	ID string `json:"id"`
-	// ActivityStreams type.
-	// example: Collection
-	Type string `json:"type"`
-	// ActivityStreams first property.
-	First SwaggerStatusRepliesCollectionPage `json:"first"`
-}
-
-// SwaggerStatusRepliesCollectionPage represents one page of a collection.
-// swagger:model swaggerStatusRepliesCollectionPage
-type SwaggerStatusRepliesCollectionPage struct {
-	// ActivityStreams ID.
-	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true
-	ID string `json:"id"`
-	// ActivityStreams type.
-	// example: CollectionPage
-	Type string `json:"type"`
-	// Link to the next page.
-	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true
-	Next string `json:"next"`
-	// Collection this page belongs to.
-	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
-	PartOf string `json:"partOf"`
-	// Items on this page.
-	// example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"]
-	Items []string `json:"items"`
-}
diff --git a/internal/api/s2s/user/user.go b/internal/api/s2s/user/user.go
index ef0057f98..56b940f1d 100644
--- a/internal/api/s2s/user/user.go
+++ b/internal/api/s2s/user/user.go
@@ -37,6 +37,8 @@ const (
 	OnlyOtherAccountsKey = "only_other_accounts"
 	// MinIDKey is for filtering status responses.
 	MinIDKey = "min_id"
+	// MaxIDKey is for filtering status responses.
+	MaxIDKey = "max_id"
 	// PageKey is for filtering status responses.
 	PageKey = "page"
 
@@ -50,6 +52,8 @@ const (
 	UsersPublicKeyPath = UsersBasePathWithUsername + "/" + util.PublicKeyPath
 	// UsersInboxPath is for serving POST requests to a user's inbox with the given username key.
 	UsersInboxPath = UsersBasePathWithUsername + "/" + util.InboxPath
+	// UsersOutboxPath is for serving GET requests to a user's outbox with the given username key.
+	UsersOutboxPath = UsersBasePathWithUsername + "/" + util.OutboxPath
 	// UsersFollowersPath is for serving GET request's to a user's followers list, with the given username key.
 	UsersFollowersPath = UsersBasePathWithUsername + "/" + util.FollowersPath
 	// UsersFollowingPath is for serving GET request's to a user's following list, with the given username key.
@@ -83,5 +87,6 @@ func (m *Module) Route(s router.Router) error {
 	s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler)
 	s.AttachHandler(http.MethodGet, UsersPublicKeyPath, m.PublicKeyGETHandler)
 	s.AttachHandler(http.MethodGet, UsersStatusRepliesPath, m.StatusRepliesGETHandler)
+	s.AttachHandler(http.MethodGet, UsersOutboxPath, m.OutboxGETHandler)
 	return nil
 }