Ap (#14)

Big restructuring and initial work on activitypub

3 files changed, 339 insertions, 0 deletions

diff --git a/internal/api/client/fileserver/fileserver.go b/internal/api/client/fileserver/fileserver.go
new file mode 100644
index 000000000..63d323a01
--- /dev/null
+++ b/internal/api/client/fileserver/fileserver.go
@@ -0,0 +1,82 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/message"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+)
+
+const (
+	// AccountIDKey is the url key for account id (an account uuid)
+	AccountIDKey = "account_id"
+	// MediaTypeKey is the url key for media type (usually something like attachment or header etc)
+	MediaTypeKey = "media_type"
+	// MediaSizeKey is the url key for the desired media size--original/small/static
+	MediaSizeKey = "media_size"
+	// FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg
+	FileNameKey = "file_name"
+)
+
+// FileServer implements the RESTAPIModule interface.
+// The goal here is to serve requested media files if the gotosocial server is configured to use local storage.
+type FileServer struct {
+	config      *config.Config
+	processor   message.Processor
+	log         *logrus.Logger
+	storageBase string
+}
+
+// New returns a new fileServer module
+func New(config *config.Config, processor message.Processor, log *logrus.Logger) api.ClientModule {
+	return &FileServer{
+		config:      config,
+		processor:   processor,
+		log:         log,
+		storageBase: config.StorageConfig.ServeBasePath,
+	}
+}
+
+// Route satisfies the RESTAPIModule interface
+func (m *FileServer) Route(s router.Router) error {
+	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, AccountIDKey, MediaTypeKey, MediaSizeKey, FileNameKey), m.ServeFile)
+	return nil
+}
+
+// CreateTables populates necessary tables in the given DB
+func (m *FileServer) CreateTables(db db.DB) error {
+	models := []interface{}{
+		&gtsmodel.MediaAttachment{},
+	}
+
+	for _, m := range models {
+		if err := db.CreateTable(m); err != nil {
+			return fmt.Errorf("error creating table: %s", err)
+		}
+	}
+	return nil
+}
diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go
new file mode 100644
index 000000000..9823eb387
--- /dev/null
+++ b/internal/api/client/fileserver/servefile.go
@@ -0,0 +1,94 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage.
+//
+// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found".
+// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything.
+func (m *FileServer) ServeFile(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "ServeFile",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+	l.Trace("received request")
+
+	authed, err := oauth.Authed(c, false, false, false, false)
+	if err != nil {
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows:
+	// "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]"
+	// "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension.
+	accountID := c.Param(AccountIDKey)
+	if accountID == "" {
+		l.Debug("missing accountID from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	mediaType := c.Param(MediaTypeKey)
+	if mediaType == "" {
+		l.Debug("missing mediaType from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	mediaSize := c.Param(MediaSizeKey)
+	if mediaSize == "" {
+		l.Debug("missing mediaSize from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	fileName := c.Param(FileNameKey)
+	if fileName == "" {
+		l.Debug("missing fileName from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	content, err := m.processor.MediaGet(authed, &model.GetContentRequestForm{
+		AccountID: accountID,
+		MediaType: mediaType,
+		MediaSize: mediaSize,
+		FileName:  fileName,
+	})
+	if err != nil {
+		l.Debug(err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	c.DataFromReader(http.StatusOK, content.ContentLength, content.ContentType, bytes.NewReader(content.Content), nil)
+}
diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go
new file mode 100644
index 000000000..09fd8ea43
--- /dev/null
+++ b/internal/api/client/fileserver/servefile_test.go
@@ -0,0 +1,163 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver_test
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/message"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ServeFileTestSuite struct {
+	// standard suite interfaces
+	suite.Suite
+	config       *config.Config
+	db           db.DB
+	log          *logrus.Logger
+	storage      storage.Storage
+	federator    federation.Federator
+	tc           typeutils.TypeConverter
+	processor    message.Processor
+	mediaHandler media.Handler
+	oauthServer  oauth.Server
+
+	// standard suite models
+	testTokens       map[string]*oauth.Token
+	testClients      map[string]*oauth.Client
+	testApplications map[string]*gtsmodel.Application
+	testUsers        map[string]*gtsmodel.User
+	testAccounts     map[string]*gtsmodel.Account
+	testAttachments  map[string]*gtsmodel.MediaAttachment
+
+	// item being tested
+	fileServer *fileserver.FileServer
+}
+
+/*
+	TEST INFRASTRUCTURE
+*/
+
+func (suite *ServeFileTestSuite) SetupSuite() {
+	// setup standard items
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.storage = testrig.NewTestStorage()
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
+	suite.tc = testrig.NewTestTypeConverter(suite.db)
+	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
+	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
+
+	// setup module being tested
+	suite.fileServer = fileserver.New(suite.config, suite.processor, suite.log).(*fileserver.FileServer)
+}
+
+func (suite *ServeFileTestSuite) TearDownSuite() {
+	if err := suite.db.Stop(context.Background()); err != nil {
+		logrus.Panicf("error closing db connection: %s", err)
+	}
+}
+
+func (suite *ServeFileTestSuite) SetupTest() {
+	testrig.StandardDBSetup(suite.db)
+	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+}
+
+func (suite *ServeFileTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+/*
+	ACTUAL TESTS
+*/
+
+func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
+	targetAttachment, ok := suite.testAttachments["admin_account_status_1_attachment_1"]
+	assert.True(suite.T(), ok)
+	assert.NotNil(suite.T(), targetAttachment)
+
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAttachment.URL, nil)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the ServeFile function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   fileserver.AccountIDKey,
+			Value: targetAttachment.AccountID,
+		},
+		gin.Param{
+			Key:   fileserver.MediaTypeKey,
+			Value: string(media.Attachment),
+		},
+		gin.Param{
+			Key:   fileserver.MediaSizeKey,
+			Value: string(media.Original),
+		},
+		gin.Param{
+			Key:   fileserver.FileNameKey,
+			Value: fmt.Sprintf("%s.jpeg", targetAttachment.ID),
+		},
+	}
+
+	// call the function we're testing and check status code
+	suite.fileServer.ServeFile(ctx)
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	b, err := ioutil.ReadAll(recorder.Body)
+	assert.NoError(suite.T(), err)
+	assert.NotNil(suite.T(), b)
+
+	fileInStorage, err := suite.storage.RetrieveFileFrom(targetAttachment.File.Path)
+	assert.NoError(suite.T(), err)
+	assert.NotNil(suite.T(), fileInStorage)
+	assert.Equal(suite.T(), b, fileInStorage)
+}
+
+func TestServeFileTestSuite(t *testing.T) {
+	suite.Run(t, new(ServeFileTestSuite))
+}