summaryrefslogtreecommitdiff
path: root/internal/api/fileserver
diff options
context:
space:
mode:
Diffstat (limited to 'internal/api/fileserver')
-rw-r--r--internal/api/fileserver/fileserver.go54
-rw-r--r--internal/api/fileserver/fileserver_test.go109
-rw-r--r--internal/api/fileserver/servefile.go130
-rw-r--r--internal/api/fileserver/servefile_test.go272
4 files changed, 565 insertions, 0 deletions
diff --git a/internal/api/fileserver/fileserver.go b/internal/api/fileserver/fileserver.go
new file mode 100644
index 000000000..bbfd7d200
--- /dev/null
+++ b/internal/api/fileserver/fileserver.go
@@ -0,0 +1,54 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
+)
+
+const (
+ // AccountIDKey is the url key for account id (an account ulid)
+ AccountIDKey = "account_id"
+ // MediaTypeKey is the url key for media type (usually something like attachment or header etc)
+ MediaTypeKey = "media_type"
+ // MediaSizeKey is the url key for the desired media size--original/small/static
+ MediaSizeKey = "media_size"
+ // FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg
+ FileNameKey = "file_name"
+ // FileServePath is the fileserve path minus the 'fileserver' prefix.
+ FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey
+)
+
+type Module struct {
+ processor processing.Processor
+}
+
+func New(processor processing.Processor) *Module {
+ return &Module{
+ processor: processor,
+ }
+}
+
+func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) {
+ attachHandler(http.MethodGet, FileServePath, m.ServeFile)
+ attachHandler(http.MethodHead, FileServePath, m.ServeFile)
+}
diff --git a/internal/api/fileserver/fileserver_test.go b/internal/api/fileserver/fileserver_test.go
new file mode 100644
index 000000000..f6de72a27
--- /dev/null
+++ b/internal/api/fileserver/fileserver_test.go
@@ -0,0 +1,109 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver_test
+
+import (
+ "context"
+
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
+ "github.com/superseriousbusiness/gotosocial/internal/concurrency"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/email"
+ "github.com/superseriousbusiness/gotosocial/internal/federation"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/log"
+ "github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/internal/messages"
+ "github.com/superseriousbusiness/gotosocial/internal/oauth"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
+ "github.com/superseriousbusiness/gotosocial/internal/typeutils"
+ "github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type FileserverTestSuite struct {
+ // standard suite interfaces
+ suite.Suite
+ db db.DB
+ storage *storage.Driver
+ federator federation.Federator
+ tc typeutils.TypeConverter
+ processor processing.Processor
+ mediaManager media.Manager
+ oauthServer oauth.Server
+ emailSender email.Sender
+
+ // standard suite models
+ testTokens map[string]*gtsmodel.Token
+ testClients map[string]*gtsmodel.Client
+ testApplications map[string]*gtsmodel.Application
+ testUsers map[string]*gtsmodel.User
+ testAccounts map[string]*gtsmodel.Account
+ testAttachments map[string]*gtsmodel.MediaAttachment
+
+ // item being tested
+ fileServer *fileserver.Module
+}
+
+/*
+ TEST INFRASTRUCTURE
+*/
+
+func (suite *FileserverTestSuite) SetupSuite() {
+ testrig.InitTestConfig()
+ testrig.InitTestLog()
+
+ fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
+ clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
+
+ suite.db = testrig.NewTestDB()
+ suite.storage = testrig.NewInMemoryStorage()
+ suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+ suite.emailSender = testrig.NewEmailSender("../../../web/template/", nil)
+
+ suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker)
+ suite.tc = testrig.NewTestTypeConverter(suite.db)
+ suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
+ suite.oauthServer = testrig.NewTestOauthServer(suite.db)
+
+ suite.fileServer = fileserver.New(suite.processor)
+}
+
+func (suite *FileserverTestSuite) SetupTest() {
+ testrig.StandardDBSetup(suite.db, nil)
+ testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
+ suite.testTokens = testrig.NewTestTokens()
+ suite.testClients = testrig.NewTestClients()
+ suite.testApplications = testrig.NewTestApplications()
+ suite.testUsers = testrig.NewTestUsers()
+ suite.testAccounts = testrig.NewTestAccounts()
+ suite.testAttachments = testrig.NewTestAttachments()
+}
+
+func (suite *FileserverTestSuite) TearDownSuite() {
+ if err := suite.db.Stop(context.Background()); err != nil {
+ log.Panicf("error closing db connection: %s", err)
+ }
+}
+
+func (suite *FileserverTestSuite) TearDownTest() {
+ testrig.StandardDBTeardown(suite.db)
+ testrig.StandardStorageTeardown(suite.storage)
+}
diff --git a/internal/api/fileserver/servefile.go b/internal/api/fileserver/servefile.go
new file mode 100644
index 000000000..08ab3110b
--- /dev/null
+++ b/internal/api/fileserver/servefile.go
@@ -0,0 +1,130 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+ "github.com/superseriousbusiness/gotosocial/internal/gtserror"
+ "github.com/superseriousbusiness/gotosocial/internal/log"
+ "github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage.
+//
+// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found".
+// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything.
+func (m *Module) ServeFile(c *gin.Context) {
+ authed, err := oauth.Authed(c, false, false, false, false)
+ if err != nil {
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
+ return
+ }
+
+ // We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows:
+ // "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]"
+ // "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension.
+ accountID := c.Param(AccountIDKey)
+ if accountID == "" {
+ err := fmt.Errorf("missing %s from request", AccountIDKey)
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
+ return
+ }
+
+ mediaType := c.Param(MediaTypeKey)
+ if mediaType == "" {
+ err := fmt.Errorf("missing %s from request", MediaTypeKey)
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
+ return
+ }
+
+ mediaSize := c.Param(MediaSizeKey)
+ if mediaSize == "" {
+ err := fmt.Errorf("missing %s from request", MediaSizeKey)
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
+ return
+ }
+
+ fileName := c.Param(FileNameKey)
+ if fileName == "" {
+ err := fmt.Errorf("missing %s from request", FileNameKey)
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet)
+ return
+ }
+
+ content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &apimodel.GetContentRequestForm{
+ AccountID: accountID,
+ MediaType: mediaType,
+ MediaSize: mediaSize,
+ FileName: fileName,
+ })
+ if errWithCode != nil {
+ apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
+ return
+ }
+
+ defer func() {
+ // close content when we're done
+ if content.Content != nil {
+ if err := content.Content.Close(); err != nil {
+ log.Errorf("ServeFile: error closing readcloser: %s", err)
+ }
+ }
+ }()
+
+ if content.URL != nil {
+ c.Redirect(http.StatusFound, content.URL.String())
+ return
+ }
+
+ // TODO: if the requester only accepts text/html we should try to serve them *something*.
+ // This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will
+ // attempt to look up the content to provide a preview of the link, and they ask for text/html.
+ format, err := apiutil.NegotiateAccept(c, apiutil.MIME(content.ContentType))
+ if err != nil {
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet)
+ return
+ }
+
+ // if this is a head request, just return info + throw the reader away
+ if c.Request.Method == http.MethodHead {
+ c.Header("Content-Type", format)
+ c.Header("Content-Length", strconv.FormatInt(content.ContentLength, 10))
+ c.Status(http.StatusOK)
+ return
+ }
+
+ // try to slurp the first few bytes to make sure we have something
+ b := bytes.NewBuffer(make([]byte, 0, 64))
+ if _, err := io.CopyN(b, content.Content, 64); err != nil {
+ err = fmt.Errorf("ServeFile: error reading from content: %w", err)
+ apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet)
+ return
+ }
+
+ // we're good, return the slurped bytes + the rest of the content
+ c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil)
+}
diff --git a/internal/api/fileserver/servefile_test.go b/internal/api/fileserver/servefile_test.go
new file mode 100644
index 000000000..efaf3896d
--- /dev/null
+++ b/internal/api/fileserver/servefile_test.go
@@ -0,0 +1,272 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver_test
+
+import (
+ "context"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ServeFileTestSuite struct {
+ FileserverTestSuite
+}
+
+// GetFile is just a convenience function to save repetition in this test suite.
+// It takes the required params to serve a file, calls the handler, and returns
+// the http status code, the response headers, and the parsed body bytes.
+func (suite *ServeFileTestSuite) GetFile(
+ accountID string,
+ mediaType media.Type,
+ mediaSize media.Size,
+ filename string,
+) (code int, headers http.Header, body []byte) {
+ recorder := httptest.NewRecorder()
+
+ ctx, _ := testrig.CreateGinTestContext(recorder, nil)
+ ctx.Request = httptest.NewRequest(http.MethodGet, "http://localhost:8080/whatever", nil)
+ ctx.Request.Header.Set("accept", "*/*")
+ ctx.AddParam(fileserver.AccountIDKey, accountID)
+ ctx.AddParam(fileserver.MediaTypeKey, string(mediaType))
+ ctx.AddParam(fileserver.MediaSizeKey, string(mediaSize))
+ ctx.AddParam(fileserver.FileNameKey, filename)
+
+ suite.fileServer.ServeFile(ctx)
+ code = recorder.Code
+ headers = recorder.Result().Header
+
+ var err error
+ body, err = ioutil.ReadAll(recorder.Body)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ return
+}
+
+// UncacheAttachment is a convenience function that uncaches the targetAttachment by
+// removing its associated files from storage, and updating the database.
+func (suite *ServeFileTestSuite) UncacheAttachment(targetAttachment *gtsmodel.MediaAttachment) {
+ ctx := context.Background()
+
+ cached := false
+ targetAttachment.Cached = &cached
+
+ if err := suite.db.UpdateByID(ctx, targetAttachment, targetAttachment.ID, "cached"); err != nil {
+ suite.FailNow(err.Error())
+ }
+ if err := suite.storage.Delete(ctx, targetAttachment.File.Path); err != nil {
+ suite.FailNow(err.Error())
+ }
+ if err := suite.storage.Delete(ctx, targetAttachment.Thumbnail.Path); err != nil {
+ suite.FailNow(err.Error())
+ }
+}
+
+func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeOriginal,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeSmall,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeOriginal,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeSmall,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // uncache the attachment so we'll have to refetch it from the 'remote' instance
+ suite.UncacheAttachment(targetAttachment)
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeOriginal,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+ fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // uncache the attachment so we'll have to refetch it from the 'remote' instance
+ suite.UncacheAttachment(targetAttachment)
+
+ code, headers, body := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeSmall,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusOK, code)
+ suite.Equal("image/jpeg", headers.Get("content-type"))
+ suite.Equal(fileInStorage, body)
+}
+
+func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+
+ // uncache the attachment *and* set the remote URL to something that will return a 404
+ suite.UncacheAttachment(targetAttachment)
+ targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee"
+ if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, _, _ := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeOriginal,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusNotFound, code)
+}
+
+func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() {
+ targetAttachment := &gtsmodel.MediaAttachment{}
+ *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"]
+
+ // uncache the attachment *and* set the remote URL to something that will return a 404
+ suite.UncacheAttachment(targetAttachment)
+ targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee"
+ if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ code, _, _ := suite.GetFile(
+ targetAttachment.AccountID,
+ media.TypeAttachment,
+ media.SizeSmall,
+ targetAttachment.ID+".jpeg",
+ )
+
+ suite.Equal(http.StatusNotFound, code)
+}
+
+// Callers trying to get some random-ass file that doesn't exist should just get a 404
+func (suite *ServeFileTestSuite) TestServeFileNotFound() {
+ code, _, _ := suite.GetFile(
+ "01GMMY4G9B0QEG0PQK5Q5JGJWZ",
+ media.TypeAttachment,
+ media.SizeOriginal,
+ "01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg",
+ )
+
+ suite.Equal(http.StatusNotFound, code)
+}
+
+func TestServeFileTestSuite(t *testing.T) {
+ suite.Run(t, new(ServeFileTestSuite))
+}