diff options
Diffstat (limited to 'internal/api/fileserver')
-rw-r--r-- | internal/api/fileserver/fileserver.go | 54 | ||||
-rw-r--r-- | internal/api/fileserver/fileserver_test.go | 109 | ||||
-rw-r--r-- | internal/api/fileserver/servefile.go | 130 | ||||
-rw-r--r-- | internal/api/fileserver/servefile_test.go | 272 |
4 files changed, 565 insertions, 0 deletions
diff --git a/internal/api/fileserver/fileserver.go b/internal/api/fileserver/fileserver.go new file mode 100644 index 000000000..bbfd7d200 --- /dev/null +++ b/internal/api/fileserver/fileserver.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package fileserver + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/superseriousbusiness/gotosocial/internal/processing" +) + +const ( + // AccountIDKey is the url key for account id (an account ulid) + AccountIDKey = "account_id" + // MediaTypeKey is the url key for media type (usually something like attachment or header etc) + MediaTypeKey = "media_type" + // MediaSizeKey is the url key for the desired media size--original/small/static + MediaSizeKey = "media_size" + // FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg + FileNameKey = "file_name" + // FileServePath is the fileserve path minus the 'fileserver' prefix. + FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey +) + +type Module struct { + processor processing.Processor +} + +func New(processor processing.Processor) *Module { + return &Module{ + processor: processor, + } +} + +func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) { + attachHandler(http.MethodGet, FileServePath, m.ServeFile) + attachHandler(http.MethodHead, FileServePath, m.ServeFile) +} diff --git a/internal/api/fileserver/fileserver_test.go b/internal/api/fileserver/fileserver_test.go new file mode 100644 index 000000000..f6de72a27 --- /dev/null +++ b/internal/api/fileserver/fileserver_test.go @@ -0,0 +1,109 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package fileserver_test + +import ( + "context" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/fileserver" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/email" + "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type FileserverTestSuite struct { + // standard suite interfaces + suite.Suite + db db.DB + storage *storage.Driver + federator federation.Federator + tc typeutils.TypeConverter + processor processing.Processor + mediaManager media.Manager + oauthServer oauth.Server + emailSender email.Sender + + // standard suite models + testTokens map[string]*gtsmodel.Token + testClients map[string]*gtsmodel.Client + testApplications map[string]*gtsmodel.Application + testUsers map[string]*gtsmodel.User + testAccounts map[string]*gtsmodel.Account + testAttachments map[string]*gtsmodel.MediaAttachment + + // item being tested + fileServer *fileserver.Module +} + +/* + TEST INFRASTRUCTURE +*/ + +func (suite *FileserverTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() + + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewInMemoryStorage() + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.emailSender = testrig.NewEmailSender("../../../web/template/", nil) + + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker) + suite.tc = testrig.NewTestTypeConverter(suite.db) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.oauthServer = testrig.NewTestOauthServer(suite.db) + + suite.fileServer = fileserver.New(suite.processor) +} + +func (suite *FileserverTestSuite) SetupTest() { + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() +} + +func (suite *FileserverTestSuite) TearDownSuite() { + if err := suite.db.Stop(context.Background()); err != nil { + log.Panicf("error closing db connection: %s", err) + } +} + +func (suite *FileserverTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} diff --git a/internal/api/fileserver/servefile.go b/internal/api/fileserver/servefile.go new file mode 100644 index 000000000..08ab3110b --- /dev/null +++ b/internal/api/fileserver/servefile.go @@ -0,0 +1,130 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package fileserver + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage. +// +// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found". +// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything. +func (m *Module) ServeFile(c *gin.Context) { + authed, err := oauth.Authed(c, false, false, false, false) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) + return + } + + // We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows: + // "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]" + // "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension. + accountID := c.Param(AccountIDKey) + if accountID == "" { + err := fmt.Errorf("missing %s from request", AccountIDKey) + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) + return + } + + mediaType := c.Param(MediaTypeKey) + if mediaType == "" { + err := fmt.Errorf("missing %s from request", MediaTypeKey) + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) + return + } + + mediaSize := c.Param(MediaSizeKey) + if mediaSize == "" { + err := fmt.Errorf("missing %s from request", MediaSizeKey) + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) + return + } + + fileName := c.Param(FileNameKey) + if fileName == "" { + err := fmt.Errorf("missing %s from request", FileNameKey) + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) + return + } + + content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &apimodel.GetContentRequestForm{ + AccountID: accountID, + MediaType: mediaType, + MediaSize: mediaSize, + FileName: fileName, + }) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGet) + return + } + + defer func() { + // close content when we're done + if content.Content != nil { + if err := content.Content.Close(); err != nil { + log.Errorf("ServeFile: error closing readcloser: %s", err) + } + } + }() + + if content.URL != nil { + c.Redirect(http.StatusFound, content.URL.String()) + return + } + + // TODO: if the requester only accepts text/html we should try to serve them *something*. + // This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will + // attempt to look up the content to provide a preview of the link, and they ask for text/html. + format, err := apiutil.NegotiateAccept(c, apiutil.MIME(content.ContentType)) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet) + return + } + + // if this is a head request, just return info + throw the reader away + if c.Request.Method == http.MethodHead { + c.Header("Content-Type", format) + c.Header("Content-Length", strconv.FormatInt(content.ContentLength, 10)) + c.Status(http.StatusOK) + return + } + + // try to slurp the first few bytes to make sure we have something + b := bytes.NewBuffer(make([]byte, 0, 64)) + if _, err := io.CopyN(b, content.Content, 64); err != nil { + err = fmt.Errorf("ServeFile: error reading from content: %w", err) + apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet) + return + } + + // we're good, return the slurped bytes + the rest of the content + c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil) +} diff --git a/internal/api/fileserver/servefile_test.go b/internal/api/fileserver/servefile_test.go new file mode 100644 index 000000000..efaf3896d --- /dev/null +++ b/internal/api/fileserver/servefile_test.go @@ -0,0 +1,272 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package fileserver_test + +import ( + "context" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/fileserver" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type ServeFileTestSuite struct { + FileserverTestSuite +} + +// GetFile is just a convenience function to save repetition in this test suite. +// It takes the required params to serve a file, calls the handler, and returns +// the http status code, the response headers, and the parsed body bytes. +func (suite *ServeFileTestSuite) GetFile( + accountID string, + mediaType media.Type, + mediaSize media.Size, + filename string, +) (code int, headers http.Header, body []byte) { + recorder := httptest.NewRecorder() + + ctx, _ := testrig.CreateGinTestContext(recorder, nil) + ctx.Request = httptest.NewRequest(http.MethodGet, "http://localhost:8080/whatever", nil) + ctx.Request.Header.Set("accept", "*/*") + ctx.AddParam(fileserver.AccountIDKey, accountID) + ctx.AddParam(fileserver.MediaTypeKey, string(mediaType)) + ctx.AddParam(fileserver.MediaSizeKey, string(mediaSize)) + ctx.AddParam(fileserver.FileNameKey, filename) + + suite.fileServer.ServeFile(ctx) + code = recorder.Code + headers = recorder.Result().Header + + var err error + body, err = ioutil.ReadAll(recorder.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + return +} + +// UncacheAttachment is a convenience function that uncaches the targetAttachment by +// removing its associated files from storage, and updating the database. +func (suite *ServeFileTestSuite) UncacheAttachment(targetAttachment *gtsmodel.MediaAttachment) { + ctx := context.Background() + + cached := false + targetAttachment.Cached = &cached + + if err := suite.db.UpdateByID(ctx, targetAttachment, targetAttachment.ID, "cached"); err != nil { + suite.FailNow(err.Error()) + } + if err := suite.storage.Delete(ctx, targetAttachment.File.Path); err != nil { + suite.FailNow(err.Error()) + } + if err := suite.storage.Delete(ctx, targetAttachment.Thumbnail.Path); err != nil { + suite.FailNow(err.Error()) + } +} + +func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeOriginal, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeSmall, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeOriginal, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeSmall, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + // uncache the attachment so we'll have to refetch it from the 'remote' instance + suite.UncacheAttachment(targetAttachment) + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeOriginal, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) + if err != nil { + suite.FailNow(err.Error()) + } + + // uncache the attachment so we'll have to refetch it from the 'remote' instance + suite.UncacheAttachment(targetAttachment) + + code, headers, body := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeSmall, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusOK, code) + suite.Equal("image/jpeg", headers.Get("content-type")) + suite.Equal(fileInStorage, body) +} + +func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + + // uncache the attachment *and* set the remote URL to something that will return a 404 + suite.UncacheAttachment(targetAttachment) + targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee" + if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil { + suite.FailNow(err.Error()) + } + + code, _, _ := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeOriginal, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusNotFound, code) +} + +func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() { + targetAttachment := >smodel.MediaAttachment{} + *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] + + // uncache the attachment *and* set the remote URL to something that will return a 404 + suite.UncacheAttachment(targetAttachment) + targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee" + if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil { + suite.FailNow(err.Error()) + } + + code, _, _ := suite.GetFile( + targetAttachment.AccountID, + media.TypeAttachment, + media.SizeSmall, + targetAttachment.ID+".jpeg", + ) + + suite.Equal(http.StatusNotFound, code) +} + +// Callers trying to get some random-ass file that doesn't exist should just get a 404 +func (suite *ServeFileTestSuite) TestServeFileNotFound() { + code, _, _ := suite.GetFile( + "01GMMY4G9B0QEG0PQK5Q5JGJWZ", + media.TypeAttachment, + media.SizeOriginal, + "01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg", + ) + + suite.Equal(http.StatusNotFound, code) +} + +func TestServeFileTestSuite(t *testing.T) { + suite.Run(t, new(ServeFileTestSuite)) +} |