diff options
author | 2023-01-02 13:10:50 +0100 | |
---|---|---|
committer | 2023-01-02 12:10:50 +0000 | |
commit | 941893a774c83802afdc4cc76e1d30c59b6c5585 (patch) | |
tree | 6e7296146dedfeac8e83655157270f41e190724b /internal/api/client/fileserver | |
parent | [chore]: Bump github.com/abema/go-mp4 from 0.8.0 to 0.9.0 (#1287) (diff) | |
download | gotosocial-941893a774c83802afdc4cc76e1d30c59b6c5585.tar.xz |
[chore] The Big Middleware and API Refactor (tm) (#1250)
* interim commit: start refactoring middlewares into package under router
* another interim commit, this is becoming a big job
* another fucking massive interim commit
* refactor bookmarks to new style
* ambassador, wiz zeze commits you are spoiling uz
* she compiles, we're getting there
* we're just normal men; we're just innocent men
* apiutil
* whoopsie
* i'm glad noone reads commit msgs haha :blob_sweat:
* use that weirdo go-bytesize library for maxMultipartMemory
* fix media module paths
Diffstat (limited to 'internal/api/client/fileserver')
-rw-r--r-- | internal/api/client/fileserver/fileserver.go | 64 | ||||
-rw-r--r-- | internal/api/client/fileserver/fileserver_test.go | 109 | ||||
-rw-r--r-- | internal/api/client/fileserver/servefile.go | 135 | ||||
-rw-r--r-- | internal/api/client/fileserver/servefile_test.go | 272 |
4 files changed, 0 insertions, 580 deletions
diff --git a/internal/api/client/fileserver/fileserver.go b/internal/api/client/fileserver/fileserver.go deleted file mode 100644 index dcb54f986..000000000 --- a/internal/api/client/fileserver/fileserver.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package fileserver - -import ( - "fmt" - "net/http" - - "github.com/superseriousbusiness/gotosocial/internal/api" - "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/router" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -const ( - // FileServeBasePath forms the first part of the fileserver path. - FileServeBasePath = "/" + uris.FileserverPath - // AccountIDKey is the url key for account id (an account ulid) - AccountIDKey = "account_id" - // MediaTypeKey is the url key for media type (usually something like attachment or header etc) - MediaTypeKey = "media_type" - // MediaSizeKey is the url key for the desired media size--original/small/static - MediaSizeKey = "media_size" - // FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg - FileNameKey = "file_name" -) - -// FileServer implements the RESTAPIModule interface. -// The goal here is to serve requested media files if the gotosocial server is configured to use local storage. -type FileServer struct { - processor processing.Processor -} - -// New returns a new fileServer module -func New(processor processing.Processor) api.ClientModule { - return &FileServer{ - processor: processor, - } -} - -// Route satisfies the RESTAPIModule interface -func (m *FileServer) Route(s router.Router) error { - // something like "/fileserver/:account_id/:media_type/:media_size/:file_name" - fileServePath := fmt.Sprintf("%s/:%s/:%s/:%s/:%s", FileServeBasePath, AccountIDKey, MediaTypeKey, MediaSizeKey, FileNameKey) - s.AttachHandler(http.MethodGet, fileServePath, m.ServeFile) - s.AttachHandler(http.MethodHead, fileServePath, m.ServeFile) - return nil -} diff --git a/internal/api/client/fileserver/fileserver_test.go b/internal/api/client/fileserver/fileserver_test.go deleted file mode 100644 index f1fab5672..000000000 --- a/internal/api/client/fileserver/fileserver_test.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package fileserver_test - -import ( - "context" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver" - "github.com/superseriousbusiness/gotosocial/internal/concurrency" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/email" - "github.com/superseriousbusiness/gotosocial/internal/federation" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/storage" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -type FileserverTestSuite struct { - // standard suite interfaces - suite.Suite - db db.DB - storage *storage.Driver - federator federation.Federator - tc typeutils.TypeConverter - processor processing.Processor - mediaManager media.Manager - oauthServer oauth.Server - emailSender email.Sender - - // standard suite models - testTokens map[string]*gtsmodel.Token - testClients map[string]*gtsmodel.Client - testApplications map[string]*gtsmodel.Application - testUsers map[string]*gtsmodel.User - testAccounts map[string]*gtsmodel.Account - testAttachments map[string]*gtsmodel.MediaAttachment - - // item being tested - fileServer *fileserver.FileServer -} - -/* - TEST INFRASTRUCTURE -*/ - -func (suite *FileserverTestSuite) SetupSuite() { - testrig.InitTestConfig() - testrig.InitTestLog() - - fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) - - suite.db = testrig.NewTestDB() - suite.storage = testrig.NewInMemoryStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) - suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker) - suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.oauthServer = testrig.NewTestOauthServer(suite.db) - - suite.fileServer = fileserver.New(suite.processor).(*fileserver.FileServer) -} - -func (suite *FileserverTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db, nil) - testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") - suite.testTokens = testrig.NewTestTokens() - suite.testClients = testrig.NewTestClients() - suite.testApplications = testrig.NewTestApplications() - suite.testUsers = testrig.NewTestUsers() - suite.testAccounts = testrig.NewTestAccounts() - suite.testAttachments = testrig.NewTestAttachments() -} - -func (suite *FileserverTestSuite) TearDownSuite() { - if err := suite.db.Stop(context.Background()); err != nil { - log.Panicf("error closing db connection: %s", err) - } -} - -func (suite *FileserverTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.db) - testrig.StandardStorageTeardown(suite.storage) -} diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go deleted file mode 100644 index d2328a5fc..000000000 --- a/internal/api/client/fileserver/servefile.go +++ /dev/null @@ -1,135 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package fileserver - -import ( - "bytes" - "fmt" - "io" - "net/http" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/superseriousbusiness/gotosocial/internal/api" - "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/oauth" -) - -// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage. -// -// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found". -// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything. -func (m *FileServer) ServeFile(c *gin.Context) { - authed, err := oauth.Authed(c, false, false, false, false) - if err != nil { - api.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) - return - } - - // We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows: - // "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]" - // "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension. - accountID := c.Param(AccountIDKey) - if accountID == "" { - err := fmt.Errorf("missing %s from request", AccountIDKey) - api.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) - return - } - - mediaType := c.Param(MediaTypeKey) - if mediaType == "" { - err := fmt.Errorf("missing %s from request", MediaTypeKey) - api.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) - return - } - - mediaSize := c.Param(MediaSizeKey) - if mediaSize == "" { - err := fmt.Errorf("missing %s from request", MediaSizeKey) - api.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) - return - } - - fileName := c.Param(FileNameKey) - if fileName == "" { - err := fmt.Errorf("missing %s from request", FileNameKey) - api.ErrorHandler(c, gtserror.NewErrorNotFound(err), m.processor.InstanceGet) - return - } - - content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &model.GetContentRequestForm{ - AccountID: accountID, - MediaType: mediaType, - MediaSize: mediaSize, - FileName: fileName, - }) - if errWithCode != nil { - api.ErrorHandler(c, errWithCode, m.processor.InstanceGet) - return - } - - defer func() { - // close content when we're done - if content.Content != nil { - if err := content.Content.Close(); err != nil { - log.Errorf("ServeFile: error closing readcloser: %s", err) - } - } - }() - - if content.URL != nil { - c.Redirect(http.StatusFound, content.URL.String()) - return - } - - // TODO: if the requester only accepts text/html we should try to serve them *something*. - // This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will - // attempt to look up the content to provide a preview of the link, and they ask for text/html. - format, err := api.NegotiateAccept(c, api.MIME(content.ContentType)) - if err != nil { - api.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet) - return - } - - // since we'll never host different files at the same - // URL (bc the ULIDs are generated per piece of media), - // it's sensible and safe to use a long cache here, so - // that clients don't keep fetching files over + over again - c.Header("Cache-Control", "max-age=604800") - - if c.Request.Method == http.MethodHead { - c.Header("Content-Type", format) - c.Header("Content-Length", strconv.FormatInt(content.ContentLength, 10)) - c.Status(http.StatusOK) - return - } - - // try to slurp the first few bytes to make sure we have something - b := bytes.NewBuffer(make([]byte, 0, 64)) - if _, err := io.CopyN(b, content.Content, 64); err != nil { - err = fmt.Errorf("ServeFile: error reading from content: %w", err) - api.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet) - return - } - - // we're good, return the slurped bytes + the rest of the content - c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil) -} diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go deleted file mode 100644 index 1ca0c60d6..000000000 --- a/internal/api/client/fileserver/servefile_test.go +++ /dev/null @@ -1,272 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package fileserver_test - -import ( - "context" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -type ServeFileTestSuite struct { - FileserverTestSuite -} - -// GetFile is just a convenience function to save repetition in this test suite. -// It takes the required params to serve a file, calls the handler, and returns -// the http status code, the response headers, and the parsed body bytes. -func (suite *ServeFileTestSuite) GetFile( - accountID string, - mediaType media.Type, - mediaSize media.Size, - filename string, -) (code int, headers http.Header, body []byte) { - recorder := httptest.NewRecorder() - - ctx, _ := testrig.CreateGinTestContext(recorder, nil) - ctx.Request = httptest.NewRequest(http.MethodGet, "http://localhost:8080/whatever", nil) - ctx.Request.Header.Set("accept", "*/*") - ctx.AddParam(fileserver.AccountIDKey, accountID) - ctx.AddParam(fileserver.MediaTypeKey, string(mediaType)) - ctx.AddParam(fileserver.MediaSizeKey, string(mediaSize)) - ctx.AddParam(fileserver.FileNameKey, filename) - - suite.fileServer.ServeFile(ctx) - code = recorder.Code - headers = recorder.Result().Header - - var err error - body, err = ioutil.ReadAll(recorder.Body) - if err != nil { - suite.FailNow(err.Error()) - } - - return -} - -// UncacheAttachment is a convenience function that uncaches the targetAttachment by -// removing its associated files from storage, and updating the database. -func (suite *ServeFileTestSuite) UncacheAttachment(targetAttachment *gtsmodel.MediaAttachment) { - ctx := context.Background() - - cached := false - targetAttachment.Cached = &cached - - if err := suite.db.UpdateByID(ctx, targetAttachment, targetAttachment.ID, "cached"); err != nil { - suite.FailNow(err.Error()) - } - if err := suite.storage.Delete(ctx, targetAttachment.File.Path); err != nil { - suite.FailNow(err.Error()) - } - if err := suite.storage.Delete(ctx, targetAttachment.Thumbnail.Path); err != nil { - suite.FailNow(err.Error()) - } -} - -func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeOriginal, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["admin_account_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeSmall, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeOriginal, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeSmall, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.File.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - // uncache the attachment so we'll have to refetch it from the 'remote' instance - suite.UncacheAttachment(targetAttachment) - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeOriginal, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - fileInStorage, err := suite.storage.Get(context.Background(), targetAttachment.Thumbnail.Path) - if err != nil { - suite.FailNow(err.Error()) - } - - // uncache the attachment so we'll have to refetch it from the 'remote' instance - suite.UncacheAttachment(targetAttachment) - - code, headers, body := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeSmall, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusOK, code) - suite.Equal("image/jpeg", headers.Get("content-type")) - suite.Equal(fileInStorage, body) -} - -func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - - // uncache the attachment *and* set the remote URL to something that will return a 404 - suite.UncacheAttachment(targetAttachment) - targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee" - if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil { - suite.FailNow(err.Error()) - } - - code, _, _ := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeOriginal, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusNotFound, code) -} - -func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() { - targetAttachment := >smodel.MediaAttachment{} - *targetAttachment = *suite.testAttachments["remote_account_1_status_1_attachment_1"] - - // uncache the attachment *and* set the remote URL to something that will return a 404 - suite.UncacheAttachment(targetAttachment) - targetAttachment.RemoteURL = "http://nothing.at.this.url/weeeeeeeee" - if err := suite.db.UpdateByID(context.Background(), targetAttachment, targetAttachment.ID, "remote_url"); err != nil { - suite.FailNow(err.Error()) - } - - code, _, _ := suite.GetFile( - targetAttachment.AccountID, - media.TypeAttachment, - media.SizeSmall, - targetAttachment.ID+".jpeg", - ) - - suite.Equal(http.StatusNotFound, code) -} - -// Callers trying to get some random-ass file that doesn't exist should just get a 404 -func (suite *ServeFileTestSuite) TestServeFileNotFound() { - code, _, _ := suite.GetFile( - "01GMMY4G9B0QEG0PQK5Q5JGJWZ", - media.TypeAttachment, - media.SizeOriginal, - "01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg", - ) - - suite.Equal(http.StatusNotFound, code) -} - -func TestServeFileTestSuite(t *testing.T) { - suite.Run(t, new(ServeFileTestSuite)) -} |