diff options
author | 2022-07-03 12:08:30 +0200 | |
---|---|---|
committer | 2022-07-03 12:08:30 +0200 | |
commit | 9d0df426da59275f7aeaf46004befe5a778da274 (patch) | |
tree | 82c6bb98597e44c4f70b731336dcdfc839412c1c /internal | |
parent | [chore] Re-enable source tar but name it clearly as source (#683) (diff) | |
download | gotosocial-9d0df426da59275f7aeaf46004befe5a778da274.tar.xz |
[feature] S3 support (#674)
* feat: vendor minio client
* feat: introduce storage package with s3 support
* feat: serve s3 files directly
this saves a lot of bandwith as the files are fetched from the object
store directly
* fix: use explicit local storage in tests
* feat: integrate s3 storage with the main server
* fix: add s3 config to cli tests
* docs: explicitly set values in example config
also adds license header to the storage package
* fix: use better http status code on s3 redirect
HTTP 302 Found is the best fit, as it signifies that the resource
requested was found but not under its presumed URL
307/TemporaryRedirect would mean that this resource is usually located
here, not in this case
303/SeeOther indicates that the redirection does not link to the
requested resource but to another page
* refactor: use context in storage driver interface
Diffstat (limited to 'internal')
42 files changed, 496 insertions, 130 deletions
diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go index ed49e9158..39d942ee3 100644 --- a/internal/api/client/account/account_test.go +++ b/internal/api/client/account/account_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/account" @@ -20,6 +19,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -27,7 +27,7 @@ type AccountStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -65,7 +65,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go index 6b906f43a..c5035c798 100644 --- a/internal/api/client/admin/admin_test.go +++ b/internal/api/client/admin/admin_test.go @@ -24,7 +24,6 @@ import ( "net/http" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" @@ -38,6 +37,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -45,7 +45,7 @@ type AdminStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -83,7 +83,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 872b18a75..fa93ce564 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { suite.Empty(dbEmoji.CategoryID) // emoji should be in storage - emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath) + emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath) suite.NoError(err) suite.Len(emojiBytes, dbEmoji.ImageFileSize) - emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath) + emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath) suite.NoError(err) suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize) } diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go index 726b2be9d..188c40b3a 100644 --- a/internal/api/client/auth/auth_test.go +++ b/internal/api/client/auth/auth_test.go @@ -24,7 +24,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions/memstore" "github.com/gin-gonic/gin" @@ -42,13 +41,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oidc" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/router" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type AuthStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -88,7 +88,7 @@ func (suite *AuthStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go index 0372de9d8..ee2ff51ad 100644 --- a/internal/api/client/fileserver/servefile.go +++ b/internal/api/client/fileserver/servefile.go @@ -84,6 +84,11 @@ func (m *FileServer) ServeFile(c *gin.Context) { return } + if content.URL != nil { + c.Redirect(http.StatusFound, content.URL.String()) + return + } + defer func() { // if the content is a ReadCloser, close it when we're done if closer, ok := content.Content.(io.ReadCloser); ok { diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index e4db9a704..a36a79a58 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -26,7 +26,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -40,6 +39,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -48,7 +48,7 @@ type ServeFileTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver federator federation.Federator tc typeutils.TypeConverter processor processing.Processor @@ -81,7 +81,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) @@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.File.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) @@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go index a8c4090f6..e98d75c77 100644 --- a/internal/api/client/followrequest/followrequest_test.go +++ b/internal/api/client/followrequest/followrequest_test.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest" @@ -37,13 +36,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type FollowRequestStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -80,7 +80,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go index 1e5c59c97..645e70474 100644 --- a/internal/api/client/instance/instance_test.go +++ b/internal/api/client/instance/instance_test.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" @@ -37,6 +36,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -82,7 +82,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 2edf8be9f..153cde90f 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -30,7 +30,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -46,6 +45,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -54,7 +54,7 @@ type MediaCreateTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage *storage.Local mediaManager media.Manager federator federation.Federator tc typeutils.TypeConverter @@ -87,7 +87,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) @@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // see what's in storage *before* the request storageKeysBeforeRequest := []string{} - iter, err := suite.storage.Iterator(nil) + iter, err := suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } @@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // check what's in storage *after* the request storageKeysAfterRequest := []string{} - iter, err = suite.storage.Iterator(nil) + iter, err = suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index 0aebc435e..4457426de 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -28,7 +28,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -44,6 +43,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver federator federation.Federator tc typeutils.TypeConverter mediaManager media.Manager @@ -85,7 +85,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go index 049a5069c..ad6901d9d 100644 --- a/internal/api/client/status/status_test.go +++ b/internal/api/client/status/status_test.go @@ -19,7 +19,6 @@ package status_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -43,7 +43,7 @@ type StatusStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver // standard suite models testTokens map[string]*gtsmodel.Token @@ -76,7 +76,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index 7f5479715..116f3711b 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -19,7 +19,6 @@ package user_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/user" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -42,7 +42,7 @@ type UserStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver testTokens map[string]*gtsmodel.Token testClients map[string]*gtsmodel.Client @@ -66,7 +66,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.testUsers = testrig.NewTestUsers() suite.testAccounts = testrig.NewTestAccounts() suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/model/content.go b/internal/api/model/content.go index 2f38b2351..aa02a99c3 100644 --- a/internal/api/model/content.go +++ b/internal/api/model/content.go @@ -18,7 +18,10 @@ package model -import "io" +import ( + "io" + "net/url" +) // Content wraps everything needed to serve a blob of content (some kind of media) through the API. type Content struct { @@ -28,6 +31,8 @@ type Content struct { ContentLength int64 // Actual content Content io.Reader + // Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL) + URL *url.URL } // GetContentRequestForm describes a piece of content desired by the caller of the fileserver API. diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 024fb907e..28ca2fa67 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -19,7 +19,6 @@ package user_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" "github.com/superseriousbusiness/gotosocial/internal/api/security" @@ -32,6 +31,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -45,7 +45,7 @@ type UserStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver oauthServer oauth.Server securityModule *security.Module @@ -83,7 +83,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go index 9758a6be7..19a627a35 100644 --- a/internal/api/s2s/webfinger/webfinger_test.go +++ b/internal/api/s2s/webfinger/webfinger_test.go @@ -23,7 +23,6 @@ import ( "crypto/rsa" "time" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" @@ -37,6 +36,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -50,7 +50,7 @@ type WebfingerStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver oauthServer oauth.Server securityModule *security.Module @@ -86,7 +86,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/config/config.go b/internal/config/config.go index 0b7fc3d31..d746bd12a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -82,6 +82,11 @@ type Configuration struct { StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"` StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` + StorageS3Endpoint string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"` + StorageS3AccessKey string `name:"storage-s3-access-key" usage:"S3 Access Key"` + StorageS3SecretKey string `name:"storage-s3-secret-key" usage:"S3 Secret Key"` + StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"` + StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"` StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"` StatusesCWMaxChars int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 5bfd7aa53..d332efc04 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -61,6 +61,7 @@ var Defaults = Configuration{ StorageBackend: "local", StorageLocalBasePath: "/gotosocial/storage", + StorageS3UseSSL: true, StatusesMaxChars: 5000, StatusesCWMaxChars: 100, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a4c2adebf..a5dcc4c1c 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -843,6 +843,131 @@ func GetStorageLocalBasePath() string { return global.GetStorageLocalBasePath() // SetStorageLocalBasePath safely sets the value for global configuration 'StorageLocalBasePath' field func SetStorageLocalBasePath(v string) { global.SetStorageLocalBasePath(v) } +// GetStorageS3Endpoint safely fetches the Configuration value for state's 'StorageS3Endpoint' field +func (st *ConfigState) GetStorageS3Endpoint() (v string) { + st.mutex.Lock() + v = st.config.StorageS3Endpoint + st.mutex.Unlock() + return +} + +// SetStorageS3Endpoint safely sets the Configuration value for state's 'StorageS3Endpoint' field +func (st *ConfigState) SetStorageS3Endpoint(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3Endpoint = v + st.reloadToViper() +} + +// StorageS3EndpointFlag returns the flag name for the 'StorageS3Endpoint' field +func StorageS3EndpointFlag() string { return "storage-s3-endpoint" } + +// GetStorageS3Endpoint safely fetches the value for global configuration 'StorageS3Endpoint' field +func GetStorageS3Endpoint() string { return global.GetStorageS3Endpoint() } + +// SetStorageS3Endpoint safely sets the value for global configuration 'StorageS3Endpoint' field +func SetStorageS3Endpoint(v string) { global.SetStorageS3Endpoint(v) } + +// GetStorageS3AccessKey safely fetches the Configuration value for state's 'StorageS3AccessKey' field +func (st *ConfigState) GetStorageS3AccessKey() (v string) { + st.mutex.Lock() + v = st.config.StorageS3AccessKey + st.mutex.Unlock() + return +} + +// SetStorageS3AccessKey safely sets the Configuration value for state's 'StorageS3AccessKey' field +func (st *ConfigState) SetStorageS3AccessKey(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3AccessKey = v + st.reloadToViper() +} + +// StorageS3AccessKeyFlag returns the flag name for the 'StorageS3AccessKey' field +func StorageS3AccessKeyFlag() string { return "storage-s3-access-key" } + +// GetStorageS3AccessKey safely fetches the value for global configuration 'StorageS3AccessKey' field +func GetStorageS3AccessKey() string { return global.GetStorageS3AccessKey() } + +// SetStorageS3AccessKey safely sets the value for global configuration 'StorageS3AccessKey' field +func SetStorageS3AccessKey(v string) { global.SetStorageS3AccessKey(v) } + +// GetStorageS3SecretKey safely fetches the Configuration value for state's 'StorageS3SecretKey' field +func (st *ConfigState) GetStorageS3SecretKey() (v string) { + st.mutex.Lock() + v = st.config.StorageS3SecretKey + st.mutex.Unlock() + return +} + +// SetStorageS3SecretKey safely sets the Configuration value for state's 'StorageS3SecretKey' field +func (st *ConfigState) SetStorageS3SecretKey(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3SecretKey = v + st.reloadToViper() +} + +// StorageS3SecretKeyFlag returns the flag name for the 'StorageS3SecretKey' field +func StorageS3SecretKeyFlag() string { return "storage-s3-secret-key" } + +// GetStorageS3SecretKey safely fetches the value for global configuration 'StorageS3SecretKey' field +func GetStorageS3SecretKey() string { return global.GetStorageS3SecretKey() } + +// SetStorageS3SecretKey safely sets the value for global configuration 'StorageS3SecretKey' field +func SetStorageS3SecretKey(v string) { global.SetStorageS3SecretKey(v) } + +// GetStorageS3UseSSL safely fetches the Configuration value for state's 'StorageS3UseSSL' field +func (st *ConfigState) GetStorageS3UseSSL() (v bool) { + st.mutex.Lock() + v = st.config.StorageS3UseSSL + st.mutex.Unlock() + return +} + +// SetStorageS3UseSSL safely sets the Configuration value for state's 'StorageS3UseSSL' field +func (st *ConfigState) SetStorageS3UseSSL(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3UseSSL = v + st.reloadToViper() +} + +// StorageS3UseSSLFlag returns the flag name for the 'StorageS3UseSSL' field +func StorageS3UseSSLFlag() string { return "storage-s3-use-ssl" } + +// GetStorageS3UseSSL safely fetches the value for global configuration 'StorageS3UseSSL' field +func GetStorageS3UseSSL() bool { return global.GetStorageS3UseSSL() } + +// SetStorageS3UseSSL safely sets the value for global configuration 'StorageS3UseSSL' field +func SetStorageS3UseSSL(v bool) { global.SetStorageS3UseSSL(v) } + +// GetStorageS3BucketName safely fetches the Configuration value for state's 'StorageS3BucketName' field +func (st *ConfigState) GetStorageS3BucketName() (v string) { + st.mutex.Lock() + v = st.config.StorageS3BucketName + st.mutex.Unlock() + return +} + +// SetStorageS3BucketName safely sets the Configuration value for state's 'StorageS3BucketName' field +func (st *ConfigState) SetStorageS3BucketName(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3BucketName = v + st.reloadToViper() +} + +// StorageS3BucketNameFlag returns the flag name for the 'StorageS3BucketName' field +func StorageS3BucketNameFlag() string { return "storage-s3-bucket" } + +// GetStorageS3BucketName safely fetches the value for global configuration 'StorageS3BucketName' field +func GetStorageS3BucketName() string { return global.GetStorageS3BucketName() } + +// SetStorageS3BucketName safely sets the value for global configuration 'StorageS3BucketName' field +func SetStorageS3BucketName(v string) { global.SetStorageS3BucketName(v) } + // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field func (st *ConfigState) GetStatusesMaxChars() (v int) { st.mutex.Lock() diff --git a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go index 6139fd77d..1a3ae07aa 100644 --- a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go +++ b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go @@ -59,6 +59,11 @@ func init() { up := func(ctx context.Context, db *bun.DB) error { l := logrus.WithField("migration", "20220612091800_duplicated_media_cleanup") + if config.GetStorageBackend() != "local" { + // this migration only affects versions which only supported local storage + return nil + } + storageBasePath := config.GetStorageLocalBasePath() if storageBasePath == "" { return fmt.Errorf("%s must be set to do storage migration", config.StorageLocalBasePathFlag()) diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index 0f4732187..c0343a6b8 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -19,7 +19,6 @@ package dereferencing_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -27,13 +26,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type DereferencerStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver testRemoteStatuses map[string]vocab.ActivityStreamsNote testRemotePeople map[string]vocab.ActivityStreamsPerson @@ -57,7 +57,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media") suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 0984b405b..c93957098 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -19,11 +19,11 @@ package federation_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -31,7 +31,7 @@ import ( type FederatorStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver tc typeutils.TypeConverter testAccounts map[string]*gtsmodel.Account testStatuses map[string]*gtsmodel.Status @@ -41,7 +41,7 @@ type FederatorStandardTestSuite struct { // SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout func (suite *FederatorStandardTestSuite) SetupSuite() { // setup standard items - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.testAccounts = testrig.NewTestAccounts() suite.testStatuses = testrig.NewTestStatuses() diff --git a/internal/media/manager.go b/internal/media/manager.go index aacf607cc..b14288324 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -23,12 +23,12 @@ import ( "fmt" "time" - "codeberg.org/gruf/go-store/kv" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/storage" ) // selectPruneLimit is the amount of media entries to select at a time from the db when pruning @@ -98,7 +98,7 @@ type Manager interface { type manager struct { db db.DB - storage *kv.KVStore + storage storage.Driver emojiWorker *concurrency.WorkerPool[*ProcessingEmoji] mediaWorker *concurrency.WorkerPool[*ProcessingMedia] stopCronJobs func() error @@ -110,7 +110,7 @@ type manager struct { // a limited number of media will be processed in parallel. The numbers of workers // is determined from the $GOMAXPROCS environment variable (usually no. CPU cores). // See internal/concurrency.NewWorkerPool() documentation for further information. -func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { +func NewManager(database db.DB, storage storage.Driver) (Manager, error) { m := &manager{ db: database, storage: storage, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 3a6110f2b..f55763439 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/suite" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage" ) type ManagerTestSuite struct { @@ -87,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -100,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -159,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -172,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -231,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -244,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -314,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -327,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -393,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -406,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -474,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -487,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -523,7 +524,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { panic(err) } - diskManager, err := media.NewManager(suite.db, diskStorage) + diskManager, err := media.NewManager(suite.db, >sstorage.Local{KVStore: diskStorage}) if err != nil { panic(err) } diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 1b5011801..fda1963a7 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -19,11 +19,11 @@ package media_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/db" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -31,7 +31,7 @@ type MediaStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver manager media.Manager testAttachments map[string]*gtsmodel.MediaAttachment testAccounts map[string]*gtsmodel.Account @@ -42,7 +42,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() { testrig.InitTestLog() suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() } func (suite *MediaStandardTestSuite) SetupTest() { diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index c8c8d18c8..ffac56052 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -28,10 +28,10 @@ import ( "sync/atomic" "time" - "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -64,7 +64,7 @@ type ProcessingEmoji struct { */ database db.DB - storage *kv.KVStore + storage storage.Driver err error // error created during processing, if any @@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { switch processState(staticState) { case received: // stream the original file out of storage... - stored, err := p.storage.GetStream(p.emoji.ImagePath) + stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath) if err != nil { p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) @@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { } // put the static in storage - if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { + if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil { p.err = fmt.Errorf("loadStatic: error storing static: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) return p.err @@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { + if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 677437052..17fddddb7 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -28,12 +28,12 @@ import ( "sync/atomic" "time" - "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" terminator "github.com/superseriousbusiness/exif-terminator" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -61,7 +61,7 @@ type ProcessingMedia struct { */ database db.DB - storage *kv.KVStore + storage storage.Driver err error // error created during processing, if any @@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // stream the original file out of storage logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL) - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) @@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // put the thumbnail in storage logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL) - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { + if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil { p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err @@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { var decoded *imageMeta // stream the original file out of storage... - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) atomic.StoreInt32(&p.fullSizeState, int32(errored)) @@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.attachment.File.FileSize = fileSize // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil { + if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } p.attachment.Cached = true diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go index aa838d2a4..33391beaf 100644 --- a/internal/media/prunemeta.go +++ b/internal/media/prunemeta.go @@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } } @@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } } diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go index 1358208a8..8b250e7a5 100644 --- a/internal/media/prunemeta_test.go +++ b/internal/media/prunemeta_test.go @@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db @@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index a01995740..6cad7fbf8 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -67,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false @@ -76,7 +76,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index 31c5128ff..f5ed8a618 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(testAttachment.File.Path) + _, err = suite.storage.Get(ctx, testAttachment.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(testAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // now recache the image.... @@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same // recached files should be back in storage - _, err = suite.storage.Get(recachedAttachment.File.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) suite.NoError(err) - _, err = suite.storage.Get(recachedAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) suite.NoError(err) } @@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) suite.NoError(err) suite.True(media.Cached) - err = suite.storage.Delete(media.File.Path) + err = suite.storage.Delete(ctx, media.File.Path) suite.NoError(err) // Now attempt to prune remote for item with db entry no file diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 8ea1665ac..0755f01c2 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -21,7 +21,6 @@ package account_test import ( "context" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -33,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/account" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -43,7 +43,7 @@ type AccountStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager oauthServer oauth.Server fromClientAPIChan chan messages.FromClientAPI @@ -91,7 +91,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go index e99b4e950..45c218397 100644 --- a/internal/processing/media/delete.go +++ b/internal/processing/media/delete.go @@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr // delete the thumbnail from storage if attachment.Thumbnail.Path != "" { - if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil { errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err)) } } // delete the file from storage if attachment.File.Path != "" { - if err := p.storage.Delete(attachment.File.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.File.Path); err != nil { errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err)) } } diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index c74951e38..3227cb8c8 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount // if we have the media cached on our server already, we can now simply return it from storage if a.Cached { - return p.streamFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } // if we don't have it cached, then we can assume two things: @@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err)) } // ... so now we can safely return it - return p.streamFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } return attachmentContent, nil @@ -253,11 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize)) } - return p.streamFromStorage(storagePath, emojiContent) + return p.retrieveFromStorage(ctx, storagePath, emojiContent) } -func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { - reader, err := p.storage.GetStream(storagePath) +func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { + if url := p.storage.URL(ctx, storagePath); url != nil { + content.URL = url + return content, nil + } + reader, err := p.storage.GetStream(ctx, storagePath) if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err)) } diff --git a/internal/processing/media/getfile_test.go b/internal/processing/media/getfile_test.go index 4d7bc4621..7c6525abe 100644 --- a/internal/processing/media/getfile_test.go +++ b/internal/processing/media/getfile_test.go @@ -70,9 +70,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -106,7 +106,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -119,9 +119,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -156,7 +156,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -166,16 +166,16 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() { testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] // fetch the existing thumbnail bytes from storage first - thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path) + thumbnailBytes, err := suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // uncache the file from local testAttachment.Cached = false err = suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch the thumbnail diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go index 50cbc1b3c..66575facc 100644 --- a/internal/processing/media/media.go +++ b/internal/processing/media/media.go @@ -21,12 +21,12 @@ package media import ( "context" - "codeberg.org/gruf/go-store/kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -51,12 +51,12 @@ type processor struct { tc typeutils.TypeConverter mediaManager media.Manager transportController transport.Controller - storage *kv.KVStore + storage storage.Driver db db.DB } // New returns a new media processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage storage.Driver) Processor { return &processor{ tc: tc, mediaManager: mediaManager, diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go index c8c8736be..cf73af4e8 100644 --- a/internal/processing/media/media_test.go +++ b/internal/processing/media/media_test.go @@ -19,7 +19,6 @@ package media_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -27,6 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -37,7 +37,7 @@ type MediaStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager transportController transport.Controller @@ -72,7 +72,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)) suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage) diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 8572b583c..b8d0daf99 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -23,7 +23,6 @@ import ( "net/http" "net/url" - "codeberg.org/gruf/go-store/kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -41,6 +40,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/processing/status" "github.com/superseriousbusiness/gotosocial/internal/processing/streaming" "github.com/superseriousbusiness/gotosocial/internal/processing/user" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/stream" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -251,7 +251,7 @@ type processor struct { tc typeutils.TypeConverter oauthServer oauth.Server mediaManager media.Manager - storage *kv.KVStore + storage storage.Driver statusTimelines timeline.Manager db db.DB filter visibility.Filter @@ -275,7 +275,7 @@ func NewProcessor( federator federation.Federator, oauthServer oauth.Server, mediaManager media.Manager, - storage *kv.KVStore, + storage storage.Driver, db db.DB, emailSender email.Sender, clientWorker *concurrency.WorkerPool[messages.FromClientAPI], diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index c60ff5c97..54271cc86 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -19,7 +19,6 @@ package processing_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -39,7 +39,7 @@ type ProcessingStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager typeconverter typeutils.TypeConverter httpClient *testrig.MockHTTPClient @@ -91,7 +91,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.testActivities = testrig.NewTestActivities(suite.testAccounts) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.typeconverter = testrig.NewTestTypeConverter(suite.db) suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media") diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go index f8834e2c2..199c36404 100644 --- a/internal/processing/status/status_test.go +++ b/internal/processing/status/status_test.go @@ -19,7 +19,6 @@ package status_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -29,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/status" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -39,7 +39,7 @@ type StatusStandardTestSuite struct { db db.DB typeConverter typeutils.TypeConverter tc transport.Controller - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator clientWorker *concurrency.WorkerPool[messages.FromClientAPI] @@ -81,7 +81,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.typeConverter = testrig.NewTestTypeConverter(suite.db) suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker) suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator)) diff --git a/internal/storage/local.go b/internal/storage/local.go new file mode 100644 index 000000000..da57631f9 --- /dev/null +++ b/internal/storage/local.go @@ -0,0 +1,50 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "context" + "io" + "net/url" + + "codeberg.org/gruf/go-store/kv" +) + +type Local struct { + KVStore *kv.KVStore +} + +func (l *Local) Get(ctx context.Context, key string) ([]byte, error) { + return l.KVStore.Get(key) +} +func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + return l.KVStore.GetStream(key) +} +func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error { + return l.KVStore.PutStream(key, r) +} +func (l *Local) Put(ctx context.Context, key string, value []byte) error { + return l.KVStore.Put(key, value) +} +func (l *Local) Delete(ctx context.Context, key string) error { + return l.KVStore.Delete(key) +} +func (l *Local) URL(ctx context.Context, key string) *url.URL { + return nil +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go new file mode 100644 index 000000000..a15114488 --- /dev/null +++ b/internal/storage/s3.go @@ -0,0 +1,87 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "mime" + "net/url" + "path" + "time" + + "github.com/minio/minio-go/v7" +) + +type S3 struct { + mc *minio.Client + bucket string +} + +func NewS3(mc *minio.Client, bucket string) *S3 { + return &S3{ + mc: mc, + bucket: bucket, + } +} + +func (s *S3) Get(ctx context.Context, key string) ([]byte, error) { + r, err := s.GetStream(ctx, key) + if err != nil { + return nil, err + } + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("reading data from s3: %w", err) + } + return b, nil +} +func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + err = fmt.Errorf("retrieving object from s3: %w", err) + } + return o, err +} +func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil { + return fmt.Errorf("uploading data stream: %w", err) + } + return nil +} +func (s *S3) Put(ctx context.Context, key string, value []byte) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil { + return fmt.Errorf("uploading data slice: %w", err) + } + return nil +} +func (s *S3) Delete(ctx context.Context, key string) error { + return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} +func (s *S3) URL(ctx context.Context, key string) *url.URL { + // it's safe to ignore the error here, as we just fall back to fetching the + // file if the url request fails + url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{ + "response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, + }) + return url +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 000000000..88577442f --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,78 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "path" + + "codeberg.org/gruf/go-store/kv" + "codeberg.org/gruf/go-store/storage" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/superseriousbusiness/gotosocial/internal/config" +) + +var ( + ErrNotSupported = errors.New("driver does not suppport functionality") +) + +// Driver implements the functionality to store and retrieve blobs +// (images,video,audio) +type Driver interface { + Get(ctx context.Context, key string) ([]byte, error) + GetStream(ctx context.Context, key string) (io.ReadCloser, error) + PutStream(ctx context.Context, key string, r io.Reader) error + Put(ctx context.Context, key string, value []byte) error + Delete(ctx context.Context, key string) error + URL(ctx context.Context, key string) *url.URL +} + +func AutoConfig() (Driver, error) { + switch config.GetStorageBackend() { + case "s3": + mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{ + Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""), + Secure: config.GetStorageS3UseSSL(), + }) + if err != nil { + return nil, fmt.Errorf("creating minio client: %w", err) + } + return NewS3(mc, config.GetStorageS3BucketName()), nil + case "local": + storageBasePath := config.GetStorageLocalBasePath() + storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{ + // Put the store lockfile in the storage dir itself. + // Normally this would not be safe, since we could end up + // overwriting the lockfile if we store a file called 'store.lock'. + // However, in this case it's OK because the keys are set by + // GtS and not the user, so we know we're never going to overwrite it. + LockFile: path.Join(storageBasePath, "store.lock"), + }) + if err != nil { + return nil, fmt.Errorf("error creating storage backend: %s", err) + } + return &Local{KVStore: storage}, nil + } + return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend()) +} |