diff options
Diffstat (limited to 'internal')
42 files changed, 496 insertions, 130 deletions
diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go index ed49e9158..39d942ee3 100644 --- a/internal/api/client/account/account_test.go +++ b/internal/api/client/account/account_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/account" @@ -20,6 +19,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -27,7 +27,7 @@ type AccountStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -65,7 +65,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go index 6b906f43a..c5035c798 100644 --- a/internal/api/client/admin/admin_test.go +++ b/internal/api/client/admin/admin_test.go @@ -24,7 +24,6 @@ import ( "net/http" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" @@ -38,6 +37,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -45,7 +45,7 @@ type AdminStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -83,7 +83,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 872b18a75..fa93ce564 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { suite.Empty(dbEmoji.CategoryID) // emoji should be in storage - emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath) + emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath) suite.NoError(err) suite.Len(emojiBytes, dbEmoji.ImageFileSize) - emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath) + emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath) suite.NoError(err) suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize) } diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go index 726b2be9d..188c40b3a 100644 --- a/internal/api/client/auth/auth_test.go +++ b/internal/api/client/auth/auth_test.go @@ -24,7 +24,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions/memstore" "github.com/gin-gonic/gin" @@ -42,13 +41,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oidc" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/router" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type AuthStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -88,7 +88,7 @@ func (suite *AuthStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go index 0372de9d8..ee2ff51ad 100644 --- a/internal/api/client/fileserver/servefile.go +++ b/internal/api/client/fileserver/servefile.go @@ -84,6 +84,11 @@ func (m *FileServer) ServeFile(c *gin.Context) { return } + if content.URL != nil { + c.Redirect(http.StatusFound, content.URL.String()) + return + } + defer func() { // if the content is a ReadCloser, close it when we're done if closer, ok := content.Content.(io.ReadCloser); ok { diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index e4db9a704..a36a79a58 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -26,7 +26,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -40,6 +39,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -48,7 +48,7 @@ type ServeFileTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver federator federation.Federator tc typeutils.TypeConverter processor processing.Processor @@ -81,7 +81,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) @@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.File.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) @@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go index a8c4090f6..e98d75c77 100644 --- a/internal/api/client/followrequest/followrequest_test.go +++ b/internal/api/client/followrequest/followrequest_test.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest" @@ -37,13 +36,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type FollowRequestStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -80,7 +80,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go index 1e5c59c97..645e70474 100644 --- a/internal/api/client/instance/instance_test.go +++ b/internal/api/client/instance/instance_test.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http/httptest" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" @@ -37,6 +36,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator processor processing.Processor @@ -82,7 +82,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 2edf8be9f..153cde90f 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -30,7 +30,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -46,6 +45,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -54,7 +54,7 @@ type MediaCreateTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage *storage.Local mediaManager media.Manager federator federation.Federator tc typeutils.TypeConverter @@ -87,7 +87,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) @@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // see what's in storage *before* the request storageKeysBeforeRequest := []string{} - iter, err := suite.storage.Iterator(nil) + iter, err := suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } @@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // check what's in storage *after* the request storageKeysAfterRequest := []string{} - iter, err = suite.storage.Iterator(nil) + iter, err = suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index 0aebc435e..4457426de 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -28,7 +28,6 @@ import ( "net/http/httptest" "testing" - "codeberg.org/gruf/go-store/kv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" @@ -44,6 +43,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver federator federation.Federator tc typeutils.TypeConverter mediaManager media.Manager @@ -85,7 +85,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go index 049a5069c..ad6901d9d 100644 --- a/internal/api/client/status/status_test.go +++ b/internal/api/client/status/status_test.go @@ -19,7 +19,6 @@ package status_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -43,7 +43,7 @@ type StatusStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver // standard suite models testTokens map[string]*gtsmodel.Token @@ -76,7 +76,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index 7f5479715..116f3711b 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -19,7 +19,6 @@ package user_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/user" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -42,7 +42,7 @@ type UserStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver testTokens map[string]*gtsmodel.Token testClients map[string]*gtsmodel.Client @@ -66,7 +66,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.testUsers = testrig.NewTestUsers() suite.testAccounts = testrig.NewTestAccounts() suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/model/content.go b/internal/api/model/content.go index 2f38b2351..aa02a99c3 100644 --- a/internal/api/model/content.go +++ b/internal/api/model/content.go @@ -18,7 +18,10 @@ package model -import "io" +import ( + "io" + "net/url" +) // Content wraps everything needed to serve a blob of content (some kind of media) through the API. type Content struct { @@ -28,6 +31,8 @@ type Content struct { ContentLength int64 // Actual content Content io.Reader + // Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL) + URL *url.URL } // GetContentRequestForm describes a piece of content desired by the caller of the fileserver API. diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 024fb907e..28ca2fa67 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -19,7 +19,6 @@ package user_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" "github.com/superseriousbusiness/gotosocial/internal/api/security" @@ -32,6 +31,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -45,7 +45,7 @@ type UserStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver oauthServer oauth.Server securityModule *security.Module @@ -83,7 +83,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go index 9758a6be7..19a627a35 100644 --- a/internal/api/s2s/webfinger/webfinger_test.go +++ b/internal/api/s2s/webfinger/webfinger_test.go @@ -23,7 +23,6 @@ import ( "crypto/rsa" "time" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" @@ -37,6 +36,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -50,7 +50,7 @@ type WebfingerStandardTestSuite struct { federator federation.Federator emailSender email.Sender processor processing.Processor - storage *kv.KVStore + storage storage.Driver oauthServer oauth.Server securityModule *security.Module @@ -86,7 +86,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/config/config.go b/internal/config/config.go index 0b7fc3d31..d746bd12a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -82,6 +82,11 @@ type Configuration struct { StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"` StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` + StorageS3Endpoint string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"` + StorageS3AccessKey string `name:"storage-s3-access-key" usage:"S3 Access Key"` + StorageS3SecretKey string `name:"storage-s3-secret-key" usage:"S3 Secret Key"` + StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"` + StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"` StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"` StatusesCWMaxChars int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 5bfd7aa53..d332efc04 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -61,6 +61,7 @@ var Defaults = Configuration{ StorageBackend: "local", StorageLocalBasePath: "/gotosocial/storage", + StorageS3UseSSL: true, StatusesMaxChars: 5000, StatusesCWMaxChars: 100, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a4c2adebf..a5dcc4c1c 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -843,6 +843,131 @@ func GetStorageLocalBasePath() string { return global.GetStorageLocalBasePath() // SetStorageLocalBasePath safely sets the value for global configuration 'StorageLocalBasePath' field func SetStorageLocalBasePath(v string) { global.SetStorageLocalBasePath(v) } +// GetStorageS3Endpoint safely fetches the Configuration value for state's 'StorageS3Endpoint' field +func (st *ConfigState) GetStorageS3Endpoint() (v string) { + st.mutex.Lock() + v = st.config.StorageS3Endpoint + st.mutex.Unlock() + return +} + +// SetStorageS3Endpoint safely sets the Configuration value for state's 'StorageS3Endpoint' field +func (st *ConfigState) SetStorageS3Endpoint(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3Endpoint = v + st.reloadToViper() +} + +// StorageS3EndpointFlag returns the flag name for the 'StorageS3Endpoint' field +func StorageS3EndpointFlag() string { return "storage-s3-endpoint" } + +// GetStorageS3Endpoint safely fetches the value for global configuration 'StorageS3Endpoint' field +func GetStorageS3Endpoint() string { return global.GetStorageS3Endpoint() } + +// SetStorageS3Endpoint safely sets the value for global configuration 'StorageS3Endpoint' field +func SetStorageS3Endpoint(v string) { global.SetStorageS3Endpoint(v) } + +// GetStorageS3AccessKey safely fetches the Configuration value for state's 'StorageS3AccessKey' field +func (st *ConfigState) GetStorageS3AccessKey() (v string) { + st.mutex.Lock() + v = st.config.StorageS3AccessKey + st.mutex.Unlock() + return +} + +// SetStorageS3AccessKey safely sets the Configuration value for state's 'StorageS3AccessKey' field +func (st *ConfigState) SetStorageS3AccessKey(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3AccessKey = v + st.reloadToViper() +} + +// StorageS3AccessKeyFlag returns the flag name for the 'StorageS3AccessKey' field +func StorageS3AccessKeyFlag() string { return "storage-s3-access-key" } + +// GetStorageS3AccessKey safely fetches the value for global configuration 'StorageS3AccessKey' field +func GetStorageS3AccessKey() string { return global.GetStorageS3AccessKey() } + +// SetStorageS3AccessKey safely sets the value for global configuration 'StorageS3AccessKey' field +func SetStorageS3AccessKey(v string) { global.SetStorageS3AccessKey(v) } + +// GetStorageS3SecretKey safely fetches the Configuration value for state's 'StorageS3SecretKey' field +func (st *ConfigState) GetStorageS3SecretKey() (v string) { + st.mutex.Lock() + v = st.config.StorageS3SecretKey + st.mutex.Unlock() + return +} + +// SetStorageS3SecretKey safely sets the Configuration value for state's 'StorageS3SecretKey' field +func (st *ConfigState) SetStorageS3SecretKey(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3SecretKey = v + st.reloadToViper() +} + +// StorageS3SecretKeyFlag returns the flag name for the 'StorageS3SecretKey' field +func StorageS3SecretKeyFlag() string { return "storage-s3-secret-key" } + +// GetStorageS3SecretKey safely fetches the value for global configuration 'StorageS3SecretKey' field +func GetStorageS3SecretKey() string { return global.GetStorageS3SecretKey() } + +// SetStorageS3SecretKey safely sets the value for global configuration 'StorageS3SecretKey' field +func SetStorageS3SecretKey(v string) { global.SetStorageS3SecretKey(v) } + +// GetStorageS3UseSSL safely fetches the Configuration value for state's 'StorageS3UseSSL' field +func (st *ConfigState) GetStorageS3UseSSL() (v bool) { + st.mutex.Lock() + v = st.config.StorageS3UseSSL + st.mutex.Unlock() + return +} + +// SetStorageS3UseSSL safely sets the Configuration value for state's 'StorageS3UseSSL' field +func (st *ConfigState) SetStorageS3UseSSL(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3UseSSL = v + st.reloadToViper() +} + +// StorageS3UseSSLFlag returns the flag name for the 'StorageS3UseSSL' field +func StorageS3UseSSLFlag() string { return "storage-s3-use-ssl" } + +// GetStorageS3UseSSL safely fetches the value for global configuration 'StorageS3UseSSL' field +func GetStorageS3UseSSL() bool { return global.GetStorageS3UseSSL() } + +// SetStorageS3UseSSL safely sets the value for global configuration 'StorageS3UseSSL' field +func SetStorageS3UseSSL(v bool) { global.SetStorageS3UseSSL(v) } + +// GetStorageS3BucketName safely fetches the Configuration value for state's 'StorageS3BucketName' field +func (st *ConfigState) GetStorageS3BucketName() (v string) { + st.mutex.Lock() + v = st.config.StorageS3BucketName + st.mutex.Unlock() + return +} + +// SetStorageS3BucketName safely sets the Configuration value for state's 'StorageS3BucketName' field +func (st *ConfigState) SetStorageS3BucketName(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3BucketName = v + st.reloadToViper() +} + +// StorageS3BucketNameFlag returns the flag name for the 'StorageS3BucketName' field +func StorageS3BucketNameFlag() string { return "storage-s3-bucket" } + +// GetStorageS3BucketName safely fetches the value for global configuration 'StorageS3BucketName' field +func GetStorageS3BucketName() string { return global.GetStorageS3BucketName() } + +// SetStorageS3BucketName safely sets the value for global configuration 'StorageS3BucketName' field +func SetStorageS3BucketName(v string) { global.SetStorageS3BucketName(v) } + // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field func (st *ConfigState) GetStatusesMaxChars() (v int) { st.mutex.Lock() diff --git a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go index 6139fd77d..1a3ae07aa 100644 --- a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go +++ b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go @@ -59,6 +59,11 @@ func init() { up := func(ctx context.Context, db *bun.DB) error { l := logrus.WithField("migration", "20220612091800_duplicated_media_cleanup") + if config.GetStorageBackend() != "local" { + // this migration only affects versions which only supported local storage + return nil + } + storageBasePath := config.GetStorageLocalBasePath() if storageBasePath == "" { return fmt.Errorf("%s must be set to do storage migration", config.StorageLocalBasePathFlag()) diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index 0f4732187..c0343a6b8 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -19,7 +19,6 @@ package dereferencing_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/concurrency" @@ -27,13 +26,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) type DereferencerStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver testRemoteStatuses map[string]vocab.ActivityStreamsNote testRemotePeople map[string]vocab.ActivityStreamsPerson @@ -57,7 +57,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media") suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 0984b405b..c93957098 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -19,11 +19,11 @@ package federation_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -31,7 +31,7 @@ import ( type FederatorStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver tc typeutils.TypeConverter testAccounts map[string]*gtsmodel.Account testStatuses map[string]*gtsmodel.Status @@ -41,7 +41,7 @@ type FederatorStandardTestSuite struct { // SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout func (suite *FederatorStandardTestSuite) SetupSuite() { // setup standard items - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.testAccounts = testrig.NewTestAccounts() suite.testStatuses = testrig.NewTestStatuses() diff --git a/internal/media/manager.go b/internal/media/manager.go index aacf607cc..b14288324 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -23,12 +23,12 @@ import ( "fmt" "time" - "codeberg.org/gruf/go-store/kv" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/storage" ) // selectPruneLimit is the amount of media entries to select at a time from the db when pruning @@ -98,7 +98,7 @@ type Manager interface { type manager struct { db db.DB - storage *kv.KVStore + storage storage.Driver emojiWorker *concurrency.WorkerPool[*ProcessingEmoji] mediaWorker *concurrency.WorkerPool[*ProcessingMedia] stopCronJobs func() error @@ -110,7 +110,7 @@ type manager struct { // a limited number of media will be processed in parallel. The numbers of workers // is determined from the $GOMAXPROCS environment variable (usually no. CPU cores). // See internal/concurrency.NewWorkerPool() documentation for further information. -func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { +func NewManager(database db.DB, storage storage.Driver) (Manager, error) { m := &manager{ db: database, storage: storage, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 3a6110f2b..f55763439 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/suite" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage" ) type ManagerTestSuite struct { @@ -87,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -100,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -159,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -172,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -231,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -244,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -314,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -327,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -393,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -406,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -474,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -487,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -523,7 +524,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { panic(err) } - diskManager, err := media.NewManager(suite.db, diskStorage) + diskManager, err := media.NewManager(suite.db, >sstorage.Local{KVStore: diskStorage}) if err != nil { panic(err) } diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 1b5011801..fda1963a7 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -19,11 +19,11 @@ package media_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/db" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -31,7 +31,7 @@ type MediaStandardTestSuite struct { suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver manager media.Manager testAttachments map[string]*gtsmodel.MediaAttachment testAccounts map[string]*gtsmodel.Account @@ -42,7 +42,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() { testrig.InitTestLog() suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() } func (suite *MediaStandardTestSuite) SetupTest() { diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index c8c8d18c8..ffac56052 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -28,10 +28,10 @@ import ( "sync/atomic" "time" - "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -64,7 +64,7 @@ type ProcessingEmoji struct { */ database db.DB - storage *kv.KVStore + storage storage.Driver err error // error created during processing, if any @@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { switch processState(staticState) { case received: // stream the original file out of storage... - stored, err := p.storage.GetStream(p.emoji.ImagePath) + stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath) if err != nil { p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) @@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { } // put the static in storage - if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { + if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil { p.err = fmt.Errorf("loadStatic: error storing static: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) return p.err @@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { + if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 677437052..17fddddb7 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -28,12 +28,12 @@ import ( "sync/atomic" "time" - "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" terminator "github.com/superseriousbusiness/exif-terminator" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -61,7 +61,7 @@ type ProcessingMedia struct { */ database db.DB - storage *kv.KVStore + storage storage.Driver err error // error created during processing, if any @@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // stream the original file out of storage logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL) - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) @@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // put the thumbnail in storage logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL) - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { + if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil { p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err @@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { var decoded *imageMeta // stream the original file out of storage... - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) atomic.StoreInt32(&p.fullSizeState, int32(errored)) @@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.attachment.File.FileSize = fileSize // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil { + if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } p.attachment.Cached = true diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go index aa838d2a4..33391beaf 100644 --- a/internal/media/prunemeta.go +++ b/internal/media/prunemeta.go @@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } } @@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } } diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go index 1358208a8..8b250e7a5 100644 --- a/internal/media/prunemeta_test.go +++ b/internal/media/prunemeta_test.go @@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db @@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index a01995740..6cad7fbf8 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -67,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false @@ -76,7 +76,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index 31c5128ff..f5ed8a618 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(testAttachment.File.Path) + _, err = suite.storage.Get(ctx, testAttachment.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(testAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // now recache the image.... @@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same // recached files should be back in storage - _, err = suite.storage.Get(recachedAttachment.File.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) suite.NoError(err) - _, err = suite.storage.Get(recachedAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) suite.NoError(err) } @@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) suite.NoError(err) suite.True(media.Cached) - err = suite.storage.Delete(media.File.Path) + err = suite.storage.Delete(ctx, media.File.Path) suite.NoError(err) // Now attempt to prune remote for item with db entry no file diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 8ea1665ac..0755f01c2 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -21,7 +21,6 @@ package account_test import ( "context" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -33,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/account" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -43,7 +43,7 @@ type AccountStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager oauthServer oauth.Server fromClientAPIChan chan messages.FromClientAPI @@ -91,7 +91,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go index e99b4e950..45c218397 100644 --- a/internal/processing/media/delete.go +++ b/internal/processing/media/delete.go @@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr // delete the thumbnail from storage if attachment.Thumbnail.Path != "" { - if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil { errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err)) } } // delete the file from storage if attachment.File.Path != "" { - if err := p.storage.Delete(attachment.File.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.File.Path); err != nil { errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err)) } } diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index c74951e38..3227cb8c8 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount // if we have the media cached on our server already, we can now simply return it from storage if a.Cached { - return p.streamFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } // if we don't have it cached, then we can assume two things: @@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err)) } // ... so now we can safely return it - return p.streamFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } return attachmentContent, nil @@ -253,11 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize)) } - return p.streamFromStorage(storagePath, emojiContent) + return p.retrieveFromStorage(ctx, storagePath, emojiContent) } -func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { - reader, err := p.storage.GetStream(storagePath) +func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { + if url := p.storage.URL(ctx, storagePath); url != nil { + content.URL = url + return content, nil + } + reader, err := p.storage.GetStream(ctx, storagePath) if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err)) } diff --git a/internal/processing/media/getfile_test.go b/internal/processing/media/getfile_test.go index 4d7bc4621..7c6525abe 100644 --- a/internal/processing/media/getfile_test.go +++ b/internal/processing/media/getfile_test.go @@ -70,9 +70,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -106,7 +106,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -119,9 +119,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -156,7 +156,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -166,16 +166,16 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() { testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] // fetch the existing thumbnail bytes from storage first - thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path) + thumbnailBytes, err := suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // uncache the file from local testAttachment.Cached = false err = suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch the thumbnail diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go index 50cbc1b3c..66575facc 100644 --- a/internal/processing/media/media.go +++ b/internal/processing/media/media.go @@ -21,12 +21,12 @@ package media import ( "context" - "codeberg.org/gruf/go-store/kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -51,12 +51,12 @@ type processor struct { tc typeutils.TypeConverter mediaManager media.Manager transportController transport.Controller - storage *kv.KVStore + storage storage.Driver db db.DB } // New returns a new media processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage storage.Driver) Processor { return &processor{ tc: tc, mediaManager: mediaManager, diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go index c8c8736be..cf73af4e8 100644 --- a/internal/processing/media/media_test.go +++ b/internal/processing/media/media_test.go @@ -19,7 +19,6 @@ package media_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -27,6 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -37,7 +37,7 @@ type MediaStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager transportController transport.Controller @@ -72,7 +72,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)) suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage) diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 8572b583c..b8d0daf99 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -23,7 +23,6 @@ import ( "net/http" "net/url" - "codeberg.org/gruf/go-store/kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -41,6 +40,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/processing/status" "github.com/superseriousbusiness/gotosocial/internal/processing/streaming" "github.com/superseriousbusiness/gotosocial/internal/processing/user" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/stream" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -251,7 +251,7 @@ type processor struct { tc typeutils.TypeConverter oauthServer oauth.Server mediaManager media.Manager - storage *kv.KVStore + storage storage.Driver statusTimelines timeline.Manager db db.DB filter visibility.Filter @@ -275,7 +275,7 @@ func NewProcessor( federator federation.Federator, oauthServer oauth.Server, mediaManager media.Manager, - storage *kv.KVStore, + storage storage.Driver, db db.DB, emailSender email.Sender, clientWorker *concurrency.WorkerPool[messages.FromClientAPI], diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index c60ff5c97..54271cc86 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -19,7 +19,6 @@ package processing_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -30,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -39,7 +39,7 @@ type ProcessingStandardTestSuite struct { // standard suite interfaces suite.Suite db db.DB - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager typeconverter typeutils.TypeConverter httpClient *testrig.MockHTTPClient @@ -91,7 +91,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.testActivities = testrig.NewTestActivities(suite.testAccounts) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.typeconverter = testrig.NewTestTypeConverter(suite.db) suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media") diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go index f8834e2c2..199c36404 100644 --- a/internal/processing/status/status_test.go +++ b/internal/processing/status/status_test.go @@ -19,7 +19,6 @@ package status_test import ( - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -29,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/status" + "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -39,7 +39,7 @@ type StatusStandardTestSuite struct { db db.DB typeConverter typeutils.TypeConverter tc transport.Controller - storage *kv.KVStore + storage storage.Driver mediaManager media.Manager federator federation.Federator clientWorker *concurrency.WorkerPool[messages.FromClientAPI] @@ -81,7 +81,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.typeConverter = testrig.NewTestTypeConverter(suite.db) suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker) - suite.storage = testrig.NewTestStorage() + suite.storage = testrig.NewInMemoryStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker) suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator)) diff --git a/internal/storage/local.go b/internal/storage/local.go new file mode 100644 index 000000000..da57631f9 --- /dev/null +++ b/internal/storage/local.go @@ -0,0 +1,50 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "context" + "io" + "net/url" + + "codeberg.org/gruf/go-store/kv" +) + +type Local struct { + KVStore *kv.KVStore +} + +func (l *Local) Get(ctx context.Context, key string) ([]byte, error) { + return l.KVStore.Get(key) +} +func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + return l.KVStore.GetStream(key) +} +func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error { + return l.KVStore.PutStream(key, r) +} +func (l *Local) Put(ctx context.Context, key string, value []byte) error { + return l.KVStore.Put(key, value) +} +func (l *Local) Delete(ctx context.Context, key string) error { + return l.KVStore.Delete(key) +} +func (l *Local) URL(ctx context.Context, key string) *url.URL { + return nil +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go new file mode 100644 index 000000000..a15114488 --- /dev/null +++ b/internal/storage/s3.go @@ -0,0 +1,87 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "mime" + "net/url" + "path" + "time" + + "github.com/minio/minio-go/v7" +) + +type S3 struct { + mc *minio.Client + bucket string +} + +func NewS3(mc *minio.Client, bucket string) *S3 { + return &S3{ + mc: mc, + bucket: bucket, + } +} + +func (s *S3) Get(ctx context.Context, key string) ([]byte, error) { + r, err := s.GetStream(ctx, key) + if err != nil { + return nil, err + } + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("reading data from s3: %w", err) + } + return b, nil +} +func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + err = fmt.Errorf("retrieving object from s3: %w", err) + } + return o, err +} +func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil { + return fmt.Errorf("uploading data stream: %w", err) + } + return nil +} +func (s *S3) Put(ctx context.Context, key string, value []byte) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil { + return fmt.Errorf("uploading data slice: %w", err) + } + return nil +} +func (s *S3) Delete(ctx context.Context, key string) error { + return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} +func (s *S3) URL(ctx context.Context, key string) *url.URL { + // it's safe to ignore the error here, as we just fall back to fetching the + // file if the url request fails + url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{ + "response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, + }) + return url +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 000000000..88577442f --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,78 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "path" + + "codeberg.org/gruf/go-store/kv" + "codeberg.org/gruf/go-store/storage" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/superseriousbusiness/gotosocial/internal/config" +) + +var ( + ErrNotSupported = errors.New("driver does not suppport functionality") +) + +// Driver implements the functionality to store and retrieve blobs +// (images,video,audio) +type Driver interface { + Get(ctx context.Context, key string) ([]byte, error) + GetStream(ctx context.Context, key string) (io.ReadCloser, error) + PutStream(ctx context.Context, key string, r io.Reader) error + Put(ctx context.Context, key string, value []byte) error + Delete(ctx context.Context, key string) error + URL(ctx context.Context, key string) *url.URL +} + +func AutoConfig() (Driver, error) { + switch config.GetStorageBackend() { + case "s3": + mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{ + Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""), + Secure: config.GetStorageS3UseSSL(), + }) + if err != nil { + return nil, fmt.Errorf("creating minio client: %w", err) + } + return NewS3(mc, config.GetStorageS3BucketName()), nil + case "local": + storageBasePath := config.GetStorageLocalBasePath() + storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{ + // Put the store lockfile in the storage dir itself. + // Normally this would not be safe, since we could end up + // overwriting the lockfile if we store a file called 'store.lock'. + // However, in this case it's OK because the keys are set by + // GtS and not the user, so we know we're never going to overwrite it. + LockFile: path.Join(storageBasePath, "store.lock"), + }) + if err != nil { + return nil, fmt.Errorf("error creating storage backend: %s", err) + } + return &Local{KVStore: storage}, nil + } + return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend()) +} |