summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
authorLibravatar Dominik Süß <dominik@suess.wtf>2022-07-03 12:08:30 +0200
committerLibravatar GitHub <noreply@github.com>2022-07-03 12:08:30 +0200
commit9d0df426da59275f7aeaf46004befe5a778da274 (patch)
tree82c6bb98597e44c4f70b731336dcdfc839412c1c /internal
parent[chore] Re-enable source tar but name it clearly as source (#683) (diff)
downloadgotosocial-9d0df426da59275f7aeaf46004befe5a778da274.tar.xz
[feature] S3 support (#674)
* feat: vendor minio client * feat: introduce storage package with s3 support * feat: serve s3 files directly this saves a lot of bandwith as the files are fetched from the object store directly * fix: use explicit local storage in tests * feat: integrate s3 storage with the main server * fix: add s3 config to cli tests * docs: explicitly set values in example config also adds license header to the storage package * fix: use better http status code on s3 redirect HTTP 302 Found is the best fit, as it signifies that the resource requested was found but not under its presumed URL 307/TemporaryRedirect would mean that this resource is usually located here, not in this case 303/SeeOther indicates that the redirection does not link to the requested resource but to another page * refactor: use context in storage driver interface
Diffstat (limited to 'internal')
-rw-r--r--internal/api/client/account/account_test.go6
-rw-r--r--internal/api/client/admin/admin_test.go6
-rw-r--r--internal/api/client/admin/emojicreate_test.go4
-rw-r--r--internal/api/client/auth/auth_test.go6
-rw-r--r--internal/api/client/fileserver/servefile.go5
-rw-r--r--internal/api/client/fileserver/servefile_test.go10
-rw-r--r--internal/api/client/followrequest/followrequest_test.go6
-rw-r--r--internal/api/client/instance/instance_test.go6
-rw-r--r--internal/api/client/media/mediacreate_test.go10
-rw-r--r--internal/api/client/media/mediaupdate_test.go6
-rw-r--r--internal/api/client/status/status_test.go6
-rw-r--r--internal/api/client/user/user_test.go6
-rw-r--r--internal/api/model/content.go7
-rw-r--r--internal/api/s2s/user/user_test.go6
-rw-r--r--internal/api/s2s/webfinger/webfinger_test.go6
-rw-r--r--internal/config/config.go5
-rw-r--r--internal/config/defaults.go1
-rw-r--r--internal/config/helpers.gen.go125
-rw-r--r--internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go5
-rw-r--r--internal/federation/dereferencing/dereferencer_test.go6
-rw-r--r--internal/federation/federator_test.go6
-rw-r--r--internal/media/manager.go6
-rw-r--r--internal/media/manager_test.go27
-rw-r--r--internal/media/media_test.go6
-rw-r--r--internal/media/processingemoji.go10
-rw-r--r--internal/media/processingmedia.go12
-rw-r--r--internal/media/prunemeta.go4
-rw-r--r--internal/media/prunemeta_test.go16
-rw-r--r--internal/media/pruneremote.go4
-rw-r--r--internal/media/pruneremote_test.go10
-rw-r--r--internal/processing/account/account_test.go6
-rw-r--r--internal/processing/media/delete.go4
-rw-r--r--internal/processing/media/getfile.go14
-rw-r--r--internal/processing/media/getfile_test.go18
-rw-r--r--internal/processing/media/media.go6
-rw-r--r--internal/processing/media/media_test.go6
-rw-r--r--internal/processing/processor.go6
-rw-r--r--internal/processing/processor_test.go6
-rw-r--r--internal/processing/status/status_test.go6
-rw-r--r--internal/storage/local.go50
-rw-r--r--internal/storage/s3.go87
-rw-r--r--internal/storage/storage.go78
42 files changed, 496 insertions, 130 deletions
diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go
index ed49e9158..39d942ee3 100644
--- a/internal/api/client/account/account_test.go
+++ b/internal/api/client/account/account_test.go
@@ -6,7 +6,6 @@ import (
"net/http"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
@@ -20,6 +19,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -27,7 +27,7 @@ type AccountStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -65,7 +65,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go
index 6b906f43a..c5035c798 100644
--- a/internal/api/client/admin/admin_test.go
+++ b/internal/api/client/admin/admin_test.go
@@ -24,7 +24,6 @@ import (
"net/http"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
@@ -38,6 +37,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -45,7 +45,7 @@ type AdminStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -83,7 +83,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go
index 872b18a75..fa93ce564 100644
--- a/internal/api/client/admin/emojicreate_test.go
+++ b/internal/api/client/admin/emojicreate_test.go
@@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
suite.Empty(dbEmoji.CategoryID)
// emoji should be in storage
- emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath)
+ emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
- emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath)
+ emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
}
diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go
index 726b2be9d..188c40b3a 100644
--- a/internal/api/client/auth/auth_test.go
+++ b/internal/api/client/auth/auth_test.go
@@ -24,7 +24,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/memstore"
"github.com/gin-gonic/gin"
@@ -42,13 +41,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oidc"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AuthStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -88,7 +88,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go
index 0372de9d8..ee2ff51ad 100644
--- a/internal/api/client/fileserver/servefile.go
+++ b/internal/api/client/fileserver/servefile.go
@@ -84,6 +84,11 @@ func (m *FileServer) ServeFile(c *gin.Context) {
return
}
+ if content.URL != nil {
+ c.Redirect(http.StatusFound, content.URL.String())
+ return
+ }
+
defer func() {
// if the content is a ReadCloser, close it when we're done
if closer, ok := content.Content.(io.ReadCloser); ok {
diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go
index e4db9a704..a36a79a58 100644
--- a/internal/api/client/fileserver/servefile_test.go
+++ b/internal/api/client/fileserver/servefile_test.go
@@ -26,7 +26,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -40,6 +39,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -48,7 +48,7 @@ type ServeFileTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
processor processing.Processor
@@ -81,7 +81,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
@@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
- fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
+ fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
@@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
- fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path)
+ fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go
index a8c4090f6..e98d75c77 100644
--- a/internal/api/client/followrequest/followrequest_test.go
+++ b/internal/api/client/followrequest/followrequest_test.go
@@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest"
@@ -37,13 +36,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type FollowRequestStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -80,7 +80,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go
index 1e5c59c97..645e70474 100644
--- a/internal/api/client/instance/instance_test.go
+++ b/internal/api/client/instance/instance_test.go
@@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
@@ -37,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -82,7 +82,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go
index 2edf8be9f..153cde90f 100644
--- a/internal/api/client/media/mediacreate_test.go
+++ b/internal/api/client/media/mediacreate_test.go
@@ -30,7 +30,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -46,6 +45,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -54,7 +54,7 @@ type MediaCreateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage *storage.Local
mediaManager media.Manager
federator federation.Federator
tc typeutils.TypeConverter
@@ -87,7 +87,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
@@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// see what's in storage *before* the request
storageKeysBeforeRequest := []string{}
- iter, err := suite.storage.Iterator(nil)
+ iter, err := suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
@@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// check what's in storage *after* the request
storageKeysAfterRequest := []string{}
- iter, err = suite.storage.Iterator(nil)
+ iter, err = suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go
index 0aebc435e..4457426de 100644
--- a/internal/api/client/media/mediaupdate_test.go
+++ b/internal/api/client/media/mediaupdate_test.go
@@ -28,7 +28,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -44,6 +43,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
mediaManager media.Manager
@@ -85,7 +85,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go
index 049a5069c..ad6901d9d 100644
--- a/internal/api/client/status/status_test.go
+++ b/internal/api/client/status/status_test.go
@@ -19,7 +19,6 @@
package status_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -43,7 +43,7 @@ type StatusStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
// standard suite models
testTokens map[string]*gtsmodel.Token
@@ -76,7 +76,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go
index 7f5479715..116f3711b 100644
--- a/internal/api/client/user/user_test.go
+++ b/internal/api/client/user/user_test.go
@@ -19,7 +19,6 @@
package user_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -42,7 +42,7 @@ type UserStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
@@ -66,7 +66,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
diff --git a/internal/api/model/content.go b/internal/api/model/content.go
index 2f38b2351..aa02a99c3 100644
--- a/internal/api/model/content.go
+++ b/internal/api/model/content.go
@@ -18,7 +18,10 @@
package model
-import "io"
+import (
+ "io"
+ "net/url"
+)
// Content wraps everything needed to serve a blob of content (some kind of media) through the API.
type Content struct {
@@ -28,6 +31,8 @@ type Content struct {
ContentLength int64
// Actual content
Content io.Reader
+ // Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL)
+ URL *url.URL
}
// GetContentRequestForm describes a piece of content desired by the caller of the fileserver API.
diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go
index 024fb907e..28ca2fa67 100644
--- a/internal/api/s2s/user/user_test.go
+++ b/internal/api/s2s/user/user_test.go
@@ -19,7 +19,6 @@
package user_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
"github.com/superseriousbusiness/gotosocial/internal/api/security"
@@ -32,6 +31,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -45,7 +45,7 @@ type UserStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
oauthServer oauth.Server
securityModule *security.Module
@@ -83,7 +83,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go
index 9758a6be7..19a627a35 100644
--- a/internal/api/s2s/webfinger/webfinger_test.go
+++ b/internal/api/s2s/webfinger/webfinger_test.go
@@ -23,7 +23,6 @@ import (
"crypto/rsa"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
@@ -37,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -50,7 +50,7 @@ type WebfingerStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
oauthServer oauth.Server
securityModule *security.Module
@@ -86,7 +86,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/config/config.go b/internal/config/config.go
index 0b7fc3d31..d746bd12a 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -82,6 +82,11 @@ type Configuration struct {
StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
+ StorageS3Endpoint string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"`
+ StorageS3AccessKey string `name:"storage-s3-access-key" usage:"S3 Access Key"`
+ StorageS3SecretKey string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
+ StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
+ StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"`
StatusesCWMaxChars int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"`
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 5bfd7aa53..d332efc04 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -61,6 +61,7 @@ var Defaults = Configuration{
StorageBackend: "local",
StorageLocalBasePath: "/gotosocial/storage",
+ StorageS3UseSSL: true,
StatusesMaxChars: 5000,
StatusesCWMaxChars: 100,
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index a4c2adebf..a5dcc4c1c 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -843,6 +843,131 @@ func GetStorageLocalBasePath() string { return global.GetStorageLocalBasePath()
// SetStorageLocalBasePath safely sets the value for global configuration 'StorageLocalBasePath' field
func SetStorageLocalBasePath(v string) { global.SetStorageLocalBasePath(v) }
+// GetStorageS3Endpoint safely fetches the Configuration value for state's 'StorageS3Endpoint' field
+func (st *ConfigState) GetStorageS3Endpoint() (v string) {
+ st.mutex.Lock()
+ v = st.config.StorageS3Endpoint
+ st.mutex.Unlock()
+ return
+}
+
+// SetStorageS3Endpoint safely sets the Configuration value for state's 'StorageS3Endpoint' field
+func (st *ConfigState) SetStorageS3Endpoint(v string) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.StorageS3Endpoint = v
+ st.reloadToViper()
+}
+
+// StorageS3EndpointFlag returns the flag name for the 'StorageS3Endpoint' field
+func StorageS3EndpointFlag() string { return "storage-s3-endpoint" }
+
+// GetStorageS3Endpoint safely fetches the value for global configuration 'StorageS3Endpoint' field
+func GetStorageS3Endpoint() string { return global.GetStorageS3Endpoint() }
+
+// SetStorageS3Endpoint safely sets the value for global configuration 'StorageS3Endpoint' field
+func SetStorageS3Endpoint(v string) { global.SetStorageS3Endpoint(v) }
+
+// GetStorageS3AccessKey safely fetches the Configuration value for state's 'StorageS3AccessKey' field
+func (st *ConfigState) GetStorageS3AccessKey() (v string) {
+ st.mutex.Lock()
+ v = st.config.StorageS3AccessKey
+ st.mutex.Unlock()
+ return
+}
+
+// SetStorageS3AccessKey safely sets the Configuration value for state's 'StorageS3AccessKey' field
+func (st *ConfigState) SetStorageS3AccessKey(v string) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.StorageS3AccessKey = v
+ st.reloadToViper()
+}
+
+// StorageS3AccessKeyFlag returns the flag name for the 'StorageS3AccessKey' field
+func StorageS3AccessKeyFlag() string { return "storage-s3-access-key" }
+
+// GetStorageS3AccessKey safely fetches the value for global configuration 'StorageS3AccessKey' field
+func GetStorageS3AccessKey() string { return global.GetStorageS3AccessKey() }
+
+// SetStorageS3AccessKey safely sets the value for global configuration 'StorageS3AccessKey' field
+func SetStorageS3AccessKey(v string) { global.SetStorageS3AccessKey(v) }
+
+// GetStorageS3SecretKey safely fetches the Configuration value for state's 'StorageS3SecretKey' field
+func (st *ConfigState) GetStorageS3SecretKey() (v string) {
+ st.mutex.Lock()
+ v = st.config.StorageS3SecretKey
+ st.mutex.Unlock()
+ return
+}
+
+// SetStorageS3SecretKey safely sets the Configuration value for state's 'StorageS3SecretKey' field
+func (st *ConfigState) SetStorageS3SecretKey(v string) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.StorageS3SecretKey = v
+ st.reloadToViper()
+}
+
+// StorageS3SecretKeyFlag returns the flag name for the 'StorageS3SecretKey' field
+func StorageS3SecretKeyFlag() string { return "storage-s3-secret-key" }
+
+// GetStorageS3SecretKey safely fetches the value for global configuration 'StorageS3SecretKey' field
+func GetStorageS3SecretKey() string { return global.GetStorageS3SecretKey() }
+
+// SetStorageS3SecretKey safely sets the value for global configuration 'StorageS3SecretKey' field
+func SetStorageS3SecretKey(v string) { global.SetStorageS3SecretKey(v) }
+
+// GetStorageS3UseSSL safely fetches the Configuration value for state's 'StorageS3UseSSL' field
+func (st *ConfigState) GetStorageS3UseSSL() (v bool) {
+ st.mutex.Lock()
+ v = st.config.StorageS3UseSSL
+ st.mutex.Unlock()
+ return
+}
+
+// SetStorageS3UseSSL safely sets the Configuration value for state's 'StorageS3UseSSL' field
+func (st *ConfigState) SetStorageS3UseSSL(v bool) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.StorageS3UseSSL = v
+ st.reloadToViper()
+}
+
+// StorageS3UseSSLFlag returns the flag name for the 'StorageS3UseSSL' field
+func StorageS3UseSSLFlag() string { return "storage-s3-use-ssl" }
+
+// GetStorageS3UseSSL safely fetches the value for global configuration 'StorageS3UseSSL' field
+func GetStorageS3UseSSL() bool { return global.GetStorageS3UseSSL() }
+
+// SetStorageS3UseSSL safely sets the value for global configuration 'StorageS3UseSSL' field
+func SetStorageS3UseSSL(v bool) { global.SetStorageS3UseSSL(v) }
+
+// GetStorageS3BucketName safely fetches the Configuration value for state's 'StorageS3BucketName' field
+func (st *ConfigState) GetStorageS3BucketName() (v string) {
+ st.mutex.Lock()
+ v = st.config.StorageS3BucketName
+ st.mutex.Unlock()
+ return
+}
+
+// SetStorageS3BucketName safely sets the Configuration value for state's 'StorageS3BucketName' field
+func (st *ConfigState) SetStorageS3BucketName(v string) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.StorageS3BucketName = v
+ st.reloadToViper()
+}
+
+// StorageS3BucketNameFlag returns the flag name for the 'StorageS3BucketName' field
+func StorageS3BucketNameFlag() string { return "storage-s3-bucket" }
+
+// GetStorageS3BucketName safely fetches the value for global configuration 'StorageS3BucketName' field
+func GetStorageS3BucketName() string { return global.GetStorageS3BucketName() }
+
+// SetStorageS3BucketName safely sets the value for global configuration 'StorageS3BucketName' field
+func SetStorageS3BucketName(v string) { global.SetStorageS3BucketName(v) }
+
// GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
func (st *ConfigState) GetStatusesMaxChars() (v int) {
st.mutex.Lock()
diff --git a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go
index 6139fd77d..1a3ae07aa 100644
--- a/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go
+++ b/internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go
@@ -59,6 +59,11 @@ func init() {
up := func(ctx context.Context, db *bun.DB) error {
l := logrus.WithField("migration", "20220612091800_duplicated_media_cleanup")
+ if config.GetStorageBackend() != "local" {
+ // this migration only affects versions which only supported local storage
+ return nil
+ }
+
storageBasePath := config.GetStorageLocalBasePath()
if storageBasePath == "" {
return fmt.Errorf("%s must be set to do storage migration", config.StorageLocalBasePathFlag())
diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go
index 0f4732187..c0343a6b8 100644
--- a/internal/federation/dereferencing/dereferencer_test.go
+++ b/internal/federation/dereferencing/dereferencer_test.go
@@ -19,7 +19,6 @@
package dereferencing_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@@ -27,13 +26,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/messages"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type DereferencerStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
testRemoteStatuses map[string]vocab.ActivityStreamsNote
testRemotePeople map[string]vocab.ActivityStreamsPerson
@@ -57,7 +57,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage))
testrig.StandardDBSetup(suite.db, nil)
}
diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go
index 0984b405b..c93957098 100644
--- a/internal/federation/federator_test.go
+++ b/internal/federation/federator_test.go
@@ -19,11 +19,11 @@
package federation_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -31,7 +31,7 @@ import (
type FederatorStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
tc typeutils.TypeConverter
testAccounts map[string]*gtsmodel.Account
testStatuses map[string]*gtsmodel.Status
@@ -41,7 +41,7 @@ type FederatorStandardTestSuite struct {
// SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout
func (suite *FederatorStandardTestSuite) SetupSuite() {
// setup standard items
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.testAccounts = testrig.NewTestAccounts()
suite.testStatuses = testrig.NewTestStatuses()
diff --git a/internal/media/manager.go b/internal/media/manager.go
index aacf607cc..b14288324 100644
--- a/internal/media/manager.go
+++ b/internal/media/manager.go
@@ -23,12 +23,12 @@ import (
"fmt"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
)
// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
@@ -98,7 +98,7 @@ type Manager interface {
type manager struct {
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
emojiWorker *concurrency.WorkerPool[*ProcessingEmoji]
mediaWorker *concurrency.WorkerPool[*ProcessingMedia]
stopCronJobs func() error
@@ -110,7 +110,7 @@ type manager struct {
// a limited number of media will be processed in parallel. The numbers of workers
// is determined from the $GOMAXPROCS environment variable (usually no. CPU cores).
// See internal/concurrency.NewWorkerPool() documentation for further information.
-func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
+func NewManager(database db.DB, storage storage.Driver) (Manager, error) {
m := &manager{
db: database,
storage: storage,
diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
index 3a6110f2b..f55763439 100644
--- a/internal/media/manager_test.go
+++ b/internal/media/manager_test.go
@@ -33,6 +33,7 @@ import (
"github.com/stretchr/testify/suite"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
)
type ManagerTestSuite struct {
@@ -87,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -100,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -159,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -172,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -231,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -244,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -314,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -327,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -393,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -406,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -474,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -487,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -523,7 +524,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
panic(err)
}
- diskManager, err := media.NewManager(suite.db, diskStorage)
+ diskManager, err := media.NewManager(suite.db, &gtsstorage.Local{KVStore: diskStorage})
if err != nil {
panic(err)
}
diff --git a/internal/media/media_test.go b/internal/media/media_test.go
index 1b5011801..fda1963a7 100644
--- a/internal/media/media_test.go
+++ b/internal/media/media_test.go
@@ -19,11 +19,11 @@
package media_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -31,7 +31,7 @@ type MediaStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
testAccounts map[string]*gtsmodel.Account
@@ -42,7 +42,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() {
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
}
func (suite *MediaStandardTestSuite) SetupTest() {
diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go
index c8c8d18c8..ffac56052 100644
--- a/internal/media/processingemoji.go
+++ b/internal/media/processingemoji.go
@@ -28,10 +28,10 @@ import (
"sync/atomic"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@@ -64,7 +64,7 @@ type ProcessingEmoji struct {
*/
database db.DB
- storage *kv.KVStore
+ storage storage.Driver
err error // error created during processing, if any
@@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
switch processState(staticState) {
case received:
// stream the original file out of storage...
- stored, err := p.storage.GetStream(p.emoji.ImagePath)
+ stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
@@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
}
// put the static in storage
- if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil {
+ if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
@@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// store this for now -- other processes can pull it out of storage as they please
- if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
+ if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index 677437052..17fddddb7 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -28,12 +28,12 @@ import (
"sync/atomic"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@@ -61,7 +61,7 @@ type ProcessingMedia struct {
*/
database db.DB
- storage *kv.KVStore
+ storage storage.Driver
err error // error created during processing, if any
@@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// stream the original file out of storage
logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL)
- stored, err := p.storage.GetStream(p.attachment.File.Path)
+ stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
@@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// put the thumbnail in storage
logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL)
- if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil {
+ if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
@@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
var decoded *imageMeta
// stream the original file out of storage...
- stored, err := p.storage.GetStream(p.attachment.File.Path)
+ stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
@@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.attachment.File.FileSize = fileSize
// store this for now -- other processes can pull it out of storage as they please
- if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
+ if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.attachment.Cached = true
diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go
index aa838d2a4..33391beaf 100644
--- a/internal/media/prunemeta.go
+++ b/internal/media/prunemeta.go
@@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
- if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
@@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
- if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go
index 1358208a8..8b250e7a5 100644
--- a/internal/media/prunemeta_test.go
+++ b/internal/media/prunemeta_test.go
@@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
@@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go
index a01995740..6cad7fbf8 100644
--- a/internal/media/pruneremote.go
+++ b/internal/media/pruneremote.go
@@ -67,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
- if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
@@ -76,7 +76,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
- if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go
index 31c5128ff..f5ed8a618 100644
--- a/internal/media/pruneremote_test.go
+++ b/internal/media/pruneremote_test.go
@@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(testAttachment.File.Path)
+ _, err = suite.storage.Get(ctx, testAttachment.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(testAttachment.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
@@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
// recached files should be back in storage
- _, err = suite.storage.Get(recachedAttachment.File.Path)
+ _, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
suite.NoError(err)
- _, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}
@@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(media.Cached)
- err = suite.storage.Delete(media.File.Path)
+ err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)
// Now attempt to prune remote for item with db entry no file
diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
index 8ea1665ac..0755f01c2 100644
--- a/internal/processing/account/account_test.go
+++ b/internal/processing/account/account_test.go
@@ -21,7 +21,6 @@ package account_test
import (
"context"
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -33,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/account"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@@ -43,7 +43,7 @@ type AccountStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
oauthServer oauth.Server
fromClientAPIChan chan messages.FromClientAPI
@@ -91,7 +91,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go
index e99b4e950..45c218397 100644
--- a/internal/processing/media/delete.go
+++ b/internal/processing/media/delete.go
@@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
// delete the thumbnail from storage
if attachment.Thumbnail.Path != "" {
- if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil {
+ if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err))
}
}
// delete the file from storage
if attachment.File.Path != "" {
- if err := p.storage.Delete(attachment.File.Path); err != nil {
+ if err := p.storage.Delete(ctx, attachment.File.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err))
}
}
diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go
index c74951e38..3227cb8c8 100644
--- a/internal/processing/media/getfile.go
+++ b/internal/processing/media/getfile.go
@@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
// if we have the media cached on our server already, we can now simply return it from storage
if a.Cached {
- return p.streamFromStorage(storagePath, attachmentContent)
+ return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
// if we don't have it cached, then we can assume two things:
@@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
// ... so now we can safely return it
- return p.streamFromStorage(storagePath, attachmentContent)
+ return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
return attachmentContent, nil
@@ -253,11 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
}
- return p.streamFromStorage(storagePath, emojiContent)
+ return p.retrieveFromStorage(ctx, storagePath, emojiContent)
}
-func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
- reader, err := p.storage.GetStream(storagePath)
+func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
+ if url := p.storage.URL(ctx, storagePath); url != nil {
+ content.URL = url
+ return content, nil
+ }
+ reader, err := p.storage.GetStream(ctx, storagePath)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
}
diff --git a/internal/processing/media/getfile_test.go b/internal/processing/media/getfile_test.go
index 4d7bc4621..7c6525abe 100644
--- a/internal/processing/media/getfile_test.go
+++ b/internal/processing/media/getfile_test.go
@@ -70,9 +70,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.File.Path)
+ err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+ err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
@@ -106,7 +106,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
- refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
+ refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@@ -119,9 +119,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.File.Path)
+ err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+ err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
@@ -156,7 +156,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
- refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
+ refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@@ -166,16 +166,16 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
// fetch the existing thumbnail bytes from storage first
- thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path)
+ thumbnailBytes, err := suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// uncache the file from local
testAttachment.Cached = false
err = suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.File.Path)
+ err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
- err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+ err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch the thumbnail
diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go
index 50cbc1b3c..66575facc 100644
--- a/internal/processing/media/media.go
+++ b/internal/processing/media/media.go
@@ -21,12 +21,12 @@ package media
import (
"context"
- "codeberg.org/gruf/go-store/kv"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@@ -51,12 +51,12 @@ type processor struct {
tc typeutils.TypeConverter
mediaManager media.Manager
transportController transport.Controller
- storage *kv.KVStore
+ storage storage.Driver
db db.DB
}
// New returns a new media processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage storage.Driver) Processor {
return &processor{
tc: tc,
mediaManager: mediaManager,
diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go
index c8c8736be..cf73af4e8 100644
--- a/internal/processing/media/media_test.go
+++ b/internal/processing/media/media_test.go
@@ -19,7 +19,6 @@
package media_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -27,6 +26,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@@ -37,7 +37,7 @@ type MediaStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
transportController transport.Controller
@@ -72,7 +72,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1))
suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index 8572b583c..b8d0daf99 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -23,7 +23,6 @@ import (
"net/http"
"net/url"
- "codeberg.org/gruf/go-store/kv"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -41,6 +40,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
"github.com/superseriousbusiness/gotosocial/internal/processing/streaming"
"github.com/superseriousbusiness/gotosocial/internal/processing/user"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/stream"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -251,7 +251,7 @@ type processor struct {
tc typeutils.TypeConverter
oauthServer oauth.Server
mediaManager media.Manager
- storage *kv.KVStore
+ storage storage.Driver
statusTimelines timeline.Manager
db db.DB
filter visibility.Filter
@@ -275,7 +275,7 @@ func NewProcessor(
federator federation.Federator,
oauthServer oauth.Server,
mediaManager media.Manager,
- storage *kv.KVStore,
+ storage storage.Driver,
db db.DB,
emailSender email.Sender,
clientWorker *concurrency.WorkerPool[messages.FromClientAPI],
diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
index c60ff5c97..54271cc86 100644
--- a/internal/processing/processor_test.go
+++ b/internal/processing/processor_test.go
@@ -19,7 +19,6 @@
package processing_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@@ -39,7 +39,7 @@ type ProcessingStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
typeconverter typeutils.TypeConverter
httpClient *testrig.MockHTTPClient
@@ -91,7 +91,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")
diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go
index f8834e2c2..199c36404 100644
--- a/internal/processing/status/status_test.go
+++ b/internal/processing/status/status_test.go
@@ -19,7 +19,6 @@
package status_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -29,6 +28,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@@ -39,7 +39,7 @@ type StatusStandardTestSuite struct {
db db.DB
typeConverter typeutils.TypeConverter
tc transport.Controller
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
clientWorker *concurrency.WorkerPool[messages.FromClientAPI]
@@ -81,7 +81,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)
suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator))
diff --git a/internal/storage/local.go b/internal/storage/local.go
new file mode 100644
index 000000000..da57631f9
--- /dev/null
+++ b/internal/storage/local.go
@@ -0,0 +1,50 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package storage
+
+import (
+ "context"
+ "io"
+ "net/url"
+
+ "codeberg.org/gruf/go-store/kv"
+)
+
+type Local struct {
+ KVStore *kv.KVStore
+}
+
+func (l *Local) Get(ctx context.Context, key string) ([]byte, error) {
+ return l.KVStore.Get(key)
+}
+func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
+ return l.KVStore.GetStream(key)
+}
+func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error {
+ return l.KVStore.PutStream(key, r)
+}
+func (l *Local) Put(ctx context.Context, key string, value []byte) error {
+ return l.KVStore.Put(key, value)
+}
+func (l *Local) Delete(ctx context.Context, key string) error {
+ return l.KVStore.Delete(key)
+}
+func (l *Local) URL(ctx context.Context, key string) *url.URL {
+ return nil
+}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
new file mode 100644
index 000000000..a15114488
--- /dev/null
+++ b/internal/storage/s3.go
@@ -0,0 +1,87 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package storage
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "mime"
+ "net/url"
+ "path"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+)
+
+type S3 struct {
+ mc *minio.Client
+ bucket string
+}
+
+func NewS3(mc *minio.Client, bucket string) *S3 {
+ return &S3{
+ mc: mc,
+ bucket: bucket,
+ }
+}
+
+func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
+ r, err := s.GetStream(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+ b, err := io.ReadAll(r)
+ if err != nil {
+ return nil, fmt.Errorf("reading data from s3: %w", err)
+ }
+ return b, nil
+}
+func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
+ o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
+ if err != nil {
+ err = fmt.Errorf("retrieving object from s3: %w", err)
+ }
+ return o, err
+}
+func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error {
+ if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil {
+ return fmt.Errorf("uploading data stream: %w", err)
+ }
+ return nil
+}
+func (s *S3) Put(ctx context.Context, key string, value []byte) error {
+ if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil {
+ return fmt.Errorf("uploading data slice: %w", err)
+ }
+ return nil
+}
+func (s *S3) Delete(ctx context.Context, key string) error {
+ return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
+}
+func (s *S3) URL(ctx context.Context, key string) *url.URL {
+ // it's safe to ignore the error here, as we just fall back to fetching the
+ // file if the url request fails
+ url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{
+ "response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
+ })
+ return url
+}
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
new file mode 100644
index 000000000..88577442f
--- /dev/null
+++ b/internal/storage/storage.go
@@ -0,0 +1,78 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/url"
+ "path"
+
+ "codeberg.org/gruf/go-store/kv"
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+)
+
+var (
+ ErrNotSupported = errors.New("driver does not suppport functionality")
+)
+
+// Driver implements the functionality to store and retrieve blobs
+// (images,video,audio)
+type Driver interface {
+ Get(ctx context.Context, key string) ([]byte, error)
+ GetStream(ctx context.Context, key string) (io.ReadCloser, error)
+ PutStream(ctx context.Context, key string, r io.Reader) error
+ Put(ctx context.Context, key string, value []byte) error
+ Delete(ctx context.Context, key string) error
+ URL(ctx context.Context, key string) *url.URL
+}
+
+func AutoConfig() (Driver, error) {
+ switch config.GetStorageBackend() {
+ case "s3":
+ mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{
+ Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""),
+ Secure: config.GetStorageS3UseSSL(),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("creating minio client: %w", err)
+ }
+ return NewS3(mc, config.GetStorageS3BucketName()), nil
+ case "local":
+ storageBasePath := config.GetStorageLocalBasePath()
+ storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{
+ // Put the store lockfile in the storage dir itself.
+ // Normally this would not be safe, since we could end up
+ // overwriting the lockfile if we store a file called 'store.lock'.
+ // However, in this case it's OK because the keys are set by
+ // GtS and not the user, so we know we're never going to overwrite it.
+ LockFile: path.Join(storageBasePath, "store.lock"),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error creating storage backend: %s", err)
+ }
+ return &Local{KVStore: storage}, nil
+ }
+ return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())
+}