summaryrefslogtreecommitdiff
path: root/internal/api/client
diff options
context:
space:
mode:
authorLibravatar Dominik Süß <dominik@suess.wtf>2022-07-03 12:08:30 +0200
committerLibravatar GitHub <noreply@github.com>2022-07-03 12:08:30 +0200
commit9d0df426da59275f7aeaf46004befe5a778da274 (patch)
tree82c6bb98597e44c4f70b731336dcdfc839412c1c /internal/api/client
parent[chore] Re-enable source tar but name it clearly as source (#683) (diff)
downloadgotosocial-9d0df426da59275f7aeaf46004befe5a778da274.tar.xz
[feature] S3 support (#674)
* feat: vendor minio client * feat: introduce storage package with s3 support * feat: serve s3 files directly this saves a lot of bandwith as the files are fetched from the object store directly * fix: use explicit local storage in tests * feat: integrate s3 storage with the main server * fix: add s3 config to cli tests * docs: explicitly set values in example config also adds license header to the storage package * fix: use better http status code on s3 redirect HTTP 302 Found is the best fit, as it signifies that the resource requested was found but not under its presumed URL 307/TemporaryRedirect would mean that this resource is usually located here, not in this case 303/SeeOther indicates that the redirection does not link to the requested resource but to another page * refactor: use context in storage driver interface
Diffstat (limited to 'internal/api/client')
-rw-r--r--internal/api/client/account/account_test.go6
-rw-r--r--internal/api/client/admin/admin_test.go6
-rw-r--r--internal/api/client/admin/emojicreate_test.go4
-rw-r--r--internal/api/client/auth/auth_test.go6
-rw-r--r--internal/api/client/fileserver/servefile.go5
-rw-r--r--internal/api/client/fileserver/servefile_test.go10
-rw-r--r--internal/api/client/followrequest/followrequest_test.go6
-rw-r--r--internal/api/client/instance/instance_test.go6
-rw-r--r--internal/api/client/media/mediacreate_test.go10
-rw-r--r--internal/api/client/media/mediaupdate_test.go6
-rw-r--r--internal/api/client/status/status_test.go6
-rw-r--r--internal/api/client/user/user_test.go6
12 files changed, 41 insertions, 36 deletions
diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go
index ed49e9158..39d942ee3 100644
--- a/internal/api/client/account/account_test.go
+++ b/internal/api/client/account/account_test.go
@@ -6,7 +6,6 @@ import (
"net/http"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
@@ -20,6 +19,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -27,7 +27,7 @@ type AccountStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -65,7 +65,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go
index 6b906f43a..c5035c798 100644
--- a/internal/api/client/admin/admin_test.go
+++ b/internal/api/client/admin/admin_test.go
@@ -24,7 +24,6 @@ import (
"net/http"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
@@ -38,6 +37,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -45,7 +45,7 @@ type AdminStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -83,7 +83,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go
index 872b18a75..fa93ce564 100644
--- a/internal/api/client/admin/emojicreate_test.go
+++ b/internal/api/client/admin/emojicreate_test.go
@@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
suite.Empty(dbEmoji.CategoryID)
// emoji should be in storage
- emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath)
+ emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
- emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath)
+ emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
}
diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go
index 726b2be9d..188c40b3a 100644
--- a/internal/api/client/auth/auth_test.go
+++ b/internal/api/client/auth/auth_test.go
@@ -24,7 +24,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/memstore"
"github.com/gin-gonic/gin"
@@ -42,13 +41,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oidc"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AuthStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -88,7 +88,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/api/client/fileserver/servefile.go b/internal/api/client/fileserver/servefile.go
index 0372de9d8..ee2ff51ad 100644
--- a/internal/api/client/fileserver/servefile.go
+++ b/internal/api/client/fileserver/servefile.go
@@ -84,6 +84,11 @@ func (m *FileServer) ServeFile(c *gin.Context) {
return
}
+ if content.URL != nil {
+ c.Redirect(http.StatusFound, content.URL.String())
+ return
+ }
+
defer func() {
// if the content is a ReadCloser, close it when we're done
if closer, ok := content.Content.(io.ReadCloser); ok {
diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go
index e4db9a704..a36a79a58 100644
--- a/internal/api/client/fileserver/servefile_test.go
+++ b/internal/api/client/fileserver/servefile_test.go
@@ -26,7 +26,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -40,6 +39,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -48,7 +48,7 @@ type ServeFileTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
processor processing.Processor
@@ -81,7 +81,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
@@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
- fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
+ fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
@@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
- fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path)
+ fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go
index a8c4090f6..e98d75c77 100644
--- a/internal/api/client/followrequest/followrequest_test.go
+++ b/internal/api/client/followrequest/followrequest_test.go
@@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest"
@@ -37,13 +36,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type FollowRequestStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -80,7 +80,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go
index 1e5c59c97..645e70474 100644
--- a/internal/api/client/instance/instance_test.go
+++ b/internal/api/client/instance/instance_test.go
@@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
@@ -37,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@@ -82,7 +82,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go
index 2edf8be9f..153cde90f 100644
--- a/internal/api/client/media/mediacreate_test.go
+++ b/internal/api/client/media/mediacreate_test.go
@@ -30,7 +30,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -46,6 +45,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -54,7 +54,7 @@ type MediaCreateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage *storage.Local
mediaManager media.Manager
federator federation.Federator
tc typeutils.TypeConverter
@@ -87,7 +87,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
@@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// see what's in storage *before* the request
storageKeysBeforeRequest := []string{}
- iter, err := suite.storage.Iterator(nil)
+ iter, err := suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
@@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// check what's in storage *after* the request
storageKeysAfterRequest := []string{}
- iter, err = suite.storage.Iterator(nil)
+ iter, err = suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go
index 0aebc435e..4457426de 100644
--- a/internal/api/client/media/mediaupdate_test.go
+++ b/internal/api/client/media/mediaupdate_test.go
@@ -28,7 +28,6 @@ import (
"net/http/httptest"
"testing"
- "codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@@ -44,6 +43,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
mediaManager media.Manager
@@ -85,7 +85,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go
index 049a5069c..ad6901d9d 100644
--- a/internal/api/client/status/status_test.go
+++ b/internal/api/client/status/status_test.go
@@ -19,7 +19,6 @@
package status_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -43,7 +43,7 @@ type StatusStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
// standard suite models
testTokens map[string]*gtsmodel.Token
@@ -76,7 +76,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go
index 7f5479715..116f3711b 100644
--- a/internal/api/client/user/user_test.go
+++ b/internal/api/client/user/user_test.go
@@ -19,7 +19,6 @@
package user_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -42,7 +42,7 @@ type UserStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
- storage *kv.KVStore
+ storage storage.Driver
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
@@ -66,7 +66,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
generated by cgit v1.2.3 (git 2.46.0) at 2025-08-31 11:08:08 +0000