summaryrefslogtreecommitdiff
path: root/internal/media
diff options
context:
space:
mode:
Diffstat (limited to 'internal/media')
-rw-r--r--internal/media/manager.go6
-rw-r--r--internal/media/manager_test.go27
-rw-r--r--internal/media/media_test.go6
-rw-r--r--internal/media/processingemoji.go10
-rw-r--r--internal/media/processingmedia.go12
-rw-r--r--internal/media/prunemeta.go4
-rw-r--r--internal/media/prunemeta_test.go16
-rw-r--r--internal/media/pruneremote.go4
-rw-r--r--internal/media/pruneremote_test.go10
9 files changed, 48 insertions, 47 deletions
diff --git a/internal/media/manager.go b/internal/media/manager.go
index aacf607cc..b14288324 100644
--- a/internal/media/manager.go
+++ b/internal/media/manager.go
@@ -23,12 +23,12 @@ import (
"fmt"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
)
// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
@@ -98,7 +98,7 @@ type Manager interface {
type manager struct {
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
emojiWorker *concurrency.WorkerPool[*ProcessingEmoji]
mediaWorker *concurrency.WorkerPool[*ProcessingMedia]
stopCronJobs func() error
@@ -110,7 +110,7 @@ type manager struct {
// a limited number of media will be processed in parallel. The numbers of workers
// is determined from the $GOMAXPROCS environment variable (usually no. CPU cores).
// See internal/concurrency.NewWorkerPool() documentation for further information.
-func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
+func NewManager(database db.DB, storage storage.Driver) (Manager, error) {
m := &manager{
db: database,
storage: storage,
diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
index 3a6110f2b..f55763439 100644
--- a/internal/media/manager_test.go
+++ b/internal/media/manager_test.go
@@ -33,6 +33,7 @@ import (
"github.com/stretchr/testify/suite"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
)
type ManagerTestSuite struct {
@@ -87,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -100,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -159,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -172,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -231,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -244,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -314,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -327,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -393,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -406,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -474,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
- processedFullBytes, err := suite.storage.Get(attachment.File.Path)
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@@ -487,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
- processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@@ -523,7 +524,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
panic(err)
}
- diskManager, err := media.NewManager(suite.db, diskStorage)
+ diskManager, err := media.NewManager(suite.db, &gtsstorage.Local{KVStore: diskStorage})
if err != nil {
panic(err)
}
diff --git a/internal/media/media_test.go b/internal/media/media_test.go
index 1b5011801..fda1963a7 100644
--- a/internal/media/media_test.go
+++ b/internal/media/media_test.go
@@ -19,11 +19,11 @@
package media_test
import (
- "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -31,7 +31,7 @@ type MediaStandardTestSuite struct {
suite.Suite
db db.DB
- storage *kv.KVStore
+ storage storage.Driver
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
testAccounts map[string]*gtsmodel.Account
@@ -42,7 +42,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() {
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
- suite.storage = testrig.NewTestStorage()
+ suite.storage = testrig.NewInMemoryStorage()
}
func (suite *MediaStandardTestSuite) SetupTest() {
diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go
index c8c8d18c8..ffac56052 100644
--- a/internal/media/processingemoji.go
+++ b/internal/media/processingemoji.go
@@ -28,10 +28,10 @@ import (
"sync/atomic"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@@ -64,7 +64,7 @@ type ProcessingEmoji struct {
*/
database db.DB
- storage *kv.KVStore
+ storage storage.Driver
err error // error created during processing, if any
@@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
switch processState(staticState) {
case received:
// stream the original file out of storage...
- stored, err := p.storage.GetStream(p.emoji.ImagePath)
+ stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
@@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
}
// put the static in storage
- if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil {
+ if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
@@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// store this for now -- other processes can pull it out of storage as they please
- if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
+ if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index 677437052..17fddddb7 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -28,12 +28,12 @@ import (
"sync/atomic"
"time"
- "codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
+ "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@@ -61,7 +61,7 @@ type ProcessingMedia struct {
*/
database db.DB
- storage *kv.KVStore
+ storage storage.Driver
err error // error created during processing, if any
@@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// stream the original file out of storage
logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL)
- stored, err := p.storage.GetStream(p.attachment.File.Path)
+ stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
@@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// put the thumbnail in storage
logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL)
- if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil {
+ if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
@@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
var decoded *imageMeta
// stream the original file out of storage...
- stored, err := p.storage.GetStream(p.attachment.File.Path)
+ stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
@@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.attachment.File.FileSize = fileSize
// store this for now -- other processes can pull it out of storage as they please
- if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
+ if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.attachment.Cached = true
diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go
index aa838d2a4..33391beaf 100644
--- a/internal/media/prunemeta.go
+++ b/internal/media/prunemeta.go
@@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
- if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
@@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
- if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go
index 1358208a8..8b250e7a5 100644
--- a/internal/media/prunemeta_test.go
+++ b/internal/media/prunemeta_test.go
@@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
@@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go
index a01995740..6cad7fbf8 100644
--- a/internal/media/pruneremote.go
+++ b/internal/media/pruneremote.go
@@ -67,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
- if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
@@ -76,7 +76,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
- if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go
index 31c5128ff..f5ed8a618 100644
--- a/internal/media/pruneremote_test.go
+++ b/internal/media/pruneremote_test.go
@@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.Equal(2, totalPruned)
// media should no longer be stored
- _, err = suite.storage.Get(testAttachment.File.Path)
+ _, err = suite.storage.Get(ctx, testAttachment.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
- _, err = suite.storage.Get(testAttachment.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
@@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
// recached files should be back in storage
- _, err = suite.storage.Get(recachedAttachment.File.Path)
+ _, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
suite.NoError(err)
- _, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
+ _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}
@@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(media.Cached)
- err = suite.storage.Delete(media.File.Path)
+ err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)
// Now attempt to prune remote for item with db entry no file
generated by cgit v1.2.3 (git 2.46.0) at 2025-08-05 02:43:05 +0000