Merge pull request #214 from NyaaaWhatsUpDoc/improvement/update-storage-library

add git.iim.gay/grufwub/go-store for storage backend, replacing blob.Storage

20 files changed, 62 insertions, 212 deletions

diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go
index 56f981ab1..0acc611df 100644
--- a/internal/api/client/account/account_test.go
+++ b/internal/api/client/account/account_test.go
@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"net/http/httptest"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -28,7 +28,7 @@ type AccountStandardTestSuite struct {
 	db        db.DB
 	log       *logrus.Logger
 	tc        typeutils.TypeConverter
-	storage   blob.Storage
+	storage   *kv.KVStore
 	federator federation.Federator
 	processor processing.Processor
 
diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go
index 579bb9606..6d5f2d39e 100644
--- a/internal/api/client/fileserver/servefile_test.go
+++ b/internal/api/client/fileserver/servefile_test.go
@@ -26,12 +26,12 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -49,7 +49,7 @@ type ServeFileTestSuite struct {
 	config       *config.Config
 	db           db.DB
 	log          *logrus.Logger
-	storage      blob.Storage
+	storage      *kv.KVStore
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	processor    processing.Processor
@@ -152,7 +152,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
 	assert.NoError(suite.T(), err)
 	assert.NotNil(suite.T(), b)
 
-	fileInStorage, err := suite.storage.RetrieveFileFrom(targetAttachment.File.Path)
+	fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
 	assert.NoError(suite.T(), err)
 	assert.NotNil(suite.T(), fileInStorage)
 	assert.Equal(suite.T(), b, fileInStorage)
diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go
index 1b2c84cf9..53c1ce996 100644
--- a/internal/api/client/media/mediacreate_test.go
+++ b/internal/api/client/media/mediacreate_test.go
@@ -28,13 +28,13 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -52,7 +52,7 @@ type MediaCreateTestSuite struct {
 	config       *config.Config
 	db           db.DB
 	log          *logrus.Logger
-	storage      blob.Storage
+	storage      *kv.KVStore
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	mediaHandler media.Handler
@@ -118,7 +118,6 @@ func (suite *MediaCreateTestSuite) TearDownTest() {
 */
 
 func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful() {
-
 	// set up the context for the request
 	t := suite.testTokens["local_account_1"]
 	oauthToken := oauth.DBTokenToToken(t)
@@ -130,10 +129,15 @@ func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful()
 	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
 
 	// see what's in storage *before* the request
-	storageKeysBeforeRequest, err := suite.storage.ListKeys()
+	storageKeysBeforeRequest := []string{}
+	iter, err := suite.storage.Iterator(nil)
 	if err != nil {
 		panic(err)
 	}
+	for iter.Next() {
+		storageKeysBeforeRequest = append(storageKeysBeforeRequest, iter.Key())
+	}
+	iter.Release()
 
 	// create the request
 	buf, w, err := testrig.CreateMultipartFormData("file", "../../../../testrig/media/test-jpeg.jpg", map[string]string{
@@ -150,10 +154,15 @@ func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful()
 	suite.mediaModule.MediaCreatePOSTHandler(ctx)
 
 	// check what's in storage *after* the request
-	storageKeysAfterRequest, err := suite.storage.ListKeys()
+	storageKeysAfterRequest := []string{}
+	iter, err = suite.storage.Iterator(nil)
 	if err != nil {
 		panic(err)
 	}
+	for iter.Next() {
+		storageKeysAfterRequest = append(storageKeysAfterRequest, iter.Key())
+	}
+	iter.Release()
 
 	// check response
 	suite.EqualValues(http.StatusOK, recorder.Code)
diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go
index f9b4e3671..214a00790 100644
--- a/internal/api/client/status/status_test.go
+++ b/internal/api/client/status/status_test.go
@@ -19,10 +19,10 @@
 package status_test
 
 import (
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -41,7 +41,7 @@ type StatusStandardTestSuite struct {
 	tc        typeutils.TypeConverter
 	federator federation.Federator
 	processor processing.Processor
-	storage   blob.Storage
+	storage   *kv.KVStore
 
 	// standard suite models
 	testTokens       map[string]*gtsmodel.Token
diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go
index ecd5fadc8..269ca4bab 100644
--- a/internal/api/s2s/user/user_test.go
+++ b/internal/api/s2s/user/user_test.go
@@ -1,11 +1,11 @@
 package user_test
 
 import (
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
 	"github.com/superseriousbusiness/gotosocial/internal/api/security"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -24,7 +24,7 @@ type UserStandardTestSuite struct {
 	tc             typeutils.TypeConverter
 	federator      federation.Federator
 	processor      processing.Processor
-	storage        blob.Storage
+	storage        *kv.KVStore
 	securityModule *security.Module
 
 	// standard suite models
diff --git a/internal/blob/inmem.go b/internal/blob/inmem.go
deleted file mode 100644
index 6ea64bcfe..000000000
--- a/internal/blob/inmem.go
+++ /dev/null
@@ -1,55 +0,0 @@
-package blob
-
-import (
-	"fmt"
-
-	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
-)
-
-// NewInMem returns an in-memory implementation of the Storage interface.
-// This is good for testing and whatnot but ***SHOULD ABSOLUTELY NOT EVER
-// BE USED IN A PRODUCTION SETTING***, because A) everything will be wiped out
-// if you restart the server and B) if you store lots of images your RAM use
-// will absolutely go through the roof.
-func NewInMem(c *config.Config, log *logrus.Logger) (Storage, error) {
-	return &inMemStorage{
-		stored: make(map[string][]byte),
-		log:    log,
-	}, nil
-}
-
-type inMemStorage struct {
-	stored map[string][]byte
-	log    *logrus.Logger
-}
-
-func (s *inMemStorage) StoreFileAt(path string, data []byte) error {
-	l := s.log.WithField("func", "StoreFileAt")
-	l.Debugf("storing at path %s", path)
-	s.stored[path] = data
-	return nil
-}
-
-func (s *inMemStorage) RetrieveFileFrom(path string) ([]byte, error) {
-	l := s.log.WithField("func", "RetrieveFileFrom")
-	l.Debugf("retrieving from path %s", path)
-	d, ok := s.stored[path]
-	if !ok || len(d) == 0 {
-		return nil, fmt.Errorf("no data found at path %s", path)
-	}
-	return d, nil
-}
-
-func (s *inMemStorage) ListKeys() ([]string, error) {
-	keys := []string{}
-	for k := range s.stored {
-		keys = append(keys, k)
-	}
-	return keys, nil
-}
-
-func (s *inMemStorage) RemoveFileAt(path string) error {
-	delete(s.stored, path)
-	return nil
-}
diff --git a/internal/blob/local.go b/internal/blob/local.go
deleted file mode 100644
index 01f82f8d2..000000000
--- a/internal/blob/local.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package blob
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
-)
-
-// NewLocal returns an implementation of the Storage interface that uses
-// the local filesystem for storing and retrieving files, attachments, etc.
-func NewLocal(c *config.Config, log *logrus.Logger) (Storage, error) {
-	return &localStorage{
-		config: c,
-		log:    log,
-	}, nil
-}
-
-type localStorage struct {
-	config *config.Config
-	log    *logrus.Logger
-}
-
-func (s *localStorage) StoreFileAt(path string, data []byte) error {
-	l := s.log.WithField("func", "StoreFileAt")
-	l.Debugf("storing at path %s", path)
-	components := strings.Split(path, "/")
-	dir := strings.Join(components[0:len(components)-1], "/")
-	if err := os.MkdirAll(dir, 0777); err != nil {
-		return fmt.Errorf("error writing file at %s: %s", path, err)
-	}
-	if err := os.WriteFile(path, data, 0777); err != nil {
-		return fmt.Errorf("error writing file at %s: %s", path, err)
-	}
-	return nil
-}
-
-func (s *localStorage) RetrieveFileFrom(path string) ([]byte, error) {
-	l := s.log.WithField("func", "RetrieveFileFrom")
-	l.Debugf("retrieving from path %s", path)
-	b, err := os.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("error reading file at %s: %s", path, err)
-	}
-	return b, nil
-}
-
-func (s *localStorage) ListKeys() ([]string, error) {
-	keys := []string{}
-	err := filepath.Walk(s.config.StorageConfig.BasePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !info.IsDir() {
-			keys = append(keys, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return keys, nil
-}
-
-func (s *localStorage) RemoveFileAt(path string) error {
-	return os.Remove(path)
-}
diff --git a/internal/blob/storage.go b/internal/blob/storage.go
deleted file mode 100644
index d1deb740f..000000000
--- a/internal/blob/storage.go
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package blob
-
-// Storage is an interface for storing and retrieving blobs
-// such as images, videos, and any other attachments/documents
-// that shouldn't be stored in a database.
-type Storage interface {
-	StoreFileAt(path string, data []byte) error
-	RetrieveFileFrom(path string) ([]byte, error)
-	ListKeys() ([]string, error)
-	RemoveFileAt(path string) error
-}
diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go
index 3ef714fb0..e2f6a24b4 100644
--- a/internal/cliactions/server/server.go
+++ b/internal/cliactions/server/server.go
@@ -8,6 +8,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
@@ -32,7 +33,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/api/security"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
@@ -76,7 +76,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 		return fmt.Errorf("error creating router: %s", err)
 	}
 
-	storageBackend, err := blob.NewLocal(c, log)
+	// Open the storage backend
+	storage, err := kv.OpenFile(c.StorageConfig.BasePath, nil)
 	if err != nil {
 		return fmt.Errorf("error creating storage backend: %s", err)
 	}
@@ -86,11 +87,11 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 	timelineManager := timelineprocessing.NewManager(dbService, typeConverter, c, log)
 
 	// build backend handlers
-	mediaHandler := media.New(c, dbService, storageBackend, log)
+	mediaHandler := media.New(c, dbService, storage, log)
 	oauthServer := oauth.New(dbService, log)
 	transportController := transport.NewController(c, dbService, &federation.Clock{}, http.DefaultClient, log)
 	federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler)
-	processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log)
+	processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storage, timelineManager, dbService, log)
 	if err := processor.Start(ctx); err != nil {
 		return fmt.Errorf("error starting processor: %s", err)
 	}
diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go
index b4cb68e25..41909ec4d 100644
--- a/internal/federation/dereferencing/dereferencer_test.go
+++ b/internal/federation/dereferencing/dereferencer_test.go
@@ -24,11 +24,11 @@ import (
 	"io"
 	"net/http"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
@@ -42,7 +42,7 @@ type DereferencerStandardTestSuite struct {
 	config  *config.Config
 	db      db.DB
 	log     *logrus.Logger
-	storage blob.Storage
+	storage *kv.KVStore
 
 	testRemoteStatuses    map[string]vocab.ActivityStreamsNote
 	testRemoteAccounts    map[string]vocab.ActivityStreamsPerson
diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go
index cb21d44c2..7a76089d7 100644
--- a/internal/federation/federator_test.go
+++ b/internal/federation/federator_test.go
@@ -24,13 +24,13 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/httpsig"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -45,7 +45,7 @@ type ProtocolTestSuite struct {
 	config        *config.Config
 	db            db.DB
 	log           *logrus.Logger
-	storage       blob.Storage
+	storage       *kv.KVStore
 	typeConverter typeutils.TypeConverter
 	accounts      map[string]*gtsmodel.Account
 	activities    map[string]testrig.ActivityWithSignature
@@ -65,7 +65,6 @@ func (suite *ProtocolTestSuite) SetupSuite() {
 
 func (suite *ProtocolTestSuite) SetupTest() {
 	testrig.StandardDBSetup(suite.db, suite.accounts)
-
 }
 
 // TearDownTest drops tables to make sure there's no data in the db
@@ -75,7 +74,6 @@ func (suite *ProtocolTestSuite) TearDownTest() {
 
 // make sure PostInboxRequestBodyHook properly sets the inbox username and activity on the context
 func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
-
 	// the activity we're gonna use
 	activity := suite.activities["dm_for_zork"]
 
@@ -106,7 +104,6 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 }
 
 func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
-
 	// the activity we're gonna use
 	activity := suite.activities["dm_for_zork"]
 	sendingAccount := suite.accounts["remote_account_1"]
diff --git a/internal/media/handler.go b/internal/media/handler.go
index 9c1d3227e..70c5c0826 100644
--- a/internal/media/handler.go
+++ b/internal/media/handler.go
@@ -26,8 +26,8 @@ import (
 	"strings"
 	"time"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -86,12 +86,12 @@ type Handler interface {
 type mediaHandler struct {
 	config  *config.Config
 	db      db.DB
-	storage blob.Storage
+	storage *kv.KVStore
 	log     *logrus.Logger
 }
 
 // New returns a new handler with the given config, db, storage, and logger
-func New(config *config.Config, database db.DB, storage blob.Storage, log *logrus.Logger) Handler {
+func New(config *config.Config, database db.DB, storage *kv.KVStore, log *logrus.Logger) Handler {
 	return &mediaHandler{
 		config:  config,
 		db:      database,
@@ -250,19 +250,19 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte
 
 	// serve url and storage path for the original emoji -- can be png or gif
 	emojiURL := fmt.Sprintf("%s/%s/%s/%s/%s.%s", URLbase, instanceAccount.ID, Emoji, Original, newEmojiID, extension)
-	emojiPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, instanceAccount.ID, Emoji, Original, newEmojiID, extension)
+	emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, Emoji, Original, newEmojiID, extension)
 
 	// serve url and storage path for the static version -- will always be png
 	emojiStaticURL := fmt.Sprintf("%s/%s/%s/%s/%s.png", URLbase, instanceAccount.ID, Emoji, Static, newEmojiID)
-	emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s/%s.png", mh.config.StorageConfig.BasePath, instanceAccount.ID, Emoji, Static, newEmojiID)
+	emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, Emoji, Static, newEmojiID)
 
-	// store the original
-	if err := mh.storage.StoreFileAt(emojiPath, original.image); err != nil {
+	// Store the original emoji
+	if err := mh.storage.Put(emojiPath, original.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
-	// store the static
-	if err := mh.storage.StoreFileAt(emojiStaticPath, static.image); err != nil {
+	// Store the static emoji
+	if err := mh.storage.Put(emojiStaticPath, static.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
@@ -293,7 +293,6 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte
 }
 
 func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) {
-
 	if !currentAttachment.Header && !currentAttachment.Avatar {
 		return nil, errors.New("provided attachment was set to neither header nor avatar")
 	}
diff --git a/internal/media/processicon.go b/internal/media/processicon.go
index 5fae63198..17e48fec2 100644
--- a/internal/media/processicon.go
+++ b/internal/media/processicon.go
@@ -84,14 +84,14 @@ func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string
 	smallURL := fmt.Sprintf("%s/%s/%s/small/%s.%s", URLbase, accountID, mediaType, newMediaID, extension)
 
 	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Original, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
+	originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, Original, newMediaID, extension)
+	if err := mh.storage.Put(originalPath, original.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
 	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Small, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
+	smallPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, Small, newMediaID, extension)
+	if err := mh.storage.Put(smallPath, small.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
diff --git a/internal/media/processimage.go b/internal/media/processimage.go
index a8a6d0716..03c820cc5 100644
--- a/internal/media/processimage.go
+++ b/internal/media/processimage.go
@@ -72,14 +72,14 @@ func (mh *mediaHandler) processImageAttachment(data []byte, minAttachment *gtsmo
 	smallURL := fmt.Sprintf("%s/%s/attachment/small/%s.jpeg", URLbase, minAttachment.AccountID, newMediaID) // all thumbnails/smalls are encoded as jpeg
 
 	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, minAttachment.AccountID, Attachment, Original, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
+	originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, Attachment, Original, newMediaID, extension)
+	if err := mh.storage.Put(originalPath, original.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
 	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.jpeg", mh.config.StorageConfig.BasePath, minAttachment.AccountID, Attachment, Small, newMediaID) // all thumbnails/smalls are encoded as jpeg
-	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
+	smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, Attachment, Small, newMediaID) // all thumbnails/smalls are encoded as jpeg
+	if err := mh.storage.Put(smallPath, small.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
@@ -130,5 +130,4 @@ func (mh *mediaHandler) processImageAttachment(data []byte, minAttachment *gtsmo
 	}
 
 	return attachment, nil
-
 }
diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
index 1884f6057..cfc7130ca 100644
--- a/internal/processing/account/account_test.go
+++ b/internal/processing/account/account_test.go
@@ -19,10 +19,10 @@
 package account_test
 
 import (
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -43,7 +43,7 @@ type AccountStandardTestSuite struct {
 	db                  db.DB
 	log                 *logrus.Logger
 	tc                  typeutils.TypeConverter
-	storage             blob.Storage
+	storage             *kv.KVStore
 	mediaHandler        media.Handler
 	oauthServer         oauth.Server
 	fromClientAPIChan   chan messages.FromClientAPI
diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go
index 281ddba03..e99b4e950 100644
--- a/internal/processing/media/delete.go
+++ b/internal/processing/media/delete.go
@@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
 
 	// delete the thumbnail from storage
 	if attachment.Thumbnail.Path != "" {
-		if err := p.storage.RemoveFileAt(attachment.Thumbnail.Path); err != nil {
+		if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil {
 			errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err))
 		}
 	}
 
 	// delete the file from storage
 	if attachment.File.Path != "" {
-		if err := p.storage.RemoveFileAt(attachment.File.Path); err != nil {
+		if err := p.storage.Delete(attachment.File.Path); err != nil {
 			errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err))
 		}
 	}
diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go
index c9c9b556d..3cfdbe56b 100644
--- a/internal/processing/media/getfile.go
+++ b/internal/processing/media/getfile.go
@@ -110,7 +110,7 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form
 		}
 	}
 
-	bytes, err := p.storage.RetrieveFileFrom(storagePath)
+	bytes, err := p.storage.Get(storagePath)
 	if err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
 	}
diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go
index 6b88143e2..4c8416483 100644
--- a/internal/processing/media/media.go
+++ b/internal/processing/media/media.go
@@ -21,9 +21,9 @@ package media
 import (
 	"context"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -47,13 +47,13 @@ type processor struct {
 	tc           typeutils.TypeConverter
 	config       *config.Config
 	mediaHandler media.Handler
-	storage      blob.Storage
+	storage      *kv.KVStore
 	db           db.DB
 	log          *logrus.Logger
 }
 
 // New returns a new media processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, storage blob.Storage, config *config.Config, log *logrus.Logger) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, storage *kv.KVStore, config *config.Config, log *logrus.Logger) Processor {
 	return &processor{
 		tc:           tc,
 		config:       config,
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index 5dcdca152..bc6e6511b 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -23,9 +23,9 @@ import (
 	"net/http"
 	"net/url"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -234,7 +234,7 @@ type processor struct {
 	tc              typeutils.TypeConverter
 	oauthServer     oauth.Server
 	mediaHandler    media.Handler
-	storage         blob.Storage
+	storage         *kv.KVStore
 	timelineManager timeline.Manager
 	db              db.DB
 	filter          visibility.Filter
@@ -251,8 +251,7 @@ type processor struct {
 }
 
 // NewProcessor returns a new Processor that uses the given federator and logger
-func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, mediaHandler media.Handler, storage blob.Storage, timelineManager timeline.Manager, db db.DB, log *logrus.Logger) Processor {
-
+func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, mediaHandler media.Handler, storage *kv.KVStore, timelineManager timeline.Manager, db db.DB, log *logrus.Logger) Processor {
 	fromClientAPI := make(chan messages.FromClientAPI, 1000)
 	fromFederator := make(chan messages.FromFederator, 1000)
 
diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
index 7335b686d..daaf46726 100644
--- a/internal/processing/processor_test.go
+++ b/internal/processing/processor_test.go
@@ -21,9 +21,9 @@ package processing_test
 import (
 	"context"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -43,7 +43,7 @@ type ProcessingStandardTestSuite struct {
 	config              *config.Config
 	db                  db.DB
 	log                 *logrus.Logger
-	storage             blob.Storage
+	storage             *kv.KVStore
 	typeconverter       typeutils.TypeConverter
 	transportController transport.Controller
 	federator           federation.Federator