[chore] media and emoji refactoring (#3000)

* start updating media manager interface ready for storing attachments / emoji right away

* store emoji and media as uncached immediately, then (re-)cache on Processing{}.Load()

* remove now unused media workers

* fix tests and issues

* fix another test!

* fix emoji activitypub uri setting behaviour, fix remainder of test compilation issues

* fix more tests

* fix (most of) remaining tests, add debouncing to repeatedly failing media / emojis

* whoops, rebase issue

* remove kim's whacky experiments

* do some reshuffling, ensure emoji uri gets set

* ensure marked as not cached on cleanup

* tweaks to media / emoji processing to handle context canceled better

* ensure newly fetched emojis actually get set in returned slice

* use different varnames to be a bit more obvious

* move emoji refresh rate limiting to dereferencer

* add exported dereferencer functions for remote media, use these for recaching in processor

* add check for nil attachment in updateAttachment()

* remove unused emoji and media fields + columns

* see previous commit

* fix old migrations expecting image_updated_at to exists (from copies of old models)

* remove freshness checking code (seems to be broken...)

* fix error arg causing nil ptr exception

* finish documentating functions with comments, slight tweaks to media / emoji deref error logic

* remove some extra unneeded boolean checking

* finish writing documentation (code comments) for exported media manager methods

* undo changes to migration snapshot gtsmodels, updated failing migration to have its own snapshot

* move doesColumnExist() to util.go in migrations package

2 files changed, 55 insertions, 49 deletions

diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
index 556f4d91f..8eec1f9dd 100644
--- a/internal/processing/account/account_test.go
+++ b/internal/processing/account/account_test.go
@@ -111,7 +111,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
 
 	filter := visibility.NewFilter(&suite.state)
-	common := common.New(&suite.state, suite.tc, suite.federator, filter)
+	common := common.New(&suite.state, suite.mediaManager, suite.tc, suite.federator, filter)
 	suite.accountProcessor = account.New(&common, &suite.state, suite.tc, suite.mediaManager, suite.federator, filter, processing.GetParseMentionFunc(&suite.state, suite.federator))
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go
index ea6abed6e..61e88501f 100644
--- a/internal/processing/account/update.go
+++ b/internal/processing/account/update.go
@@ -19,10 +19,12 @@ package account
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"mime/multipart"
 
+	"codeberg.org/gruf/go-bytesize"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -203,9 +205,13 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
 	}
 
 	if form.Avatar != nil && form.Avatar.Size != 0 {
-		avatarInfo, err := p.UpdateAvatar(ctx, form.Avatar, nil, account.ID)
-		if err != nil {
-			return nil, gtserror.NewErrorBadRequest(err)
+		avatarInfo, errWithCode := p.UpdateAvatar(ctx,
+			account,
+			form.Avatar,
+			nil,
+		)
+		if errWithCode != nil {
+			return nil, errWithCode
 		}
 		account.AvatarMediaAttachmentID = avatarInfo.ID
 		account.AvatarMediaAttachment = avatarInfo
@@ -213,9 +219,13 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
 	}
 
 	if form.Header != nil && form.Header.Size != 0 {
-		headerInfo, err := p.UpdateHeader(ctx, form.Header, nil, account.ID)
-		if err != nil {
-			return nil, gtserror.NewErrorBadRequest(err)
+		headerInfo, errWithCode := p.UpdateHeader(ctx,
+			account,
+			form.Header,
+			nil,
+		)
+		if errWithCode != nil {
+			return nil, errWithCode
 		}
 		account.HeaderMediaAttachmentID = headerInfo.ID
 		account.HeaderMediaAttachment = headerInfo
@@ -316,35 +326,33 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
 // for this to become the account's new avatar.
 func (p *Processor) UpdateAvatar(
 	ctx context.Context,
+	account *gtsmodel.Account,
 	avatar *multipart.FileHeader,
 	description *string,
-	accountID string,
-) (*gtsmodel.MediaAttachment, error) {
-	maxImageSize := config.GetMediaImageMaxSize()
-	if avatar.Size > int64(maxImageSize) {
-		return nil, gtserror.Newf("size %d exceeded max media size of %d bytes", avatar.Size, maxImageSize)
+) (
+	*gtsmodel.MediaAttachment,
+	gtserror.WithCode,
+) {
+	max := config.GetMediaImageMaxSize()
+	if sz := bytesize.Size(avatar.Size); sz > max {
+		text := fmt.Sprintf("size %s exceeds max media size %s", sz, max)
+		return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
 	}
 
-	data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
 		f, err := avatar.Open()
 		return f, avatar.Size, err
 	}
 
-	// Process the media attachment and load it immediately.
-	media := p.mediaManager.PreProcessMedia(data, accountID, &media.AdditionalMediaInfo{
-		Avatar:      util.Ptr(true),
-		Description: description,
-	})
-
-	attachment, err := media.LoadAttachment(ctx)
-	if err != nil {
-		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
-	} else if attachment.Type == gtsmodel.FileTypeUnknown {
-		err = gtserror.Newf("could not process uploaded file with extension %s", attachment.File.ContentType)
-		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
-	}
-
-	return attachment, nil
+	// Write to instance storage.
+	return p.c.StoreLocalMedia(ctx,
+		account.ID,
+		data,
+		media.AdditionalMediaInfo{
+			Avatar:      util.Ptr(true),
+			Description: description,
+		},
+	)
 }
 
 // UpdateHeader does the dirty work of checking the header
@@ -353,33 +361,31 @@ func (p *Processor) UpdateAvatar(
 // for this to become the account's new header.
 func (p *Processor) UpdateHeader(
 	ctx context.Context,
+	account *gtsmodel.Account,
 	header *multipart.FileHeader,
 	description *string,
-	accountID string,
-) (*gtsmodel.MediaAttachment, error) {
-	maxImageSize := config.GetMediaImageMaxSize()
-	if header.Size > int64(maxImageSize) {
-		return nil, gtserror.Newf("size %d exceeded max media size of %d bytes", header.Size, maxImageSize)
+) (
+	*gtsmodel.MediaAttachment,
+	gtserror.WithCode,
+) {
+	max := config.GetMediaImageMaxSize()
+	if sz := bytesize.Size(header.Size); sz > max {
+		text := fmt.Sprintf("size %s exceeds max media size %s", sz, max)
+		return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
 	}
 
-	data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
 		f, err := header.Open()
 		return f, header.Size, err
 	}
 
-	// Process the media attachment and load it immediately.
-	media := p.mediaManager.PreProcessMedia(data, accountID, &media.AdditionalMediaInfo{
-		Header:      util.Ptr(true),
-		Description: description,
-	})
-
-	attachment, err := media.LoadAttachment(ctx)
-	if err != nil {
-		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
-	} else if attachment.Type == gtsmodel.FileTypeUnknown {
-		err = gtserror.Newf("could not process uploaded file with extension %s", attachment.File.ContentType)
-		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
-	}
-
-	return attachment, nil
+	// Write to instance storage.
+	return p.c.StoreLocalMedia(ctx,
+		account.ID,
+		data,
+		media.AdditionalMediaInfo{
+			Header:      util.Ptr(true),
+			Description: description,
+		},
+	)
 }