[chore] media and emoji refactoring (#3000)

* start updating media manager interface ready for storing attachments / emoji right away

* store emoji and media as uncached immediately, then (re-)cache on Processing{}.Load()

* remove now unused media workers

* fix tests and issues

* fix another test!

* fix emoji activitypub uri setting behaviour, fix remainder of test compilation issues

* fix more tests

* fix (most of) remaining tests, add debouncing to repeatedly failing media / emojis

* whoops, rebase issue

* remove kim's whacky experiments

* do some reshuffling, ensure emoji uri gets set

* ensure marked as not cached on cleanup

* tweaks to media / emoji processing to handle context canceled better

* ensure newly fetched emojis actually get set in returned slice

* use different varnames to be a bit more obvious

* move emoji refresh rate limiting to dereferencer

* add exported dereferencer functions for remote media, use these for recaching in processor

* add check for nil attachment in updateAttachment()

* remove unused emoji and media fields + columns

* see previous commit

* fix old migrations expecting image_updated_at to exists (from copies of old models)

* remove freshness checking code (seems to be broken...)

* fix error arg causing nil ptr exception

* finish documentating functions with comments, slight tweaks to media / emoji deref error logic

* remove some extra unneeded boolean checking

* finish writing documentation (code comments) for exported media manager methods

* undo changes to migration snapshot gtsmodels, updated failing migration to have its own snapshot

* move doesColumnExist() to util.go in migrations package

2 files changed, 102 insertions, 0 deletions

diff --git a/internal/processing/common/common.go b/internal/processing/common/common.go
index e4a49cc45..942cecc59 100644
--- a/internal/processing/common/common.go
+++ b/internal/processing/common/common.go
@@ -20,6 +20,7 @@ package common
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -29,6 +30,7 @@ import (
 // processing subsection of the codebase.
 type Processor struct {
 	state     *state.State
+	media     *media.Manager
 	converter *typeutils.Converter
 	federator *federation.Federator
 	filter    *visibility.Filter
@@ -37,12 +39,14 @@ type Processor struct {
 // New returns a new Processor instance.
 func New(
 	state *state.State,
+	media *media.Manager,
 	converter *typeutils.Converter,
 	federator *federation.Federator,
 	filter *visibility.Filter,
 ) Processor {
 	return Processor{
 		state:     state,
+		media:     media,
 		converter: converter,
 		federator: federator,
 		filter:    filter,
diff --git a/internal/processing/common/media.go b/internal/processing/common/media.go
new file mode 100644
index 000000000..7baf30345
--- /dev/null
+++ b/internal/processing/common/media.go
@@ -0,0 +1,98 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package common
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+// StoreLocalMedia is a wrapper around CreateMedia() and
+// ProcessingMedia{}.Load() with appropriate error responses.
+func (p *Processor) StoreLocalMedia(
+	ctx context.Context,
+	accountID string,
+	data media.DataFunc,
+	info media.AdditionalMediaInfo,
+) (
+	*gtsmodel.MediaAttachment,
+	gtserror.WithCode,
+) {
+	// Create a new processing media attachment.
+	processing, err := p.media.CreateMedia(ctx,
+		accountID,
+		data,
+		info,
+	)
+	if err != nil {
+		err := gtserror.Newf("error creating media: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Immediately trigger write to storage.
+	attachment, err := processing.Load(ctx)
+	if err != nil {
+		const text = "error processing emoji"
+		err := gtserror.Newf("error processing media: %w", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, text)
+	} else if attachment.Type == gtsmodel.FileTypeUnknown {
+		text := fmt.Sprintf("could not process %s type media", attachment.File.ContentType)
+		return nil, gtserror.NewErrorUnprocessableEntity(errors.New(text), text)
+	}
+
+	return attachment, nil
+}
+
+// StoreLocalMedia is a wrapper around CreateMedia() and
+// ProcessingMedia{}.Load() with appropriate error responses.
+func (p *Processor) StoreLocalEmoji(
+	ctx context.Context,
+	shortcode string,
+	data media.DataFunc,
+	info media.AdditionalEmojiInfo,
+) (
+	*gtsmodel.Emoji,
+	gtserror.WithCode,
+) {
+	// Create a new processing emoji media.
+	processing, err := p.media.CreateEmoji(ctx,
+		shortcode,
+		"", // domain = "" -> local
+		data,
+		info,
+	)
+	if err != nil {
+		err := gtserror.Newf("error creating emoji: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Immediately write to storage.
+	emoji, err := processing.Load(ctx)
+	if err != nil {
+		const text = "error processing emoji"
+		err := gtserror.Newf("error processing emoji %s: %w", shortcode, err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, text)
+	}
+
+	return emoji, nil
+}