From 21bb324156f582e918a097ea744e52fc21b2ddf4 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:01:16 +0000 Subject: [chore] media and emoji refactoring (#3000) * start updating media manager interface ready for storing attachments / emoji right away * store emoji and media as uncached immediately, then (re-)cache on Processing{}.Load() * remove now unused media workers * fix tests and issues * fix another test! * fix emoji activitypub uri setting behaviour, fix remainder of test compilation issues * fix more tests * fix (most of) remaining tests, add debouncing to repeatedly failing media / emojis * whoops, rebase issue * remove kim's whacky experiments * do some reshuffling, ensure emoji uri gets set * ensure marked as not cached on cleanup * tweaks to media / emoji processing to handle context canceled better * ensure newly fetched emojis actually get set in returned slice * use different varnames to be a bit more obvious * move emoji refresh rate limiting to dereferencer * add exported dereferencer functions for remote media, use these for recaching in processor * add check for nil attachment in updateAttachment() * remove unused emoji and media fields + columns * see previous commit * fix old migrations expecting image_updated_at to exists (from copies of old models) * remove freshness checking code (seems to be broken...) * fix error arg causing nil ptr exception * finish documentating functions with comments, slight tweaks to media / emoji deref error logic * remove some extra unneeded boolean checking * finish writing documentation (code comments) for exported media manager methods * undo changes to migration snapshot gtsmodels, updated failing migration to have its own snapshot * move doesColumnExist() to util.go in migrations package --- internal/processing/common/common.go | 4 ++ internal/processing/common/media.go | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 internal/processing/common/media.go (limited to 'internal/processing/common') diff --git a/internal/processing/common/common.go b/internal/processing/common/common.go index e4a49cc45..942cecc59 100644 --- a/internal/processing/common/common.go +++ b/internal/processing/common/common.go @@ -20,6 +20,7 @@ package common import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -29,6 +30,7 @@ import ( // processing subsection of the codebase. type Processor struct { state *state.State + media *media.Manager converter *typeutils.Converter federator *federation.Federator filter *visibility.Filter @@ -37,12 +39,14 @@ type Processor struct { // New returns a new Processor instance. func New( state *state.State, + media *media.Manager, converter *typeutils.Converter, federator *federation.Federator, filter *visibility.Filter, ) Processor { return Processor{ state: state, + media: media, converter: converter, federator: federator, filter: filter, diff --git a/internal/processing/common/media.go b/internal/processing/common/media.go new file mode 100644 index 000000000..7baf30345 --- /dev/null +++ b/internal/processing/common/media.go @@ -0,0 +1,98 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package common + +import ( + "context" + "errors" + "fmt" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" +) + +// StoreLocalMedia is a wrapper around CreateMedia() and +// ProcessingMedia{}.Load() with appropriate error responses. +func (p *Processor) StoreLocalMedia( + ctx context.Context, + accountID string, + data media.DataFunc, + info media.AdditionalMediaInfo, +) ( + *gtsmodel.MediaAttachment, + gtserror.WithCode, +) { + // Create a new processing media attachment. + processing, err := p.media.CreateMedia(ctx, + accountID, + data, + info, + ) + if err != nil { + err := gtserror.Newf("error creating media: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Immediately trigger write to storage. + attachment, err := processing.Load(ctx) + if err != nil { + const text = "error processing emoji" + err := gtserror.Newf("error processing media: %w", err) + return nil, gtserror.NewErrorUnprocessableEntity(err, text) + } else if attachment.Type == gtsmodel.FileTypeUnknown { + text := fmt.Sprintf("could not process %s type media", attachment.File.ContentType) + return nil, gtserror.NewErrorUnprocessableEntity(errors.New(text), text) + } + + return attachment, nil +} + +// StoreLocalMedia is a wrapper around CreateMedia() and +// ProcessingMedia{}.Load() with appropriate error responses. +func (p *Processor) StoreLocalEmoji( + ctx context.Context, + shortcode string, + data media.DataFunc, + info media.AdditionalEmojiInfo, +) ( + *gtsmodel.Emoji, + gtserror.WithCode, +) { + // Create a new processing emoji media. + processing, err := p.media.CreateEmoji(ctx, + shortcode, + "", // domain = "" -> local + data, + info, + ) + if err != nil { + err := gtserror.Newf("error creating emoji: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Immediately write to storage. + emoji, err := processing.Load(ctx) + if err != nil { + const text = "error processing emoji" + err := gtserror.Newf("error processing emoji %s: %w", shortcode, err) + return nil, gtserror.NewErrorUnprocessableEntity(err, text) + } + + return emoji, nil +} -- cgit v1.2.3