From 6803c1682b26a0d78326dbe45b06f61ac0476d0d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 27 Dec 2021 18:03:36 +0100 Subject: start refactor of media package --- internal/media/types.go | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 internal/media/types.go (limited to 'internal/media/types.go') diff --git a/internal/media/types.go b/internal/media/types.go new file mode 100644 index 000000000..f1608f880 --- /dev/null +++ b/internal/media/types.go @@ -0,0 +1,149 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "bytes" + "errors" + "fmt" + + "github.com/h2non/filetype" +) + +// mime consts +const ( + mimeImage = "image" + + mimeJpeg = "jpeg" + mimeImageJpeg = mimeImage + "/" + mimeJpeg + + mimeGif = "gif" + mimeImageGif = mimeImage + "/" + mimeGif + + mimePng = "png" + mimeImagePng = mimeImage + "/" + mimePng +) + + +// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) +// const EmojiMaxBytes = 51200 + +// maxFileHeaderBytes represents the maximum amount of bytes we want +// to examine from the beginning of a file to determine its type. +// +// See: https://en.wikipedia.org/wiki/File_format#File_header +// and https://github.com/h2non/filetype +const maxFileHeaderBytes = 262 + +type Size string + +const ( + SizeSmall Size = "small" // SizeSmall is the key for small/thumbnail versions of media + SizeOriginal Size = "original" // SizeOriginal is the key for original/fullsize versions of media and emoji + SizeStatic Size = "static" // SizeStatic is the key for static (non-animated) versions of emoji +) + +type Type string + +const ( + TypeAttachment Type = "attachment" // TypeAttachment is the key for media attachments + TypeHeader Type = "header" // TypeHeader is the key for profile header requests + TypeAvatar Type = "avatar" // TypeAvatar is the key for profile avatar requests + TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests +) + +// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). +// Returns an error if the content type is not something we can process. +func parseContentType(content []byte) (string, error) { + + // read in the first bytes of the file + fileHeader := make([]byte, maxFileHeaderBytes) + if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { + return "", fmt.Errorf("could not read first magic bytes of file: %s", err) + } + + kind, err := filetype.Match(fileHeader) + if err != nil { + return "", err + } + + if kind == filetype.Unknown { + return "", errors.New("filetype unknown") + } + + return kind.MIME.Value, nil +} + +// supportedImage checks mime type of an image against a slice of accepted types, +// and returns True if the mime type is accepted. +func supportedImage(mimeType string) bool { + acceptedImageTypes := []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedImageTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. +func supportedEmoji(mimeType string) bool { + acceptedEmojiTypes := []string{ + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedEmojiTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized +func ParseMediaType(s string) (Type, error) { + switch s { + case string(TypeAttachment): + return TypeAttachment, nil + case string(TypeHeader): + return TypeHeader, nil + case string(TypeAvatar): + return TypeAvatar, nil + case string(TypeEmoji): + return TypeEmoji, nil + } + return "", fmt.Errorf("%s not a recognized MediaType", s) +} + +// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized +func ParseMediaSize(s string) (Size, error) { + switch s { + case string(SizeSmall): + return SizeSmall, nil + case string(SizeOriginal): + return SizeOriginal, nil + case string(SizeStatic): + return SizeStatic, nil + } + return "", fmt.Errorf("%s not a recognized MediaSize", s) +} -- cgit v1.2.3 From c4d63d125b5a44c150a00b0b20b3638cad9221f8 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 28 Dec 2021 16:36:00 +0100 Subject: more refactoring, media handler => manager --- cmd/gotosocial/action/server/server.go | 6 +- internal/api/client/fileserver/servefile_test.go | 4 +- internal/api/client/media/mediacreate_test.go | 4 +- internal/api/s2s/webfinger/webfingerget_test.go | 4 +- internal/federation/dereferencing/account.go | 4 +- internal/federation/dereferencing/attachment.go | 2 +- internal/federation/dereferencing/dereferencer.go | 6 +- .../federation/dereferencing/dereferencer_test.go | 2 +- internal/federation/federator.go | 8 +- internal/federation/federator_test.go | 4 +- internal/media/handler.go | 289 --------------------- internal/media/image.go | 241 +++++++++++++++-- internal/media/manager.go | 274 +++++++++++++++++++ internal/media/processicon.go | 143 ---------- internal/media/processing.go | 190 -------------- internal/media/processvideo.go | 2 +- internal/media/types.go | 1 - internal/media/util_test.go | 2 +- internal/processing/account/account.go | 6 +- internal/processing/account/account_test.go | 6 +- internal/processing/account/update.go | 4 +- internal/processing/admin/admin.go | 6 +- internal/processing/admin/emoji.go | 4 +- internal/processing/media/create.go | 4 +- internal/processing/media/media.go | 6 +- internal/processing/processor.go | 12 +- internal/processing/processor_test.go | 6 +- testrig/federator.go | 2 +- testrig/mediahandler.go | 4 +- testrig/processor.go | 2 +- 30 files changed, 545 insertions(+), 703 deletions(-) delete mode 100644 internal/media/handler.go create mode 100644 internal/media/manager.go delete mode 100644 internal/media/processicon.go delete mode 100644 internal/media/processing.go (limited to 'internal/media/types.go') diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index d55f38fc5..05c2e8974 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,10 +105,10 @@ var Start action.GTSAction = func(ctx context.Context) error { } // build backend handlers - mediaHandler := media.New(dbService, storage) + mediaManager := media.New(dbService, storage) oauthServer := oauth.New(ctx, dbService) transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient) - federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaHandler) + federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) // decide whether to create a noop email sender (won't send emails) or a real one var emailSender email.Sender @@ -128,7 +128,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } // create and start the message processor using the other services we've created so far - processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, timelineManager, dbService, emailSender) + processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, timelineManager, dbService, emailSender) if err := processor.Start(ctx); err != nil { return fmt.Errorf("error starting processor: %s", err) } diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index 18f54542a..109ed4eba 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -51,7 +51,7 @@ type ServeFileTestSuite struct { federator federation.Federator tc typeutils.TypeConverter processor processing.Processor - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender @@ -82,7 +82,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) // setup module being tested diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 2897d786b..72a377c25 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -53,7 +53,7 @@ type MediaCreateTestSuite struct { storage *kv.KVStore federator federation.Federator tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender processor processing.Processor @@ -81,7 +81,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go index 4b27ada42..c5df1f7e5 100644 --- a/internal/api/s2s/webfinger/webfingerget_test.go +++ b/internal/api/s2s/webfinger/webfingerget_test.go @@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() { func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() @@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index d06ad21c1..19c98e203 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -246,7 +246,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ + a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ RemoteURL: targetAccount.AvatarRemoteURL, Avatar: true, }, targetAccount.ID) @@ -257,7 +257,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ + a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ RemoteURL: targetAccount.HeaderRemoteURL, Header: true, }, targetAccount.ID) diff --git a/internal/federation/dereferencing/attachment.go b/internal/federation/dereferencing/attachment.go index 0c7005e23..30ab6da10 100644 --- a/internal/federation/dereferencing/attachment.go +++ b/internal/federation/dereferencing/attachment.go @@ -93,7 +93,7 @@ func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) } - a, err := d.mediaHandler.ProcessAttachment(ctx, attachmentBytes, minAttachment) + a, err := d.mediaManager.ProcessAttachment(ctx, attachmentBytes, minAttachment) if err != nil { return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index d0b653920..4f977b8c8 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -80,18 +80,18 @@ type deref struct { db db.DB typeConverter typeutils.TypeConverter transportController transport.Controller - mediaHandler media.Handler + mediaManager media.Manager handshakes map[string][]*url.URL handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewDereferencer returns a Dereferencer initialized with the given parameters. -func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler) Dereferencer { +func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer { return &deref{ db: db, typeConverter: typeConverter, transportController: transportController, - mediaHandler: mediaHandler, + mediaManager: mediaManager, handshakeSync: &sync.Mutex{}, } } diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index 569e8e93b..fe66abce4 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -64,7 +64,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaHandler(suite.db, suite.storage)) + suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 0a82f12bc..7edff1118 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -78,13 +78,13 @@ type federator struct { typeConverter typeutils.TypeConverter transportController transport.Controller dereferencer dereferencing.Dereferencer - mediaHandler media.Handler + mediaManager media.Manager actor pub.FederatingActor } // NewFederator returns a new federator -func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator { - dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaHandler) +func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaManager media.Manager) Federator { + dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaManager) clock := &Clock{} f := &federator{ @@ -94,7 +94,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr typeConverter: typeConverter, transportController: transportController, dereferencer: dereferencer, - mediaHandler: mediaHandler, + mediaManager: mediaManager, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 43f4904a5..6dac76c05 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -78,7 +78,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { return nil, nil }), suite.db) // setup module being tested - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) // setup request ctx := context.Background() @@ -107,7 +107,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) // now setup module being tested, with the mock transport controller - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil) // we need these headers for the request to be validated diff --git a/internal/media/handler.go b/internal/media/handler.go deleted file mode 100644 index b64e583b3..000000000 --- a/internal/media/handler.go +++ /dev/null @@ -1,289 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "context" - "errors" - "fmt" - "net/url" - "strings" - "time" - - "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - - - -type ProcessedCallback func(*gtsmodel.MediaAttachment) error - -// Handler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs. -type Handler interface { - ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessAvatar(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessEmoji(ctx context.Context, data []byte, shortcode string) (*gtsmodel.Emoji, error) -} - -type mediaHandler struct { - db db.DB - storage *kv.KVStore -} - -// New returns a new handler with the given db and storage -func New(database db.DB, storage *kv.KVStore) Handler { - return &mediaHandler{ - db: database, - storage: storage, - } -} - -/* - INTERFACE FUNCTIONS -*/ - -// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, -// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, -// and then returns information to the caller about the new header. -func (mh *mediaHandler) ProcessHeaderOrAvatar(ctx context.Context, attachment []byte, accountID string, mediaType Type, remoteURL string) (*gtsmodel.MediaAttachment, error) { - l := logrus.WithField("func", "SetHeaderForAccountID") - - if mediaType != TypeHeader && mediaType != TypeAvatar { - return nil, errors.New("header or avatar not selected") - } - - // make sure we have a type we can handle - contentType, err := parseContentType(attachment) - if err != nil { - return nil, err - } - if !supportedImage(contentType) { - return nil, fmt.Errorf("%s is not an accepted image type", contentType) - } - - if len(attachment) == 0 { - return nil, fmt.Errorf("passed reader was of size 0") - } - l.Tracef("read %d bytes of file", len(attachment)) - - // process it - ma, err := mh.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL) - if err != nil { - return nil, fmt.Errorf("error processing %s: %s", mediaType, err) - } - - // set it in the database - if err := mh.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil { - return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err) - } - - return ma, nil -} - -// ProcessAttachment takes a new attachment and the owning account, checks it out, removes exif data from it, -// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media, -// and then returns information to the caller about the attachment. -func (mh *mediaHandler) ProcessAttachment(ctx context.Context, attachmentBytes []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - contentType, err := parseContentType(attachmentBytes) - if err != nil { - return nil, err - } - - minAttachment.File.ContentType = contentType - - mainType := strings.Split(contentType, "/")[0] - switch mainType { - // case MIMEVideo: - // if !SupportedVideoType(contentType) { - // return nil, fmt.Errorf("video type %s not supported", contentType) - // } - // if len(attachment) == 0 { - // return nil, errors.New("video was of size 0") - // } - // return mh.processVideoAttachment(attachment, accountID, contentType, remoteURL) - case mimeImage: - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - if len(attachmentBytes) == 0 { - return nil, errors.New("image was of size 0") - } - return mh.processImageAttachment(attachmentBytes, minAttachment) - default: - break - } - return nil, fmt.Errorf("content type %s not (yet) supported", contentType) -} - -// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new -// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct -// in the database. -func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) { - var clean []byte - var err error - var original *imageAndMeta - var static *imageAndMeta - - // check content type of the submitted emoji and make sure it's supported by us - contentType, err := parseContentType(emojiBytes) - if err != nil { - return nil, err - } - if !supportedEmoji(contentType) { - return nil, fmt.Errorf("content type %s not supported for emojis", contentType) - } - - if len(emojiBytes) == 0 { - return nil, errors.New("emoji was of size 0") - } - if len(emojiBytes) > EmojiMaxBytes { - return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) - } - - // clean any exif data from png but leave gifs alone - switch contentType { - case mimePng: - if clean, err = purgeExif(emojiBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - case mimeGif: - clean = emojiBytes - default: - return nil, errors.New("media type unrecognized") - } - - // unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc - original = &imageAndMeta{ - image: clean, - } - - static, err = deriveStaticEmoji(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error deriving static emoji: %s", err) - } - - // since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver, - // (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created - // with the same username as the instance hostname, which doesn't belong to any particular user. - instanceAccount, err := mh.db.GetInstanceAccount(ctx, "") - if err != nil { - return nil, fmt.Errorf("error fetching instance account: %s", err) - } - - // the file extension (either png or gif) - extension := strings.Split(contentType, "/")[1] - - // generate a ulid for the new emoji - newEmojiID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - // activitypub uri for the emoji -- unrelated to actually serving the image - // will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC - emojiURI := uris.GenerateURIForEmoji(newEmojiID) - - // serve url and storage path for the original emoji -- can be png or gif - emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension) - emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension) - - // serve url and storage path for the static version -- will always be png - emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png") - emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID) - - // Store the original emoji - if err := mh.storage.Put(emojiPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // Store the static emoji - if err := mh.storage.Put(emojiStaticPath, static.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and finally return the new emoji data to the caller -- it's up to them what to do with it - e := >smodel.Emoji{ - ID: newEmojiID, - Shortcode: shortcode, - Domain: "", // empty because this is a local emoji - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - ImageRemoteURL: "", // empty because this is a local emoji - ImageStaticRemoteURL: "", // empty because this is a local emoji - ImageURL: emojiURL, - ImageStaticURL: emojiStaticURL, - ImagePath: emojiPath, - ImageStaticPath: emojiStaticPath, - ImageContentType: contentType, - ImageStaticContentType: mimePng, // static version will always be a png - ImageFileSize: len(original.image), - ImageStaticFileSize: len(static.image), - ImageUpdatedAt: time.Now(), - Disabled: false, - URI: emojiURI, - VisibleInPicker: true, - CategoryID: "", // empty because this is a new emoji -- no category yet - } - return e, nil -} - -func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { - if !currentAttachment.Header && !currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to neither header nor avatar") - } - - if currentAttachment.Header && currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to both header and avatar") - } - - var headerOrAvi Type - if currentAttachment.Header { - headerOrAvi = TypeHeader - } else if currentAttachment.Avatar { - headerOrAvi = TypeAvatar - } - - if currentAttachment.RemoteURL == "" { - return nil, errors.New("no remote URL on media attachment to dereference") - } - remoteIRI, err := url.Parse(currentAttachment.RemoteURL) - if err != nil { - return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) - } - - // for content type, we assume we don't know what to expect... - expectedContentType := "*/*" - if currentAttachment.File.ContentType != "" { - // ... and then narrow it down if we do - expectedContentType = currentAttachment.File.ContentType - } - - attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType) - if err != nil { - return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) - } - - return mh.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL) -} diff --git a/internal/media/image.go b/internal/media/image.go index f1cc03bb6..87b5d70b7 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -19,67 +19,88 @@ package media import ( + "bytes" "errors" "fmt" + "image" + "image/gif" + "image/jpeg" + "image/png" "strings" "time" + "github.com/buckket/go-blurhash" + "github.com/nfnt/resize" + "github.com/superseriousbusiness/exifremove/pkg/exifremove" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/uris" ) -func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { +const ( + thumbnailMaxWidth = 512 + thumbnailMaxHeight = 512 +) + +type imageAndMeta struct { + image []byte + width int + height int + size int + aspect float64 + blurhash string +} + +func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.MediaAttachment, error) { var clean []byte var err error var original *imageAndMeta var small *imageAndMeta - contentType := minAttachment.File.ContentType - switch contentType { - case mimeJpeg, mimePng: - if clean, err = purgeExif(data); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(data); exifErr != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) } - original, err = deriveImage(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing image: %s", err) - } - case mimeGif: + original, err = decodeImage(clean, contentType) + case mimeImageGif: + // gifs are already clean - no exif data to remove clean = data - original, err = deriveGif(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing gif: %s", err) - } + original, err = decodeGif(clean, contentType) default: - return nil, errors.New("media type unrecognized") + err = fmt.Errorf("content type %s not a recognized image type", contentType) + } + + if err != nil { + return nil, err } - small, err = deriveThumbnail(clean, contentType, 512, 512) + small, err = deriveThumbnail(clean, contentType, thumbnailMaxWidth, thumbnailMaxHeight) if err != nil { return nil, fmt.Errorf("error deriving thumbnail: %s", err) } // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it extension := strings.Split(contentType, "/")[1] - newMediaID, err := id.NewRandomULID() + attachmentID, err := id.NewRandomULID() if err != nil { return nil, err } - originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), newMediaID, extension) - smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), newMediaID, "jpeg") // all thumbnails/smalls are encoded as jpeg + originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), attachmentID, extension) + smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), attachmentID, "jpeg") // all thumbnails/smalls are encoded as jpeg // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, newMediaID, extension) - if err := mh.storage.Put(originalPath, original.image); err != nil { + originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, attachmentID, extension) + if err := m.storage.Put(originalPath, original.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, newMediaID) // all thumbnails/smalls are encoded as jpeg - if err := mh.storage.Put(smallPath, small.image); err != nil { + smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, attachmentID) // all thumbnails/smalls are encoded as jpeg + if err := m.storage.Put(smallPath, small.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } @@ -98,7 +119,7 @@ func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaA } attachment := >smodel.MediaAttachment{ - ID: newMediaID, + ID: attachmentID, StatusID: minAttachment.StatusID, URL: originalURL, RemoteURL: minAttachment.RemoteURL, @@ -131,3 +152,173 @@ func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaA return attachment, nil } + +func decodeGif(b []byte, extension string) (*imageAndMeta, error) { + var g *gif.GIF + var err error + + switch extension { + case mimeGif: + g, err = gif.DecodeAll(bytes.NewReader(b)) + default: + err = fmt.Errorf("extension %s not recognised", extension) + } + + if err != nil { + return nil, err + } + + // use the first frame to get the static characteristics + width := g.Config.Width + height := g.Config.Height + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +func decodeImage(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + default: + err = fmt.Errorf("content type %s not recognised", contentType) + } + + if err != nil { + return nil, err + } + + if i == nil { + return nil, errors.New("processed image was nil") + } + + width := i.Bounds().Size().X + height := i.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, +// of a given jpeg, png, or gif, or an error if something goes wrong. +// +// Note that the aspect ratio of the image will be retained, +// so it will not necessarily be a square, even if x and y are set as the same value. +func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not recognised", contentType) + } + + thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) + width := thumb.Bounds().Size().X + height := thumb.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) + bh, err := blurhash.Encode(4, 3, tiny) + if err != nil { + return nil, err + } + + out := &bytes.Buffer{} + if err := jpeg.Encode(out, thumb, &jpeg.Options{ + Quality: 75, + }); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + width: width, + height: height, + size: size, + aspect: aspect, + blurhash: bh, + }, nil +} + +// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. +func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) + } + + out := &bytes.Buffer{} + if err := png.Encode(out, i); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + }, nil +} + +// purgeExif is a little wrapper for the action of removing exif data from an image. +// Only pass pngs or jpegs to this function. +func purgeExif(data []byte) ([]byte, error) { + if len(data) == 0 { + return nil, errors.New("passed image was not valid") + } + + clean, err := exifremove.Remove(data) + if err != nil { + return nil, fmt.Errorf("could not purge exif from image: %s", err) + } + + if len(clean) == 0 { + return nil, errors.New("purged image was not valid") + } + + return clean, nil +} diff --git a/internal/media/manager.go b/internal/media/manager.go new file mode 100644 index 000000000..782542ca9 --- /dev/null +++ b/internal/media/manager.go @@ -0,0 +1,274 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +// ProcessCallback is triggered by the media manager when an attachment has finished undergoing +// image processing (generation of a blurhash, thumbnail etc) but hasn't yet been inserted into +// the database. It is provided to allow callers to a) access the processed media attachment and b) +// make any last-minute changes to the media attachment before it enters the database. +type ProcessCallback func(*gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment + +// defaultCB will be used when a nil ProcessCallback is passed to one of the manager's interface functions. +// It just returns the processed media attachment with no additional changes. +var defaultCB ProcessCallback = func(a *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment { + return a +} + +// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. +type Manager interface { + ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) +} + +type manager struct { + db db.DB + storage *kv.KVStore +} + +// New returns a media manager with the given db and underlying storage. +func New(database db.DB, storage *kv.KVStore) Manager { + return &manager{ + db: database, + storage: storage, + } +} + +/* + INTERFACE FUNCTIONS +*/ + +func (m *manager) ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { + contentType, err := parseContentType(data) + if err != nil { + return nil, err + } + + mainType := strings.Split(contentType, "/")[0] + switch mainType { + case mimeImage: + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + return m.processImage(attachmentBytes, minAttachment) + default: + return nil, fmt.Errorf("content type %s not (yet) supported", contentType) + } +} + +// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, +// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, +// and then returns information to the caller about the new header. +func (m *manager) ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { + + // make sure we have a type we can handle + contentType, err := parseContentType(data) + if err != nil { + return nil, err + } + + if !supportedImage(contentType) { + return nil, fmt.Errorf("%s is not an accepted image type", contentType) + } + + if len(data) == 0 { + return nil, fmt.Errorf("passed reader was of size 0") + } + + // process it + ma, err := m.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL) + if err != nil { + return nil, fmt.Errorf("error processing %s: %s", mediaType, err) + } + + // set it in the database + if err := m.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil { + return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err) + } + + return ma, nil +} + +// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new +// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct +// in the database. +func (m *manager) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) { + var clean []byte + var err error + var original *imageAndMeta + var static *imageAndMeta + + // check content type of the submitted emoji and make sure it's supported by us + contentType, err := parseContentType(emojiBytes) + if err != nil { + return nil, err + } + if !supportedEmoji(contentType) { + return nil, fmt.Errorf("content type %s not supported for emojis", contentType) + } + + if len(emojiBytes) == 0 { + return nil, errors.New("emoji was of size 0") + } + if len(emojiBytes) > EmojiMaxBytes { + return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) + } + + // clean any exif data from png but leave gifs alone + switch contentType { + case mimePng: + if clean, err = purgeExif(emojiBytes); err != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", err) + } + case mimeGif: + clean = emojiBytes + default: + return nil, errors.New("media type unrecognized") + } + + // unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc + original = &imageAndMeta{ + image: clean, + } + + static, err = deriveStaticEmoji(clean, contentType) + if err != nil { + return nil, fmt.Errorf("error deriving static emoji: %s", err) + } + + // since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver, + // (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created + // with the same username as the instance hostname, which doesn't belong to any particular user. + instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + if err != nil { + return nil, fmt.Errorf("error fetching instance account: %s", err) + } + + // the file extension (either png or gif) + extension := strings.Split(contentType, "/")[1] + + // generate a ulid for the new emoji + newEmojiID, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + // activitypub uri for the emoji -- unrelated to actually serving the image + // will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC + emojiURI := uris.GenerateURIForEmoji(newEmojiID) + + // serve url and storage path for the original emoji -- can be png or gif + emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension) + emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension) + + // serve url and storage path for the static version -- will always be png + emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png") + emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID) + + // Store the original emoji + if err := m.storage.Put(emojiPath, original.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + // Store the static emoji + if err := m.storage.Put(emojiStaticPath, static.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + // and finally return the new emoji data to the caller -- it's up to them what to do with it + e := >smodel.Emoji{ + ID: newEmojiID, + Shortcode: shortcode, + Domain: "", // empty because this is a local emoji + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + ImageRemoteURL: "", // empty because this is a local emoji + ImageStaticRemoteURL: "", // empty because this is a local emoji + ImageURL: emojiURL, + ImageStaticURL: emojiStaticURL, + ImagePath: emojiPath, + ImageStaticPath: emojiStaticPath, + ImageContentType: contentType, + ImageStaticContentType: mimePng, // static version will always be a png + ImageFileSize: len(original.image), + ImageStaticFileSize: len(static.image), + ImageUpdatedAt: time.Now(), + Disabled: false, + URI: emojiURI, + VisibleInPicker: true, + CategoryID: "", // empty because this is a new emoji -- no category yet + } + return e, nil +} + +func (m *manager) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { + if !currentAttachment.Header && !currentAttachment.Avatar { + return nil, errors.New("provided attachment was set to neither header nor avatar") + } + + if currentAttachment.Header && currentAttachment.Avatar { + return nil, errors.New("provided attachment was set to both header and avatar") + } + + var headerOrAvi Type + if currentAttachment.Header { + headerOrAvi = TypeHeader + } else if currentAttachment.Avatar { + headerOrAvi = TypeAvatar + } + + if currentAttachment.RemoteURL == "" { + return nil, errors.New("no remote URL on media attachment to dereference") + } + remoteIRI, err := url.Parse(currentAttachment.RemoteURL) + if err != nil { + return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) + } + + // for content type, we assume we don't know what to expect... + expectedContentType := "*/*" + if currentAttachment.File.ContentType != "" { + // ... and then narrow it down if we do + expectedContentType = currentAttachment.File.ContentType + } + + attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType) + if err != nil { + return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) + } + + return m.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL) +} diff --git a/internal/media/processicon.go b/internal/media/processicon.go deleted file mode 100644 index faeae0ee6..000000000 --- a/internal/media/processicon.go +++ /dev/null @@ -1,143 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string, mediaType Type, accountID string, remoteURL string) (*gtsmodel.MediaAttachment, error) { - var isHeader bool - var isAvatar bool - - switch mediaType { - case TypeHeader: - isHeader = true - case TypeAvatar: - isAvatar = true - default: - return nil, errors.New("header or avatar not selected") - } - - var clean []byte - var err error - - var original *imageAndMeta - switch contentType { - case mimeJpeg: - if clean, err = purgeExif(imageBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - original, err = deriveImage(clean, contentType) - case mimePng: - if clean, err = purgeExif(imageBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - original, err = deriveImage(clean, contentType) - case mimeGif: - clean = imageBytes - original, err = deriveGif(clean, contentType) - default: - return nil, errors.New("media type unrecognized") - } - - if err != nil { - return nil, fmt.Errorf("error parsing image: %s", err) - } - - small, err := deriveThumbnail(clean, contentType, 256, 256) - if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) - } - - // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - extension := strings.Split(contentType, "/")[1] - newMediaID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - originalURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeOriginal), newMediaID, extension) - smallURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeSmall), newMediaID, extension) - // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeOriginal, newMediaID, extension) - if err := mh.storage.Put(originalPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeSmall, newMediaID, extension) - if err := mh.storage.Put(smallPath, small.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - ma := >smodel.MediaAttachment{ - ID: newMediaID, - StatusID: "", - URL: originalURL, - RemoteURL: remoteURL, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Original: gtsmodel.Original{ - Width: original.width, - Height: original.height, - Size: original.size, - Aspect: original.aspect, - }, - Small: gtsmodel.Small{ - Width: small.width, - Height: small.height, - Size: small.size, - Aspect: small.aspect, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: small.blurhash, - Processing: 2, - File: gtsmodel.File{ - Path: originalPath, - ContentType: contentType, - FileSize: len(original.image), - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - Path: smallPath, - ContentType: contentType, - FileSize: len(small.image), - UpdatedAt: time.Now(), - URL: smallURL, - RemoteURL: "", - }, - Avatar: isAvatar, - Header: isHeader, - } - - return ma, nil -} diff --git a/internal/media/processing.go b/internal/media/processing.go deleted file mode 100644 index ccd9ebfdb..000000000 --- a/internal/media/processing.go +++ /dev/null @@ -1,190 +0,0 @@ -package media - -import ( - "bytes" - "errors" - "fmt" - "image" - "image/gif" - "image/jpeg" - "image/png" - - "github.com/buckket/go-blurhash" - "github.com/nfnt/resize" - "github.com/superseriousbusiness/exifremove/pkg/exifremove" -) - -// purgeExif is a little wrapper for the action of removing exif data from an image. -// Only pass pngs or jpegs to this function. -func purgeExif(data []byte) ([]byte, error) { - if len(data) == 0 { - return nil, errors.New("passed image was not valid") - } - - clean, err := exifremove.Remove(data) - if err != nil { - return nil, fmt.Errorf("could not purge exif from image: %s", err) - } - - if len(clean) == 0 { - return nil, errors.New("purged image was not valid") - } - - return clean, nil -} - -func deriveGif(b []byte, extension string) (*imageAndMeta, error) { - var g *gif.GIF - var err error - switch extension { - case mimeGif: - g, err = gif.DecodeAll(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("extension %s not recognised", extension) - } - - // use the first frame to get the static characteristics - width := g.Config.Width - height := g.Config.Height - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -func deriveImage(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - width := i.Bounds().Size().X - height := i.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, -// of a given jpeg, png, or gif, or an error if something goes wrong. -// -// Note that the aspect ratio of the image will be retained, -// so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) - width := thumb.Bounds().Size().X - height := thumb.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, err - } - - out := &bytes.Buffer{} - if err := jpeg.Encode(out, thumb, &jpeg.Options{ - Quality: 75, - }); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - width: width, - height: height, - size: size, - aspect: aspect, - blurhash: bh, - }, nil -} - -// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) - } - - out := &bytes.Buffer{} - if err := png.Encode(out, i); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - }, nil -} - -type imageAndMeta struct { - image []byte - width int - height int - size int - aspect float64 - blurhash string -} diff --git a/internal/media/processvideo.go b/internal/media/processvideo.go index d0d11f779..e829c68c0 100644 --- a/internal/media/processvideo.go +++ b/internal/media/processvideo.go @@ -18,6 +18,6 @@ package media -// func (mh *mediaHandler) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) { +// func (mh *mediaManager) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) { // return nil, nil // } diff --git a/internal/media/types.go b/internal/media/types.go index f1608f880..d40f402d2 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -40,7 +40,6 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) - // EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) // const EmojiMaxBytes = 51200 diff --git a/internal/media/util_test.go b/internal/media/util_test.go index 817b597cb..1180bf2aa 100644 --- a/internal/media/util_test.go +++ b/internal/media/util_test.go @@ -103,7 +103,7 @@ func (suite *MediaUtilTestSuite) TestDeriveImageFromJPEG() { suite.NoError(err) // clean it up and validate the clean version - imageAndMeta, err := deriveImage(b, "image/jpeg") + imageAndMeta, err := decodeImage(b, "image/jpeg") suite.NoError(err) suite.Equal(1920, imageAndMeta.width) diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go index ae005f4f6..b2321f414 100644 --- a/internal/processing/account/account.go +++ b/internal/processing/account/account.go @@ -77,7 +77,7 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager fromClientAPI chan messages.FromClientAPI oauthServer oauth.Server filter visibility.Filter @@ -87,10 +87,10 @@ type processor struct { } // New returns a new account processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, fromClientAPI: fromClientAPI, oauthServer: oauthServer, filter: visibility.NewFilter(db), diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index e4611ba23..9c7f0fe67 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -41,7 +41,7 @@ type AccountStandardTestSuite struct { db db.DB tc typeutils.TypeConverter storage *kv.KVStore - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server fromClientAPIChan chan messages.FromClientAPI httpClient pub.HttpClient @@ -80,7 +80,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) suite.httpClient = testrig.NewMockHTTPClient(nil) @@ -88,7 +88,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) - suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaHandler, suite.oauthServer, suite.fromClientAPIChan, suite.federator) + suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") } diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index a32dd9ac0..8de6c83f0 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -159,7 +159,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead } // do the setting - avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "") + avatarInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "") if err != nil { return nil, fmt.Errorf("error processing avatar: %s", err) } @@ -193,7 +193,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead } // do the setting - headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "") + headerInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "") if err != nil { return nil, fmt.Errorf("error processing header: %s", err) } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 217d10dfe..27a7da47a 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -43,16 +43,16 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager fromClientAPI chan messages.FromClientAPI db db.DB } // New returns a new admin processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, fromClientAPI chan messages.FromClientAPI) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, fromClientAPI chan messages.FromClientAPI) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, fromClientAPI: fromClientAPI, db: db, } diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 4989d8e8d..5620374b8 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -49,8 +49,8 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, errors.New("could not read provided emoji: size 0 bytes") } - // allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using - emoji, err := p.mediaHandler.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode) + // allow the mediaManager to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using + emoji, err := p.mediaManager.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode) if err != nil { return nil, fmt.Errorf("error reading emoji: %s", err) } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index de15d3162..68a011683 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -65,8 +65,8 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form }, } - // allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using - attachment, err := p.mediaHandler.ProcessAttachment(ctx, buf.Bytes(), minAttachment) + // allow the mediaManager to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using + attachment, err := p.mediaManager.ProcessAttachment(ctx, buf.Bytes(), minAttachment) if err != nil { return nil, fmt.Errorf("error reading attachment: %s", err) } diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go index 9e050fe84..3d4ae5009 100644 --- a/internal/processing/media/media.go +++ b/internal/processing/media/media.go @@ -43,16 +43,16 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager storage *kv.KVStore db db.DB } // New returns a new media processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, storage *kv.KVStore) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, storage: storage, db: db, } diff --git a/internal/processing/processor.go b/internal/processing/processor.go index f5334a1ef..2626c1fea 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -235,7 +235,7 @@ type processor struct { stop chan interface{} tc typeutils.TypeConverter oauthServer oauth.Server - mediaHandler media.Handler + mediaManager media.Manager storage *kv.KVStore timelineManager timeline.Manager db db.DB @@ -259,7 +259,7 @@ func NewProcessor( tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, - mediaHandler media.Handler, + mediaManager media.Manager, storage *kv.KVStore, timelineManager timeline.Manager, db db.DB, @@ -269,9 +269,9 @@ func NewProcessor( statusProcessor := status.New(db, tc, fromClientAPI) streamingProcessor := streaming.New(db, oauthServer) - accountProcessor := account.New(db, tc, mediaHandler, oauthServer, fromClientAPI, federator) - adminProcessor := admin.New(db, tc, mediaHandler, fromClientAPI) - mediaProcessor := mediaProcessor.New(db, tc, mediaHandler, storage) + accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator) + adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI) + mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage) userProcessor := user.New(db, emailSender) federationProcessor := federationProcessor.New(db, tc, federator, fromFederator) @@ -282,7 +282,7 @@ func NewProcessor( stop: make(chan interface{}), tc: tc, oauthServer: oauthServer, - mediaHandler: mediaHandler, + mediaManager: mediaManager, storage: storage, timelineManager: timelineManager, db: db, diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index dc7562a2e..04793898f 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -51,7 +51,7 @@ type ProcessingStandardTestSuite struct { transportController transport.Controller federator federation.Federator oauthServer oauth.Server - mediaHandler media.Handler + mediaManager media.Manager timelineManager timeline.Manager emailSender email.Sender @@ -218,11 +218,11 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.transportController = testrig.NewTestTransportController(httpClient, suite.db) suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.timelineManager = testrig.NewTestTimelineManager(suite.db) suite.emailSender = testrig.NewEmailSender("../../web/template/", nil) - suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.timelineManager, suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.timelineManager, suite.db, suite.emailSender) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/testrig/federator.go b/testrig/federator.go index b04c01d63..b9c742832 100644 --- a/testrig/federator.go +++ b/testrig/federator.go @@ -27,5 +27,5 @@ import ( // NewTestFederator returns a federator with the given database and (mock!!) transport controller. func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore) federation.Federator { - return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaHandler(db, storage)) + return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaManager(db, storage)) } diff --git a/testrig/mediahandler.go b/testrig/mediahandler.go index ab7fee621..ba2148655 100644 --- a/testrig/mediahandler.go +++ b/testrig/mediahandler.go @@ -24,7 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -// NewTestMediaHandler returns a media handler with the default test config, and the given db and storage. -func NewTestMediaHandler(db db.DB, storage *kv.KVStore) media.Handler { +// NewTestMediaManager returns a media handler with the default test config, and the given db and storage. +func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager { return media.New(db, storage) } diff --git a/testrig/processor.go b/testrig/processor.go index 0baffd35b..d86dd8411 100644 --- a/testrig/processor.go +++ b/testrig/processor.go @@ -28,5 +28,5 @@ import ( // NewTestProcessor returns a Processor suitable for testing purposes func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor { - return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, NewTestTimelineManager(db), db, emailSender) + return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaManager(db, storage), storage, NewTestTimelineManager(db), db, emailSender) } -- cgit v1.2.3 From dccf21dd87638320a687a0556c973cced541c945 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 9 Jan 2022 18:41:22 +0100 Subject: tests are passing, but there's still much to be done --- cmd/gotosocial/action/server/server.go | 5 +- internal/ap/extract.go | 18 ++--- internal/ap/extractattachments_test.go | 2 +- internal/ap/interfaces.go | 8 ++- internal/federation/dereferencing/account.go | 25 +++---- internal/federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/media.go | 4 +- internal/federation/dereferencing/media_test.go | 17 +++-- internal/federation/dereferencing/status.go | 9 ++- internal/media/image.go | 32 +++++---- internal/media/manager.go | 80 +++++++++++++++++++---- internal/media/manager_test.go | 50 ++++++++++++++ internal/media/media.go | 51 ++++++--------- internal/media/media_test.go | 65 ------------------ internal/media/types.go | 26 ++++++++ internal/processing/account/update.go | 23 ++++--- internal/processing/admin/emoji.go | 2 +- internal/processing/media/create.go | 12 +++- 18 files changed, 260 insertions(+), 171 deletions(-) delete mode 100644 internal/media/media_test.go (limited to 'internal/media/types.go') diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 05c2e8974..8a227edb7 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,7 +105,10 @@ var Start action.GTSAction = func(ctx context.Context) error { } // build backend handlers - mediaManager := media.New(dbService, storage) + mediaManager, err := media.New(dbService, storage) + if err != nil { + return fmt.Errorf("error creating media manager: %s", err) + } oauthServer := oauth.New(ctx, dbService) transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient) federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) diff --git a/internal/ap/extract.go b/internal/ap/extract.go index ed61faf1e..49dac7186 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -395,20 +395,20 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment.Description = name } + attachment.Blurhash = ExtractBlurhash(i) + attachment.Processing = gtsmodel.ProcessingStatusReceived return attachment, nil } -// func extractBlurhash(i withBlurhash) (string, error) { -// if i.GetTootBlurhashProperty() == nil { -// return "", errors.New("blurhash property was nil") -// } -// if i.GetTootBlurhashProperty().Get() == "" { -// return "", errors.New("empty blurhash string") -// } -// return i.GetTootBlurhashProperty().Get(), nil -// } +// ExtractBlurhash extracts the blurhash value (if present) from a WithBlurhash interface. +func ExtractBlurhash(i WithBlurhash) string { + if i.GetTootBlurhash() == nil { + return "" + } + return i.GetTootBlurhash().Get() +} // ExtractHashtags returns a slice of tags on the interface. func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { diff --git a/internal/ap/extractattachments_test.go b/internal/ap/extractattachments_test.go index 3cee98faa..b937911d2 100644 --- a/internal/ap/extractattachments_test.go +++ b/internal/ap/extractattachments_test.go @@ -42,7 +42,7 @@ func (suite *ExtractAttachmentsTestSuite) TestExtractAttachments() { suite.Equal("image/jpeg", attachment1.File.ContentType) suite.Equal("https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg", attachment1.RemoteURL) suite.Equal("It's a cute plushie.", attachment1.Description) - suite.Empty(attachment1.Blurhash) // atm we discard blurhashes and generate them ourselves during processing + suite.Equal("UxQ0EkRP_4tRxtRjWBt7%hozM_ayV@oLf6WB", attachment1.Blurhash) } func (suite *ExtractAttachmentsTestSuite) TestExtractNoAttachments() { diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index 582465ec3..6edaa42ba 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -70,6 +70,7 @@ type Attachmentable interface { WithMediaType WithURL WithName + WithBlurhash } // Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. @@ -284,9 +285,10 @@ type WithMediaType interface { GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty } -// type withBlurhash interface { -// GetTootBlurhashProperty() vocab.TootBlurhashProperty -// } +// WithBlurhash represents an activity with TootBlurhashProperty +type WithBlurhash interface { + GetTootBlurhash() vocab.TootBlurhashProperty +} // type withFocalPoint interface { // // TODO diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 5912ff29a..d83fc3bac 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -256,16 +257,16 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.AvatarRemoteURL) + avatar := true + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + RemoteURL: &targetAccount.AvatarRemoteURL, + Avatar: &avatar, + }) if err != nil { return err } - if err := media.SetAsAvatar(ctx); err != nil { - return err - } - - targetAccount.AvatarMediaAttachmentID = media.AttachmentID() + targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { @@ -279,16 +280,16 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.HeaderRemoteURL) + header := true + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + RemoteURL: &targetAccount.HeaderRemoteURL, + Header: &header, + }) if err != nil { return err } - if err := media.SetAsHeader(ctx); err != nil { - return err - } - - targetAccount.HeaderMediaAttachmentID = media.AttachmentID() + targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() } return nil } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index d4786f62d..787a39739 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 4d62fe0a6..0ddab7ae0 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -26,7 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) { +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) { if accountID == "" { return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") } @@ -46,7 +46,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) } - m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, remoteURL) + m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) if err != nil { return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) } diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index cc158c9a9..8fb28d196 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" ) type AttachmentTestSuite struct { @@ -32,7 +33,7 @@ type AttachmentTestSuite struct { func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { ctx := context.Background() - + fetchingAccount := suite.testAccounts["local_account_1"] attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" @@ -40,8 +41,14 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { attachmentContentType := "image/jpeg" attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" attachmentDescription := "It's a cute plushie." - - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL) + attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" + + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + StatusID: &attachmentStatus, + RemoteURL: &attachmentURL, + Description: &attachmentDescription, + Blurhash: &attachmentBlurhash, + }) suite.NoError(err) attachment, err := media.LoadAttachment(ctx) @@ -61,7 +68,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { suite.Equal(2071680, attachment.FileMeta.Original.Size) suite.Equal(1245, attachment.FileMeta.Original.Height) suite.Equal(1664, attachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) + suite.Equal(attachmentBlurhash, attachment.Blurhash) suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) suite.NotEmpty(attachment.File.Path) suite.Equal(attachmentContentType, attachment.File.ContentType) @@ -87,7 +94,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) suite.Equal(1245, dbAttachment.FileMeta.Original.Height) suite.Equal(1664, dbAttachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) + suite.Equal(attachmentBlurhash, dbAttachment.Blurhash) suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) suite.NotEmpty(dbAttachment.File.Path) suite.Equal(attachmentContentType, dbAttachment.File.ContentType) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index e184b585f..47ce087a2 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/media" ) // EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form, @@ -393,7 +394,13 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL) + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalInfo{ + CreatedAt: &a.CreatedAt, + StatusID: &a.StatusID, + RemoteURL: &a.RemoteURL, + Description: &a.Description, + Blurhash: &a.Blurhash, + }) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err) continue diff --git a/internal/media/image.go b/internal/media/image.go index acc62a28b..4c0b28c02 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -108,7 +108,7 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { // // Note that the aspect ratio of the image will be retained, // so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { +func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageMeta, error) { var i image.Image var err error @@ -138,10 +138,20 @@ func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, err + im := &ImageMeta{ + width: width, + height: height, + size: size, + aspect: aspect, + } + + if createBlurhash { + tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) + bh, err := blurhash.Encode(4, 3, tiny) + if err != nil { + return nil, err + } + im.blurhash = bh } out := &bytes.Buffer{} @@ -150,14 +160,10 @@ func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { }); err != nil { return nil, err } - return &ImageMeta{ - image: out.Bytes(), - width: width, - height: height, - size: size, - aspect: aspect, - blurhash: bh, - }, nil + + im.image = out.Bytes() + + return im, nil } // deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. diff --git a/internal/media/manager.go b/internal/media/manager.go index 9ca450141..5e62b39b2 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -45,9 +45,9 @@ type Manager interface { // // RemoteURL is optional, and can be an empty string. Setting this to a non-empty string indicates that // the piece of media originated on a remote instance and has been dereferenced to be cached locally. - ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) + ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) - ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) + ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) } type manager struct { @@ -80,7 +80,7 @@ func New(database db.DB, storage *kv.KVStore) (Manager, error) { INTERFACE FUNCTIONS */ -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { +func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) { contentType, err := parseContentType(data) if err != nil { return nil, err @@ -95,7 +95,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin switch mainType { case mimeImage: - media, err := m.preProcessImage(ctx, data, contentType, accountID, remoteURL) + media, err := m.preProcessImage(ctx, data, contentType, accountID, ai) if err != nil { return nil, err } @@ -117,12 +117,12 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin } } -func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { +func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) { return nil, nil } // preProcessImage initializes processing -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, remoteURL string) (*Media, error) { +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Media, error) { if !supportedImage(contentType) { return nil, fmt.Errorf("image type %s not supported", contentType) } @@ -139,13 +139,24 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType extension := strings.Split(contentType, "/")[1] attachment := >smodel.MediaAttachment{ - ID: id, - UpdatedAt: time.Now(), - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - RemoteURL: remoteURL, - Type: gtsmodel.FileTypeImage, - AccountID: accountID, - Processing: 0, + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StatusID: "", + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + RemoteURL: "", + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: 0, File: gtsmodel.File{ Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), ContentType: contentType, @@ -161,6 +172,49 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType Header: false, } + // check if we have additional info to add to the attachment + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + media := &Media{ attachment: attachment, rawData: data, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 45428fbba..aad7f46d0 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -1,4 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package media_test +import ( + "codeberg.org/gruf/go-store/kv" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type MediaManagerStandardTestSuite struct { + suite.Suite + + db db.DB + storage *kv.KVStore + manager media.Manager +} + +func (suite *MediaManagerStandardTestSuite) SetupSuite() { + testrig.InitTestLog() + testrig.InitTestConfig() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() +} +func (suite *MediaManagerStandardTestSuite) SetupTest() { + testrig.StandardStorageSetup(suite.storage, "../../testrig/media") + testrig.StandardDBSetup(suite.db, nil) + suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) +} +func (suite *MediaManagerStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} diff --git a/internal/media/media.go b/internal/media/media.go index e19997391..e0cfe09b7 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -47,7 +47,8 @@ type Media struct { attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment emoji *gtsmodel.Emoji // will only be set if the media is an emoji - rawData []byte + + rawData []byte /* below fields represent the processing state of the media thumbnail @@ -81,7 +82,15 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { switch m.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now - thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) + + // check if we need to create a blurhash or if there's already one set + var createBlurhash bool + if m.attachment.Blurhash == "" { + // no blurhash created yet + createBlurhash = true + } + + thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType, createBlurhash) if err != nil { m.err = fmt.Errorf("error deriving thumbnail: %s", err) m.thumbstate = errored @@ -96,7 +105,10 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { } // set appropriate fields on the attachment based on the thumbnail we derived - m.attachment.Blurhash = thumb.blurhash + if createBlurhash { + m.attachment.Blurhash = thumb.blurhash + } + m.attachment.FileMeta.Small = gtsmodel.Small{ Width: thumb.width, Height: thumb.height, @@ -105,7 +117,6 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { } m.attachment.Thumbnail.FileSize = thumb.size - // put or update the attachment in the database if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { m.err = err m.thumbstate = errored @@ -177,8 +188,8 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { } m.attachment.File.FileSize = decoded.size m.attachment.File.UpdatedAt = time.Now() + m.attachment.Processing = gtsmodel.ProcessingStatusProcessed - // put or update the attachment in the database if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { m.err = err m.fullSizeState = errored @@ -200,30 +211,6 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) } -func (m *Media) SetAsAvatar(ctx context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.Avatar = true - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - -func (m *Media) SetAsHeader(ctx context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.Header = true - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - -func (m *Media) SetStatusID(ctx context.Context, statusID string) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.StatusID = statusID - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - // AttachmentID returns the ID of the underlying media attachment without blocking processing. func (m *Media) AttachmentID() string { return m.attachment.ID @@ -237,8 +224,8 @@ func (m *Media) preLoad(ctx context.Context) { go m.FullSize(ctx) } -// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image -// have been processed, then it returns the full-size image. +// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size +// image have been processed, then it returns the completed attachment. func (m *Media) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { if _, err := m.Thumb(ctx); err != nil { return nil, err @@ -255,6 +242,8 @@ func (m *Media) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { return nil, nil } +// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, +// and then if that doesn't work because the attachment already exists, updating it instead. func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { if err := database.Put(ctx, attachment); err != nil { if err != db.ErrAlreadyExists { diff --git a/internal/media/media_test.go b/internal/media/media_test.go deleted file mode 100644 index 7e820c9ea..000000000 --- a/internal/media/media_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media_test - -import ( - "testing" - - "codeberg.org/gruf/go-store/kv" - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -type MediaStandardTestSuite struct { - suite.Suite - - db db.DB - storage *kv.KVStore - manager media.Manager -} - -func (suite *MediaStandardTestSuite) SetupSuite() { - testrig.InitTestLog() - testrig.InitTestConfig() - - suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() -} - -func (suite *MediaStandardTestSuite) SetupTest() { - testrig.StandardStorageSetup(suite.storage, "../../testrig/media") - testrig.StandardDBSetup(suite.db, nil) - - m, err := media.New(suite.db, suite.storage) - if err != nil { - panic(err) - } - suite.manager = m -} - -func (suite *MediaStandardTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.db) - testrig.StandardStorageTeardown(suite.storage) -} - -func TestMediaStandardTestSuite(t *testing.T) { - suite.Run(t, &MediaStandardTestSuite{}) -} diff --git a/internal/media/types.go b/internal/media/types.go index d40f402d2..aaf423682 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -22,6 +22,7 @@ import ( "bytes" "errors" "fmt" + "time" "github.com/h2non/filetype" ) @@ -67,6 +68,31 @@ const ( TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests ) +// AdditionalInfo represents additional information that should be added to an attachment +// when processing a piece of media. +type AdditionalInfo struct { + // Time that this media was created; defaults to time.Now(). + CreatedAt *time.Time + // ID of the status to which this media is attached; defaults to "". + StatusID *string + // URL of the media on a remote instance; defaults to "". + RemoteURL *string + // Image description of this media; defaults to "". + Description *string + // Blurhash of this media; defaults to "". + Blurhash *string + // ID of the scheduled status to which this media is attached; defaults to "". + ScheduledStatusID *string + // Mark this media as in-use as an avatar; defaults to false. + Avatar *bool + // Mark this media as in-use as a header; defaults to false. + Header *bool + // X focus coordinate for this media; defaults to 0. + FocusX *float32 + // Y focus coordinate for this media; defaults to 0. + FocusY *float32 +} + // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. func parseContentType(content []byte) (string, error) { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 6e74a0ccd..e0dd493e1 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -33,6 +33,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -163,16 +164,15 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead } // do the setting - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") + isAvatar := true + processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + Avatar: &isAvatar, + }) if err != nil { return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } - if err := media.SetAsAvatar(ctx); err != nil { - return nil, fmt.Errorf("UpdateAvatar: error setting media as avatar: %s", err) - } - - return media.LoadAttachment(ctx) + return processingMedia.LoadAttachment(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, @@ -206,16 +206,15 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead } // do the setting - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") + isHeader := true + processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + Header: &isHeader, + }) if err != nil { return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } - if err := media.SetAsHeader(ctx); err != nil { - return nil, fmt.Errorf("UpdateHeader: error setting media as header: %s", err) - } - - return media.LoadAttachment(ctx) + return processingMedia.LoadAttachment(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 6fb2ca8c5..737a4ebe2 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -48,7 +48,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, errors.New("could not read provided emoji: size 0 bytes") } - media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID, "") + media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID) if err != nil { return nil, err } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index d1840196a..093a3d2be 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -27,6 +27,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { @@ -44,8 +45,17 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, errors.New("could not read provided attachment: size 0 bytes") } + focusX, focusY, err := parseFocus(form.Focus) + if err != nil { + return nil, fmt.Errorf("could not parse focus value %s: %s", form.Focus, err) + } + // process the media attachment and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, "") + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, &media.AdditionalInfo{ + Description: &form.Description, + FocusX: &focusX, + FocusY: &focusY, + }) if err != nil { return nil, err } -- cgit v1.2.3 From 113f9d9ab4797de6ae17819c96ae866992214021 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 11 Jan 2022 17:49:14 +0100 Subject: pass a function into the manager, start work on emoji --- internal/api/client/admin/admin.go | 2 +- internal/api/client/admin/admin_test.go | 123 +++++++ internal/api/client/admin/emojicreate.go | 4 +- internal/api/client/admin/emojicreate_test.go | 50 +++ internal/federation/dereferencing/account.go | 15 +- internal/federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/media.go | 17 +- internal/federation/dereferencing/media_test.go | 20 +- internal/federation/dereferencing/status.go | 4 +- internal/media/image.go | 108 ------ internal/media/manager.go | 81 +++-- internal/media/manager_test.go | 32 +- internal/media/processing.go | 256 -------------- internal/media/processingemoji.go | 382 ++++++++++++++++++++ internal/media/processingmedia.go | 411 ++++++++++++++++++++++ internal/media/types.go | 12 +- internal/processing/account/update.go | 95 ++--- internal/processing/admin/emoji.go | 34 +- internal/processing/media/create.go | 31 +- internal/transport/transport.go | 2 +- testrig/testmodels.go | 10 + 21 files changed, 1164 insertions(+), 527 deletions(-) create mode 100644 internal/api/client/admin/admin_test.go create mode 100644 internal/api/client/admin/emojicreate_test.go delete mode 100644 internal/media/processing.go create mode 100644 internal/media/processingemoji.go create mode 100644 internal/media/processingmedia.go (limited to 'internal/media/types.go') diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index f8ea03cc6..f5256c996 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -58,7 +58,7 @@ func New(processor processing.Processor) api.ClientModule { // Route attaches all routes from this module to the given router func (m *Module) Route(r router.Router) error { - r.AttachHandler(http.MethodPost, EmojiPath, m.emojiCreatePOSTHandler) + r.AttachHandler(http.MethodPost, EmojiPath, m.EmojiCreatePOSTHandler) r.AttachHandler(http.MethodPost, DomainBlocksPath, m.DomainBlocksPOSTHandler) r.AttachHandler(http.MethodGet, DomainBlocksPath, m.DomainBlocksGETHandler) r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go new file mode 100644 index 000000000..da5b03949 --- /dev/null +++ b/internal/api/client/admin/admin_test.go @@ -0,0 +1,123 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin_test + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + + "codeberg.org/gruf/go-store/kv" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/email" + "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type AdminStandardTestSuite struct { + // standard suite interfaces + suite.Suite + db db.DB + tc typeutils.TypeConverter + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator + processor processing.Processor + emailSender email.Sender + sentEmails map[string]string + + // standard suite models + testTokens map[string]*gtsmodel.Token + testClients map[string]*gtsmodel.Client + testApplications map[string]*gtsmodel.Application + testUsers map[string]*gtsmodel.User + testAccounts map[string]*gtsmodel.Account + testAttachments map[string]*gtsmodel.MediaAttachment + testStatuses map[string]*gtsmodel.Status + + // module being tested + adminModule *admin.Module +} + +func (suite *AdminStandardTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *AdminStandardTestSuite) SetupTest() { + testrig.InitTestConfig() + testrig.InitTestLog() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.sentEmails = make(map[string]string) + suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.adminModule = admin.New(suite.processor).(*admin.Module) + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") +} + +func (suite *AdminStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func (suite *AdminStandardTestSuite) newContext(recorder *httptest.ResponseRecorder, requestMethod string, requestBody []byte, requestPath string, bodyContentType string) *gin.Context { + ctx, _ := gin.CreateTestContext(recorder) + + ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["admin_account"]) + ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(suite.testTokens["admin_account"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["admin_account"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["admin_account"]) + + protocol := viper.GetString(config.Keys.Protocol) + host := viper.GetString(config.Keys.Host) + + baseURI := fmt.Sprintf("%s://%s", protocol, host) + requestURI := fmt.Sprintf("%s/%s", baseURI, requestPath) + + ctx.Request = httptest.NewRequest(http.MethodPatch, requestURI, bytes.NewReader(requestBody)) // the endpoint we're hitting + + if bodyContentType != "" { + ctx.Request.Header.Set("Content-Type", bodyContentType) + } + + ctx.Request.Header.Set("accept", "application/json") + + return ctx +} diff --git a/internal/api/client/admin/emojicreate.go b/internal/api/client/admin/emojicreate.go index 882654ea9..de7a2979a 100644 --- a/internal/api/client/admin/emojicreate.go +++ b/internal/api/client/admin/emojicreate.go @@ -31,7 +31,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/validate" ) -// emojiCreateRequest swagger:operation POST /api/v1/admin/custom_emojis emojiCreate +// EmojiCreatePOSTHandler swagger:operation POST /api/v1/admin/custom_emojis emojiCreate // // Upload and create a new instance emoji. // @@ -73,7 +73,7 @@ import ( // description: forbidden // '400': // description: bad request -func (m *Module) emojiCreatePOSTHandler(c *gin.Context) { +func (m *Module) EmojiCreatePOSTHandler(c *gin.Context) { l := logrus.WithFields(logrus.Fields{ "func": "emojiCreatePOSTHandler", "request_uri": c.Request.RequestURI, diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go new file mode 100644 index 000000000..290b478f7 --- /dev/null +++ b/internal/api/client/admin/emojicreate_test.go @@ -0,0 +1,50 @@ +package admin_test + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type EmojiCreateTestSuite struct { + AdminStandardTestSuite +} + +func (suite *EmojiCreateTestSuite) TestEmojiCreate() { + // set up the request + requestBody, w, err := testrig.CreateMultipartFormData( + "image", "../../../../testrig/media/rainbow-original.png", + map[string]string{ + "shortcode": "rainbow", + }) + if err != nil { + panic(err) + } + bodyBytes := requestBody.Bytes() + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPath, w.FormDataContentType()) + + // call the handler + suite.adminModule.EmojiCreatePOSTHandler(ctx) + + // 1. we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + // 2. we should have no error message in the result body + result := recorder.Result() + defer result.Body.Close() + + // check the response + b, err := ioutil.ReadAll(result.Body) + suite.NoError(err) + suite.NotEmpty(b) +} + +func TestEmojiCreateTestSuite(t *testing.T) { + suite.Run(t, &EmojiCreateTestSuite{}) +} diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 27d9f39ce..b9efbfa45 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -119,7 +119,6 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc } else { // take the id we already have and do an update gtsAccount.ID = maybeAccount.ID -aaaaaaaaaaaaaaaaaa if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil { return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) } @@ -252,13 +251,12 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data, err := t.DereferenceMedia(ctx, avatarIRI) - if err != nil { - return err + data := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, avatarIRI) } avatar := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ RemoteURL: &targetAccount.AvatarRemoteURL, Avatar: &avatar, }) @@ -275,13 +273,12 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data, err := t.DereferenceMedia(ctx, headerIRI) - if err != nil { - return err + data := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, headerIRI) } header := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ RemoteURL: &targetAccount.HeaderRemoteURL, Header: &header, }) diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 1e6f781b8..800da6c04 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index f02303aa1..46cb4a5f4 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -26,29 +26,28 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) { +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) { if accountID == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") + return nil, fmt.Errorf("GetRemoteMedia: account ID was empty") } t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) + return nil, fmt.Errorf("GetRemoteMedia: error creating transport: %s", err) } derefURI, err := url.Parse(remoteURL) if err != nil { - return nil, err + return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - data, err := t.DereferenceMedia(ctx, derefURI) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) + dataFunc := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, derefURI) } - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) + processingMedia, err := d.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) + return nil, fmt.Errorf("GetRemoteMedia: error processing attachment: %s", err) } return processingMedia, nil diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index 61ee6edb6..26d5c0c49 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -20,6 +20,7 @@ package dereferencing_test import ( "context" + "fmt" "testing" "time" @@ -44,7 +45,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { attachmentDescription := "It's a cute plushie." attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, RemoteURL: &attachmentURL, Description: &attachmentDescription, @@ -53,7 +54,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { suite.NoError(err) // make a blocking call to load the attachment from the in-process media - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) @@ -118,18 +119,21 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { attachmentDescription := "It's a cute plushie." attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, RemoteURL: &attachmentURL, Description: &attachmentDescription, Blurhash: &attachmentBlurhash, }) suite.NoError(err) - attachmentID := media.AttachmentID() - - // wait 5 seconds to let the image process in the background - // it probably won't really take this long but hey let's be sure - time.Sleep(5 * time.Second) + attachmentID := processingMedia.AttachmentID() + + // wait for the media to finish processing + for finished := processingMedia.Finished(); !finished; finished = processingMedia.Finished() { + time.Sleep(10 * time.Millisecond) + fmt.Printf("\n\nnot finished yet...\n\n") + } + fmt.Printf("\n\nfinished!\n\n") // now get the attachment from the database attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 041cfa6b4..004d648b5 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -394,7 +394,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalInfo{ + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{ CreatedAt: &a.CreatedAt, StatusID: &a.StatusID, RemoteURL: &a.RemoteURL, @@ -406,7 +406,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. continue } - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue diff --git a/internal/media/image.go b/internal/media/image.go index 074dd3839..a5a818206 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -27,7 +27,6 @@ import ( "image/gif" "image/jpeg" "image/png" - "strings" "time" "github.com/buckket/go-blurhash" @@ -52,113 +51,6 @@ type ImageMeta struct { blurhash string } -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Processing, error) { - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - extension := strings.Split(contentType, "/")[1] - - // populate initial fields on the media attachment -- some of these will be overwritten as we proceed - attachment := >smodel.MediaAttachment{ - ID: id, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - StatusID: "", - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - RemoteURL: "", - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Focus: gtsmodel.Focus{ - X: 0, - Y: 0, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: "", - Processing: gtsmodel.ProcessingStatusReceived, - File: gtsmodel.File{ - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), - ContentType: contentType, - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeJpeg, - UpdatedAt: time.Now(), - }, - Avatar: false, - Header: false, - } - - // check if we have additional info to add to the attachment, - // and overwrite some of the attachment fields if so - if ai != nil { - if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description - } - - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID - } - - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash - } - - if ai.Avatar != nil { - attachment.Avatar = *ai.Avatar - } - - if ai.Header != nil { - attachment.Header = *ai.Header - } - - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX - } - - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY - } - } - - media := &Processing{ - attachment: attachment, - rawData: data, - thumbstate: received, - fullSizeState: received, - database: m.db, - storage: m.storage, - } - - return media, nil -} - func decodeGif(b []byte) (*ImageMeta, error) { gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { diff --git a/internal/media/manager.go b/internal/media/manager.go index c8642fcb4..e34471591 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -21,9 +21,7 @@ package media import ( "context" "errors" - "fmt" "runtime" - "strings" "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" @@ -33,15 +31,17 @@ import ( // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { - // ProcessMedia begins the process of decoding and storing the given data as a piece of media (aka an attachment). + // ProcessMedia begins the process of decoding and storing the given data as an attachment. // It will return a pointer to a Media struct upon which further actions can be performed, such as getting // the finished media, thumbnail, attachment, etc. // + // data should be a function that the media manager can call to return raw bytes of a piece of media. + // // accountID should be the account that the media belongs to. // // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. - ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) - ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) + ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) + ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) // NumWorkers returns the total number of workers available to this manager. NumWorkers() int // QueueSize returns the total capacity of the queue. @@ -101,49 +101,52 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { return m, nil } -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) { - contentType, err := parseContentType(data) +func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { + processingMedia, err := m.preProcessMedia(ctx, data, accountID, ai) if err != nil { return nil, err } - split := strings.Split(contentType, "/") - if len(split) != 2 { - return nil, fmt.Errorf("content type %s malformed", contentType) - } - - mainType := split[0] - - switch mainType { - case mimeImage: - media, err := m.preProcessImage(ctx, data, contentType, accountID, ai) - if err != nil { - return nil, err + logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue()) + m.pool.Enqueue(func(innerCtx context.Context) { + select { + case <-innerCtx.Done(): + // if the inner context is done that means the worker pool is closing, so we should just return + return + default: + // start loading the media already for the caller's convenience + if _, err := processingMedia.LoadAttachment(innerCtx); err != nil { + logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", processingMedia.AttachmentID(), err) + } } + }) + logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue()) - logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) - m.pool.Enqueue(func(innerCtx context.Context) { - select { - case <-innerCtx.Done(): - // if the inner context is done that means the worker pool is closing, so we should just return - return - default: - // start loading the media already for the caller's convenience - if _, err := media.Load(innerCtx); err != nil { - logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", media.AttachmentID(), err) - } - } - }) - logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) + return processingMedia, nil +} - return media, nil - default: - return nil, fmt.Errorf("content type %s not (yet) supported", contentType) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, ai) + if err != nil { + return nil, err } -} -func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) { - return nil, nil + logrus.Tracef("ProcessEmoji: about to enqueue emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue()) + m.pool.Enqueue(func(innerCtx context.Context) { + select { + case <-innerCtx.Done(): + // if the inner context is done that means the worker pool is closing, so we should just return + return + default: + // start loading the emoji already for the caller's convenience + if _, err := processingEmoji.LoadEmoji(innerCtx); err != nil { + logrus.Errorf("ProcessEmoji: error processing emoji with id %s: %s", processingEmoji.EmojiID(), err) + } + } + }) + logrus.Tracef("ProcessEmoji: succesfully queued emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue()) + + return processingEmoji, nil } func (m *manager) NumWorkers() int { diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 74d0c3008..0fadceb37 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -37,21 +37,21 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + // load bytes from a test image + return os.ReadFile("./test/test-jpeg.jpg") + } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() // do a blocking call to fetch the attachment - attachment, err := processingMedia.Load(ctx) + attachment, err := processingMedia.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) @@ -103,15 +103,15 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + // load bytes from a test image + return os.ReadFile("./test/test-jpeg.jpg") + } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() @@ -183,13 +183,17 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.NoError(err) suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + return testBytes, nil + } + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" spam := 50 - inProcess := []*media.Processing{} + inProcess := []*media.ProcessingMedia{} for i := 0; i < spam; i++ { // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) inProcess = append(inProcess, processingMedia) } @@ -201,7 +205,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { attachmentID := processingMedia.AttachmentID() // do a blocking call to fetch the attachment - attachment, err := processingMedia.Load(ctx) + attachment, err := processingMedia.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) diff --git a/internal/media/processing.go b/internal/media/processing.go deleted file mode 100644 index 3f9fc2bfc..000000000 --- a/internal/media/processing.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "context" - "fmt" - "sync" - "time" - - "codeberg.org/gruf/go-store/kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type processState int - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - -// Processing represents a piece of media that is currently being processed. It exposes -// various functions for retrieving data from the process. -type Processing struct { - mu sync.Mutex - - /* - below fields should be set on newly created media; - attachment will be updated incrementally as media goes through processing - */ - - attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment - emoji *gtsmodel.Emoji // will only be set if the media is an emoji - - rawData []byte - - /* - below fields represent the processing state of the media thumbnail - */ - - thumbstate processState - thumb *ImageMeta - - /* - below fields represent the processing state of the full-sized media - */ - - fullSizeState processState - fullSize *ImageMeta - - /* - below pointers to database and storage are maintained so that - the media can store and update itself during processing steps - */ - - database db.DB - storage *kv.KVStore - - err error // error created during processing, if any -} - -func (p *Processing) Thumb(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.thumbstate { - case received: - // we haven't processed a thumbnail for this media yet so do it now - - // check if we need to create a blurhash or if there's already one set - var createBlurhash bool - if p.attachment.Blurhash == "" { - // no blurhash created yet - createBlurhash = true - } - - thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) - if err != nil { - p.err = fmt.Errorf("error deriving thumbnail: %s", err) - p.thumbstate = errored - return nil, p.err - } - - // put the thumbnail in storage - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { - p.err = fmt.Errorf("error storing thumbnail: %s", err) - p.thumbstate = errored - return nil, p.err - } - - // set appropriate fields on the attachment based on the thumbnail we derived - if createBlurhash { - p.attachment.Blurhash = thumb.blurhash - } - - p.attachment.FileMeta.Small = gtsmodel.Small{ - Width: thumb.width, - Height: thumb.height, - Size: thumb.size, - Aspect: thumb.aspect, - } - p.attachment.Thumbnail.FileSize = thumb.size - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.thumbstate = errored - return nil, err - } - - // set the thumbnail of this media - p.thumb = thumb - - // we're done processing the thumbnail! - p.thumbstate = complete - fallthrough - case complete: - return p.thumb, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) -} - -func (p *Processing) FullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.fullSizeState { - case received: - var clean []byte - var err error - var decoded *ImageMeta - - ct := p.attachment.File.ContentType - switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", ct) - } - - if err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) - p.fullSizeState = errored - return nil, p.err - } - - // set appropriate fields on the attachment based on the image we derived - p.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, - } - p.attachment.File.FileSize = decoded.size - p.attachment.File.UpdatedAt = time.Now() - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - - // set the fullsize of this media - p.fullSize = decoded - - // we're done processing the full-size image - p.fullSizeState = complete - fallthrough - case complete: - return p.fullSize, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) -} - -// AttachmentID returns the ID of the underlying media attachment without blocking processing. -func (p *Processing) AttachmentID() string { - return p.attachment.ID -} - -// Load blocks until the thumbnail and fullsize content has been processed, and then -// returns the completed attachment. -func (p *Processing) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - if _, err := p.Thumb(ctx); err != nil { - return nil, err - } - - if _, err := p.FullSize(ctx); err != nil { - return nil, err - } - - return p.attachment, nil -} - -func (p *Processing) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - return nil, nil -} - -func (p *Processing) Finished() bool { - return p.thumbstate == complete && p.fullSizeState == complete -} - -// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, -// and then if that doesn't work because the attachment already exists, updating it instead. -func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { - if err := database.Put(ctx, attachment); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { - return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) - } - } - - return nil -} diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go new file mode 100644 index 000000000..7e2d4f31f --- /dev/null +++ b/internal/media/processingemoji.go @@ -0,0 +1,382 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +// ProcessingEmoji represents an emoji currently processing. It exposes +// various functions for retrieving data from the process. +type ProcessingEmoji struct { + mu sync.Mutex + + /* + below fields should be set on newly created media; + emoji will be updated incrementally as media goes through processing + */ + + emoji *gtsmodel.Emoji + data DataFunc + + rawData []byte // will be set once the fetchRawData function has been called + + /* + below fields represent the processing state of the static version of the emoji + */ + + staticState processState + static *ImageMeta + + /* + below fields represent the processing state of the emoji image + */ + + fullSizeState processState + fullSize *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any +} + +// EmojiID returns the ID of the underlying emoji without blocking processing. +func (p *ProcessingEmoji) EmojiID() string { + return p.emoji.ID +} + +// LoadEmoji blocks until the static and fullsize image +// has been processed, and then returns the completed emoji. +func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + if err := p.fetchRawData(ctx); err != nil { + return nil, err + } + + if _, err := p.loadStatic(ctx); err != nil { + return nil, err + } + + if _, err := p.loadFullSize(ctx); err != nil { + return nil, err + } + + return p.emoji, nil +} + +// Finished returns true if processing has finished for both the thumbnail +// and full fized version of this piece of media. +func (p *ProcessingEmoji) Finished() bool { + return p.staticState == complete && p.fullSizeState == complete +} + +func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.staticState { + case received: + // we haven't processed a static version of this emoji yet so do it now + static, err := deriveStaticEmoji(p.rawData, p.emoji.ImageContentType) + if err != nil { + p.err = fmt.Errorf("error deriving static: %s", err) + p.staticState = errored + return nil, p.err + } + + // put the static in storage + if err := p.storage.Put(p.emoji.ImageStaticPath, static.image); err != nil { + p.err = fmt.Errorf("error storing static: %s", err) + p.staticState = errored + return nil, p.err + } + + // set appropriate fields on the emoji based on the static version we derived + p.attachment.FileMeta.Small = gtsmodel.Small{ + Width: static.width, + Height: static.height, + Size: static.size, + Aspect: static.aspect, + } + p.attachment.Thumbnail.FileSize = static.size + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.thumbstate = errored + return nil, err + } + + // set the thumbnail of this media + p.thumb = static + + // we're done processing the thumbnail! + p.thumbstate = complete + fallthrough + case complete: + return p.thumb, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) +} + +func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := p.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(p.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = p.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + + if err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // put the full size in storage + if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size image: %s", err) + p.fullSizeState = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the image we derived + p.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + p.attachment.File.FileSize = decoded.size + p.attachment.File.UpdatedAt = time.Now() + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + p.fullSize = decoded + + // we're done processing the full-size image + p.fullSizeState = complete + fallthrough + case complete: + return p.fullSize, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) +} + +// fetchRawData calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. +// It should only be called from within a function that already has a lock on p! +func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { + // check if we've already done this and bail early if we have + if p.rawData != nil { + return nil + } + + // execute the data function and pin the raw bytes for further processing + b, err := p.data(ctx) + if err != nil { + return fmt.Errorf("fetchRawData: error executing data function: %s", err) + } + p.rawData = b + + // now we have the data we can work out the content type + contentType, err := parseContentType(p.rawData) + if err != nil { + return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + } + + if !supportedEmoji(contentType) { + return fmt.Errorf("fetchRawData: content type %s was not valid for an emoji", contentType) + } + + split := strings.Split(contentType, "/") + mainType := split[0] // something like 'image' + extension := split[1] // something like 'gif' + + // set some additional fields on the emoji now that + // we know more about what the underlying image actually is + p.emoji.ImageURL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) + p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) + p.attachment.File.ContentType = contentType + + switch mainType { + case mimeImage: + if extension == mimeGif { + p.attachment.Type = gtsmodel.FileTypeGif + } else { + p.attachment.Type = gtsmodel.FileTypeImage + } + default: + return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) + } + + return nil +} + +// putOrUpdateEmoji is just a convenience function for first trying to PUT the emoji in the database, +// and then if that doesn't work because the emoji already exists, updating it instead. +func putOrUpdateEmoji(ctx context.Context, database db.DB, emoji *gtsmodel.Emoji) error { + if err := database.Put(ctx, emoji); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateEmoji: proper error while putting emoji: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, emoji); err != nil { + return fmt.Errorf("putOrUpdateEmoji: error while updating emoji: %s", err) + } + } + + return nil +} + +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) + } + + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + // populate initial fields on the emoji -- some of these will be overwritten as we proceed + emoji := >smodel.Emoji{ + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Shortcode: shortcode, + Domain: "", // assume our own domain unless told otherwise + ImageRemoteURL: "", + ImageStaticRemoteURL: "", + ImageURL: "", // we don't know yet + ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), id, mimePng), // all static emojis are encoded as png + ImagePath: "", // we don't know yet + ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, id, mimePng), // all static emojis are encoded as png + ImageContentType: "", // we don't know yet + ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png + ImageFileSize: 0, + ImageStaticFileSize: 0, + ImageUpdatedAt: time.Now(), + Disabled: false, + URI: "", // we don't know yet + VisibleInPicker: true, + CategoryID: "", + } + + // check if we have additional info to add to the emoji, + // and overwrite some of the emoji fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + + processingEmoji := &ProcessingEmoji{ + emoji: emoji, + data: data, + staticState: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return processingEmoji, nil +} diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go new file mode 100644 index 000000000..a6e45034f --- /dev/null +++ b/internal/media/processingmedia.go @@ -0,0 +1,411 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + +// ProcessingMedia represents a piece of media that is currently being processed. It exposes +// various functions for retrieving data from the process. +type ProcessingMedia struct { + mu sync.Mutex + + /* + below fields should be set on newly created media; + attachment will be updated incrementally as media goes through processing + */ + + attachment *gtsmodel.MediaAttachment + data DataFunc + + rawData []byte // will be set once the fetchRawData function has been called + + /* + below fields represent the processing state of the media thumbnail + */ + + thumbstate processState + thumb *ImageMeta + + /* + below fields represent the processing state of the full-sized media + */ + + fullSizeState processState + fullSize *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any +} + +// AttachmentID returns the ID of the underlying media attachment without blocking processing. +func (p *ProcessingMedia) AttachmentID() string { + return p.attachment.ID +} + +// LoadAttachment blocks until the thumbnail and fullsize content +// has been processed, and then returns the completed attachment. +func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { + if err := p.fetchRawData(ctx); err != nil { + return nil, err + } + + if _, err := p.loadThumb(ctx); err != nil { + return nil, err + } + + if _, err := p.loadFullSize(ctx); err != nil { + return nil, err + } + + return p.attachment, nil +} + +func (p *ProcessingMedia) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + return nil, nil +} + +// Finished returns true if processing has finished for both the thumbnail +// and full fized version of this piece of media. +func (p *ProcessingMedia) Finished() bool { + return p.thumbstate == complete && p.fullSizeState == complete +} + +func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.thumbstate { + case received: + // we haven't processed a thumbnail for this media yet so do it now + + // check if we need to create a blurhash or if there's already one set + var createBlurhash bool + if p.attachment.Blurhash == "" { + // no blurhash created yet + createBlurhash = true + } + + thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) + if err != nil { + p.err = fmt.Errorf("error deriving thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // put the thumbnail in storage + if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { + p.err = fmt.Errorf("error storing thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the thumbnail we derived + if createBlurhash { + p.attachment.Blurhash = thumb.blurhash + } + + p.attachment.FileMeta.Small = gtsmodel.Small{ + Width: thumb.width, + Height: thumb.height, + Size: thumb.size, + Aspect: thumb.aspect, + } + p.attachment.Thumbnail.FileSize = thumb.size + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.thumbstate = errored + return nil, err + } + + // set the thumbnail of this media + p.thumb = thumb + + // we're done processing the thumbnail! + p.thumbstate = complete + fallthrough + case complete: + return p.thumb, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) +} + +func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := p.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(p.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = p.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + + if err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // put the full size in storage + if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size image: %s", err) + p.fullSizeState = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the image we derived + p.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + p.attachment.File.FileSize = decoded.size + p.attachment.File.UpdatedAt = time.Now() + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + p.fullSize = decoded + + // we're done processing the full-size image + p.fullSizeState = complete + fallthrough + case complete: + return p.fullSize, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) +} + +// fetchRawData calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. +// It should only be called from within a function that already has a lock on p! +func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { + // check if we've already done this and bail early if we have + if p.rawData != nil { + return nil + } + + // execute the data function and pin the raw bytes for further processing + b, err := p.data(ctx) + if err != nil { + return fmt.Errorf("fetchRawData: error executing data function: %s", err) + } + p.rawData = b + + // now we have the data we can work out the content type + contentType, err := parseContentType(p.rawData) + if err != nil { + return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + } + + split := strings.Split(contentType, "/") + if len(split) != 2 { + return fmt.Errorf("fetchRawData: content type %s was not valid", contentType) + } + + mainType := split[0] // something like 'image' + extension := split[1] // something like 'jpeg' + + // set some additional fields on the attachment now that + // we know more about what the underlying media actually is + p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) + p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) + p.attachment.File.ContentType = contentType + + switch mainType { + case mimeImage: + if extension == mimeGif { + p.attachment.Type = gtsmodel.FileTypeGif + } else { + p.attachment.Type = gtsmodel.FileTypeImage + } + default: + return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) + } + + return nil +} + +// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, +// and then if that doesn't work because the attachment already exists, updating it instead. +func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { + if err := database.Put(ctx, attachment); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { + return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) + } + } + + return nil +} + +func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + file := gtsmodel.File{ + Path: "", // we don't know yet because it depends on the uncalled DataFunc + ContentType: "", // we don't know yet because it depends on the uncalled DataFunc + UpdatedAt: time.Now(), + } + + thumbnail := gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + } + + // populate initial fields on the media attachment -- some of these will be overwritten as we proceed + attachment := >smodel.MediaAttachment{ + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StatusID: "", + URL: "", // we don't know yet because it depends on the uncalled DataFunc + RemoteURL: "", + Type: gtsmodel.FileTypeUnknown, // we don't know yet because it depends on the uncalled DataFunc + FileMeta: gtsmodel.FileMeta{}, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: gtsmodel.ProcessingStatusReceived, + File: file, + Thumbnail: thumbnail, + Avatar: false, + Header: false, + } + + // check if we have additional info to add to the attachment, + // and overwrite some of the attachment fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + + processingMedia := &ProcessingMedia{ + attachment: attachment, + data: data, + thumbstate: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return processingMedia, nil +} diff --git a/internal/media/types.go b/internal/media/types.go index aaf423682..6426223d1 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,6 +20,7 @@ package media import ( "bytes" + "context" "errors" "fmt" "time" @@ -68,9 +69,9 @@ const ( TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests ) -// AdditionalInfo represents additional information that should be added to an attachment +// AdditionalMediaInfo represents additional information that should be added to an attachment // when processing a piece of media. -type AdditionalInfo struct { +type AdditionalMediaInfo struct { // Time that this media was created; defaults to time.Now(). CreatedAt *time.Time // ID of the status to which this media is attached; defaults to "". @@ -93,6 +94,13 @@ type AdditionalInfo struct { FocusY *float32 } +type AdditionalEmojiInfo struct { + +} + +// DataFunc represents a function used to retrieve the raw bytes of a piece of media. +type DataFunc func(ctx context.Context) ([]byte, error) + // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. func parseContentType(content []byte) (string, error) { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 6d15b5afb..7b305dc95 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -137,84 +137,87 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form // parsing and checking the image, and doing the necessary updates in the database for this to become // the account's new avatar image. func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { - var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(avatar.Size) > maxImageSize { - err = fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) - return nil, err - } - f, err := avatar.Open() - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") - } + dataFunc := func(ctx context.Context) ([]byte, error) { + // pop open the fileheader + f, err := avatar.Open() + if err != nil { + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + } - // we're done with the FileHeader now - if err := f.Close(); err != nil { - return nil, fmt.Errorf("UpdateAvatar: error closing multipart fileheader: %s", err) + // extract the bytes + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + } + if size == 0 { + return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") + } + + return buf.Bytes(), f.Close() } - // do the setting isAvatar := true - processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + ai := &media.AdditionalMediaInfo{ Avatar: &isAvatar, - }) + } + + processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) if err != nil { return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } - return processingMedia.Load(ctx) + return processingMedia.LoadAttachment(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, // parsing and checking the image, and doing the necessary updates in the database for this to become // the account's new header image. func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { - var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(header.Size) > maxImageSize { - err = fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) - return nil, err - } - f, err := header.Open() - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") - } + dataFunc := func(ctx context.Context) ([]byte, error) { + // pop open the fileheader + f, err := header.Open() + if err != nil { + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + } + + // extract the bytes + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + } + if size == 0 { + return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") + } - // we're done with the FileHeader now - if err := f.Close(); err != nil { - return nil, fmt.Errorf("UpdateHeader: error closing multipart fileheader: %s", err) + return buf.Bytes(), f.Close() } - // do the setting isHeader := true - processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + ai := &media.AdditionalMediaInfo{ Header: &isHeader, - }) + } + + processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) + if err != nil { + return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) + } if err != nil { return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } - return processingMedia.Load(ctx) + return processingMedia.LoadAttachment(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 737a4ebe2..8858dbd02 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -30,30 +30,34 @@ import ( ) func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { - if user.Admin { + if !user.Admin { return nil, fmt.Errorf("user %s not an admin", user.ID) } - // open the emoji and extract the bytes from it - f, err := form.Image.Open() - if err != nil { - return nil, fmt.Errorf("error opening emoji: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading emoji: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided emoji: size 0 bytes") + data := func(innerCtx context.Context) ([]byte, error) { + // open the emoji and extract the bytes from it + f, err := form.Image.Open() + if err != nil { + return nil, fmt.Errorf("error opening emoji: %s", err) + } + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("error reading emoji: %s", err) + } + if size == 0 { + return nil, errors.New("could not read provided emoji: size 0 bytes") + } + + return buf.Bytes(), f.Close() } - media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID) + processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, nil) if err != nil { return nil, err } - emoji, err := media.LoadEmoji(ctx) + emoji, err := processingEmoji.LoadEmoji(ctx) if err != nil { return nil, err } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 9df5c7c1f..0896315b1 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -31,18 +31,21 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - // open the attachment and extract the bytes from it - f, err := form.File.Open() - if err != nil { - return nil, fmt.Errorf("error opening attachment: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided attachment: size 0 bytes") + data := func(innerCtx context.Context) ([]byte, error) { + // open the attachment and extract the bytes from it + f, err := form.File.Open() + if err != nil { + return nil, fmt.Errorf("error opening attachment: %s", err) + } + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("error reading attachment: %s", err) + } + if size == 0 { + return nil, errors.New("could not read provided attachment: size 0 bytes") + } + return buf.Bytes(), f.Close() } focusX, focusY, err := parseFocus(form.Focus) @@ -51,7 +54,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form } // process the media attachment and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, &media.AdditionalInfo{ + media, err := p.mediaManager.ProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{ Description: &form.Description, FocusX: &focusX, FocusY: &focusY, @@ -60,7 +63,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, err } - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) if err != nil { return nil, err } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index b470b289a..c43515a42 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -33,7 +33,7 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType. + // DereferenceMedia fetches the bytes of the given media attachment IRI. DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 203aaef96..9a9ea5d2f 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -66,6 +66,16 @@ func NewTestTokens() map[string]*gtsmodel.Token { AccessCreateAt: time.Now(), AccessExpiresAt: time.Now().Add(72 * time.Hour), }, + "admin_account": { + ID: "01FS4TP8ANA5VE92EAPA9E0M7Q", + ClientID: "01F8MGWSJCND9BWBD4WGJXBM93", + UserID: "01F8MGWYWKVKS3VS8DV1AMYPGE", + RedirectURI: "http://localhost:8080", + Scope: "read write follow push admin", + Access: "AININALKNENFNF98717NAMG4LWE4NJITMWUXM2M4MTRHZDEX", + AccessCreateAt: time.Now(), + AccessExpiresAt: time.Now().Add(72 * time.Hour), + }, } return tokens } -- cgit v1.2.3 From c4a533db72505ca5303d8da637f54fae12b137a2 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 15 Jan 2022 14:33:58 +0100 Subject: start fixing up emoji processing code --- internal/api/client/admin/emojicreate_test.go | 40 +++++++ internal/media/image.go | 5 - internal/media/manager.go | 6 +- internal/media/processingemoji.go | 151 +++++++------------------- internal/media/processingmedia.go | 35 +----- internal/media/types.go | 124 ++++++--------------- internal/media/util.go | 123 +++++++++++++++++++++ internal/processing/admin/emoji.go | 11 +- 8 files changed, 249 insertions(+), 246 deletions(-) create mode 100644 internal/media/util.go (limited to 'internal/media/types.go') diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 290b478f7..14b83b534 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -1,6 +1,8 @@ package admin_test import ( + "context" + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -8,6 +10,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -43,6 +48,41 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { b, err := ioutil.ReadAll(result.Body) suite.NoError(err) suite.NotEmpty(b) + + // response should be an api model emoji + apiEmoji := &apimodel.Emoji{} + err = json.Unmarshal(b, apiEmoji) + suite.NoError(err) + + // appropriate fields should be set + suite.Equal("rainbow", apiEmoji.Shortcode) + suite.NotEmpty(apiEmoji.URL) + suite.NotEmpty(apiEmoji.StaticURL) + suite.True(apiEmoji.VisibleInPicker) + + // emoji should be in the db + dbEmoji := >smodel.Emoji{} + err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "rainbow"}}, dbEmoji) + suite.NoError(err) + + // check fields on the emoji + suite.NotEmpty(dbEmoji.ID) + suite.Equal("rainbow", dbEmoji.Shortcode) + suite.Empty(dbEmoji.Domain) + suite.Empty(dbEmoji.ImageRemoteURL) + suite.Empty(dbEmoji.ImageStaticRemoteURL) + suite.Equal(apiEmoji.URL, dbEmoji.ImageURL) + suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageURL) + suite.NotEmpty(dbEmoji.ImagePath) + suite.NotEmpty(dbEmoji.ImageStaticPath) + suite.Equal("image/png", dbEmoji.ImageContentType) + suite.Equal("image/png", dbEmoji.ImageStaticContentType) + suite.Equal(36702, dbEmoji.ImageFileSize) + suite.Equal(10413, dbEmoji.ImageStaticFileSize) + suite.False(dbEmoji.Disabled) + suite.NotEmpty(dbEmoji.URI) + suite.True(dbEmoji.VisibleInPicker) + suite.Empty(dbEmoji.CategoryID)aaaaaaaaa } func TestEmojiCreateTestSuite(t *testing.T) { diff --git a/internal/media/image.go b/internal/media/image.go index a5a818206..de4b71210 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,21 +20,16 @@ package media import ( "bytes" - "context" "errors" "fmt" "image" "image/gif" "image/jpeg" "image/png" - "time" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" "github.com/superseriousbusiness/exifremove/pkg/exifremove" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) const ( diff --git a/internal/media/manager.go b/internal/media/manager.go index e34471591..7f626271a 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -41,7 +41,7 @@ type Manager interface { // // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) - ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) + ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) // NumWorkers returns the total number of workers available to this manager. NumWorkers() int // QueueSize returns the total capacity of the queue. @@ -125,8 +125,8 @@ func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, accountID str return processingMedia, nil } -func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { - processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, ai) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, id, uri, ai) if err != nil { return nil, err } diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 41754830f..eeccdb281 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -28,7 +28,6 @@ import ( "codeberg.org/gruf/go-store/kv" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -126,33 +125,28 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { } // set appropriate fields on the emoji based on the static version we derived - p.attachment.FileMeta.Small = gtsmodel.Small{ - Width: static.width, - Height: static.height, - Size: static.size, - Aspect: static.aspect, - } - p.attachment.Thumbnail.FileSize = static.size + p.emoji.ImageStaticFileSize = len(static.image) - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + // update the emoji in the db + if err := putOrUpdate(ctx, p.database, p.emoji); err != nil { p.err = err - p.thumbstate = errored + p.staticState = errored return nil, err } - // set the thumbnail of this media - p.thumb = static + // set the static on the processing emoji + p.static = static - // we're done processing the thumbnail! - p.thumbstate = complete + // we're done processing the static version of the emoji! + p.staticState = complete fallthrough case complete: - return p.thumb, nil + return p.static, nil case errored: return nil, p.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) + return nil, fmt.Errorf("static processing status %d unknown", p.staticState) } func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { @@ -161,26 +155,17 @@ func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) switch p.fullSizeState { case received: - var clean []byte var err error var decoded *ImageMeta - ct := p.attachment.File.ContentType + ct := p.emoji.ImageContentType switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) + case mimeImagePng: + decoded, err = decodeImage(p.rawData, ct) case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) + decoded, err = decodeGif(p.rawData) default: - err = fmt.Errorf("content type %s not a processible image type", ct) + err = fmt.Errorf("content type %s not a processible emoji type", ct) } if err != nil { @@ -189,34 +174,17 @@ func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) return nil, err } - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) + // put the full size emoji in storage + if err := p.storage.Put(p.emoji.ImagePath, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size emoji: %s", err) p.fullSizeState = errored return nil, p.err } - // set appropriate fields on the attachment based on the image we derived - p.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, - } - p.attachment.File.FileSize = decoded.size - p.attachment.File.UpdatedAt = time.Now() - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - // set the fullsize of this media p.fullSize = decoded - // we're done processing the full-size image + // we're done processing the full-size emoji p.fullSizeState = complete fallthrough case complete: @@ -255,55 +223,24 @@ func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { } split := strings.Split(contentType, "/") - mainType := split[0] // something like 'image' extension := split[1] // something like 'gif' // set some additional fields on the emoji now that // we know more about what the underlying image actually is - p.emoji.ImageURL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) - p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) - p.attachment.File.ContentType = contentType - - switch mainType { - case mimeImage: - if extension == mimeGif { - p.attachment.Type = gtsmodel.FileTypeGif - } else { - p.attachment.Type = gtsmodel.FileTypeImage - } - default: - return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) - } - - return nil -} - -// putOrUpdateEmoji is just a convenience function for first trying to PUT the emoji in the database, -// and then if that doesn't work because the emoji already exists, updating it instead. -func putOrUpdateEmoji(ctx context.Context, database db.DB, emoji *gtsmodel.Emoji) error { - if err := database.Put(ctx, emoji); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateEmoji: proper error while putting emoji: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, emoji); err != nil { - return fmt.Errorf("putOrUpdateEmoji: error while updating emoji: %s", err) - } - } + p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) + p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) + p.emoji.ImageContentType = contentType + p.emoji.ImageFileSize = len(p.rawData) return nil } -func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { instanceAccount, err := m.db.GetInstanceAccount(ctx, "") if err != nil { return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) } - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - // populate initial fields on the emoji -- some of these will be overwritten as we proceed emoji := >smodel.Emoji{ ID: id, @@ -323,7 +260,7 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode ImageStaticFileSize: 0, ImageUpdatedAt: time.Now(), Disabled: false, - URI: "", // we don't know yet + URI: uri, VisibleInPicker: true, CategoryID: "", } @@ -332,43 +269,31 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode // and overwrite some of the emoji fields if so if ai != nil { if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description + emoji.CreatedAt = *ai.CreatedAt } - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID + if ai.Domain != nil { + emoji.Domain = *ai.Domain } - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash + if ai.ImageRemoteURL != nil { + emoji.ImageRemoteURL = *ai.ImageRemoteURL } - if ai.Avatar != nil { - attachment.Avatar = *ai.Avatar + if ai.ImageStaticRemoteURL != nil { + emoji.ImageStaticRemoteURL = *ai.ImageStaticRemoteURL } - if ai.Header != nil { - attachment.Header = *ai.Header + if ai.Disabled != nil { + emoji.Disabled = *ai.Disabled } - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX + if ai.VisibleInPicker != nil { + emoji.VisibleInPicker = *ai.VisibleInPicker } - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY + if ai.CategoryID != nil { + emoji.CategoryID = *ai.CategoryID } } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index a6e45034f..1bfd7b629 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -32,14 +32,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/uris" ) -type processState int - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - // ProcessingMedia represents a piece of media that is currently being processed. It exposes // various functions for retrieving data from the process. type ProcessingMedia struct { @@ -103,10 +95,6 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt return p.attachment, nil } -func (p *ProcessingMedia) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - return nil, nil -} - // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingMedia) Finished() bool { @@ -153,9 +141,9 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { Size: thumb.size, Aspect: thumb.aspect, } - p.attachment.Thumbnail.FileSize = thumb.size + p.attachment.Thumbnail.FileSize = len(thumb.image) - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { p.err = err p.thumbstate = errored return nil, err @@ -224,11 +212,11 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) Size: decoded.size, Aspect: decoded.aspect, } - p.attachment.File.FileSize = decoded.size + p.attachment.File.FileSize = len(decoded.image) p.attachment.File.UpdatedAt = time.Now() p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { p.err = err p.fullSizeState = errored return nil, err @@ -299,21 +287,6 @@ func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { return nil } -// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, -// and then if that doesn't work because the attachment already exists, updating it instead. -func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { - if err := database.Put(ctx, attachment); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { - return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) - } - } - - return nil -} - func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { id, err := id.NewRandomULID() if err != nil { diff --git a/internal/media/types.go b/internal/media/types.go index 6426223d1..5b3fe4a41 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -19,15 +19,17 @@ package media import ( - "bytes" "context" - "errors" - "fmt" "time" - - "github.com/h2non/filetype" ) +// maxFileHeaderBytes represents the maximum amount of bytes we want +// to examine from the beginning of a file to determine its type. +// +// See: https://en.wikipedia.org/wiki/File_format#File_header +// and https://github.com/h2non/filetype +const maxFileHeaderBytes = 262 + // mime consts const ( mimeImage = "image" @@ -42,16 +44,17 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + // EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) // const EmojiMaxBytes = 51200 -// maxFileHeaderBytes represents the maximum amount of bytes we want -// to examine from the beginning of a file to determine its type. -// -// See: https://en.wikipedia.org/wiki/File_format#File_header -// and https://github.com/h2non/filetype -const maxFileHeaderBytes = 262 - type Size string const ( @@ -94,89 +97,24 @@ type AdditionalMediaInfo struct { FocusY *float32 } +// AdditionalMediaInfo represents additional information +// that should be added to an emoji when processing it. type AdditionalEmojiInfo struct { - + // Time that this emoji was created; defaults to time.Now(). + CreatedAt *time.Time + // Domain the emoji originated from. Blank for this instance's domain. Defaults to "". + Domain *string + // URL of this emoji on a remote instance; defaults to "". + ImageRemoteURL *string + // URL of the static version of this emoji on a remote instance; defaults to "". + ImageStaticRemoteURL *string + // Whether this emoji should be disabled (not shown) on this instance; defaults to false. + Disabled *bool + // Whether this emoji should be visible in the instance's emoji picker; defaults to true. + VisibleInPicker *bool + // ID of the category this emoji should be placed in; defaults to "". + CategoryID *string } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. type DataFunc func(ctx context.Context) ([]byte, error) - -// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). -// Returns an error if the content type is not something we can process. -func parseContentType(content []byte) (string, error) { - - // read in the first bytes of the file - fileHeader := make([]byte, maxFileHeaderBytes) - if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { - return "", fmt.Errorf("could not read first magic bytes of file: %s", err) - } - - kind, err := filetype.Match(fileHeader) - if err != nil { - return "", err - } - - if kind == filetype.Unknown { - return "", errors.New("filetype unknown") - } - - return kind.MIME.Value, nil -} - -// supportedImage checks mime type of an image against a slice of accepted types, -// and returns True if the mime type is accepted. -func supportedImage(mimeType string) bool { - acceptedImageTypes := []string{ - mimeImageJpeg, - mimeImageGif, - mimeImagePng, - } - for _, accepted := range acceptedImageTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. -func supportedEmoji(mimeType string) bool { - acceptedEmojiTypes := []string{ - mimeImageGif, - mimeImagePng, - } - for _, accepted := range acceptedEmojiTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized -func ParseMediaType(s string) (Type, error) { - switch s { - case string(TypeAttachment): - return TypeAttachment, nil - case string(TypeHeader): - return TypeHeader, nil - case string(TypeAvatar): - return TypeAvatar, nil - case string(TypeEmoji): - return TypeEmoji, nil - } - return "", fmt.Errorf("%s not a recognized MediaType", s) -} - -// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized -func ParseMediaSize(s string) (Size, error) { - switch s { - case string(SizeSmall): - return SizeSmall, nil - case string(SizeOriginal): - return SizeOriginal, nil - case string(SizeStatic): - return SizeStatic, nil - } - return "", fmt.Errorf("%s not a recognized MediaSize", s) -} diff --git a/internal/media/util.go b/internal/media/util.go new file mode 100644 index 000000000..16e874a99 --- /dev/null +++ b/internal/media/util.go @@ -0,0 +1,123 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/h2non/filetype" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). +// Returns an error if the content type is not something we can process. +func parseContentType(content []byte) (string, error) { + // read in the first bytes of the file + fileHeader := make([]byte, maxFileHeaderBytes) + if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { + return "", fmt.Errorf("could not read first magic bytes of file: %s", err) + } + + kind, err := filetype.Match(fileHeader) + if err != nil { + return "", err + } + + if kind == filetype.Unknown { + return "", errors.New("filetype unknown") + } + + return kind.MIME.Value, nil +} + +// supportedImage checks mime type of an image against a slice of accepted types, +// and returns True if the mime type is accepted. +func supportedImage(mimeType string) bool { + acceptedImageTypes := []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedImageTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. +func supportedEmoji(mimeType string) bool { + acceptedEmojiTypes := []string{ + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedEmojiTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized +func ParseMediaType(s string) (Type, error) { + switch s { + case string(TypeAttachment): + return TypeAttachment, nil + case string(TypeHeader): + return TypeHeader, nil + case string(TypeAvatar): + return TypeAvatar, nil + case string(TypeEmoji): + return TypeEmoji, nil + } + return "", fmt.Errorf("%s not a recognized MediaType", s) +} + +// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized +func ParseMediaSize(s string) (Size, error) { + switch s { + case string(SizeSmall): + return SizeSmall, nil + case string(SizeOriginal): + return SizeOriginal, nil + case string(SizeStatic): + return SizeStatic, nil + } + return "", fmt.Errorf("%s not a recognized MediaSize", s) +} + +// putOrUpdate is just a convenience function for first trying to PUT the attachment or emoji in the database, +// and then if that doesn't work because the attachment/emoji already exists, updating it instead. +func putOrUpdate(ctx context.Context, database db.DB, i interface{}) error { + if err := database.Put(ctx, i); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdate: proper error while putting: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, i); err != nil { + return fmt.Errorf("putOrUpdate: error while updating: %s", err) + } + } + + return nil +} diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 8858dbd02..77fa5102b 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -27,6 +27,8 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { @@ -52,7 +54,14 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return buf.Bytes(), f.Close() } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, nil) + emojiID, err := id.NewRandomULID() + if err != nil { + return nil, fmt.Errorf("error creating id for new emoji: %s", err) + } + + emojiURI := uris.GenerateURIForEmoji(emojiID) + + processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, emojiID, emojiURI, nil) if err != nil { return nil, err } -- cgit v1.2.3 From 589bb9df0275457b5f9c3790e67517ec1be1745d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 16 Jan 2022 18:52:55 +0100 Subject: pass reader around instead of []byte --- internal/federation/dereferencing/account.go | 5 +- internal/federation/dereferencing/media.go | 3 +- internal/media/image.go | 46 ++++--- internal/media/manager_test.go | 32 +++-- internal/media/processingemoji.go | 168 +++++++++++-------------- internal/media/processingmedia.go | 179 ++++++++++++++------------- internal/media/types.go | 5 +- internal/media/util.go | 11 +- internal/processing/account/update.go | 42 +------ internal/processing/admin/emoji.go | 20 +-- internal/processing/media/create.go | 19 +-- internal/transport/derefmedia.go | 7 +- internal/transport/transport.go | 5 +- testrig/storage.go | 82 +----------- 14 files changed, 238 insertions(+), 386 deletions(-) (limited to 'internal/media/types.go') diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index b9efbfa45..6ea8256d5 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -23,6 +23,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/url" "strings" @@ -251,7 +252,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) ([]byte, error) { + data := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, avatarIRI) } @@ -273,7 +274,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) ([]byte, error) { + data := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, headerIRI) } diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 46cb4a5f4..c427f2507 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -21,6 +21,7 @@ package dereferencing import ( "context" "fmt" + "io" "net/url" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -41,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) ([]byte, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/media/image.go b/internal/media/image.go index de4b71210..b8f00024f 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -26,6 +26,7 @@ import ( "image/gif" "image/jpeg" "image/png" + "io" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" @@ -37,17 +38,17 @@ const ( thumbnailMaxHeight = 512 ) -type ImageMeta struct { - image []byte +type imageMeta struct { width int height int size int aspect float64 - blurhash string + blurhash string // defined only for calls to deriveThumbnail if createBlurhash is true + small []byte // defined only for calls to deriveStaticEmoji or deriveThumbnail } -func decodeGif(b []byte) (*ImageMeta, error) { - gif, err := gif.DecodeAll(bytes.NewReader(b)) +func decodeGif(r io.Reader) (*imageMeta, error) { + gif, err := gif.DecodeAll(r) if err != nil { return nil, err } @@ -58,8 +59,7 @@ func decodeGif(b []byte) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &ImageMeta{ - image: b, + return &imageMeta{ width: width, height: height, size: size, @@ -67,15 +67,15 @@ func decodeGif(b []byte) (*ImageMeta, error) { }, nil } -func decodeImage(b []byte, contentType string) (*ImageMeta, error) { +func decodeImage(r io.Reader, contentType string) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) + i, err = jpeg.Decode(r) case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) default: err = fmt.Errorf("content type %s not recognised", contentType) } @@ -93,8 +93,7 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &ImageMeta{ - image: b, + return &imageMeta{ width: width, height: height, size: size, @@ -111,17 +110,17 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { // // If createBlurhash is false, then the blurhash field on the returned ImageAndMeta // will be an empty string. -func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageMeta, error) { +func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) + i, err = jpeg.Decode(r) case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) + i, err = gif.Decode(r) default: err = fmt.Errorf("content type %s can't be thumbnailed", contentType) } @@ -140,7 +139,7 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM size := width * height aspect := float64(width) / float64(height) - im := &ImageMeta{ + im := &imageMeta{ width: width, height: height, size: size, @@ -165,25 +164,24 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM }); err != nil { return nil, err } - - im.image = out.Bytes() + im.small = out.Bytes() return im, nil } // deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*ImageMeta, error) { +func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) if err != nil { return nil, err } case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) + i, err = gif.Decode(r) if err != nil { return nil, err } @@ -195,8 +193,8 @@ func deriveStaticEmoji(b []byte, contentType string) (*ImageMeta, error) { if err := png.Encode(out, i); err != nil { return nil, err } - return &ImageMeta{ - image: out.Bytes(), + return &imageMeta{ + small: out.Bytes(), }, nil } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 0fadceb37..5380b83b1 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -19,8 +19,10 @@ package media_test import ( + "bytes" "context" "fmt" + "io" "os" "testing" "time" @@ -37,9 +39,13 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) ([]byte, error) { + data := func(_ context.Context) (io.Reader, error) { // load bytes from a test image - return os.ReadFile("./test/test-jpeg.jpg") + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -103,9 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - data := func(_ context.Context) ([]byte, error) { + data := func(_ context.Context) (io.Reader, error) { // load bytes from a test image - return os.ReadFile("./test/test-jpeg.jpg") + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -175,16 +185,16 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { // in this test, we spam the manager queue with 50 new media requests, just to see how it holds up - ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } - data := func(_ context.Context) ([]byte, error) { - return testBytes, nil + data := func(_ context.Context) (io.Reader, error) { + // load bytes from a test image + return bytes.NewReader(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 467b500fc..147b6b5b3 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -19,8 +19,10 @@ package media import ( + "bytes" "context" "fmt" + "io" "strings" "sync" "time" @@ -46,22 +48,14 @@ type ProcessingEmoji struct { emoji *gtsmodel.Emoji data DataFunc - - rawData []byte // will be set once the fetchRawData function has been called + read bool // bool indicating that data function has been triggered already /* - below fields represent the processing state of the static version of the emoji - */ - - staticState processState - static *ImageMeta - - /* - below fields represent the processing state of the emoji image + below fields represent the processing state of the static of the emoji */ + staticState processState fullSizeState processState - fullSize *ImageMeta /* below pointers to database and storage are maintained so that @@ -85,21 +79,18 @@ func (p *ProcessingEmoji) EmojiID() string { // LoadEmoji blocks until the static and fullsize image // has been processed, and then returns the completed emoji. func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - if err := p.fetchRawData(ctx); err != nil { - return nil, err - } + p.mu.Lock() + defer p.mu.Unlock() - if _, err := p.loadStatic(ctx); err != nil { + if err := p.store(ctx); err != nil { return nil, err } - if _, err := p.loadFullSize(ctx); err != nil { + if err := p.loadStatic(ctx); err != nil { return nil, err } // store the result in the database before returning it - p.mu.Lock() - defer p.mu.Unlock() if !p.insertedInDB { if err := p.database.Put(ctx, p.emoji); err != nil { return nil, err @@ -116,118 +107,85 @@ func (p *ProcessingEmoji) Finished() bool { return p.staticState == complete && p.fullSizeState == complete } -func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { switch p.staticState { case received: - // we haven't processed a static version of this emoji yet so do it now - static, err := deriveStaticEmoji(p.rawData, p.emoji.ImageContentType) + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.emoji.ImagePath) if err != nil { - p.err = fmt.Errorf("error deriving static: %s", err) + p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) p.staticState = errored - return nil, p.err + return p.err } - // put the static in storage - if err := p.storage.Put(p.emoji.ImageStaticPath, static.image); err != nil { - p.err = fmt.Errorf("error storing static: %s", err) + // we haven't processed a static version of this emoji yet so do it now + static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType) + if err != nil { + p.err = fmt.Errorf("loadStatic: error deriving static: %s", err) p.staticState = errored - return nil, p.err + return p.err } - // set appropriate fields on the emoji based on the static version we derived - p.emoji.ImageStaticFileSize = len(static.image) - - // set the static on the processing emoji - p.static = static - - // we're done processing the static version of the emoji! - p.staticState = complete - fallthrough - case complete: - return p.static, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("static processing status %d unknown", p.staticState) -} - -func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.fullSizeState { - case received: - var err error - var decoded *ImageMeta - - ct := p.emoji.ImageContentType - switch ct { - case mimeImagePng: - decoded, err = decodeImage(p.rawData, ct) - case mimeImageGif: - decoded, err = decodeGif(p.rawData) - default: - err = fmt.Errorf("content type %s not a processible emoji type", ct) - } - - if err != nil { - p.err = err - p.fullSizeState = errored - return nil, err + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadStatic: error closing stored full size: %s", err) + p.staticState = errored + return p.err } - // put the full size emoji in storage - if err := p.storage.Put(p.emoji.ImagePath, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size emoji: %s", err) - p.fullSizeState = errored - return nil, p.err + // put the static in storage + if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { + p.err = fmt.Errorf("loadStatic: error storing static: %s", err) + p.staticState = errored + return p.err } - // set the fullsize of this media - p.fullSize = decoded + p.emoji.ImageStaticFileSize = len(static.small) - // we're done processing the full-size emoji - p.fullSizeState = complete + // we're done processing the static version of the emoji! + p.staticState = complete fallthrough case complete: - return p.fullSize, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) + return fmt.Errorf("static processing status %d unknown", p.staticState) } -// fetchRawData calls the data function attached to p if it hasn't been called yet, -// and updates the underlying emoji fields as necessary. -// It should only be called from within a function that already has a lock on p! -func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { +// store calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. It will then stream +// bytes from p's reader directly into storage so that it can be retrieved later. +func (p *ProcessingEmoji) store(ctx context.Context) error { // check if we've already done this and bail early if we have - if p.rawData != nil { + if p.read { return nil } - // execute the data function and pin the raw bytes for further processing - b, err := p.data(ctx) + // execute the data function to get the reader out of it + reader, err := p.data(ctx) if err != nil { - return fmt.Errorf("fetchRawData: error executing data function: %s", err) + return fmt.Errorf("store: error executing data function: %s", err) + } + + // extract no more than 261 bytes from the beginning of the file -- this is the header + firstBytes := make([]byte, maxFileHeaderBytes) + if _, err := reader.Read(firstBytes); err != nil { + return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) } - p.rawData = b - // now we have the data we can work out the content type - contentType, err := parseContentType(p.rawData) + // now we have the file header we can work out the content type from it + contentType, err := parseContentType(firstBytes) if err != nil { - return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + return fmt.Errorf("store: error parsing content type: %s", err) } + // bail if this is a type we can't process if !supportedEmoji(contentType) { - return fmt.Errorf("fetchRawData: content type %s was not valid for an emoji", contentType) + return fmt.Errorf("store: content type %s was not valid for an emoji", contentType) } + // extract the file extension split := strings.Split(contentType, "/") extension := split[1] // something like 'gif' @@ -236,8 +194,24 @@ func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) p.emoji.ImageContentType = contentType - p.emoji.ImageFileSize = len(p.rawData) + // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // store this for now -- other processes can pull it out of storage as they please + if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { + return fmt.Errorf("store: error storing stream: %s", err) + } + p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream + + // if the original reader is a readcloser, close it since we're done with it now + if rc, ok := reader.(io.ReadCloser); ok { + if err := rc.Close(); err != nil { + return fmt.Errorf("store: error closing readcloser: %s", err) + } + } + + p.read = true return nil } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 082c58607..82db863e0 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -19,8 +19,10 @@ package media import ( + "bytes" "context" "fmt" + "io" "strings" "sync" "time" @@ -44,22 +46,10 @@ type ProcessingMedia struct { attachment *gtsmodel.MediaAttachment data DataFunc + read bool // bool indicating that data function has been triggered already - rawData []byte // will be set once the fetchRawData function has been called - - /* - below fields represent the processing state of the media thumbnail - */ - - thumbstate processState - thumb *ImageMeta - - /* - below fields represent the processing state of the full-sized media - */ - - fullSizeState processState - fullSize *ImageMeta + thumbstate processState // the processing state of the media thumbnail + fullSizeState processState // the processing state of the full-sized media /* below pointers to database and storage are maintained so that @@ -83,21 +73,22 @@ func (p *ProcessingMedia) AttachmentID() string { // LoadAttachment blocks until the thumbnail and fullsize content // has been processed, and then returns the completed attachment. func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - if err := p.fetchRawData(ctx); err != nil { + p.mu.Lock() + defer p.mu.Unlock() + + if err := p.store(ctx); err != nil { return nil, err } - if _, err := p.loadThumb(ctx); err != nil { + if err := p.loadThumb(ctx); err != nil { return nil, err } - if _, err := p.loadFullSize(ctx); err != nil { + if err := p.loadFullSize(ctx); err != nil { return nil, err } // store the result in the database before returning it - p.mu.Lock() - defer p.mu.Unlock() if !p.insertedInDB { if err := p.database.Put(ctx, p.attachment); err != nil { return nil, err @@ -114,10 +105,7 @@ func (p *ProcessingMedia) Finished() bool { return p.thumbstate == complete && p.fullSizeState == complete } -func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingMedia) loadThumb(ctx context.Context) error { switch p.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now @@ -129,87 +117,94 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { createBlurhash = true } - thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.attachment.File.Path) + if err != nil { + p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) + p.thumbstate = errored + return p.err + } + + // ... and into the derive thumbnail function + thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash) if err != nil { - p.err = fmt.Errorf("error deriving thumbnail: %s", err) + p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) + p.thumbstate = errored + return p.err + } + + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadThumb: error closing stored full size: %s", err) p.thumbstate = errored - return nil, p.err + return p.err } // put the thumbnail in storage - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { - p.err = fmt.Errorf("error storing thumbnail: %s", err) + if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { + p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) p.thumbstate = errored - return nil, p.err + return p.err } // set appropriate fields on the attachment based on the thumbnail we derived if createBlurhash { p.attachment.Blurhash = thumb.blurhash } - p.attachment.FileMeta.Small = gtsmodel.Small{ Width: thumb.width, Height: thumb.height, Size: thumb.size, Aspect: thumb.aspect, } - p.attachment.Thumbnail.FileSize = len(thumb.image) - - // set the thumbnail of this media - p.thumb = thumb + p.attachment.Thumbnail.FileSize = len(thumb.small) // we're done processing the thumbnail! p.thumbstate = complete fallthrough case complete: - return p.thumb, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) + return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbstate) } -func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { switch p.fullSizeState { case received: - var clean []byte var err error - var decoded *ImageMeta + var decoded *imageMeta + + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.attachment.File.Path) + if err != nil { + p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) + p.fullSizeState = errored + return p.err + } + // decode the image ct := p.attachment.File.ContentType switch ct { case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) + decoded, err = decodeImage(stored, ct) case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) + decoded, err = decodeGif(stored) default: - err = fmt.Errorf("content type %s not a processible image type", ct) + err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct) } if err != nil { p.err = err p.fullSizeState = errored - return nil, err + return p.err } - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) - p.fullSizeState = errored - return nil, p.err + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadFullSize: error closing stored full size: %s", err) + p.thumbstate = errored + return p.err } // set appropriate fields on the attachment based on the image we derived @@ -219,56 +214,58 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) Size: decoded.size, Aspect: decoded.aspect, } - p.attachment.File.FileSize = len(decoded.image) p.attachment.File.UpdatedAt = time.Now() p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - // set the fullsize of this media - p.fullSize = decoded - // we're done processing the full-size image p.fullSizeState = complete fallthrough case complete: - return p.fullSize, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) + return fmt.Errorf("loadFullSize: full size processing status %d unknown", p.fullSizeState) } -// fetchRawData calls the data function attached to p if it hasn't been called yet, -// and updates the underlying attachment fields as necessary. -// It should only be called from within a function that already has a lock on p! -func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { +// store calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. It will then stream +// bytes from p's reader directly into storage so that it can be retrieved later. +func (p *ProcessingMedia) store(ctx context.Context) error { // check if we've already done this and bail early if we have - if p.rawData != nil { + if p.read { return nil } - // execute the data function and pin the raw bytes for further processing - b, err := p.data(ctx) + // execute the data function to get the reader out of it + reader, err := p.data(ctx) if err != nil { - return fmt.Errorf("fetchRawData: error executing data function: %s", err) + return fmt.Errorf("store: error executing data function: %s", err) + } + + // extract no more than 261 bytes from the beginning of the file -- this is the header + firstBytes := make([]byte, maxFileHeaderBytes) + if _, err := reader.Read(firstBytes); err != nil { + return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) } - p.rawData = b - // now we have the data we can work out the content type - contentType, err := parseContentType(p.rawData) + // now we have the file header we can work out the content type from it + contentType, err := parseContentType(firstBytes) if err != nil { - return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + return fmt.Errorf("store: error parsing content type: %s", err) } + // bail if this is a type we can't process if !supportedImage(contentType) { - return fmt.Errorf("fetchRawData: media type %s not (yet) supported", contentType) + return fmt.Errorf("store: media type %s not (yet) supported", contentType) } + // extract the file extension split := strings.Split(contentType, "/") if len(split) != 2 { - return fmt.Errorf("fetchRawData: content type %s was not valid", contentType) + return fmt.Errorf("store: content type %s was not valid", contentType) } - extension := split[1] // something like 'jpeg' // set some additional fields on the attachment now that @@ -282,6 +279,22 @@ func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) p.attachment.File.ContentType = contentType + // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // store this for now -- other processes can pull it out of storage as they please + if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil { + return fmt.Errorf("store: error storing stream: %s", err) + } + + // if the original reader is a readcloser, close it since we're done with it now + if rc, ok := reader.(io.ReadCloser); ok { + if err := rc.Close(); err != nil { + return fmt.Errorf("store: error closing readcloser: %s", err) + } + } + + p.read = true return nil } diff --git a/internal/media/types.go b/internal/media/types.go index 5b3fe4a41..0a7f60d66 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,6 +20,7 @@ package media import ( "context" + "io" "time" ) @@ -28,7 +29,7 @@ import ( // // See: https://en.wikipedia.org/wiki/File_format#File_header // and https://github.com/h2non/filetype -const maxFileHeaderBytes = 262 +const maxFileHeaderBytes = 261 // mime consts const ( @@ -117,4 +118,4 @@ type AdditionalEmojiInfo struct { } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. -type DataFunc func(ctx context.Context) ([]byte, error) +type DataFunc func(ctx context.Context) (io.Reader, error) diff --git a/internal/media/util.go b/internal/media/util.go index 7a3d81c0f..248d5fb19 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -19,7 +19,6 @@ package media import ( - "bytes" "errors" "fmt" @@ -28,11 +27,11 @@ import ( // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. -func parseContentType(content []byte) (string, error) { - // read in the first bytes of the file - fileHeader := make([]byte, maxFileHeaderBytes) - if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { - return "", fmt.Errorf("could not read first magic bytes of file: %s", err) +// +// Fileheader should be no longer than 262 bytes; anything more than this is inefficient. +func parseContentType(fileHeader []byte) (string, error) { + if fhLength := len(fileHeader); fhLength > maxFileHeaderBytes { + return "", fmt.Errorf("parseContentType requires %d bytes max, we got %d", maxFileHeaderBytes, fhLength) } kind, err := filetype.Match(fileHeader) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 7b305dc95..5a0a3e5a1 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -19,9 +19,7 @@ package account import ( - "bytes" "context" - "errors" "fmt" "io" "mime/multipart" @@ -142,24 +140,8 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(ctx context.Context) ([]byte, error) { - // pop open the fileheader - f, err := avatar.Open() - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") - } - - return buf.Bytes(), f.Close() + dataFunc := func(ctx context.Context) (io.Reader, error) { + return avatar.Open() } isAvatar := true @@ -184,24 +166,8 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(ctx context.Context) ([]byte, error) { - // pop open the fileheader - f, err := header.Open() - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") - } - - return buf.Bytes(), f.Close() + dataFunc := func(ctx context.Context) (io.Reader, error) { + return header.Open() } isHeader := true diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index fcc17c4be..e0068858b 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -19,9 +19,7 @@ package admin import ( - "bytes" "context" - "errors" "fmt" "io" @@ -38,22 +36,8 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } - data := func(innerCtx context.Context) ([]byte, error) { - // open the emoji and extract the bytes from it - f, err := form.Image.Open() - if err != nil { - return nil, fmt.Errorf("error opening emoji: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading emoji: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided emoji: size 0 bytes") - } - - return buf.Bytes(), f.Close() + data := func(innerCtx context.Context) (io.Reader, error) { + return form.Image.Open() } emojiID, err := id.NewRandomULID() diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 0896315b1..0fda4c27b 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -19,9 +19,7 @@ package media import ( - "bytes" "context" - "errors" "fmt" "io" @@ -31,21 +29,8 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - data := func(innerCtx context.Context) ([]byte, error) { - // open the attachment and extract the bytes from it - f, err := form.File.Open() - if err != nil { - return nil, fmt.Errorf("error opening attachment: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided attachment: size 0 bytes") - } - return buf.Bytes(), f.Close() + data := func(innerCtx context.Context) (io.Reader, error) { + return form.File.Open() } focusX, focusY, err := parseFocus(form.Focus) diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index 3fa4a89e4..ed32f20c6 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -21,14 +21,14 @@ package transport import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "github.com/sirupsen/logrus" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) { l := logrus.WithField("func", "DereferenceMedia") l.Debugf("performing GET to %s", iri.String()) req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil) @@ -50,9 +50,8 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, if err != nil { return nil, err } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) } - return ioutil.ReadAll(resp.Body) + return resp.Body, nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index c43515a42..d9650d952 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -21,6 +21,7 @@ package transport import ( "context" "crypto" + "io" "net/url" "sync" @@ -33,8 +34,8 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the bytes of the given media attachment IRI. - DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) + // DereferenceMedia fetches the given media attachment IRI. + DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. diff --git a/testrig/storage.go b/testrig/storage.go index a8cf0d838..0e91d7dbe 100644 --- a/testrig/storage.go +++ b/testrig/storage.go @@ -19,20 +19,16 @@ package testrig import ( - "bytes" - "errors" "fmt" - "io" "os" "codeberg.org/gruf/go-store/kv" "codeberg.org/gruf/go-store/storage" - "codeberg.org/gruf/go-store/util" ) // NewTestStorage returns a new in memory storage with the default test config func NewTestStorage() *kv.KVStore { - storage, err := kv.OpenStorage(&inMemStorage{storage: map[string][]byte{}, overwrite: false}) + storage, err := kv.OpenStorage(storage.OpenMemory(200, false)) if err != nil { panic(err) } @@ -113,79 +109,3 @@ func StandardStorageTeardown(s *kv.KVStore) { } } } - -type inMemStorage struct { - storage map[string][]byte - overwrite bool -} - -func (s *inMemStorage) Clean() error { - return nil -} - -func (s *inMemStorage) ReadBytes(key string) ([]byte, error) { - b, ok := s.storage[key] - if !ok { - return nil, errors.New("key not found") - } - return b, nil -} - -func (s *inMemStorage) ReadStream(key string) (io.ReadCloser, error) { - b, err := s.ReadBytes(key) - if err != nil { - return nil, err - } - return util.NopReadCloser(bytes.NewReader(b)), nil -} - -func (s *inMemStorage) WriteBytes(key string, value []byte) error { - if _, ok := s.storage[key]; ok && !s.overwrite { - return errors.New("key already in storage") - } - s.storage[key] = copyBytes(value) - return nil -} - -func (s *inMemStorage) WriteStream(key string, r io.Reader) error { - b, err := io.ReadAll(r) - if err != nil { - return err - } - return s.WriteBytes(key, b) -} - -func (s *inMemStorage) Stat(key string) (bool, error) { - _, ok := s.storage[key] - return ok, nil -} - -func (s *inMemStorage) Remove(key string) error { - if _, ok := s.storage[key]; !ok { - return errors.New("key not found") - } - delete(s.storage, key) - return nil -} - -func (s *inMemStorage) WalkKeys(opts storage.WalkKeysOptions) error { - if opts.WalkFn == nil { - return errors.New("invalid walkfn") - } - for key := range s.storage { - opts.WalkFn(entry(key)) - } - return nil -} - -type entry string - -func (e entry) Key() string { - return string(e) -} - -func copyBytes(b []byte) []byte { - p := make([]byte, len(b)) - copy(p, b) - return p -} -- cgit v1.2.3 From c157b1b20b38cc331cfd1673433d077719feef3f Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 23 Jan 2022 14:41:58 +0100 Subject: rework data function to provide filesize --- internal/federation/dereferencing/account.go | 4 ++-- internal/federation/dereferencing/media.go | 2 +- internal/media/image.go | 20 ----------------- internal/media/manager_test.go | 12 +++++----- internal/media/processingemoji.go | 4 ++-- internal/media/processingmedia.go | 33 ++++++++++++++++++++-------- internal/media/types.go | 2 +- internal/processing/account/update.go | 10 +++++---- internal/processing/admin/emoji.go | 5 +++-- internal/processing/media/create.go | 5 +++-- internal/transport/derefmedia.go | 12 +++++----- internal/transport/transport.go | 4 ++-- 12 files changed, 56 insertions(+), 57 deletions(-) (limited to 'internal/media/types.go') diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 6ea8256d5..581c95de2 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -252,7 +252,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) (io.Reader, error) { + data := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, avatarIRI) } @@ -274,7 +274,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) (io.Reader, error) { + data := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, headerIRI) } diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index c427f2507..0b19570f2 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -42,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) (io.Reader, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/media/image.go b/internal/media/image.go index b8f00024f..e5390cee5 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -30,7 +30,6 @@ import ( "github.com/buckket/go-blurhash" "github.com/nfnt/resize" - "github.com/superseriousbusiness/exifremove/pkg/exifremove" ) const ( @@ -197,22 +196,3 @@ func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) { small: out.Bytes(), }, nil } - -// purgeExif is a little wrapper for the action of removing exif data from an image. -// Only pass pngs or jpegs to this function. -func purgeExif(data []byte) ([]byte, error) { - if len(data) == 0 { - return nil, errors.New("passed image was not valid") - } - - clean, err := exifremove.Remove(data) - if err != nil { - return nil, fmt.Errorf("could not purge exif from image: %s", err) - } - - if len(clean) == 0 { - return nil, errors.New("purged image was not valid") - } - - return clean, nil -} diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 5380b83b1..960f34843 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -39,13 +39,13 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), nil + return bytes.NewBuffer(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -109,13 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), nil + return bytes.NewBuffer(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -192,9 +192,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { panic(err) } - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image - return bytes.NewReader(b), nil + return bytes.NewReader(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 147b6b5b3..292712427 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -163,7 +163,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { } // execute the data function to get the reader out of it - reader, err := p.data(ctx) + reader, fileSize, err := p.data(ctx) if err != nil { return fmt.Errorf("store: error executing data function: %s", err) } @@ -194,6 +194,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) p.emoji.ImageContentType = contentType + p.emoji.ImageFileSize = fileSize // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) @@ -202,7 +203,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } - p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream // if the original reader is a readcloser, close it since we're done with it now if rc, ok := reader.(io.ReadCloser); ok { diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 82db863e0..0bbe35aee 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -28,6 +28,7 @@ import ( "time" "codeberg.org/gruf/go-store/kv" + terminator "github.com/superseriousbusiness/exif-terminator" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" @@ -239,7 +240,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { } // execute the data function to get the reader out of it - reader, err := p.data(ctx) + reader, fileSize, err := p.data(ctx) if err != nil { return fmt.Errorf("store: error executing data function: %s", err) } @@ -268,22 +269,36 @@ func (p *ProcessingMedia) store(ctx context.Context) error { } extension := split[1] // something like 'jpeg' - // set some additional fields on the attachment now that - // we know more about what the underlying media actually is - if extension == mimeGif { + // concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // we'll need to clean exif data from the first bytes; while we're + // here, we can also use the extension to derive the attachment type + var clean io.Reader + switch extension { + case mimeGif: p.attachment.Type = gtsmodel.FileTypeGif - } else { + clean = multiReader // nothing to clean from a gif + case mimeJpeg, mimePng: p.attachment.Type = gtsmodel.FileTypeImage + purged, err := terminator.Terminate(multiReader, fileSize, extension) + if err != nil { + return fmt.Errorf("store: exif error: %s", err) + } + clean = purged + default: + return fmt.Errorf("store: couldn't process %s", extension) } + + // now set some additional fields on the attachment since + // we know more about what the underlying media actually is p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) p.attachment.File.ContentType = contentType - - // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) - multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + p.attachment.File.FileSize = fileSize // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil { + if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } diff --git a/internal/media/types.go b/internal/media/types.go index 0a7f60d66..b9c79d464 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -118,4 +118,4 @@ type AdditionalEmojiInfo struct { } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. -type DataFunc func(ctx context.Context) (io.Reader, error) +type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 5a0a3e5a1..758cc6600 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -140,8 +140,9 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, error) { - return avatar.Open() + dataFunc := func(ctx context.Context) (io.Reader, int, error) { + f, err := avatar.Open() + return f, int(avatar.Size), err } isAvatar := true @@ -166,8 +167,9 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, error) { - return header.Open() + dataFunc := func(ctx context.Context) (io.Reader, int, error) { + f, err := header.Open() + return f, int(header.Size), err } isHeader := true diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index e0068858b..bb9f4ecb5 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -36,8 +36,9 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } - data := func(innerCtx context.Context) (io.Reader, error) { - return form.Image.Open() + data := func(innerCtx context.Context) (io.Reader, int, error) { + f, err := form.Image.Open() + return f, int(form.Image.Size), err } emojiID, err := id.NewRandomULID() diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 0fda4c27b..4047278eb 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -29,8 +29,9 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - data := func(innerCtx context.Context) (io.Reader, error) { - return form.File.Open() + data := func(innerCtx context.Context) (io.Reader, int, error) { + f, err := form.File.Open() + return f, int(form.File.Size), err } focusX, focusY, err := parseFocus(form.Focus) diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index ed32f20c6..e3c86ce1e 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -28,12 +28,12 @@ import ( "github.com/sirupsen/logrus" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) { l := logrus.WithField("func", "DereferenceMedia") l.Debugf("performing GET to %s", iri.String()) req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil) if err != nil { - return nil, err + return nil, 0, err } req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here @@ -44,14 +44,14 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.Read err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil) t.getSignerMu.Unlock() if err != nil { - return nil, err + return nil, 0, err } resp, err := t.client.Do(req) if err != nil { - return nil, err + return nil, 0, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) + return nil, 0, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) } - return resp.Body, nil + return resp.Body, int(resp.ContentLength), nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index d9650d952..9e8cd8213 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -34,8 +34,8 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the given media attachment IRI. - DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) + // DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize. + DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. -- cgit v1.2.3 From 8c0141d103cb70fdbe74f1d5a936860707da973f Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 13:38:44 +0100 Subject: store and retrieve processState atomically --- internal/media/processingemoji.go | 23 +++++++++++------------ internal/media/processingmedia.go | 37 ++++++++++++++++++++----------------- internal/media/types.go | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) (limited to 'internal/media/types.go') diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 292712427..741854b9b 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -25,6 +25,7 @@ import ( "io" "strings" "sync" + "sync/atomic" "time" "codeberg.org/gruf/go-store/kv" @@ -53,9 +54,7 @@ type ProcessingEmoji struct { /* below fields represent the processing state of the static of the emoji */ - - staticState processState - fullSizeState processState + staticState int32 /* below pointers to database and storage are maintained so that @@ -104,17 +103,18 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingEmoji) Finished() bool { - return p.staticState == complete && p.fullSizeState == complete + return atomic.LoadInt32(&p.staticState) == int32(complete) } func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { - switch p.staticState { + staticState := atomic.LoadInt32(&p.staticState) + switch processState(staticState) { case received: // stream the original file out of storage... stored, err := p.storage.GetStream(p.emoji.ImagePath) if err != nil { p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } @@ -122,27 +122,27 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType) if err != nil { p.err = fmt.Errorf("loadStatic: error deriving static: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadStatic: error closing stored full size: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } // put the static in storage if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { p.err = fmt.Errorf("loadStatic: error storing static: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } p.emoji.ImageStaticFileSize = len(static.small) // we're done processing the static version of the emoji! - p.staticState = complete + atomic.StoreInt32(&p.staticState, int32(complete)) fallthrough case complete: return nil @@ -281,8 +281,7 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode instanceAccountID: instanceAccount.ID, emoji: emoji, data: data, - staticState: received, - fullSizeState: received, + staticState: int32(received), database: m.db, storage: m.storage, } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 0bbe35aee..0f47ee4e6 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -25,6 +25,7 @@ import ( "io" "strings" "sync" + "sync/atomic" "time" "codeberg.org/gruf/go-store/kv" @@ -49,8 +50,8 @@ type ProcessingMedia struct { data DataFunc read bool // bool indicating that data function has been triggered already - thumbstate processState // the processing state of the media thumbnail - fullSizeState processState // the processing state of the full-sized media + thumbState int32 // the processing state of the media thumbnail + fullSizeState int32 // the processing state of the full-sized media /* below pointers to database and storage are maintained so that @@ -103,11 +104,12 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingMedia) Finished() bool { - return p.thumbstate == complete && p.fullSizeState == complete + return atomic.LoadInt32(&p.thumbState) == int32(complete) && atomic.LoadInt32(&p.fullSizeState) == int32(complete) } func (p *ProcessingMedia) loadThumb(ctx context.Context) error { - switch p.thumbstate { + thumbState := atomic.LoadInt32(&p.thumbState) + switch processState(thumbState) { case received: // we haven't processed a thumbnail for this media yet so do it now @@ -122,7 +124,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { stored, err := p.storage.GetStream(p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } @@ -130,20 +132,20 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash) if err != nil { p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadThumb: error closing stored full size: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } // put the thumbnail in storage if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } @@ -160,7 +162,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { p.attachment.Thumbnail.FileSize = len(thumb.small) // we're done processing the thumbnail! - p.thumbstate = complete + atomic.StoreInt32(&p.thumbState, int32(complete)) fallthrough case complete: return nil @@ -168,11 +170,12 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { return p.err } - return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbstate) + return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbState) } func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { - switch p.fullSizeState { + fullSizeState := atomic.LoadInt32(&p.fullSizeState) + switch processState(fullSizeState) { case received: var err error var decoded *imageMeta @@ -181,7 +184,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { stored, err := p.storage.GetStream(p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) - p.fullSizeState = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } @@ -198,13 +201,13 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { if err != nil { p.err = err - p.fullSizeState = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadFullSize: error closing stored full size: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } @@ -219,7 +222,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { p.attachment.Processing = gtsmodel.ProcessingStatusProcessed // we're done processing the full-size image - p.fullSizeState = complete + atomic.StoreInt32(&p.fullSizeState, int32(complete)) fallthrough case complete: return nil @@ -400,8 +403,8 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID processingMedia := &ProcessingMedia{ attachment: attachment, data: data, - thumbstate: received, - fullSizeState: received, + thumbState: int32(received), + fullSizeState: int32(received), database: m.db, storage: m.storage, } diff --git a/internal/media/types.go b/internal/media/types.go index b9c79d464..a6b38b467 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -45,7 +45,7 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) -type processState int +type processState int32 const ( received processState = iota // processing order has been received but not done yet -- cgit v1.2.3