diff options
author | 2023-02-11 12:48:38 +0100 | |
---|---|---|
committer | 2023-02-11 12:48:38 +0100 | |
commit | 40bc03e71789523ec0f3cc4ae9f8532430832cd4 (patch) | |
tree | 9a2baceffea0b80d1701b636eb19107b96e70fd3 /internal/media | |
parent | [performance] remove throttling timers (#1466) (diff) | |
download | gotosocial-40bc03e71789523ec0f3cc4ae9f8532430832cd4.tar.xz |
[chore/performance] Update media prune logic, add extra CLI command (#1474)v0.7.0-rc2
* start updating media prune stuff a wee bit
* continue prune / uncache work
* more tidying + consistency stuff
* add prune CLI command
* docs
* arg
Diffstat (limited to 'internal/media')
-rw-r--r-- | internal/media/cron.go | 73 | ||||
-rw-r--r-- | internal/media/manager.go | 129 | ||||
-rw-r--r-- | internal/media/png-stripper.go | 6 | ||||
-rw-r--r-- | internal/media/processingmedia.go | 2 | ||||
-rw-r--r-- | internal/media/prune.go | 353 | ||||
-rw-r--r-- | internal/media/prune_test.go | 358 | ||||
-rw-r--r-- | internal/media/prunemeta.go | 89 | ||||
-rw-r--r-- | internal/media/prunemeta_test.go | 132 | ||||
-rw-r--r-- | internal/media/pruneorphaned.go | 138 | ||||
-rw-r--r-- | internal/media/pruneorphaned_test.go | 82 | ||||
-rw-r--r-- | internal/media/pruneremote.go | 95 | ||||
-rw-r--r-- | internal/media/pruneremote_test.go | 143 | ||||
-rw-r--r-- | internal/media/pruneunusedlocal.go | 83 | ||||
-rw-r--r-- | internal/media/pruneunusedlocal_test.go | 75 | ||||
-rw-r--r-- | internal/media/util.go | 79 |
15 files changed, 826 insertions, 1011 deletions
diff --git a/internal/media/cron.go b/internal/media/cron.go new file mode 100644 index 000000000..e32a63661 --- /dev/null +++ b/internal/media/cron.go @@ -0,0 +1,73 @@ +/* + GoToSocial + Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package media + +import ( + "context" + "fmt" + "time" + + "github.com/robfig/cron/v3" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/log" +) + +type cronLogger struct{} + +func (l *cronLogger) Info(msg string, keysAndValues ...interface{}) { + log.Info("media manager cron logger: ", msg, keysAndValues) +} + +func (l *cronLogger) Error(err error, msg string, keysAndValues ...interface{}) { + log.Error("media manager cron logger: ", err, msg, keysAndValues) +} + +func scheduleCleanup(m *manager) error { + pruneCtx, pruneCancel := context.WithCancel(context.Background()) + + c := cron.New(cron.WithLogger(new(cronLogger))) + defer c.Start() + + if _, err := c.AddFunc("@midnight", func() { + if err := m.PruneAll(pruneCtx, config.GetMediaRemoteCacheDays(), true); err != nil { + log.Error(err) + return + } + }); err != nil { + pruneCancel() + return fmt.Errorf("error starting media manager cleanup job: %s", err) + } + + m.stopCronJobs = func() error { + // Try to stop jobs gracefully by waiting til they're finished. + stopCtx := c.Stop() + + select { + case <-stopCtx.Done(): + log.Infof("media manager: cron finished jobs and stopped gracefully") + case <-time.After(1 * time.Minute): + log.Warnf("media manager: cron didn't stop after 60 seconds, force closing jobs") + pruneCancel() + } + + return nil + } + + return nil +} diff --git a/internal/media/manager.go b/internal/media/manager.go index 44483787a..b770a7dcd 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -21,22 +21,24 @@ package media import ( "context" "fmt" - "time" - "github.com/robfig/cron/v3" "github.com/superseriousbusiness/gotosocial/internal/concurrency" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/storage" ) -// selectPruneLimit is the amount of media entries to select at a time from the db when pruning -const selectPruneLimit = 20 +var SupportedMIMETypes = []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + mimeImageWebp, + mimeVideoMp4, +} -// UnusedLocalAttachmentCacheDays is the amount of days to keep local media in storage if it -// is not attached to a status, or was never attached to a status. -const UnusedLocalAttachmentCacheDays = 3 +var SupportedEmojiMIMETypes = []string{ + mimeImageGif, + mimeImagePng, +} // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { @@ -85,25 +87,36 @@ type Manager interface { RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) /* - PRUNING FUNCTIONS + PRUNING/UNCACHING FUNCTIONS */ - // PruneAllRemote prunes all remote media attachments cached on this instance which are older than the given amount of days. - // 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size), - // and setting 'cached' to false on the associated attachment. + // PruneAll runs all of the below pruning/uncacheing functions, and then cleans up any resulting + // empty directories from the storage driver. It can be called as a shortcut for calling the below + // pruning functions one by one. // - // The returned int is the amount of media that was pruned by this function. - PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) - // PruneAllMeta prunes unused/out of date headers and avatars cached on this instance. + // If blocking is true, then any errors encountered during the prune will be combined + returned to + // the caller. If blocking is false, the prune is run in the background and errors are just logged + // instead. + PruneAll(ctx context.Context, mediaCacheRemoteDays int, blocking bool) error + // UncacheRemote uncaches all remote media attachments older than the given amount of days. + // + // In this context, uncacheing means deleting media files from storage and marking the attachment + // as cached=false in the database. + // + // If 'dry' is true, then only a dry run will be performed: nothing will actually be changed. + // + // The returned int is the amount of media that was/would be uncached by this function. + UncacheRemote(ctx context.Context, olderThanDays int, dry bool) (int, error) + // PruneUnusedRemote prunes unused/out of date headers and avatars cached on this instance. // // The returned int is the amount of media that was pruned by this function. - PruneAllMeta(ctx context.Context) (int, error) - // PruneUnusedLocalAttachments prunes unused media attachments that were uploaded by + PruneUnusedRemote(ctx context.Context, dry bool) (int, error) + // PruneUnusedLocal prunes unused media attachments that were uploaded by // a user on this instance, but never actually attached to a status, or attached but // later detached. // // The returned int is the amount of media that was pruned by this function. - PruneUnusedLocalAttachments(ctx context.Context) (int, error) + PruneUnusedLocal(ctx context.Context, dry bool) (int, error) // PruneOrphaned prunes files that exist in storage but which do not have a corresponding // entry in the database. // @@ -145,7 +158,7 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { storage: storage, } - // Prepare the media worker pool + // Prepare the media worker pool. m.mediaWorker = concurrency.NewWorkerPool[*ProcessingMedia](-1, 10) m.mediaWorker.SetProcessor(func(ctx context.Context, media *ProcessingMedia) error { if _, err := media.LoadAttachment(ctx); err != nil { @@ -154,7 +167,7 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { return nil }) - // Prepare the emoji worker pool + // Prepare the emoji worker pool. m.emojiWorker = concurrency.NewWorkerPool[*ProcessingEmoji](-1, 10) m.emojiWorker.SetProcessor(func(ctx context.Context, emoji *ProcessingEmoji) error { if _, err := emoji.LoadEmoji(ctx); err != nil { @@ -163,7 +176,7 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { return nil }) - // Start the worker pools + // Start the worker pools. if err := m.mediaWorker.Start(); err != nil { return nil, err } @@ -171,7 +184,8 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { return nil, err } - if err := scheduleCleanupJobs(m); err != nil { + // Schedule cron job(s) for clean up. + if err := scheduleCleanup(m); err != nil { return nil, err } @@ -206,7 +220,7 @@ func (m *manager) RecacheMedia(ctx context.Context, data DataFunc, postData Post } func (m *manager) Stop() error { - // Stop media and emoji worker pools + // Stop worker pools. mediaErr := m.mediaWorker.Stop() emojiErr := m.emojiWorker.Stop() @@ -223,70 +237,3 @@ func (m *manager) Stop() error { return cronErr } - -func scheduleCleanupJobs(m *manager) error { - // create a new cron instance for scheduling cleanup jobs - c := cron.New(cron.WithLogger(&logrusWrapper{})) - pruneCtx, pruneCancel := context.WithCancel(context.Background()) - - if _, err := c.AddFunc("@midnight", func() { - begin := time.Now() - pruned, err := m.PruneAllMeta(pruneCtx) - if err != nil { - log.Errorf("media manager: error pruning meta: %s", err) - return - } - log.Infof("media manager: pruned %d meta entries in %s", pruned, time.Since(begin)) - }); err != nil { - pruneCancel() - return fmt.Errorf("error starting media manager meta cleanup job: %s", err) - } - - if _, err := c.AddFunc("@midnight", func() { - begin := time.Now() - pruned, err := m.PruneUnusedLocalAttachments(pruneCtx) - if err != nil { - log.Errorf("media manager: error pruning unused local attachments: %s", err) - return - } - log.Infof("media manager: pruned %d unused local attachments in %s", pruned, time.Since(begin)) - }); err != nil { - pruneCancel() - return fmt.Errorf("error starting media manager unused local attachments cleanup job: %s", err) - } - - // start remote cache cleanup cronjob if configured - if mediaRemoteCacheDays := config.GetMediaRemoteCacheDays(); mediaRemoteCacheDays > 0 { - if _, err := c.AddFunc("@midnight", func() { - begin := time.Now() - pruned, err := m.PruneAllRemote(pruneCtx, mediaRemoteCacheDays) - if err != nil { - log.Errorf("media manager: error pruning remote cache: %s", err) - return - } - log.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin)) - }); err != nil { - pruneCancel() - return fmt.Errorf("error starting media manager remote cache cleanup job: %s", err) - } - } - - // try to stop any jobs gracefully by waiting til they're finished - m.stopCronJobs = func() error { - cronCtx := c.Stop() - - select { - case <-cronCtx.Done(): - log.Infof("media manager: cron finished jobs and stopped gracefully") - case <-time.After(1 * time.Minute): - log.Infof("media manager: cron didn't stop after 60 seconds, will force close jobs") - break - } - - pruneCancel() - return nil - } - - c.Start() - return nil -} diff --git a/internal/media/png-stripper.go b/internal/media/png-stripper.go index 79b0bac05..e23ac2cd0 100644 --- a/internal/media/png-stripper.go +++ b/internal/media/png-stripper.go @@ -82,10 +82,10 @@ import ( // type (the first of four ASCII letters) is lower-case. const chunkTypeAncillaryBit = 0x20000000 -// PNGAncillaryChunkStripper wraps another io.Reader to strip ancillary chunks, +// pngAncillaryChunkStripper wraps another io.Reader to strip ancillary chunks, // if the data is in the PNG file format. If the data isn't PNG, it is passed // through unmodified. -type PNGAncillaryChunkStripper struct { +type pngAncillaryChunkStripper struct { // Reader is the wrapped io.Reader. Reader io.Reader @@ -113,7 +113,7 @@ type PNGAncillaryChunkStripper struct { } // Read implements io.Reader. -func (r *PNGAncillaryChunkStripper) Read(p []byte) (int, error) { +func (r *pngAncillaryChunkStripper) Read(p []byte) (int, error) { for { // If the wrapped io.Reader returned a non-nil error, drain r.buffer // (what data we have) and return that error (if fully drained). diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 4b2ef322d..34f8dc26b 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -239,7 +239,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // .png image (requires ancillary chunk stripping) case mimeImagePng: - fullImg, err = decodeImage(&PNGAncillaryChunkStripper{ + fullImg, err = decodeImage(&pngAncillaryChunkStripper{ Reader: rc, }, imaging.AutoOrientation(true)) if err != nil { diff --git a/internal/media/prune.go b/internal/media/prune.go new file mode 100644 index 000000000..7335feb7a --- /dev/null +++ b/internal/media/prune.go @@ -0,0 +1,353 @@ +/* + GoToSocial + Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package media + +import ( + "context" + "errors" + "fmt" + "time" + + "codeberg.org/gruf/go-store/v2/storage" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/regexes" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +const ( + selectPruneLimit = 50 // Amount of media entries to select at a time from the db when pruning. + unusedLocalAttachmentDays = 3 // Number of days to keep local media in storage if not attached to a status. +) + +func (m *manager) PruneAll(ctx context.Context, mediaCacheRemoteDays int, blocking bool) error { + const dry = false + + f := func(innerCtx context.Context) error { + errs := gtserror.MultiError{} + + pruned, err := m.PruneUnusedLocal(innerCtx, dry) + if err != nil { + errs = append(errs, fmt.Sprintf("error pruning unused local media (%s)", err)) + } else { + log.Infof("pruned %d unused local media", pruned) + } + + pruned, err = m.PruneUnusedRemote(innerCtx, dry) + if err != nil { + errs = append(errs, fmt.Sprintf("error pruning unused remote media: (%s)", err)) + } else { + log.Infof("pruned %d unused remote media", pruned) + } + + pruned, err = m.UncacheRemote(innerCtx, mediaCacheRemoteDays, dry) + if err != nil { + errs = append(errs, fmt.Sprintf("error uncacheing remote media older than %d day(s): (%s)", mediaCacheRemoteDays, err)) + } else { + log.Infof("uncached %d remote media older than %d day(s)", pruned, mediaCacheRemoteDays) + } + + pruned, err = m.PruneOrphaned(innerCtx, dry) + if err != nil { + errs = append(errs, fmt.Sprintf("error pruning orphaned media: (%s)", err)) + } else { + log.Infof("pruned %d orphaned media", pruned) + } + + if err := m.storage.Storage.Clean(innerCtx); err != nil { + errs = append(errs, fmt.Sprintf("error cleaning storage: (%s)", err)) + } else { + log.Info("cleaned storage") + } + + return errs.Combine() + } + + if blocking { + return f(ctx) + } + + go func() { + if err := f(context.Background()); err != nil { + log.Error(err) + } + }() + + return nil +} + +func (m *manager) PruneUnusedRemote(ctx context.Context, dry bool) (int, error) { + var ( + totalPruned int + maxID string + attachments []*gtsmodel.MediaAttachment + err error + ) + + // We don't know in advance how many remote attachments will meet + // our criteria for being 'unused'. So a dry run in this case just + // means we iterate through as normal, but do nothing with each entry + // instead of removing it. Define this here so we don't do the 'if dry' + // check inside the loop a million times. + var f func(ctx context.Context, attachment *gtsmodel.MediaAttachment) error + if !dry { + f = m.deleteAttachment + } else { + f = func(_ context.Context, _ *gtsmodel.MediaAttachment) error { + return nil // noop + } + } + + for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { + maxID = attachments[len(attachments)-1].ID // use the id of the last attachment in the slice as the next 'maxID' value + + // Prune each attachment that meets one of the following criteria: + // - Has no owning account in the database. + // - Is a header but isn't the owning account's current header. + // - Is an avatar but isn't the owning account's current avatar. + for _, attachment := range attachments { + if attachment.Account == nil || + (*attachment.Header && attachment.ID != attachment.Account.HeaderMediaAttachmentID) || + (*attachment.Avatar && attachment.ID != attachment.Account.AvatarMediaAttachmentID) { + if err := f(ctx, attachment); err != nil { + return totalPruned, err + } + totalPruned++ + } + } + } + + // Make sure we don't have a real error when we leave the loop. + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return totalPruned, err + } + + return totalPruned, nil +} + +func (m *manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) { + // keys in storage will look like the following: + // `[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[MEDIA_ID].[EXTENSION]` + // We can filter out keys we're not interested in by + // matching through a regex. + var matchCount int + match := func(storageKey string) bool { + if regexes.FilePath.MatchString(storageKey) { + matchCount++ + return true + } + return false + } + + iterator, err := m.storage.Iterator(ctx, match) // make sure this iterator is always released + if err != nil { + return 0, fmt.Errorf("PruneOrphaned: error getting storage iterator: %w", err) + } + + // Ensure we have some keys, and also advance + // the iterator to the first non-empty key. + if !iterator.Next() { + iterator.Release() + return 0, nil // nothing else to do here + } + + // Emojis are stored under the instance account, + // so we need the ID of the instance account for + // the next part. + instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + if err != nil { + iterator.Release() + return 0, fmt.Errorf("PruneOrphaned: error getting instance account: %w", err) + } + instanceAccountID := instanceAccount.ID + + // For each key in the iterator, check if entry is orphaned. + orphanedKeys := make([]string, 0, matchCount) + for key := iterator.Key(); iterator.Next(); key = iterator.Key() { + orphaned, err := m.orphaned(ctx, key, instanceAccountID) + if err != nil { + iterator.Release() + return 0, fmt.Errorf("PruneOrphaned: checking orphaned status: %w", err) + } + + if orphaned { + orphanedKeys = append(orphanedKeys, key) + } + } + iterator.Release() + + totalPruned := len(orphanedKeys) + + if dry { + // Dry run: don't remove anything. + return totalPruned, nil + } + + // This is not a drill! + // We have to delete stuff! + return totalPruned, m.removeFiles(ctx, orphanedKeys...) +} + +func (m *manager) orphaned(ctx context.Context, key string, instanceAccountID string) (bool, error) { + pathParts := regexes.FilePath.FindStringSubmatch(key) + if len(pathParts) != 6 { + // This doesn't match our expectations so + // it wasn't created by gts; ignore it. + return false, nil + } + + var ( + mediaType = pathParts[2] + mediaID = pathParts[4] + orphaned = false + ) + + // Look for keys in storage that we don't have an attachment for. + switch Type(mediaType) { + case TypeAttachment, TypeHeader, TypeAvatar: + if _, err := m.db.GetAttachmentByID(ctx, mediaID); err != nil { + if !errors.Is(err, db.ErrNoEntries) { + return false, fmt.Errorf("error calling GetAttachmentByID: %w", err) + } + orphaned = true + } + case TypeEmoji: + // Look using the static URL for the emoji. Emoji images can change, so + // the MEDIA_ID part of the key for emojis will not necessarily correspond + // to the file that's currently being used as the emoji image. + staticURL := uris.GenerateURIForAttachment(instanceAccountID, string(TypeEmoji), string(SizeStatic), mediaID, mimePng) + if _, err := m.db.GetEmojiByStaticURL(ctx, staticURL); err != nil { + if !errors.Is(err, db.ErrNoEntries) { + return false, fmt.Errorf("error calling GetEmojiByStaticURL: %w", err) + } + orphaned = true + } + } + + return orphaned, nil +} + +func (m *manager) UncacheRemote(ctx context.Context, olderThanDays int, dry bool) (int, error) { + if olderThanDays < 0 { + return 0, nil + } + + olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays)) + + if dry { + // Dry run, just count eligible entries without removing them. + return m.db.CountRemoteOlderThan(ctx, olderThan) + } + + var ( + totalPruned int + attachments []*gtsmodel.MediaAttachment + err error + ) + + for attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { + olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value + + for _, attachment := range attachments { + if err := m.uncacheAttachment(ctx, attachment); err != nil { + return totalPruned, err + } + totalPruned++ + } + } + + // Make sure we don't have a real error when we leave the loop. + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return totalPruned, err + } + + return totalPruned, nil +} + +func (m *manager) PruneUnusedLocal(ctx context.Context, dry bool) (int, error) { + olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(unusedLocalAttachmentDays)) + + if dry { + // Dry run, just count eligible entries without removing them. + return m.db.CountLocalUnattachedOlderThan(ctx, olderThan) + } + + var ( + totalPruned int + attachments []*gtsmodel.MediaAttachment + err error + ) + + for attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit) { + olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value + + for _, attachment := range attachments { + if err := m.deleteAttachment(ctx, attachment); err != nil { + return totalPruned, err + } + totalPruned++ + } + } + + // Make sure we don't have a real error when we leave the loop. + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return totalPruned, err + } + + return totalPruned, nil +} + +/* + Handy little helpers +*/ + +func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { + if err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil { + return err + } + + // Delete attachment completely. + return m.db.DeleteByID(ctx, attachment.ID, attachment) +} + +func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { + if err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil { + return err + } + + // Update attachment to reflect that we no longer have it cached. + attachment.UpdatedAt = time.Now() + cached := false + attachment.Cached = &cached + return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") +} + +func (m *manager) removeFiles(ctx context.Context, keys ...string) error { + errs := make(gtserror.MultiError, 0, len(keys)) + + for _, key := range keys { + if err := m.storage.Delete(ctx, key); err != nil && !errors.Is(err, storage.ErrNotFound) { + errs = append(errs, "storage error removing "+key+": "+err.Error()) + } + } + + return errs.Combine() +} diff --git a/internal/media/prune_test.go b/internal/media/prune_test.go new file mode 100644 index 000000000..ed040913f --- /dev/null +++ b/internal/media/prune_test.go @@ -0,0 +1,358 @@ +/* + GoToSocial + Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package media_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "codeberg.org/gruf/go-store/v2/storage" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type PruneTestSuite struct { + MediaStandardTestSuite +} + +func (suite *PruneTestSuite) TestPruneOrphanedDry() { + // add a big orphan panda to store + b, err := os.ReadFile("./test/big-panda.gif") + if err != nil { + suite.FailNow(err.Error()) + } + + pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" + if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { + suite.FailNow(err.Error()) + } + + // dry run should show up 1 orphaned panda + totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true) + suite.NoError(err) + suite.Equal(1, totalPruned) + + // panda should still be in storage + hasKey, err := suite.storage.Has(context.Background(), pandaPath) + suite.NoError(err) + suite.True(hasKey) +} + +func (suite *PruneTestSuite) TestPruneOrphanedMoist() { + // add a big orphan panda to store + b, err := os.ReadFile("./test/big-panda.gif") + if err != nil { + suite.FailNow(err.Error()) + } + + pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" + if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { + suite.FailNow(err.Error()) + } + + // should show up 1 orphaned panda + totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false) + suite.NoError(err) + suite.Equal(1, totalPruned) + + // panda should no longer be in storage + hasKey, err := suite.storage.Has(context.Background(), pandaPath) + suite.NoError(err) + suite.False(hasKey) +} + +func (suite *PruneTestSuite) TestPruneUnusedLocal() { + testAttachment := suite.testAttachments["local_account_1_unattached_1"] + suite.True(*testAttachment.Cached) + + totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) + suite.NoError(err) + suite.Equal(1, totalPruned) + + _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func (suite *PruneTestSuite) TestPruneUnusedLocalDry() { + testAttachment := suite.testAttachments["local_account_1_unattached_1"] + suite.True(*testAttachment.Cached) + + totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true) + suite.NoError(err) + suite.Equal(1, totalPruned) + + _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.NoError(err) +} + +func (suite *PruneTestSuite) TestPruneRemoteTwice() { + totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) + suite.NoError(err) + suite.Equal(1, totalPruned) + + // final prune should prune nothing, since the first prune already happened + totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false) + suite.NoError(err) + suite.Equal(0, totalPrunedAgain) +} + +func (suite *PruneTestSuite) TestPruneOneNonExistent() { + ctx := context.Background() + testAttachment := suite.testAttachments["local_account_1_unattached_1"] + + // Delete this attachment cached on disk + media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) + suite.NoError(err) + suite.True(*media.Cached) + err = suite.storage.Delete(ctx, media.File.Path) + suite.NoError(err) + + // Now attempt to prune for item with db entry no file + totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false) + suite.NoError(err) + suite.Equal(1, totalPruned) +} + +func (suite *PruneTestSuite) TestPruneUnusedRemote() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] + zorkOldHeader := suite.testAttachments["local_account_1_avatar"] + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // media should no longer be stored + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // attachments should no longer be in the db + _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) + suite.ErrorIs(err, db.ErrNoEntries) + _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func (suite *PruneTestSuite) TestPruneUnusedRemoteTwice() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // final prune should prune nothing, since the first prune already happened + totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false) + suite.NoError(err) + suite.Equal(0, totalPruned) +} + +func (suite *PruneTestSuite) TestPruneUnusedRemoteMultipleAccounts() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] + zorkOldHeader := suite.testAttachments["local_account_1_avatar"] + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { + panic(err) + } + + // set zork's unused header as belonging to turtle + turtle := suite.testAccounts["local_account_1"] + zorkOldHeader.AccountID = turtle.ID + if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // media should no longer be stored + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // attachments should no longer be in the db + _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) + suite.ErrorIs(err, db.ErrNoEntries) + _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func (suite *PruneTestSuite) TestUncacheRemote() { + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + suite.True(*testStatusAttachment.Cached) + + testHeader := suite.testAttachments["remote_account_3_header"] + suite.True(*testHeader.Cached) + + totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) + suite.NoError(err) + suite.Equal(2, totalUncached) + + uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) + suite.NoError(err) + suite.False(*uncachedAttachment.Cached) + + uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) + suite.NoError(err) + suite.False(*uncachedAttachment.Cached) +} + +func (suite *PruneTestSuite) TestUncacheRemoteDry() { + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + suite.True(*testStatusAttachment.Cached) + + testHeader := suite.testAttachments["remote_account_3_header"] + suite.True(*testHeader.Cached) + + totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, true) + suite.NoError(err) + suite.Equal(2, totalUncached) + + uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) + suite.NoError(err) + suite.True(*uncachedAttachment.Cached) + + uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) + suite.NoError(err) + suite.True(*uncachedAttachment.Cached) +} + +func (suite *PruneTestSuite) TestUncacheRemoteTwice() { + totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) + suite.NoError(err) + suite.Equal(2, totalUncached) + + // final uncache should uncache nothing, since the first uncache already happened + totalUncachedAgain, err := suite.manager.UncacheRemote(context.Background(), 1, false) + suite.NoError(err) + suite.Equal(0, totalUncachedAgain) +} + +func (suite *PruneTestSuite) TestUncacheAndRecache() { + ctx := context.Background() + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + testHeader := suite.testAttachments["remote_account_3_header"] + + totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) + suite.NoError(err) + suite.Equal(2, totalUncached) + + // media should no longer be stored + _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // now recache the image.... + data := func(_ context.Context) (io.ReadCloser, int64, error) { + // load bytes from a test image + b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") + if err != nil { + panic(err) + } + return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil + } + + for _, original := range []*gtsmodel.MediaAttachment{ + testStatusAttachment, + testHeader, + } { + processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID) + suite.NoError(err) + + // synchronously load the recached attachment + recachedAttachment, err := processingRecache.LoadAttachment(ctx) + suite.NoError(err) + suite.NotNil(recachedAttachment) + + // recachedAttachment should be basically the same as the old attachment + suite.True(*recachedAttachment.Cached) + suite.Equal(original.ID, recachedAttachment.ID) + suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place + suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail + suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same + + // recached files should be back in storage + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) + suite.NoError(err) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) + suite.NoError(err) + } +} + +func (suite *PruneTestSuite) TestUncacheOneNonExistent() { + ctx := context.Background() + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + + // Delete this attachment cached on disk + media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) + suite.NoError(err) + suite.True(*media.Cached) + err = suite.storage.Delete(ctx, media.File.Path) + suite.NoError(err) + + // Now attempt to uncache remote for item with db entry no file + totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) + suite.NoError(err) + suite.Equal(2, totalUncached) +} + +func TestPruneOrphanedTestSuite(t *testing.T) { + suite.Run(t, &PruneTestSuite{}) +} diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go deleted file mode 100644 index 67c4ce8b1..000000000 --- a/internal/media/prunemeta.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "context" - "errors" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (m *manager) PruneAllMeta(ctx context.Context) (int, error) { - var ( - totalPruned int - maxID string - ) - - for { - // select "selectPruneLimit" headers / avatars at a time for pruning - attachments, err := m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return totalPruned, err - } else if len(attachments) == 0 { - break - } - - // use the id of the last attachment in the slice as the next 'maxID' value - log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", len(attachments), maxID) - maxID = attachments[len(attachments)-1].ID - - // prune each attachment that meets one of the following criteria: - // - has no owning account in the database - // - is a header but isn't the owning account's current header - // - is an avatar but isn't the owning account's current avatar - for _, attachment := range attachments { - if attachment.Account == nil || - (*attachment.Header && attachment.ID != attachment.Account.HeaderMediaAttachmentID) || - (*attachment.Avatar && attachment.ID != attachment.Account.AvatarMediaAttachmentID) { - if err := m.pruneOneAvatarOrHeader(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - } - - log.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned) - return totalPruned, nil -} - -func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { - if attachment.File.Path != "" { - // delete the full size attachment from storage - log.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path) - if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { - return err - } - } - - if attachment.Thumbnail.Path != "" { - // delete the thumbnail from storage - log.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { - return err - } - } - - // delete the attachment entry completely - return m.db.DeleteByID(ctx, attachment.ID, >smodel.MediaAttachment{}) -} diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go deleted file mode 100644 index 17bb679f0..000000000 --- a/internal/media/prunemeta_test.go +++ /dev/null @@ -1,132 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media_test - -import ( - "context" - "testing" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" -) - -type PruneMetaTestSuite struct { - MediaStandardTestSuite -} - -func (suite *PruneMetaTestSuite) TestPruneMeta() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] - zorkOldHeader := suite.testAttachments["local_account_1_avatar"] - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneAllMeta(ctx) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // attachments should no longer be in the db - _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) - suite.ErrorIs(err, db.ErrNoEntries) - _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func (suite *PruneMetaTestSuite) TestPruneMetaTwice() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneAllMeta(ctx) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // final prune should prune nothing, since the first prune already happened - totalPruned, err = suite.manager.PruneAllMeta(ctx) - suite.NoError(err) - suite.Equal(0, totalPruned) -} - -func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] - zorkOldHeader := suite.testAttachments["local_account_1_avatar"] - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - // set zork's unused header as belonging to turtle - turtle := suite.testAccounts["local_account_1"] - zorkOldHeader.AccountID = turtle.ID - if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneAllMeta(ctx) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // attachments should no longer be in the db - _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) - suite.ErrorIs(err, db.ErrNoEntries) - _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func TestPruneMetaTestSuite(t *testing.T) { - suite.Run(t, &PruneMetaTestSuite{}) -} diff --git a/internal/media/pruneorphaned.go b/internal/media/pruneorphaned.go deleted file mode 100644 index dd4c4e35f..000000000 --- a/internal/media/pruneorphaned.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "context" - "errors" - "fmt" - - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/regexes" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -func (m *manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) { - var totalPruned int - - // keys in storage will look like the following: - // `[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[MEDIA_ID].[EXTENSION]` - // we can filter out keys we're not interested in by - // matching through a regex - var matchCount int - match := func(storageKey string) bool { - if regexes.FilePath.MatchString(storageKey) { - matchCount++ - return true - } - return false - } - - log.Info("checking storage keys for orphaned pruning candidates...") - iterator, err := m.storage.Iterator(ctx, match) - if err != nil { - return 0, fmt.Errorf("PruneOrphaned: error getting storage iterator: %w", err) - } - - // make sure we have some keys, and also advance - // the iterator to the first non-empty key - if !iterator.Next() { - return 0, nil - } - - instanceAccount, err := m.db.GetInstanceAccount(ctx, "") - if err != nil { - return 0, fmt.Errorf("PruneOrphaned: error getting instance account: %w", err) - } - instanceAccountID := instanceAccount.ID - - // for each key in the iterator, check if entry is orphaned - log.Info("got %d orphaned pruning candidates, checking for orphaned status, please wait...") - var checkedKeys int - orphanedKeys := make([]string, 0, matchCount) - for key := iterator.Key(); iterator.Next(); key = iterator.Key() { - if m.orphaned(ctx, key, instanceAccountID) { - orphanedKeys = append(orphanedKeys, key) - } - checkedKeys++ - if checkedKeys%50 == 0 { - log.Infof("checked %d of %d orphaned pruning candidates...", checkedKeys, matchCount) - } - } - iterator.Release() - - if !dry { - // the real deal, we have to delete stuff - for _, key := range orphanedKeys { - log.Infof("key %s corresponds to orphaned media, will remove it now", key) - if err := m.storage.Delete(ctx, key); err != nil { - log.Errorf("error deleting item with key %s from storage: %s", key, err) - continue - } - totalPruned++ - } - } else { - // just a dry run, don't delete anything - for _, key := range orphanedKeys { - log.Infof("DRY RUN: key %s corresponds to orphaned media which would be deleted", key) - totalPruned++ - } - } - - return totalPruned, nil -} - -func (m *manager) orphaned(ctx context.Context, key string, instanceAccountID string) bool { - pathParts := regexes.FilePath.FindStringSubmatch(key) - if len(pathParts) != 6 { - return false - } - - mediaType := pathParts[2] - mediaID := pathParts[4] - - var orphaned bool - switch Type(mediaType) { - case TypeAttachment, TypeHeader, TypeAvatar: - if _, err := m.db.GetAttachmentByID(ctx, mediaID); err != nil { - if errors.Is(err, db.ErrNoEntries) { - orphaned = true - } else { - log.Errorf("orphaned: error calling GetAttachmentByID: %s", err) - } - } - case TypeEmoji: - // look using the static URL for the emoji, since the MEDIA_ID part of - // the key for emojis will not necessarily correspond to the file that's - // currently being used as the emoji image - staticURI := uris.GenerateURIForAttachment(instanceAccountID, string(TypeEmoji), string(SizeStatic), mediaID, mimePng) - if _, err := m.db.GetEmojiByStaticURL(ctx, staticURI); err != nil { - if errors.Is(err, db.ErrNoEntries) { - orphaned = true - } else { - log.Errorf("orphaned: error calling GetEmojiByID: %s", err) - } - } - default: - orphaned = true - } - - return orphaned -} diff --git a/internal/media/pruneorphaned_test.go b/internal/media/pruneorphaned_test.go deleted file mode 100644 index 52976b51b..000000000 --- a/internal/media/pruneorphaned_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media_test - -import ( - "bytes" - "context" - "os" - "testing" - - "github.com/stretchr/testify/suite" -) - -type PruneOrphanedTestSuite struct { - MediaStandardTestSuite -} - -func (suite *PruneOrphanedTestSuite) TestPruneOrphanedDry() { - // add a big orphan panda to store - b, err := os.ReadFile("./test/big-panda.gif") - if err != nil { - panic(err) - } - - pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { - panic(err) - } - - // dry run should show up 1 orphaned panda - totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // panda should still be in storage - hasKey, err := suite.storage.Has(context.Background(), pandaPath) - suite.NoError(err) - suite.True(hasKey) -} - -func (suite *PruneOrphanedTestSuite) TestPruneOrphanedMoist() { - // add a big orphan panda to store - b, err := os.ReadFile("./test/big-panda.gif") - if err != nil { - panic(err) - } - - pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { - panic(err) - } - - // should show up 1 orphaned panda - totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // panda should no longer be in storage - hasKey, err := suite.storage.Has(context.Background(), pandaPath) - suite.NoError(err) - suite.False(hasKey) -} - -func TestPruneOrphanedTestSuite(t *testing.T) { - suite.Run(t, &PruneOrphanedTestSuite{}) -} diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go deleted file mode 100644 index 1cc35ffda..000000000 --- a/internal/media/pruneremote.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "context" - "errors" - "time" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) { - var totalPruned int - - olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays)) - log.Infof("PruneAllRemote: pruning media older than %s", olderThan) - - for { - // Select "selectPruneLimit" status attacchments at a time for pruning - attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return totalPruned, err - } else if len(attachments) == 0 { - break - } - - // use the age of the oldest attachment (last in slice) as the next 'olderThan' value - log.Tracef("PruneAllRemote: got %d status attachments older than %s", len(attachments), olderThan) - olderThan = attachments[len(attachments)-1].CreatedAt - - // prune each status attachment - for _, attachment := range attachments { - if err := m.pruneOneRemote(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - - log.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned) - return totalPruned, nil -} - -func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { - var changed bool - - if attachment.File.Path != "" { - // delete the full size attachment from storage - log.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) - if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && !errors.Is(err, storage.ErrNotFound) { - return err - } - cached := false - attachment.Cached = &cached - changed = true - } - - if attachment.Thumbnail.Path != "" { - // delete the thumbnail from storage - log.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && !errors.Is(err, storage.ErrNotFound) { - return err - } - cached := false - attachment.Cached = &cached - changed = true - } - - if !changed { - return nil - } - - // update the attachment to reflect that we no longer have it cached - return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") -} diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go deleted file mode 100644 index 6c0409cf9..000000000 --- a/internal/media/pruneremote_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media_test - -import ( - "bytes" - "context" - "io" - "os" - "testing" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type PruneRemoteTestSuite struct { - MediaStandardTestSuite -} - -func (suite *PruneRemoteTestSuite) TestPruneRemote() { - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - suite.True(*testStatusAttachment.Cached) - - testHeader := suite.testAttachments["remote_account_3_header"] - suite.True(*testHeader.Cached) - - totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) - suite.NoError(err) - suite.Equal(2, totalPruned) - - prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) - suite.NoError(err) - suite.False(*prunedAttachment.Cached) - - prunedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) - suite.NoError(err) - suite.False(*prunedAttachment.Cached) -} - -func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() { - totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // final prune should prune nothing, since the first prune already happened - totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1) - suite.NoError(err) - suite.Equal(0, totalPrunedAgain) -} - -func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { - ctx := context.Background() - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - testHeader := suite.testAttachments["remote_account_3_header"] - - totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // now recache the image.... - data := func(_ context.Context) (io.ReadCloser, int64, error) { - // load bytes from a test image - b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") - if err != nil { - panic(err) - } - return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil - } - - for _, original := range []*gtsmodel.MediaAttachment{ - testStatusAttachment, - testHeader, - } { - processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID) - suite.NoError(err) - - // synchronously load the recached attachment - recachedAttachment, err := processingRecache.LoadAttachment(ctx) - suite.NoError(err) - suite.NotNil(recachedAttachment) - - // recachedAttachment should be basically the same as the old attachment - suite.True(*recachedAttachment.Cached) - suite.Equal(original.ID, recachedAttachment.ID) - suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place - suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail - suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same - - // recached files should be back in storage - _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) - suite.NoError(err) - _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) - suite.NoError(err) - } -} - -func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { - ctx := context.Background() - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - - // Delete this attachment cached on disk - media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) - suite.NoError(err) - suite.True(*media.Cached) - err = suite.storage.Delete(ctx, media.File.Path) - suite.NoError(err) - - // Now attempt to prune remote for item with db entry no file - totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) - suite.NoError(err) - suite.Equal(2, totalPruned) -} - -func TestPruneRemoteTestSuite(t *testing.T) { - suite.Run(t, &PruneRemoteTestSuite{}) -} diff --git a/internal/media/pruneunusedlocal.go b/internal/media/pruneunusedlocal.go deleted file mode 100644 index 6dee92a12..000000000 --- a/internal/media/pruneunusedlocal.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "context" - "time" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error) { - var totalPruned int - var maxID string - var attachments []*gtsmodel.MediaAttachment - var err error - - olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(UnusedLocalAttachmentCacheDays)) - log.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan) - - // select 20 attachments at a time and prune them - for attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit) { - // use the id of the last attachment in the slice as the next 'maxID' value - l := len(attachments) - maxID = attachments[l-1].ID - log.Tracef("PruneUnusedLocalAttachments: got %d unused local attachments older than %s with maxID < %s", l, olderThan, maxID) - - for _, attachment := range attachments { - if err := m.pruneOneLocal(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - - // make sure we don't have a real error when we leave the loop - if err != nil && err != db.ErrNoEntries { - return totalPruned, err - } - - log.Infof("PruneUnusedLocalAttachments: finished pruning: pruned %d entries", totalPruned) - return totalPruned, nil -} - -func (m *manager) pruneOneLocal(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { - if attachment.File.Path != "" { - // delete the full size attachment from storage - log.Tracef("pruneOneLocal: deleting %s", attachment.File.Path) - if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { - return err - } - } - - if attachment.Thumbnail.Path != "" { - // delete the thumbnail from storage - log.Tracef("pruneOneLocal: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { - return err - } - } - - // delete the attachment completely - return m.db.DeleteByID(ctx, attachment.ID, attachment) -} diff --git a/internal/media/pruneunusedlocal_test.go b/internal/media/pruneunusedlocal_test.go deleted file mode 100644 index ff12e2271..000000000 --- a/internal/media/pruneunusedlocal_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" -) - -type PruneUnusedLocalTestSuite struct { - MediaStandardTestSuite -} - -func (suite *PruneUnusedLocalTestSuite) TestPruneUnusedLocal() { - testAttachment := suite.testAttachments["local_account_1_unattached_1"] - suite.True(*testAttachment.Cached) - - totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) - suite.NoError(err) - suite.Equal(1, totalPruned) - - _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func (suite *PruneUnusedLocalTestSuite) TestPruneRemoteTwice() { - totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // final prune should prune nothing, since the first prune already happened - totalPrunedAgain, err := suite.manager.PruneUnusedLocalAttachments(context.Background()) - suite.NoError(err) - suite.Equal(0, totalPrunedAgain) -} - -func (suite *PruneUnusedLocalTestSuite) TestPruneOneNonExistent() { - ctx := context.Background() - testAttachment := suite.testAttachments["local_account_1_unattached_1"] - - // Delete this attachment cached on disk - media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) - suite.NoError(err) - suite.True(*media.Cached) - err = suite.storage.Delete(ctx, media.File.Path) - suite.NoError(err) - - // Now attempt to prune for item with db entry no file - totalPruned, err := suite.manager.PruneUnusedLocalAttachments(ctx) - suite.NoError(err) - suite.Equal(1, totalPruned) -} - -func TestPruneUnusedLocalTestSuite(t *testing.T) { - suite.Run(t, &PruneUnusedLocalTestSuite{}) -} diff --git a/internal/media/util.go b/internal/media/util.go deleted file mode 100644 index b15583026..000000000 --- a/internal/media/util.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "fmt" - - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -var SupportedMIMETypes = []string{ - mimeImageJpeg, - mimeImageGif, - mimeImagePng, - mimeImageWebp, - mimeVideoMp4, -} - -var SupportedEmojiMIMETypes = []string{ - mimeImageGif, - mimeImagePng, -} - -// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized -func ParseMediaType(s string) (Type, error) { - switch s { - case string(TypeAttachment): - return TypeAttachment, nil - case string(TypeHeader): - return TypeHeader, nil - case string(TypeAvatar): - return TypeAvatar, nil - case string(TypeEmoji): - return TypeEmoji, nil - } - return "", fmt.Errorf("%s not a recognized MediaType", s) -} - -// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized -func ParseMediaSize(s string) (Size, error) { - switch s { - case string(SizeSmall): - return SizeSmall, nil - case string(SizeOriginal): - return SizeOriginal, nil - case string(SizeStatic): - return SizeStatic, nil - } - return "", fmt.Errorf("%s not a recognized MediaSize", s) -} - -// logrusWrapper is just a util for passing the logrus logger into the cron logging system. -type logrusWrapper struct{} - -// Info logs routine messages about cron's operation. -func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) { - log.Info("media manager cron logger: ", msg, keysAndValues) -} - -// Error logs an error condition. -func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) { - log.Error("media manager cron logger: ", err, msg, keysAndValues) -} |