diff options
Diffstat (limited to 'internal')
32 files changed, 1946 insertions, 1056 deletions
diff --git a/internal/api/client/search/searchget_test.go b/internal/api/client/search/searchget_test.go index 9e0a8eb67..f6a2db70a 100644 --- a/internal/api/client/search/searchget_test.go +++ b/internal/api/client/search/searchget_test.go @@ -120,7 +120,7 @@ func (suite *SearchGetTestSuite) getSearch( // Check expected code + body. if resultCode := recorder.Code; expectedHTTPStatus != resultCode { - errs = append(errs, fmt.Sprintf("expected %d got %d", expectedHTTPStatus, resultCode)) + errs = append(errs, fmt.Sprintf("expected %d got %d: %v", expectedHTTPStatus, resultCode, ctx.Errors.JSON())) } // If we got an expected body, return early. diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 510b6eb53..df7d9a7ae 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -92,6 +92,13 @@ func (c *Caches) setuphooks() { c.Visibility.Invalidate("RequesterID", block.TargetAccountID) }) + c.GTS.EmojiCategory().SetInvalidateCallback(func(category *gtsmodel.EmojiCategory) { + // Invalidate entire emoji cache, + // as we can't know which emojis + // specifically this will effect. + c.GTS.Emoji().Clear() + }) + c.GTS.Follow().SetInvalidateCallback(func(follow *gtsmodel.Follow) { // Invalidate follow origin account ID cached visibility. c.Visibility.Invalidate("ItemID", follow.AccountID) diff --git a/internal/cleaner/cleaner.go b/internal/cleaner/cleaner.go new file mode 100644 index 000000000..ee1e4785f --- /dev/null +++ b/internal/cleaner/cleaner.go @@ -0,0 +1,135 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package cleaner + +import ( + "context" + "errors" + "time" + + "codeberg.org/gruf/go-runners" + "codeberg.org/gruf/go-sched" + "codeberg.org/gruf/go-store/v2/storage" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" +) + +const ( + selectLimit = 50 +) + +type Cleaner struct { + state *state.State + emoji Emoji + media Media +} + +func New(state *state.State) *Cleaner { + c := new(Cleaner) + c.state = state + c.emoji.Cleaner = c + c.media.Cleaner = c + scheduleJobs(c) + return c +} + +// Emoji returns the emoji set of cleaner utilities. +func (c *Cleaner) Emoji() *Emoji { + return &c.emoji +} + +// Media returns the media set of cleaner utilities. +func (c *Cleaner) Media() *Media { + return &c.media +} + +// checkFiles checks for each of the provided files, and calls onMissing() if any of them are missing. Returns true if missing. +func (c *Cleaner) checkFiles(ctx context.Context, onMissing func() error, files ...string) (bool, error) { + for _, file := range files { + // Check whether each file exists in storage. + have, err := c.state.Storage.Has(ctx, file) + if err != nil { + return false, gtserror.Newf("error checking storage for %s: %w", file, err) + } else if !have { + // Missing files, perform hook. + return true, onMissing() + } + } + return false, nil +} + +// removeFiles removes the provided files, returning the number of them returned. +func (c *Cleaner) removeFiles(ctx context.Context, files ...string) (int, error) { + if gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return len(files), nil + } + + var errs gtserror.MultiError + + for _, path := range files { + // Remove each provided storage path. + log.Debugf(ctx, "removing file: %s", path) + err := c.state.Storage.Delete(ctx, path) + if err != nil && !errors.Is(err, storage.ErrNotFound) { + errs.Appendf("error removing %s: %v", path, err) + } + } + + // Calculate no. files removed. + diff := len(files) - len(errs) + + // Wrap the combined error slice. + if err := errs.Combine(); err != nil { + return diff, gtserror.Newf("error(s) removing files: %w", err) + } + + return diff, nil +} + +func scheduleJobs(c *Cleaner) { + const day = time.Hour * 24 + + // Calculate closest midnight. + now := time.Now() + midnight := now.Round(day) + + if midnight.Before(now) { + // since <= 11:59am rounds down. + midnight = midnight.Add(day) + } + + // Get ctx associated with scheduler run state. + done := c.state.Workers.Scheduler.Done() + doneCtx := runners.CancelCtx(done) + + // TODO: we'll need to do some thinking to make these + // jobs restartable if we want to implement reloads in + // the future that make call to Workers.Stop() -> Workers.Start(). + + // Schedule the cleaning tasks to execute every day at midnight. + c.state.Workers.Scheduler.Schedule(sched.NewJob(func(start time.Time) { + log.Info(nil, "starting media clean") + c.Media().All(doneCtx, config.GetMediaRemoteCacheDays()) + c.Emoji().All(doneCtx) + log.Infof(nil, "finished media clean after %s", time.Since(start)) + }).EveryAt(midnight, day)) +} diff --git a/internal/cleaner/emoji.go b/internal/cleaner/emoji.go new file mode 100644 index 000000000..35e579171 --- /dev/null +++ b/internal/cleaner/emoji.go @@ -0,0 +1,238 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package cleaner + +import ( + "context" + "errors" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" +) + +// Emoji encompasses a set of +// emoji cleanup / admin utils. +type Emoji struct { + *Cleaner +} + +// All will execute all cleaner.Emoji utilities synchronously, including output logging. +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (e *Emoji) All(ctx context.Context) { + e.LogPruneMissing(ctx) + e.LogFixBroken(ctx) +} + +// LogPruneMissing performs emoji.PruneMissing(...), logging the start and outcome. +func (e *Emoji) LogPruneMissing(ctx context.Context) { + log.Info(ctx, "start") + if n, err := e.PruneMissing(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "pruned: %d", n) + } +} + +// LogFixBroken performs emoji.FixBroken(...), logging the start and outcome. +func (e *Emoji) LogFixBroken(ctx context.Context) { + log.Info(ctx, "start") + if n, err := e.FixBroken(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "fixed: %d", n) + } +} + +// PruneMissing will delete emoji with missing files from the database and storage driver. +// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function +// should be updated to match media.FixCacheStat() if we ever support emoji uncaching. +func (e *Emoji) PruneMissing(ctx context.Context) (int, error) { + var ( + total int + maxID string + ) + + for { + // Fetch the next batch of emoji media up to next ID. + emojis, err := e.state.DB.GetEmojis(ctx, maxID, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting emojis: %w", err) + } + + if len(emojis) == 0 { + // reached end. + break + } + + // Use last as the next 'maxID' value. + maxID = emojis[len(emojis)-1].ID + + for _, emoji := range emojis { + // Check / fix missing emoji media. + fixed, err := e.pruneMissing(ctx, emoji) + if err != nil { + return total, err + } + + if fixed { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +// FixBroken will check all emojis for valid related models (e.g. category). +// Broken media will be automatically updated to remove now-missing models. +// Context will be checked for `gtscontext.DryRun()` to perform the action. +func (e *Emoji) FixBroken(ctx context.Context) (int, error) { + var ( + total int + maxID string + ) + + for { + // Fetch the next batch of emoji media up to next ID. + emojis, err := e.state.DB.GetEmojis(ctx, maxID, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting emojis: %w", err) + } + + if len(emojis) == 0 { + // reached end. + break + } + + // Use last as the next 'maxID' value. + maxID = emojis[len(emojis)-1].ID + + for _, emoji := range emojis { + // Check / fix missing broken emoji. + fixed, err := e.fixBroken(ctx, emoji) + if err != nil { + return total, err + } + + if fixed { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +func (e *Emoji) pruneMissing(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { + return e.checkFiles(ctx, func() error { + // Emoji missing files, delete it. + // NOTE: if we ever support uncaching + // of emojis, change to e.uncache(). + // In that case we should also rename + // this function to match the media + // equivalent -> fixCacheState(). + log.WithContext(ctx). + WithField("emoji", emoji.ID). + Debug("deleting due to missing emoji") + return e.delete(ctx, emoji) + }, + emoji.ImageStaticPath, + emoji.ImagePath, + ) +} + +func (e *Emoji) fixBroken(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { + // Check we have the required category for emoji. + _, missing, err := e.getRelatedCategory(ctx, emoji) + if err != nil { + return false, err + } + + if missing { + if !gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return true, nil + } + + // Remove related category. + emoji.CategoryID = "" + + // Update emoji model in the database to remove category ID. + log.Debugf(ctx, "fixing missing emoji category: %s", emoji.ID) + if err := e.state.DB.UpdateEmoji(ctx, emoji, "category_id"); err != nil { + return true, gtserror.Newf("error updating emoji: %w", err) + } + + return true, nil + } + + return false, nil +} + +func (e *Emoji) getRelatedCategory(ctx context.Context, emoji *gtsmodel.Emoji) (*gtsmodel.EmojiCategory, bool, error) { + if emoji.CategoryID == "" { + // no related category. + return nil, false, nil + } + + // Load the category related to this emoji. + category, err := e.state.DB.GetEmojiCategory( + gtscontext.SetBarebones(ctx), + emoji.CategoryID, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, false, gtserror.Newf("error fetching category by id %s: %w", emoji.CategoryID, err) + } + + if category == nil { + // Category is missing. + return nil, true, nil + } + + return category, false, nil +} + +func (e *Emoji) delete(ctx context.Context, emoji *gtsmodel.Emoji) error { + if gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return nil + } + + // Remove emoji and static files. + _, err := e.removeFiles(ctx, + emoji.ImageStaticPath, + emoji.ImagePath, + ) + if err != nil { + return gtserror.Newf("error removing emoji files: %w", err) + } + + // Delete emoji entirely from the database by its ID. + if err := e.state.DB.DeleteEmojiByID(ctx, emoji.ID); err != nil { + return gtserror.Newf("error deleting emoji: %w", err) + } + + return nil +} diff --git a/internal/cleaner/media.go b/internal/cleaner/media.go new file mode 100644 index 000000000..51a0aea6d --- /dev/null +++ b/internal/cleaner/media.go @@ -0,0 +1,547 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package cleaner + +import ( + "context" + "errors" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/regexes" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +// Media encompasses a set of +// media cleanup / admin utils. +type Media struct { + *Cleaner +} + +// All will execute all cleaner.Media utilities synchronously, including output logging. +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (m *Media) All(ctx context.Context, maxRemoteDays int) { + t := time.Now().Add(-24 * time.Hour * time.Duration(maxRemoteDays)) + m.LogUncacheRemote(ctx, t) + m.LogPruneOrphaned(ctx) + m.LogPruneUnused(ctx) + m.LogFixCacheStates(ctx) + _ = m.state.Storage.Storage.Clean(ctx) +} + +// LogUncacheRemote performs Media.UncacheRemote(...), logging the start and outcome. +func (m *Media) LogUncacheRemote(ctx context.Context, olderThan time.Time) { + log.Infof(ctx, "start older than: %s", olderThan.Format(time.Stamp)) + if n, err := m.UncacheRemote(ctx, olderThan); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "uncached: %d", n) + } +} + +// LogPruneOrphaned performs Media.PruneOrphaned(...), logging the start and outcome. +func (m *Media) LogPruneOrphaned(ctx context.Context) { + log.Info(ctx, "start") + if n, err := m.PruneOrphaned(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "pruned: %d", n) + } +} + +// LogPruneUnused performs Media.PruneUnused(...), logging the start and outcome. +func (m *Media) LogPruneUnused(ctx context.Context) { + log.Info(ctx, "start") + if n, err := m.PruneUnused(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "pruned: %d", n) + } +} + +// LogFixCacheStates performs Media.FixCacheStates(...), logging the start and outcome. +func (m *Media) LogFixCacheStates(ctx context.Context) { + log.Info(ctx, "start") + if n, err := m.FixCacheStates(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "fixed: %d", n) + } +} + +// PruneOrphaned will delete orphaned files from storage (i.e. media missing a database entry). +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (m *Media) PruneOrphaned(ctx context.Context) (int, error) { + var files []string + + // All media files in storage will have path fitting: {$account}/{$type}/{$size}/{$id}.{$ext} + if err := m.state.Storage.WalkKeys(ctx, func(ctx context.Context, path string) error { + if !regexes.FilePath.MatchString(path) { + // This is not our expected media + // path format, skip this one. + return nil + } + + // Check whether this entry is orphaned. + orphaned, err := m.isOrphaned(ctx, path) + if err != nil { + return gtserror.Newf("error checking orphaned status: %w", err) + } + + if orphaned { + // Add this orphaned entry. + files = append(files, path) + } + + return nil + }); err != nil { + return 0, gtserror.Newf("error walking storage: %w", err) + } + + // Delete all orphaned files from storage. + return m.removeFiles(ctx, files...) +} + +// PruneUnused will delete all unused media attachments from the database and storage driver. +// Media is marked as unused if not attached to any status, account or account is suspended. +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (m *Media) PruneUnused(ctx context.Context) (int, error) { + var ( + total int + maxID string + ) + + for { + // Fetch the next batch of media attachments up to next max ID. + attachments, err := m.state.DB.GetAttachments(ctx, maxID, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting attachments: %w", err) + } + + if len(attachments) == 0 { + // reached end. + break + } + + // Use last ID as the next 'maxID' value. + maxID = attachments[len(attachments)-1].ID + + for _, media := range attachments { + // Check / prune unused media attachment. + fixed, err := m.pruneUnused(ctx, media) + if err != nil { + return total, err + } + + if fixed { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +// UncacheRemote will uncache all remote media attachments older than given input time. +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (m *Media) UncacheRemote(ctx context.Context, olderThan time.Time) (int, error) { + var total int + + // Drop time by a minute to improve search, + // (i.e. make it olderThan inclusive search). + olderThan = olderThan.Add(-time.Minute) + + // Store recent time. + mostRecent := olderThan + + for { + // Fetch the next batch of attachments older than last-set time. + attachments, err := m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting remote media: %w", err) + } + + if len(attachments) == 0 { + // reached end. + break + } + + // Use last created-at as the next 'olderThan' value. + olderThan = attachments[len(attachments)-1].CreatedAt + + for _, media := range attachments { + // Check / uncache each remote media attachment. + uncached, err := m.uncacheRemote(ctx, mostRecent, media) + if err != nil { + return total, err + } + + if uncached { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +// FixCacheStatus will check all media for up-to-date cache status (i.e. in storage driver). +// Media marked as cached, with any required files missing, will be automatically uncached. +// Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (m *Media) FixCacheStates(ctx context.Context) (int, error) { + var ( + total int + maxID string + ) + + for { + // Fetch the next batch of media attachments up to next max ID. + attachments, err := m.state.DB.GetAttachments(ctx, maxID, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting avatars / headers: %w", err) + } + + if len(attachments) == 0 { + // reached end. + break + } + + // Use last ID as the next 'maxID' value. + maxID = attachments[len(attachments)-1].ID + + for _, media := range attachments { + // Check / fix required media cache states. + fixed, err := m.fixCacheState(ctx, media) + if err != nil { + return total, err + } + + if fixed { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +func (m *Media) isOrphaned(ctx context.Context, path string) (bool, error) { + pathParts := regexes.FilePath.FindStringSubmatch(path) + if len(pathParts) != 6 { + // This doesn't match our expectations so + // it wasn't created by gts; ignore it. + return false, nil + } + + var ( + // 0th -> whole match + // 1st -> account ID + mediaType = pathParts[2] + // 3rd -> media sub-type (e.g. small, static) + mediaID = pathParts[4] + // 5th -> file extension + ) + + // Start a log entry for media. + l := log.WithContext(ctx). + WithField("media", mediaID) + + switch media.Type(mediaType) { + case media.TypeAttachment: + // Look for media in database stored by ID. + media, err := m.state.DB.GetAttachmentByID( + gtscontext.SetBarebones(ctx), + mediaID, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return false, gtserror.Newf("error fetching media by id %s: %w", mediaID, err) + } + + if media == nil { + l.Debug("missing db entry for media") + return true, nil + } + + case media.TypeEmoji: + // Generate static URL for this emoji to lookup. + staticURL := uris.GenerateURIForAttachment( + pathParts[1], // instance account ID + string(media.TypeEmoji), + string(media.SizeStatic), + mediaID, + "png", + ) + + // Look for emoji in database stored by static URL. + // The media ID part of the storage key for emojis can + // change for refreshed items, so search by generated URL. + emoji, err := m.state.DB.GetEmojiByStaticURL( + gtscontext.SetBarebones(ctx), + staticURL, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return false, gtserror.Newf("error fetching emoji by url %s: %w", staticURL, err) + } + + if emoji == nil { + l.Debug("missing db entry for emoji") + return true, nil + } + } + + return false, nil +} + +func (m *Media) pruneUnused(ctx context.Context, media *gtsmodel.MediaAttachment) (bool, error) { + // Start a log entry for media. + l := log.WithContext(ctx). + WithField("media", media.ID) + + // Check whether we have the required account for media. + account, missing, err := m.getRelatedAccount(ctx, media) + if err != nil { + return false, err + } else if missing { + l.Debug("deleting due to missing account") + return true, m.delete(ctx, media) + } + + if account != nil { + // Related account exists for this media, check whether it is being used. + headerInUse := (*media.Header && media.ID == account.HeaderMediaAttachmentID) + avatarInUse := (*media.Avatar && media.ID == account.AvatarMediaAttachmentID) + if (headerInUse || avatarInUse) && account.SuspendedAt.IsZero() { + l.Debug("skipping as account media in use") + return false, nil + } + } + + // Check whether we have the required status for media. + status, missing, err := m.getRelatedStatus(ctx, media) + if err != nil { + return false, err + } else if missing { + l.Debug("deleting due to missing status") + return true, m.delete(ctx, media) + } + + if status != nil { + // Check whether still attached to status. + for _, id := range status.AttachmentIDs { + if id == media.ID { + l.Debug("skippping as attached to status") + return false, nil + } + } + } + + // Media totally unused, delete it. + l.Debug("deleting unused media") + return true, m.delete(ctx, media) +} + +func (m *Media) fixCacheState(ctx context.Context, media *gtsmodel.MediaAttachment) (bool, error) { + if !*media.Cached { + // We ignore uncached media, a + // false negative is a much better + // situation than a false positive, + // re-cache will just overwrite it. + return false, nil + } + + // Start a log entry for media. + l := log.WithContext(ctx). + WithField("media", media.ID) + + // Check whether we have the required account for media. + _, missingAccount, err := m.getRelatedAccount(ctx, media) + if err != nil { + return false, err + } else if missingAccount { + l.Debug("skipping due to missing account") + return false, nil + } + + // Check whether we have the required status for media. + _, missingStatus, err := m.getRelatedStatus(ctx, media) + if err != nil { + return false, err + } else if missingStatus { + l.Debug("skipping due to missing status") + return false, nil + } + + // So we know this a valid cached media entry. + // Check that we have the files on disk required.... + return m.checkFiles(ctx, func() error { + l.Debug("uncaching due to missing media") + return m.uncache(ctx, media) + }, + media.Thumbnail.Path, + media.File.Path, + ) +} + +func (m *Media) uncacheRemote(ctx context.Context, after time.Time, media *gtsmodel.MediaAttachment) (bool, error) { + if !*media.Cached { + // Already uncached. + return false, nil + } + + // Start a log entry for media. + l := log.WithContext(ctx). + WithField("media", media.ID) + + // Check whether we have the required account for media. + account, missing, err := m.getRelatedAccount(ctx, media) + if err != nil { + return false, err + } else if missing { + l.Debug("skipping due to missing account") + return false, nil + } + + if account != nil && account.FetchedAt.After(after) { + l.Debug("skipping due to recently fetched account") + return false, nil + } + + // Check whether we have the required status for media. + status, missing, err := m.getRelatedStatus(ctx, media) + if err != nil { + return false, err + } else if missing { + l.Debug("skipping due to missing status") + return false, nil + } + + if status != nil && status.FetchedAt.After(after) { + l.Debug("skipping due to recently fetched status") + return false, nil + } + + // This media is too old, uncache it. + l.Debug("uncaching old remote media") + return true, m.uncache(ctx, media) +} + +func (m *Media) getRelatedAccount(ctx context.Context, media *gtsmodel.MediaAttachment) (*gtsmodel.Account, bool, error) { + if media.AccountID == "" { + // no related account. + return nil, false, nil + } + + // Load the account related to this media. + account, err := m.state.DB.GetAccountByID( + gtscontext.SetBarebones(ctx), + media.AccountID, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, false, gtserror.Newf("error fetching account by id %s: %w", media.AccountID, err) + } + + if account == nil { + // account is missing. + return nil, true, nil + } + + return account, false, nil +} + +func (m *Media) getRelatedStatus(ctx context.Context, media *gtsmodel.MediaAttachment) (*gtsmodel.Status, bool, error) { + if media.StatusID == "" { + // no related status. + return nil, false, nil + } + + // Load the status related to this media. + status, err := m.state.DB.GetStatusByID( + gtscontext.SetBarebones(ctx), + media.StatusID, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, false, gtserror.Newf("error fetching status by id %s: %w", media.StatusID, err) + } + + if status == nil { + // status is missing. + return nil, true, nil + } + + return status, false, nil +} + +func (m *Media) uncache(ctx context.Context, media *gtsmodel.MediaAttachment) error { + if gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return nil + } + + // Remove media and thumbnail. + _, err := m.removeFiles(ctx, + media.File.Path, + media.Thumbnail.Path, + ) + if err != nil { + return gtserror.Newf("error removing media files: %w", err) + } + + // Update attachment to reflect that we no longer have it cached. + log.Debugf(ctx, "marking media attachment as uncached: %s", media.ID) + media.Cached = func() *bool { i := false; return &i }() + if err := m.state.DB.UpdateAttachment(ctx, media, "cached"); err != nil { + return gtserror.Newf("error updating media: %w", err) + } + + return nil +} + +func (m *Media) delete(ctx context.Context, media *gtsmodel.MediaAttachment) error { + if gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return nil + } + + // Remove media and thumbnail. + _, err := m.removeFiles(ctx, + media.File.Path, + media.Thumbnail.Path, + ) + if err != nil { + return gtserror.Newf("error removing media files: %w", err) + } + + // Delete media attachment entirely from the database. + log.Debugf(ctx, "deleting media attachment: %s", media.ID) + if err := m.state.DB.DeleteAttachment(ctx, media.ID); err != nil { + return gtserror.Newf("error deleting media: %w", err) + } + + return nil +} diff --git a/internal/cleaner/media_test.go b/internal/cleaner/media_test.go new file mode 100644 index 000000000..824df2ca5 --- /dev/null +++ b/internal/cleaner/media_test.go @@ -0,0 +1,427 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package cleaner_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/cleaner" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/visibility" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type MediaTestSuite struct { + suite.Suite + + db db.DB + storage *storage.Driver + state state.State + manager *media.Manager + cleaner *cleaner.Cleaner + transportController transport.Controller + testAttachments map[string]*gtsmodel.MediaAttachment + testAccounts map[string]*gtsmodel.Account + testEmojis map[string]*gtsmodel.Emoji +} + +func TestMediaTestSuite(t *testing.T) { + suite.Run(t, &MediaTestSuite{}) +} + +func (suite *MediaTestSuite) SetupTest() { + testrig.InitTestConfig() + testrig.InitTestLog() + + suite.state.Caches.Init() + testrig.StartWorkers(&suite.state) + + suite.db = testrig.NewTestDB(&suite.state) + suite.storage = testrig.NewInMemoryStorage() + suite.state.DB = suite.db + suite.state.Storage = suite.storage + + testrig.StandardStorageSetup(suite.storage, "../../testrig/media") + testrig.StandardDBSetup(suite.db, nil) + + testrig.StartTimelines( + &suite.state, + visibility.NewFilter(&suite.state), + testrig.NewTestTypeConverter(suite.db), + ) + + suite.testAttachments = testrig.NewTestAttachments() + suite.testAccounts = testrig.NewTestAccounts() + suite.testEmojis = testrig.NewTestEmojis() + suite.manager = testrig.NewTestMediaManager(&suite.state) + suite.cleaner = cleaner.New(&suite.state) + suite.transportController = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../testrig/media")) +} + +func (suite *MediaTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) + testrig.StopWorkers(&suite.state) +} + +// func (suite *MediaTestSuite) TestPruneOrphanedDry() { +// // add a big orphan panda to store +// b, err := os.ReadFile("../media/test/big-panda.gif") +// if err != nil { +// suite.FailNow(err.Error()) +// } + +// pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" +// if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { +// suite.FailNow(err.Error()) +// } + +// ctx := context.Background() + +// // dry run should show up 1 orphaned panda +// totalPruned, err := suite.cleaner.Media().PruneOrphaned(gtscontext.SetDryRun(ctx)) +// suite.NoError(err) +// suite.Equal(1, totalPruned) + +// // panda should still be in storage +// hasKey, err := suite.storage.Has(ctx, pandaPath) +// suite.NoError(err) +// suite.True(hasKey) +// } + +// func (suite *MediaTestSuite) TestPruneOrphanedMoist() { +// // i am not complicit in the moistness of this codebase :| + +// // add a big orphan panda to store +// b, err := os.ReadFile("../media/test/big-panda.gif") +// if err != nil { +// suite.FailNow(err.Error()) +// } + +// pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" +// if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { +// suite.FailNow(err.Error()) +// } + +// ctx := context.Background() + +// // should show up 1 orphaned panda +// totalPruned, err := suite.cleaner.Media().PruneOrphaned(ctx) +// suite.NoError(err) +// suite.Equal(1, totalPruned) + +// // panda should no longer be in storage +// hasKey, err := suite.storage.Has(ctx, pandaPath) +// suite.NoError(err) +// suite.False(hasKey) +// } + +// func (suite *MediaTestSuite) TestPruneUnusedLocal() { +// testAttachment := suite.testAttachments["local_account_1_unattached_1"] +// suite.True(*testAttachment.Cached) + +// totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) +// suite.NoError(err) +// suite.Equal(1, totalPruned) + +// _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) +// suite.ErrorIs(err, db.ErrNoEntries) +// } + +// func (suite *MediaTestSuite) TestPruneUnusedLocalDry() { +// testAttachment := suite.testAttachments["local_account_1_unattached_1"] +// suite.True(*testAttachment.Cached) + +// totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true) +// suite.NoError(err) +// suite.Equal(1, totalPruned) + +// _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) +// suite.NoError(err) +// } + +// func (suite *MediaTestSuite) TestPruneRemoteTwice() { +// totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) +// suite.NoError(err) +// suite.Equal(1, totalPruned) + +// // final prune should prune nothing, since the first prune already happened +// totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false) +// suite.NoError(err) +// suite.Equal(0, totalPrunedAgain) +// } + +// func (suite *MediaTestSuite) TestPruneOneNonExistent() { +// ctx := context.Background() +// testAttachment := suite.testAttachments["local_account_1_unattached_1"] + +// // Delete this attachment cached on disk +// media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) +// suite.NoError(err) +// suite.True(*media.Cached) +// err = suite.storage.Delete(ctx, media.File.Path) +// suite.NoError(err) + +// // Now attempt to prune for item with db entry no file +// totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false) +// suite.NoError(err) +// suite.Equal(1, totalPruned) +// } + +// func (suite *MediaTestSuite) TestPruneUnusedRemote() { +// ctx := context.Background() + +// // start by clearing zork's avatar + header +// zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] +// zorkOldHeader := suite.testAttachments["local_account_1_avatar"] +// zork := suite.testAccounts["local_account_1"] +// zork.AvatarMediaAttachmentID = "" +// zork.HeaderMediaAttachmentID = "" +// if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { +// panic(err) +// } + +// totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) +// suite.NoError(err) +// suite.Equal(2, totalPruned) + +// // media should no longer be stored +// _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) +// suite.ErrorIs(err, storage.ErrNotFound) + +// // attachments should no longer be in the db +// _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) +// suite.ErrorIs(err, db.ErrNoEntries) +// _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) +// suite.ErrorIs(err, db.ErrNoEntries) +// } + +// func (suite *MediaTestSuite) TestPruneUnusedRemoteTwice() { +// ctx := context.Background() + +// // start by clearing zork's avatar + header +// zork := suite.testAccounts["local_account_1"] +// zork.AvatarMediaAttachmentID = "" +// zork.HeaderMediaAttachmentID = "" +// if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { +// panic(err) +// } + +// totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) +// suite.NoError(err) +// suite.Equal(2, totalPruned) + +// // final prune should prune nothing, since the first prune already happened +// totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false) +// suite.NoError(err) +// suite.Equal(0, totalPruned) +// } + +// func (suite *MediaTestSuite) TestPruneUnusedRemoteMultipleAccounts() { +// ctx := context.Background() + +// // start by clearing zork's avatar + header +// zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] +// zorkOldHeader := suite.testAttachments["local_account_1_avatar"] +// zork := suite.testAccounts["local_account_1"] +// zork.AvatarMediaAttachmentID = "" +// zork.HeaderMediaAttachmentID = "" +// if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { +// panic(err) +// } + +// // set zork's unused header as belonging to turtle +// turtle := suite.testAccounts["local_account_1"] +// zorkOldHeader.AccountID = turtle.ID +// if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil { +// panic(err) +// } + +// totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) +// suite.NoError(err) +// suite.Equal(2, totalPruned) + +// // media should no longer be stored +// _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) +// suite.ErrorIs(err, storage.ErrNotFound) +// _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) +// suite.ErrorIs(err, storage.ErrNotFound) + +// // attachments should no longer be in the db +// _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) +// suite.ErrorIs(err, db.ErrNoEntries) +// _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) +// suite.ErrorIs(err, db.ErrNoEntries) +// } + +func (suite *MediaTestSuite) TestUncacheRemote() { + ctx := context.Background() + + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + suite.True(*testStatusAttachment.Cached) + + testHeader := suite.testAttachments["remote_account_3_header"] + suite.True(*testHeader.Cached) + + after := time.Now().Add(-24 * time.Hour) + totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after) + suite.NoError(err) + suite.Equal(2, totalUncached) + + uncachedAttachment, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) + suite.NoError(err) + suite.False(*uncachedAttachment.Cached) + + uncachedAttachment, err = suite.db.GetAttachmentByID(ctx, testHeader.ID) + suite.NoError(err) + suite.False(*uncachedAttachment.Cached) +} + +func (suite *MediaTestSuite) TestUncacheRemoteDry() { + ctx := context.Background() + + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + suite.True(*testStatusAttachment.Cached) + + testHeader := suite.testAttachments["remote_account_3_header"] + suite.True(*testHeader.Cached) + + after := time.Now().Add(-24 * time.Hour) + totalUncached, err := suite.cleaner.Media().UncacheRemote(gtscontext.SetDryRun(ctx), after) + suite.NoError(err) + suite.Equal(2, totalUncached) + + uncachedAttachment, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) + suite.NoError(err) + suite.True(*uncachedAttachment.Cached) + + uncachedAttachment, err = suite.db.GetAttachmentByID(ctx, testHeader.ID) + suite.NoError(err) + suite.True(*uncachedAttachment.Cached) +} + +func (suite *MediaTestSuite) TestUncacheRemoteTwice() { + ctx := context.Background() + after := time.Now().Add(-24 * time.Hour) + + totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after) + suite.NoError(err) + suite.Equal(2, totalUncached) + + // final uncache should uncache nothing, since the first uncache already happened + totalUncachedAgain, err := suite.cleaner.Media().UncacheRemote(ctx, after) + suite.NoError(err) + suite.Equal(0, totalUncachedAgain) +} + +func (suite *MediaTestSuite) TestUncacheAndRecache() { + ctx := context.Background() + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + testHeader := suite.testAttachments["remote_account_3_header"] + + after := time.Now().Add(-24 * time.Hour) + totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after) + suite.NoError(err) + suite.Equal(2, totalUncached) + + // media should no longer be stored + _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // now recache the image.... + data := func(_ context.Context) (io.ReadCloser, int64, error) { + // load bytes from a test image + b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") + if err != nil { + panic(err) + } + return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil + } + + for _, original := range []*gtsmodel.MediaAttachment{ + testStatusAttachment, + testHeader, + } { + processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, original.ID) + suite.NoError(err) + + // synchronously load the recached attachment + recachedAttachment, err := processingRecache.LoadAttachment(ctx) + suite.NoError(err) + suite.NotNil(recachedAttachment) + + // recachedAttachment should be basically the same as the old attachment + suite.True(*recachedAttachment.Cached) + suite.Equal(original.ID, recachedAttachment.ID) + suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place + suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail + suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same + + // recached files should be back in storage + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) + suite.NoError(err) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) + suite.NoError(err) + } +} + +func (suite *MediaTestSuite) TestUncacheOneNonExistent() { + ctx := context.Background() + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + + // Delete this attachment cached on disk + media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) + suite.NoError(err) + suite.True(*media.Cached) + err = suite.storage.Delete(ctx, media.File.Path) + suite.NoError(err) + + // Now attempt to uncache remote for item with db entry no file + after := time.Now().Add(-24 * time.Hour) + totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after) + suite.NoError(err) + suite.Equal(2, totalUncached) +} diff --git a/internal/db/bundb/emoji.go b/internal/db/bundb/emoji.go index 60b8fc12b..60c140264 100644 --- a/internal/db/bundb/emoji.go +++ b/internal/db/bundb/emoji.go @@ -19,6 +19,7 @@ package bundb import ( "context" + "database/sql" "errors" "strings" "time" @@ -37,19 +38,6 @@ type emojiDB struct { state *state.State } -func (e *emojiDB) newEmojiQ(emoji *gtsmodel.Emoji) *bun.SelectQuery { - return e.conn. - NewSelect(). - Model(emoji). - Relation("Category") -} - -func (e *emojiDB) newEmojiCategoryQ(emojiCategory *gtsmodel.EmojiCategory) *bun.SelectQuery { - return e.conn. - NewSelect(). - Model(emojiCategory) -} - func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error { return e.state.Caches.GTS.Emoji().Store(emoji, func() error { _, err := e.conn.NewInsert().Model(emoji).Exec(ctx) @@ -57,14 +45,15 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error }) } -func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) { +func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) error { emoji.UpdatedAt = time.Now() if len(columns) > 0 { // If we're updating by column, ensure "updated_at" is included. columns = append(columns, "updated_at") } - err := e.state.Caches.GTS.Emoji().Store(emoji, func() error { + // Update the emoji model in the database. + return e.state.Caches.GTS.Emoji().Store(emoji, func() error { _, err := e.conn. NewUpdate(). Model(emoji). @@ -73,15 +62,34 @@ func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, column Exec(ctx) return e.conn.ProcessError(err) }) - if err != nil { - return nil, err - } - - return emoji, nil } func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error { - defer e.state.Caches.GTS.Emoji().Invalidate("ID", id) + var ( + accountIDs []string + statusIDs []string + ) + + defer func() { + // Invalidate cached emoji. + e.state.Caches.GTS. + Emoji(). + Invalidate("ID", id) + + for _, id := range accountIDs { + // Invalidate cached account. + e.state.Caches.GTS. + Account(). + Invalidate("ID", id) + } + + for _, id := range statusIDs { + // Invalidate cached account. + e.state.Caches.GTS. + Status(). + Invalidate("ID", id) + } + }() // Load emoji into cache before attempting a delete, // as we need it cached in order to trigger the invalidate @@ -99,6 +107,7 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error { return e.conn.RunInTx(ctx, func(tx bun.Tx) error { // delete links between this emoji and any statuses that use it + // TODO: remove when we delete this table if _, err := tx. NewDelete(). TableExpr("? AS ?", bun.Ident("status_to_emojis"), bun.Ident("status_to_emoji")). @@ -108,6 +117,7 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error { } // delete links between this emoji and any accounts that use it + // TODO: remove when we delete this table if _, err := tx. NewDelete(). TableExpr("? AS ?", bun.Ident("account_to_emojis"), bun.Ident("account_to_emoji")). @@ -116,19 +126,91 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error { return err } - if _, err := tx. - NewDelete(). - TableExpr("? AS ?", bun.Ident("emojis"), bun.Ident("emoji")). - Where("? = ?", bun.Ident("emoji.id"), id). + // Select all accounts using this emoji. + if _, err := tx.NewSelect(). + Table("accounts"). + Column("id"). + Where("? IN (emojis)", id). + Exec(ctx, &accountIDs); err != nil { + return err + } + + for _, id := range accountIDs { + var emojiIDs []string + + // Select account with ID. + if _, err := tx.NewSelect(). + Table("accounts"). + Column("emojis"). + Where("id = ?", id). + Exec(ctx); err != nil && + err != sql.ErrNoRows { + return err + } + + // Drop ID from account emojis. + emojiIDs = dropID(emojiIDs, id) + + // Update account emoji IDs. + if _, err := tx.NewUpdate(). + Table("accounts"). + Where("id = ?", id). + Set("emojis = ?", emojiIDs). + Exec(ctx); err != nil && + err != sql.ErrNoRows { + return err + } + } + + // Select all statuses using this emoji. + if _, err := tx.NewSelect(). + Table("statuses"). + Column("id"). + Where("? IN (emojis)", id). + Exec(ctx, &statusIDs); err != nil { + return err + } + + for _, id := range statusIDs { + var emojiIDs []string + + // Select statuses with ID. + if _, err := tx.NewSelect(). + Table("statuses"). + Column("emojis"). + Where("id = ?", id). + Exec(ctx); err != nil && + err != sql.ErrNoRows { + return err + } + + // Drop ID from account emojis. + emojiIDs = dropID(emojiIDs, id) + + // Update status emoji IDs. + if _, err := tx.NewUpdate(). + Table("statuses"). + Where("id = ?", id). + Set("emojis = ?", emojiIDs). + Exec(ctx); err != nil && + err != sql.ErrNoRows { + return err + } + } + + // Delete emoji from database. + if _, err := tx.NewDelete(). + Table("emojis"). + Where("id = ?", id). Exec(ctx); err != nil { - return e.conn.ProcessError(err) + return err } return nil }) } -func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) { +func (e *emojiDB) GetEmojisBy(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, error) { emojiIDs := []string{} subQuery := e.conn. @@ -245,6 +327,29 @@ func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled return e.GetEmojisByIDs(ctx, emojiIDs) } +func (e *emojiDB) GetEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error) { + emojiIDs := []string{} + + q := e.conn.NewSelect(). + Table("emojis"). + Column("id"). + Order("id DESC") + + if maxID != "" { + q = q.Where("? < ?", bun.Ident("id"), maxID) + } + + if limit != 0 { + q = q.Limit(limit) + } + + if err := q.Scan(ctx, &emojiIDs); err != nil { + return nil, e.conn.ProcessError(err) + } + + return e.GetEmojisByIDs(ctx, emojiIDs) +} + func (e *emojiDB) GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, db.Error) { emojiIDs := []string{} @@ -269,7 +374,10 @@ func (e *emojiDB) GetEmojiByID(ctx context.Context, id string) (*gtsmodel.Emoji, ctx, "ID", func(emoji *gtsmodel.Emoji) error { - return e.newEmojiQ(emoji).Where("? = ?", bun.Ident("emoji.id"), id).Scan(ctx) + return e.conn. + NewSelect(). + Model(emoji). + Where("? = ?", bun.Ident("emoji.id"), id).Scan(ctx) }, id, ) @@ -280,7 +388,10 @@ func (e *emojiDB) GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoj ctx, "URI", func(emoji *gtsmodel.Emoji) error { - return e.newEmojiQ(emoji).Where("? = ?", bun.Ident("emoji.uri"), uri).Scan(ctx) + return e.conn. + NewSelect(). + Model(emoji). + Where("? = ?", bun.Ident("emoji.uri"), uri).Scan(ctx) }, uri, ) @@ -291,7 +402,9 @@ func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode strin ctx, "Shortcode.Domain", func(emoji *gtsmodel.Emoji) error { - q := e.newEmojiQ(emoji) + q := e.conn. + NewSelect(). + Model(emoji) if domain != "" { q = q.Where("? = ?", bun.Ident("emoji.shortcode"), shortcode) @@ -313,8 +426,9 @@ func (e *emojiDB) GetEmojiByStaticURL(ctx context.Context, imageStaticURL string ctx, "ImageStaticURL", func(emoji *gtsmodel.Emoji) error { - return e. - newEmojiQ(emoji). + return e.conn. + NewSelect(). + Model(emoji). Where("? = ?", bun.Ident("emoji.image_static_url"), imageStaticURL). Scan(ctx) }, @@ -350,7 +464,10 @@ func (e *emojiDB) GetEmojiCategory(ctx context.Context, id string) (*gtsmodel.Em ctx, "ID", func(emojiCategory *gtsmodel.EmojiCategory) error { - return e.newEmojiCategoryQ(emojiCategory).Where("? = ?", bun.Ident("emoji_category.id"), id).Scan(ctx) + return e.conn. + NewSelect(). + Model(emojiCategory). + Where("? = ?", bun.Ident("emoji_category.id"), id).Scan(ctx) }, id, ) @@ -361,14 +478,18 @@ func (e *emojiDB) GetEmojiCategoryByName(ctx context.Context, name string) (*gts ctx, "Name", func(emojiCategory *gtsmodel.EmojiCategory) error { - return e.newEmojiCategoryQ(emojiCategory).Where("LOWER(?) = ?", bun.Ident("emoji_category.name"), strings.ToLower(name)).Scan(ctx) + return e.conn. + NewSelect(). + Model(emojiCategory). + Where("LOWER(?) = ?", bun.Ident("emoji_category.name"), strings.ToLower(name)).Scan(ctx) }, name, ) } func (e *emojiDB) getEmoji(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Emoji) error, keyParts ...any) (*gtsmodel.Emoji, db.Error) { - return e.state.Caches.GTS.Emoji().Load(lookup, func() (*gtsmodel.Emoji, error) { + // Fetch emoji from database cache with loader callback + emoji, err := e.state.Caches.GTS.Emoji().Load(lookup, func() (*gtsmodel.Emoji, error) { var emoji gtsmodel.Emoji // Not cached! Perform database query @@ -378,6 +499,23 @@ func (e *emojiDB) getEmoji(ctx context.Context, lookup string, dbQuery func(*gts return &emoji, nil }, keyParts...) + if err != nil { + return nil, err + } + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return emoji, nil + } + + if emoji.CategoryID != "" { + emoji.Category, err = e.GetEmojiCategory(ctx, emoji.CategoryID) + if err != nil { + log.Errorf(ctx, "error getting emoji category %s: %v", emoji.CategoryID, err) + } + } + + return emoji, nil } func (e *emojiDB) GetEmojisByIDs(ctx context.Context, emojiIDs []string) ([]*gtsmodel.Emoji, db.Error) { @@ -432,3 +570,17 @@ func (e *emojiDB) GetEmojiCategoriesByIDs(ctx context.Context, emojiCategoryIDs return emojiCategories, nil } + +// dropIDs drops given ID string from IDs slice. +func dropID(ids []string, id string) []string { + for i := 0; i < len(ids); { + if ids[i] == id { + // Remove this reference. + copy(ids[i:], ids[i+1:]) + ids = ids[:len(ids)-1] + continue + } + i++ + } + return ids +} diff --git a/internal/db/bundb/emoji_test.go b/internal/db/bundb/emoji_test.go index c71c63efb..f75334d90 100644 --- a/internal/db/bundb/emoji_test.go +++ b/internal/db/bundb/emoji_test.go @@ -59,7 +59,7 @@ func (suite *EmojiTestSuite) TestGetEmojiByStaticURL() { } func (suite *EmojiTestSuite) TestGetAllEmojis() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0) suite.NoError(err) suite.Equal(2, len(emojis)) @@ -68,7 +68,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojis() { } func (suite *EmojiTestSuite) TestGetAllEmojisLimit1() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 1) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 1) suite.NoError(err) suite.Equal(1, len(emojis)) @@ -76,7 +76,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisLimit1() { } func (suite *EmojiTestSuite) TestGetAllEmojisMaxID() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "rainbow@", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "rainbow@", "", 0) suite.NoError(err) suite.Equal(1, len(emojis)) @@ -84,7 +84,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisMaxID() { } func (suite *EmojiTestSuite) TestGetAllEmojisMinID() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "yell@fossbros-anonymous.io", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "yell@fossbros-anonymous.io", 0) suite.NoError(err) suite.Equal(1, len(emojis)) @@ -92,14 +92,14 @@ func (suite *EmojiTestSuite) TestGetAllEmojisMinID() { } func (suite *EmojiTestSuite) TestGetAllDisabledEmojis() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, false, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, false, "", "", "", 0) suite.ErrorIs(err, db.ErrNoEntries) suite.Equal(0, len(emojis)) } func (suite *EmojiTestSuite) TestGetAllEnabledEmojis() { - emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, false, true, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, false, true, "", "", "", 0) suite.NoError(err) suite.Equal(2, len(emojis)) @@ -108,7 +108,7 @@ func (suite *EmojiTestSuite) TestGetAllEnabledEmojis() { } func (suite *EmojiTestSuite) TestGetLocalEnabledEmojis() { - emojis, err := suite.db.GetEmojis(context.Background(), "", false, true, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), "", false, true, "", "", "", 0) suite.NoError(err) suite.Equal(1, len(emojis)) @@ -116,21 +116,21 @@ func (suite *EmojiTestSuite) TestGetLocalEnabledEmojis() { } func (suite *EmojiTestSuite) TestGetLocalDisabledEmojis() { - emojis, err := suite.db.GetEmojis(context.Background(), "", true, false, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), "", true, false, "", "", "", 0) suite.ErrorIs(err, db.ErrNoEntries) suite.Equal(0, len(emojis)) } func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain() { - emojis, err := suite.db.GetEmojis(context.Background(), "peepee.poopoo", true, true, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), "peepee.poopoo", true, true, "", "", "", 0) suite.ErrorIs(err, db.ErrNoEntries) suite.Equal(0, len(emojis)) } func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain2() { - emojis, err := suite.db.GetEmojis(context.Background(), "fossbros-anonymous.io", true, true, "", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), "fossbros-anonymous.io", true, true, "", "", "", 0) suite.NoError(err) suite.Equal(1, len(emojis)) @@ -138,7 +138,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain2() { } func (suite *EmojiTestSuite) TestGetSpecificEmojisFromDomain2() { - emojis, err := suite.db.GetEmojis(context.Background(), "fossbros-anonymous.io", true, true, "yell", "", "", 0) + emojis, err := suite.db.GetEmojisBy(context.Background(), "fossbros-anonymous.io", true, true, "yell", "", "", 0) suite.NoError(err) suite.Equal(1, len(emojis)) diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index b64447beb..a9b60e3ae 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -24,6 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/state" @@ -110,7 +111,7 @@ func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error { // Load media into cache before attempting a delete, // as we need it cached in order to trigger the invalidate // callback. This in turn invalidates others. - _, err := m.GetAttachmentByID(gtscontext.SetBarebones(ctx), id) + media, err := m.GetAttachmentByID(gtscontext.SetBarebones(ctx), id) if err != nil { if errors.Is(err, db.ErrNoEntries) { // not an issue. @@ -119,11 +120,115 @@ func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error { return err } - // Finally delete media from DB. - _, err = m.conn.NewDelete(). - TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")). - Where("? = ?", bun.Ident("media_attachment.id"), id). - Exec(ctx) + var ( + invalidateAccount bool + invalidateStatus bool + ) + + // Delete media attachment in new transaction. + err = m.conn.RunInTx(ctx, func(tx bun.Tx) error { + if media.AccountID != "" { + var account gtsmodel.Account + + // Get related account model. + if _, err := tx.NewSelect(). + Model(&account). + Where("? = ?", bun.Ident("id"), media.AccountID). + Exec(ctx); err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("error selecting account: %w", err) + } + + var set func(*bun.UpdateQuery) *bun.UpdateQuery + + switch { + case *media.Avatar && account.AvatarMediaAttachmentID == id: + set = func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("? = NULL", bun.Ident("avatar_media_attachment_id")) + } + case *media.Header && account.HeaderMediaAttachmentID == id: + set = func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("? = NULL", bun.Ident("header_media_attachment_id")) + } + } + + if set != nil { + // Note: this handles not found. + // + // Update the account model. + q := tx.NewUpdate(). + Table("accounts"). + Where("? = ?", bun.Ident("id"), account.ID) + if _, err := set(q).Exec(ctx); err != nil { + return gtserror.Newf("error updating account: %w", err) + } + + // Mark as needing invalidate. + invalidateAccount = true + } + } + + if media.StatusID != "" { + var status gtsmodel.Status + + // Get related status model. + if _, err := tx.NewSelect(). + Model(&status). + Where("? = ?", bun.Ident("id"), media.StatusID). + Exec(ctx); err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("error selecting status: %w", err) + } + + // Get length of attachments beforehand. + before := len(status.AttachmentIDs) + + for i := 0; i < len(status.AttachmentIDs); { + if status.AttachmentIDs[i] == id { + // Remove this reference to deleted attachment ID. + copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:]) + status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1] + continue + } + i++ + } + + if before != len(status.AttachmentIDs) { + // Note: this accounts for status not found. + // + // Attachments changed, update the status. + if _, err := tx.NewUpdate(). + Table("statuses"). + Where("? = ?", bun.Ident("id"), status.ID). + Set("? = ?", bun.Ident("attachment_ids"), status.AttachmentIDs). + Exec(ctx); err != nil { + return gtserror.Newf("error updating status: %w", err) + } + + // Mark as needing invalidate. + invalidateStatus = true + } + } + + // Finally delete this media. + if _, err := tx.NewDelete(). + Table("media_attachments"). + Where("? = ?", bun.Ident("id"), id). + Exec(ctx); err != nil { + return gtserror.Newf("error deleting media: %w", err) + } + + return nil + }) + + if invalidateAccount { + // The account for given ID will have been updated in transaction. + m.state.Caches.GTS.Account().Invalidate("ID", media.AccountID) + } + + if invalidateStatus { + // The status for given ID will have been updated in transaction. + m.state.Caches.GTS.Status().Invalidate("ID", media.StatusID) + } + return m.conn.ProcessError(err) } @@ -167,6 +272,29 @@ func (m *mediaDB) CountRemoteOlderThan(ctx context.Context, olderThan time.Time) return count, nil } +func (m *mediaDB) GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) { + attachmentIDs := []string{} + + q := m.conn.NewSelect(). + Table("media_attachments"). + Column("id"). + Order("id DESC") + + if maxID != "" { + q = q.Where("? < ?", bun.Ident("id"), maxID) + } + + if limit != 0 { + q = q.Limit(limit) + } + + if err := q.Scan(ctx, &attachmentIDs); err != nil { + return nil, m.conn.ProcessError(err) + } + + return m.GetAttachmentsByIDs(ctx, attachmentIDs) +} + func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) { attachmentIDs := []string{} diff --git a/internal/db/bundb/session.go b/internal/db/bundb/session.go index 6d900d75a..9a4256de5 100644 --- a/internal/db/bundb/session.go +++ b/internal/db/bundb/session.go @@ -20,6 +20,7 @@ package bundb import ( "context" "crypto/rand" + "io" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -52,13 +53,11 @@ func (s *sessionDB) GetSession(ctx context.Context) (*gtsmodel.RouterSession, db } func (s *sessionDB) createSession(ctx context.Context) (*gtsmodel.RouterSession, db.Error) { - auth := make([]byte, 32) - crypt := make([]byte, 32) + buf := make([]byte, 64) + auth := buf[:32] + crypt := buf[32:64] - if _, err := rand.Read(auth); err != nil { - return nil, err - } - if _, err := rand.Read(crypt); err != nil { + if _, err := io.ReadFull(rand.Reader, buf); err != nil { return nil, err } diff --git a/internal/db/emoji.go b/internal/db/emoji.go index 0c6ff4d1d..5dcad9ece 100644 --- a/internal/db/emoji.go +++ b/internal/db/emoji.go @@ -33,15 +33,17 @@ type Emoji interface { PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) Error // UpdateEmoji updates the given columns of one emoji. // If no columns are specified, every column is updated. - UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, Error) + UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) error // DeleteEmojiByID deletes one emoji by its database ID. DeleteEmojiByID(ctx context.Context, id string) Error // GetEmojisByIDs gets emojis for the given IDs. GetEmojisByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Emoji, Error) // GetUseableEmojis gets all emojis which are useable by accounts on this instance. GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, Error) - // GetEmojis gets emojis based on given parameters. Useful for admin actions. - GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, Error) + // GetEmojis ... + GetEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error) + // GetEmojisBy gets emojis based on given parameters. Useful for admin actions. + GetEmojisBy(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, error) // GetEmojiByID gets a specific emoji by its database ID. GetEmojiByID(ctx context.Context, id string) (*gtsmodel.Emoji, Error) // GetEmojiByShortcodeDomain gets an emoji based on its shortcode and domain. diff --git a/internal/db/media.go b/internal/db/media.go index 05609ba52..01bca1748 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -41,6 +41,9 @@ type Media interface { // DeleteAttachment deletes the attachment with given ID from the database. DeleteAttachment(ctx context.Context, id string) error + // GetAttachments ... + GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) + // GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given // olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words). // diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 5b0de99bc..f7e740d4b 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -116,7 +116,7 @@ func (d *deref) getAccountByURI(ctx context.Context, requestUser string, uri *ur if account == nil { // Ensure that this is isn't a search for a local account. if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() { - return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries + return nil, nil, gtserror.SetUnretrievable(err) // this will be db.ErrNoEntries } // Create and pass-through a new bare-bones model for dereferencing. @@ -179,7 +179,7 @@ func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser stri if account == nil { if domain == "" { // failed local lookup, will be db.ErrNoEntries. - return nil, nil, NewErrNotRetrievable(err) + return nil, nil, gtserror.SetUnretrievable(err) } // Create and pass-through a new bare-bones model for dereferencing. @@ -306,8 +306,10 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url. accDomain, accURI, err := d.fingerRemoteAccount(ctx, tsport, account.Username, account.Domain) if err != nil { if account.URI == "" { - // this is a new account (to us) with username@domain but failed webfinger, nothing more we can do. - return nil, nil, &ErrNotRetrievable{gtserror.Newf("error webfingering account: %w", err)} + // this is a new account (to us) with username@domain + // but failed webfinger, nothing more we can do. + err := gtserror.Newf("error webfingering account: %w", err) + return nil, nil, gtserror.SetUnretrievable(err) } // Simply log this error and move on, we already have an account URI. @@ -316,10 +318,6 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url. if err == nil { if account.Domain != accDomain { - // Domain has changed, assume the activitypub - // account data provided may not be the latest. - apubAcc = nil - // After webfinger, we now have correct account domain from which we can do a final DB check. alreadyAccount, err := d.state.DB.GetAccountByUsernameDomain(ctx, account.Username, accDomain) if err != nil && !errors.Is(err, db.ErrNoEntries) { @@ -358,31 +356,28 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url. d.startHandshake(requestUser, uri) defer d.stopHandshake(requestUser, uri) - // By default we assume that apubAcc has been passed, - // indicating that the given account is already latest. - latestAcc := account - if apubAcc == nil { // Dereference latest version of the account. b, err := tsport.Dereference(ctx, uri) if err != nil { - return nil, nil, &ErrNotRetrievable{gtserror.Newf("error deferencing %s: %w", uri, err)} + err := gtserror.Newf("error deferencing %s: %w", uri, err) + return nil, nil, gtserror.SetUnretrievable(err) } - // Attempt to resolve ActivityPub account from data. + // Attempt to resolve ActivityPub acc from data. apubAcc, err = ap.ResolveAccountable(ctx, b) if err != nil { return nil, nil, gtserror.Newf("error resolving accountable from data for account %s: %w", uri, err) } + } - // Convert the dereferenced AP account object to our GTS model. - latestAcc, err = d.typeConverter.ASRepresentationToAccount(ctx, - apubAcc, - account.Domain, - ) - if err != nil { - return nil, nil, gtserror.Newf("error converting accountable to gts model for account %s: %w", uri, err) - } + // Convert the dereferenced AP account object to our GTS model. + latestAcc, err := d.typeConverter.ASRepresentationToAccount(ctx, + apubAcc, + account.Domain, + ) + if err != nil { + return nil, nil, gtserror.Newf("error converting accountable to gts model for account %s: %w", uri, err) } if account.Username == "" { @@ -425,52 +420,14 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url. latestAcc.ID = account.ID latestAcc.FetchedAt = time.Now() - // Reuse the existing account media attachments by default. - latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID - latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID - - if (latestAcc.AvatarMediaAttachmentID == "") || - (latestAcc.AvatarRemoteURL != account.AvatarRemoteURL) { - // Reset the avatar media ID (handles removed). - latestAcc.AvatarMediaAttachmentID = "" - - if latestAcc.AvatarRemoteURL != "" { - // Avatar has changed to a new one, fetch up-to-date copy and use new ID. - latestAcc.AvatarMediaAttachmentID, err = d.fetchRemoteAccountAvatar(ctx, - tsport, - latestAcc.AvatarRemoteURL, - latestAcc.ID, - ) - if err != nil { - log.Errorf(ctx, "error fetching remote avatar for account %s: %v", uri, err) - - // Keep old avatar for now, we'll try again in $interval. - latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID - latestAcc.AvatarRemoteURL = account.AvatarRemoteURL - } - } + // Ensure the account's avatar media is populated, passing in existing to check for chages. + if err := d.fetchRemoteAccountAvatar(ctx, tsport, account, latestAcc); err != nil { + log.Errorf(ctx, "error fetching remote avatar for account %s: %v", uri, err) } - if (latestAcc.HeaderMediaAttachmentID == "") || - (latestAcc.HeaderRemoteURL != account.HeaderRemoteURL) { - // Reset the header media ID (handles removed). - latestAcc.HeaderMediaAttachmentID = "" - - if latestAcc.HeaderRemoteURL != "" { - // Header has changed to a new one, fetch up-to-date copy and use new ID. - latestAcc.HeaderMediaAttachmentID, err = d.fetchRemoteAccountHeader(ctx, - tsport, - latestAcc.HeaderRemoteURL, - latestAcc.ID, - ) - if err != nil { - log.Errorf(ctx, "error fetching remote header for account %s: %v", uri, err) - - // Keep old header for now, we'll try again in $interval. - latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID - latestAcc.HeaderRemoteURL = account.HeaderRemoteURL - } - } + // Ensure the account's avatar media is populated, passing in existing to check for chages. + if err := d.fetchRemoteAccountHeader(ctx, tsport, account, latestAcc); err != nil { + log.Errorf(ctx, "error fetching remote header for account %s: %v", uri, err) } // Fetch the latest remote account emoji IDs used in account display name/bio. @@ -515,11 +472,34 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url. return latestAcc, apubAcc, nil } -func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.Transport, avatarURL string, accountID string) (string, error) { - // Parse and validate provided media URL. - avatarURI, err := url.Parse(avatarURL) +func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.Transport, existing, account *gtsmodel.Account) error { + if account.AvatarRemoteURL == "" { + // No fetching to do. + return nil + } + + // By default we set the original media attachment ID. + account.AvatarMediaAttachmentID = existing.AvatarMediaAttachmentID + + if account.AvatarMediaAttachmentID != "" && + existing.AvatarRemoteURL == account.AvatarRemoteURL { + // Look for an existing media attachment by the known ID. + media, err := d.state.DB.GetAttachmentByID(ctx, existing.AvatarMediaAttachmentID) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("error getting attachment %s: %w", existing.AvatarMediaAttachmentID, err) + } + + if media != nil && *media.Cached { + // Media already cached, + // use this existing. + return nil + } + } + + // Parse and validate the newly provided media URL. + avatarURI, err := url.Parse(account.AvatarRemoteURL) if err != nil { - return "", err + return gtserror.Newf("error parsing url %s: %w", account.AvatarRemoteURL, err) } // Acquire lock for derefs map. @@ -527,7 +507,7 @@ func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.T defer unlock() // Look for an existing dereference in progress. - processing, ok := d.derefAvatars[avatarURL] + processing, ok := d.derefAvatars[account.AvatarRemoteURL] if !ok { var err error @@ -538,21 +518,21 @@ func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.T } // Create new media processing request from the media manager instance. - processing, err = d.mediaManager.PreProcessMedia(ctx, data, accountID, &media.AdditionalMediaInfo{ + processing, err = d.mediaManager.PreProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{ Avatar: func() *bool { v := true; return &v }(), - RemoteURL: &avatarURL, + RemoteURL: &account.AvatarRemoteURL, }) if err != nil { - return "", err + return gtserror.Newf("error preprocessing media for attachment %s: %w", account.AvatarRemoteURL, err) } // Store media in map to mark as processing. - d.derefAvatars[avatarURL] = processing + d.derefAvatars[account.AvatarRemoteURL] = processing defer func() { // On exit safely remove media from map. unlock := d.derefAvatarsMu.Lock() - delete(d.derefAvatars, avatarURL) + delete(d.derefAvatars, account.AvatarRemoteURL) unlock() }() } @@ -562,17 +542,43 @@ func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.T // Start media attachment loading (blocking call). if _, err := processing.LoadAttachment(ctx); err != nil { - return "", err + return gtserror.Newf("error loading attachment %s: %w", account.AvatarRemoteURL, err) } - return processing.AttachmentID(), nil + // Set the newly loaded avatar media attachment ID. + account.AvatarMediaAttachmentID = processing.AttachmentID() + + return nil } -func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.Transport, headerURL string, accountID string) (string, error) { - // Parse and validate provided media URL. - headerURI, err := url.Parse(headerURL) +func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.Transport, existing, account *gtsmodel.Account) error { + if account.HeaderRemoteURL == "" { + // No fetching to do. + return nil + } + + // By default we set the original media attachment ID. + account.HeaderMediaAttachmentID = existing.HeaderMediaAttachmentID + + if account.HeaderMediaAttachmentID != "" && + existing.HeaderRemoteURL == account.HeaderRemoteURL { + // Look for an existing media attachment by the known ID. + media, err := d.state.DB.GetAttachmentByID(ctx, existing.HeaderMediaAttachmentID) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("error getting attachment %s: %w", existing.HeaderMediaAttachmentID, err) + } + + if media != nil && *media.Cached { + // Media already cached, + // use this existing. + return nil + } + } + + // Parse and validate the newly provided media URL. + headerURI, err := url.Parse(account.HeaderRemoteURL) if err != nil { - return "", err + return gtserror.Newf("error parsing url %s: %w", account.HeaderRemoteURL, err) } // Acquire lock for derefs map. @@ -580,32 +586,32 @@ func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.T defer unlock() // Look for an existing dereference in progress. - processing, ok := d.derefHeaders[headerURL] + processing, ok := d.derefHeaders[account.HeaderRemoteURL] if !ok { var err error - // Set the media data function to dereference header from URI. + // Set the media data function to dereference avatar from URI. data := func(ctx context.Context) (io.ReadCloser, int64, error) { return tsport.DereferenceMedia(ctx, headerURI) } // Create new media processing request from the media manager instance. - processing, err = d.mediaManager.PreProcessMedia(ctx, data, accountID, &media.AdditionalMediaInfo{ + processing, err = d.mediaManager.PreProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{ Header: func() *bool { v := true; return &v }(), - RemoteURL: &headerURL, + RemoteURL: &account.HeaderRemoteURL, }) if err != nil { - return "", err + return gtserror.Newf("error preprocessing media for attachment %s: %w", account.HeaderRemoteURL, err) } // Store media in map to mark as processing. - d.derefHeaders[headerURL] = processing + d.derefHeaders[account.HeaderRemoteURL] = processing defer func() { // On exit safely remove media from map. unlock := d.derefHeadersMu.Lock() - delete(d.derefHeaders, headerURL) + delete(d.derefHeaders, account.HeaderRemoteURL) unlock() }() } @@ -615,10 +621,13 @@ func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.T // Start media attachment loading (blocking call). if _, err := processing.LoadAttachment(ctx); err != nil { - return "", err + return gtserror.Newf("error loading attachment %s: %w", account.HeaderRemoteURL, err) } - return processing.AttachmentID(), nil + // Set the newly loaded avatar media attachment ID. + account.HeaderMediaAttachmentID = processing.AttachmentID() + + return nil } func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) { diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index 9cff0a171..71028e342 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -174,9 +174,8 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() "thisaccountdoesnotexist", config.GetHost(), ) - var errNotRetrievable *dereferencing.ErrNotRetrievable - suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: no entries") + suite.True(gtserror.Unretrievable(err)) + suite.EqualError(err, "no entries") suite.Nil(fetchedAccount) } @@ -189,9 +188,8 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDom "thisaccountdoesnotexist", "localhost:8080", ) - var errNotRetrievable *dereferencing.ErrNotRetrievable - suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: no entries") + suite.True(gtserror.Unretrievable(err)) + suite.EqualError(err, "no entries") suite.Nil(fetchedAccount) } @@ -203,9 +201,8 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() { fetchingAccount.Username, testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"), ) - var errNotRetrievable *dereferencing.ErrNotRetrievable - suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: no entries") + suite.True(gtserror.Unretrievable(err)) + suite.EqualError(err, "no entries") suite.Nil(fetchedAccount) } diff --git a/internal/federation/dereferencing/error.go b/internal/federation/dereferencing/error.go index 1b8d90653..6a1ce0a6e 100644 --- a/internal/federation/dereferencing/error.go +++ b/internal/federation/dereferencing/error.go @@ -16,21 +16,3 @@ // along with this program. If not, see <http://www.gnu.org/licenses/>. package dereferencing - -import ( - "fmt" -) - -// ErrNotRetrievable denotes that an item could not be dereferenced -// with the given parameters. -type ErrNotRetrievable struct { - wrapped error -} - -func (err *ErrNotRetrievable) Error() string { - return fmt.Sprintf("item could not be retrieved: %v", err.wrapped) -} - -func NewErrNotRetrievable(err error) error { - return &ErrNotRetrievable{wrapped: err} -} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 11d6d7147..75adfdd6f 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -106,7 +106,7 @@ func (d *deref) getStatusByURI(ctx context.Context, requestUser string, uri *url if status == nil { // Ensure that this is isn't a search for a local status. if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() { - return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries + return nil, nil, gtserror.SetUnretrievable(err) // this will be db.ErrNoEntries } // Create and pass-through a new bare-bones model for deref. @@ -220,13 +220,12 @@ func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.U return nil, nil, gtserror.Newf("%s is blocked", uri.Host) } - var derefd bool - if apubStatus == nil { // Dereference latest version of the status. b, err := tsport.Dereference(ctx, uri) if err != nil { - return nil, nil, &ErrNotRetrievable{gtserror.Newf("error deferencing %s: %w", uri, err)} + err := gtserror.Newf("error deferencing %s: %w", uri, err) + return nil, nil, gtserror.SetUnretrievable(err) } // Attempt to resolve ActivityPub status from data. @@ -234,9 +233,6 @@ func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.U if err != nil { return nil, nil, gtserror.Newf("error resolving statusable from data for account %s: %w", uri, err) } - - // Mark as deref'd. - derefd = true } // Get the attributed-to account in order to fetch profile. @@ -256,17 +252,11 @@ func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.U log.Warnf(ctx, "status author account ID changed: old=%s new=%s", status.AccountID, author.ID) } - // By default we assume that apubStatus has been passed, - // indicating that the given status is already latest. - latestStatus := status - - if derefd { - // ActivityPub model was recently dereferenced, so assume that passed status - // may contain out-of-date information, convert AP model to our GTS model. - latestStatus, err = d.typeConverter.ASStatusToStatus(ctx, apubStatus) - if err != nil { - return nil, nil, gtserror.Newf("error converting statusable to gts model for status %s: %w", uri, err) - } + // ActivityPub model was recently dereferenced, so assume that passed status + // may contain out-of-date information, convert AP model to our GTS model. + latestStatus, err := d.typeConverter.ASStatusToStatus(ctx, apubStatus) + if err != nil { + return nil, nil, gtserror.Newf("error converting statusable to gts model for status %s: %w", uri, err) } // Use existing status ID. @@ -327,7 +317,7 @@ func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.U return latestStatus, apubStatus, nil } -func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error { +func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, existing, status *gtsmodel.Status) error { // Allocate new slice to take the yet-to-be created mention IDs. status.MentionIDs = make([]string, len(status.Mentions)) @@ -385,7 +375,7 @@ func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, exi status.MentionIDs[i] = mention.ID } - for i := 0; i < len(status.MentionIDs); i++ { + for i := 0; i < len(status.MentionIDs); { if status.MentionIDs[i] == "" { // This is a failed mention population, likely due // to invalid incoming data / now-deleted accounts. @@ -393,13 +383,15 @@ func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, exi copy(status.MentionIDs[i:], status.MentionIDs[i+1:]) status.Mentions = status.Mentions[:len(status.Mentions)-1] status.MentionIDs = status.MentionIDs[:len(status.MentionIDs)-1] + continue } + i++ } return nil } -func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing *gtsmodel.Status, status *gtsmodel.Status) error { +func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing, status *gtsmodel.Status) error { // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) @@ -408,7 +400,7 @@ func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Tra // Look for existing media attachment with remoet URL first. existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL) - if ok && existing.ID != "" { + if ok && existing.ID != "" && *existing.Cached { status.Attachments[i] = existing status.AttachmentIDs[i] = existing.ID continue @@ -447,7 +439,7 @@ func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Tra status.AttachmentIDs[i] = media.ID } - for i := 0; i < len(status.AttachmentIDs); i++ { + for i := 0; i < len(status.AttachmentIDs); { if status.AttachmentIDs[i] == "" { // This is a failed attachment population, this may // be due to us not currently supporting a media type. @@ -455,13 +447,15 @@ func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Tra copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:]) status.Attachments = status.Attachments[:len(status.Attachments)-1] status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1] + continue } + i++ } return nil } -func (d *deref) fetchStatusEmojis(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error { +func (d *deref) fetchStatusEmojis(ctx context.Context, requestUser string, existing, status *gtsmodel.Status) error { // Fetch the full-fleshed-out emoji objects for our status. emojis, err := d.populateEmojis(ctx, status.Emojis, requestUser) if err != nil { diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go index ec22c66a8..a12e537bc 100644 --- a/internal/federation/dereferencing/thread.go +++ b/internal/federation/dereferencing/thread.go @@ -279,13 +279,21 @@ stackLoop: // Get the current page's "next" property pageNext := current.page.GetActivityStreamsNext() - if pageNext == nil { + if pageNext == nil || !pageNext.IsIRI() { continue stackLoop } - // Get the "next" page property IRI + // Get the IRI of the "next" property. pageNextIRI := pageNext.GetIRI() - if pageNextIRI == nil { + + // Ensure this isn't a self-referencing page... + // We don't need to store / check against a map of IRIs + // as our getStatusByIRI() function above prevents iter'ing + // over statuses that have been dereferenced recently, due to + // the `fetched_at` field preventing frequent refetches. + if id := current.page.GetJSONLDId(); id != nil && + pageNextIRI.String() == id.Get().String() { + log.Warnf(ctx, "self referencing collection page: %s", pageNextIRI) continue stackLoop } diff --git a/internal/gtscontext/context.go b/internal/gtscontext/context.go index c8cd42208..46f2899fa 100644 --- a/internal/gtscontext/context.go +++ b/internal/gtscontext/context.go @@ -41,8 +41,23 @@ const ( httpSigVerifierKey httpSigKey httpSigPubKeyIDKey + dryRunKey ) +// DryRun returns whether the "dryrun" context key has been set. This can be +// used to indicate to functions, (that support it), that only a dry-run of +// the operation should be performed. As opposed to making any permanent changes. +func DryRun(ctx context.Context) bool { + _, ok := ctx.Value(dryRunKey).(struct{}) + return ok +} + +// SetDryRun sets the "dryrun" context flag and returns this wrapped context. +// See DryRun() for further information on the "dryrun" context flag. +func SetDryRun(ctx context.Context) context.Context { + return context.WithValue(ctx, dryRunKey, struct{}{}) +} + // RequestID returns the request ID associated with context. This value will usually // be set by the request ID middleware handler, either pulling an existing supplied // value from request headers, or generating a unique new entry. This is useful for diff --git a/internal/gtserror/error.go b/internal/gtserror/error.go index e68ed7d3b..6eaa3db63 100644 --- a/internal/gtserror/error.go +++ b/internal/gtserror/error.go @@ -33,11 +33,35 @@ const ( statusCodeKey notFoundKey errorTypeKey + unrtrvableKey + wrongTypeKey // Types returnable from Type(...). TypeSMTP ErrorType = "smtp" // smtp (mail) ) +// Unretrievable ... +func Unretrievable(err error) bool { + _, ok := errors.Value(err, unrtrvableKey).(struct{}) + return ok +} + +// SetUnretrievable ... +func SetUnretrievable(err error) error { + return errors.WithValue(err, unrtrvableKey, struct{}{}) +} + +// WrongType ... +func WrongType(err error) bool { + _, ok := errors.Value(err, wrongTypeKey).(struct{}) + return ok +} + +// SetWrongType ... +func SetWrongType(err error) error { + return errors.WithValue(err, unrtrvableKey, struct{}{}) +} + // StatusCode checks error for a stored status code value. For example // an error from an outgoing HTTP request may be stored, or an API handler // expected response status code may be stored. diff --git a/internal/media/manager.go b/internal/media/manager.go index ec95b67e9..1d673128a 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -25,10 +25,8 @@ import ( "time" "codeberg.org/gruf/go-iotools" - "codeberg.org/gruf/go-runners" - "codeberg.org/gruf/go-sched" "codeberg.org/gruf/go-store/v2/storage" - "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -61,7 +59,6 @@ type Manager struct { // See internal/concurrency.NewWorkerPool() documentation for further information. func NewManager(state *state.State) *Manager { m := &Manager{state: state} - scheduleCleanupJobs(m) return m } @@ -214,7 +211,7 @@ func (m *Manager) ProcessMedia(ctx context.Context, data DataFunc, accountID str func (m *Manager) PreProcessEmoji(ctx context.Context, data DataFunc, shortcode string, emojiID string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { instanceAccount, err := m.state.DB.GetInstanceAccount(ctx, "") if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) + return nil, gtserror.Newf("error fetching this instance account from the db: %s", err) } var ( @@ -227,7 +224,7 @@ func (m *Manager) PreProcessEmoji(ctx context.Context, data DataFunc, shortcode // Look for existing emoji by given ID. emoji, err = m.state.DB.GetEmojiByID(ctx, emojiID) if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error fetching emoji to refresh from the db: %s", err) + return nil, gtserror.Newf("error fetching emoji to refresh from the db: %s", err) } // if this is a refresh, we will end up with new images @@ -260,7 +257,7 @@ func (m *Manager) PreProcessEmoji(ctx context.Context, data DataFunc, shortcode newPathID, err = id.NewRandomULID() if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error generating alternateID for emoji refresh: %s", err) + return nil, gtserror.Newf("error generating alternateID for emoji refresh: %s", err) } // store + serve static image at new path ID @@ -356,33 +353,3 @@ func (m *Manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode str return emoji, nil } - -func scheduleCleanupJobs(m *Manager) { - const day = time.Hour * 24 - - // Calculate closest midnight. - now := time.Now() - midnight := now.Round(day) - - if midnight.Before(now) { - // since <= 11:59am rounds down. - midnight = midnight.Add(day) - } - - // Get ctx associated with scheduler run state. - done := m.state.Workers.Scheduler.Done() - doneCtx := runners.CancelCtx(done) - - // TODO: we'll need to do some thinking to make these - // jobs restartable if we want to implement reloads in - // the future that make call to Workers.Stop() -> Workers.Start(). - - // Schedule the PruneAll task to execute every day at midnight. - m.state.Workers.Scheduler.Schedule(sched.NewJob(func(now time.Time) { - err := m.PruneAll(doneCtx, config.GetMediaRemoteCacheDays(), true) - if err != nil { - log.Errorf(nil, "error during prune: %v", err) - } - log.Infof(nil, "finished pruning all in %s", time.Since(now)) - }).EveryAt(midnight, day)) -} diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 2bee1091d..7b9b66147 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -214,7 +214,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { // do a blocking call to fetch the emoji emoji, err := processingEmoji.LoadEmoji(ctx) - suite.EqualError(err, "given emoji size 630kiB greater than max allowed 50.0kiB") + suite.EqualError(err, "store: given emoji size 630kiB greater than max allowed 50.0kiB") suite.Nil(emoji) } @@ -238,7 +238,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { // do a blocking call to fetch the emoji emoji, err := processingEmoji.LoadEmoji(ctx) - suite.EqualError(err, "calculated emoji size 630kiB greater than max allowed 50.0kiB") + suite.EqualError(err, "store: calculated emoji size 630kiB greater than max allowed 50.0kiB") suite.Nil(emoji) } @@ -626,7 +626,7 @@ func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() { // we should get an error while loading attachment, err := processingMedia.LoadAttachment(ctx) - suite.EqualError(err, "error decoding video: error determining video metadata: [width height framerate]") + suite.EqualError(err, "finish: error decoding video: error determining video metadata: [width height framerate]") suite.Nil(attachment) } diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 7c3db8196..d3a1edbf8 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -28,6 +28,7 @@ import ( "codeberg.org/gruf/go-runners" "github.com/h2non/filetype" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/uris" @@ -137,7 +138,7 @@ func (p *ProcessingEmoji) load(ctx context.Context) (*gtsmodel.Emoji, bool, erro } // Existing emoji we're refreshing, so only need to update. - _, err = p.mgr.state.DB.UpdateEmoji(ctx, p.emoji, columns...) + err = p.mgr.state.DB.UpdateEmoji(ctx, p.emoji, columns...) return err } @@ -160,7 +161,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // Load media from provided data fn. rc, sz, err := p.dataFn(ctx) if err != nil { - return fmt.Errorf("error executing data function: %w", err) + return gtserror.Newf("error executing data function: %w", err) } defer func() { @@ -177,13 +178,13 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // Read the first 261 header bytes into buffer. if _, err := io.ReadFull(rc, hdrBuf); err != nil { - return fmt.Errorf("error reading incoming media: %w", err) + return gtserror.Newf("error reading incoming media: %w", err) } // Parse file type info from header buffer. info, err := filetype.Match(hdrBuf) if err != nil { - return fmt.Errorf("error parsing file type: %w", err) + return gtserror.Newf("error parsing file type: %w", err) } switch info.Extension { @@ -192,7 +193,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // unhandled default: - return fmt.Errorf("unsupported emoji filetype: %s", info.Extension) + return gtserror.Newf("unsupported emoji filetype: %s", info.Extension) } // Recombine header bytes with remaining stream @@ -211,7 +212,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // Check that provided size isn't beyond max. We check beforehand // so that we don't attempt to stream the emoji into storage if not needed. if size := bytesize.Size(sz); sz > 0 && size > maxSize { - return fmt.Errorf("given emoji size %s greater than max allowed %s", size, maxSize) + return gtserror.Newf("given emoji size %s greater than max allowed %s", size, maxSize) } var pathID string @@ -241,14 +242,14 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // Attempt to remove existing emoji at storage path (might be broken / out-of-date) if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil { - return fmt.Errorf("error removing emoji from storage: %v", err) + return gtserror.Newf("error removing emoji from storage: %v", err) } } // Write the final image reader stream to our storage. sz, err = p.mgr.state.Storage.PutStream(ctx, p.emoji.ImagePath, r) if err != nil { - return fmt.Errorf("error writing emoji to storage: %w", err) + return gtserror.Newf("error writing emoji to storage: %w", err) } // Once again check size in case none was provided previously. @@ -257,7 +258,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil { log.Errorf(ctx, "error removing too-large-emoji from storage: %v", err) } - return fmt.Errorf("calculated emoji size %s greater than max allowed %s", size, maxSize) + return gtserror.Newf("calculated emoji size %s greater than max allowed %s", size, maxSize) } // Fill in remaining attachment data now it's stored. @@ -278,19 +279,19 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { // Fetch a stream to the original file in storage. rc, err := p.mgr.state.Storage.GetStream(ctx, p.emoji.ImagePath) if err != nil { - return fmt.Errorf("error loading file from storage: %w", err) + return gtserror.Newf("error loading file from storage: %w", err) } defer rc.Close() // Decode the image from storage. staticImg, err := decodeImage(rc) if err != nil { - return fmt.Errorf("error decoding image: %w", err) + return gtserror.Newf("error decoding image: %w", err) } // The image should be in-memory by now. if err := rc.Close(); err != nil { - return fmt.Errorf("error closing file: %w", err) + return gtserror.Newf("error closing file: %w", err) } // This shouldn't already exist, but we do a check as it's worth logging. @@ -298,7 +299,7 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { log.Warnf(ctx, "static emoji already exists at storage path: %s", p.emoji.ImagePath) // Attempt to remove static existing emoji at storage path (might be broken / out-of-date) if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil { - return fmt.Errorf("error removing static emoji from storage: %v", err) + return gtserror.Newf("error removing static emoji from storage: %v", err) } } @@ -308,7 +309,7 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { // Stream-encode the PNG static image into storage. sz, err := p.mgr.state.Storage.PutStream(ctx, p.emoji.ImageStaticPath, enc) if err != nil { - return fmt.Errorf("error stream-encoding static emoji to storage: %w", err) + return gtserror.Newf("error stream-encoding static emoji to storage: %w", err) } // Set written image size. diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 5c66f561d..acfee48f3 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -30,6 +30,7 @@ import ( "github.com/disintegration/imaging" "github.com/h2non/filetype" terminator "github.com/superseriousbusiness/exif-terminator" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/uris" @@ -145,7 +146,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { // Load media from provided data fun rc, sz, err := p.dataFn(ctx) if err != nil { - return fmt.Errorf("error executing data function: %w", err) + return gtserror.Newf("error executing data function: %w", err) } defer func() { @@ -162,13 +163,13 @@ func (p *ProcessingMedia) store(ctx context.Context) error { // Read the first 261 header bytes into buffer. if _, err := io.ReadFull(rc, hdrBuf); err != nil { - return fmt.Errorf("error reading incoming media: %w", err) + return gtserror.Newf("error reading incoming media: %w", err) } // Parse file type info from header buffer. info, err := filetype.Match(hdrBuf) if err != nil { - return fmt.Errorf("error parsing file type: %w", err) + return gtserror.Newf("error parsing file type: %w", err) } // Recombine header bytes with remaining stream @@ -187,12 +188,12 @@ func (p *ProcessingMedia) store(ctx context.Context) error { // A file size was provided so we can clean exif data from image. r, err = terminator.Terminate(r, int(sz), info.Extension) if err != nil { - return fmt.Errorf("error cleaning exif data: %w", err) + return gtserror.Newf("error cleaning exif data: %w", err) } } default: - return fmt.Errorf("unsupported file type: %s", info.Extension) + return gtserror.Newf("unsupported file type: %s", info.Extension) } // Calculate attachment file path. @@ -211,14 +212,14 @@ func (p *ProcessingMedia) store(ctx context.Context) error { // Attempt to remove existing media at storage path (might be broken / out-of-date) if err := p.mgr.state.Storage.Delete(ctx, p.media.File.Path); err != nil { - return fmt.Errorf("error removing media from storage: %v", err) + return gtserror.Newf("error removing media from storage: %v", err) } } // Write the final image reader stream to our storage. sz, err = p.mgr.state.Storage.PutStream(ctx, p.media.File.Path, r) if err != nil { - return fmt.Errorf("error writing media to storage: %w", err) + return gtserror.Newf("error writing media to storage: %w", err) } // Set written image size. @@ -245,7 +246,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // Fetch a stream to the original file in storage. rc, err := p.mgr.state.Storage.GetStream(ctx, p.media.File.Path) if err != nil { - return fmt.Errorf("error loading file from storage: %w", err) + return gtserror.Newf("error loading file from storage: %w", err) } defer rc.Close() @@ -256,7 +257,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { case mimeImageJpeg, mimeImageGif, mimeImageWebp: fullImg, err = decodeImage(rc, imaging.AutoOrientation(true)) if err != nil { - return fmt.Errorf("error decoding image: %w", err) + return gtserror.Newf("error decoding image: %w", err) } // .png image (requires ancillary chunk stripping) @@ -265,14 +266,14 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { Reader: rc, }, imaging.AutoOrientation(true)) if err != nil { - return fmt.Errorf("error decoding image: %w", err) + return gtserror.Newf("error decoding image: %w", err) } // .mp4 video type case mimeVideoMp4: video, err := decodeVideoFrame(rc) if err != nil { - return fmt.Errorf("error decoding video: %w", err) + return gtserror.Newf("error decoding video: %w", err) } // Set video frame as image. @@ -286,7 +287,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // The image should be in-memory by now. if err := rc.Close(); err != nil { - return fmt.Errorf("error closing file: %w", err) + return gtserror.Newf("error closing file: %w", err) } // Set full-size dimensions in attachment info. @@ -314,7 +315,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // Blurhash needs generating from thumb. hash, err := thumbImg.Blurhash() if err != nil { - return fmt.Errorf("error generating blurhash: %w", err) + return gtserror.Newf("error generating blurhash: %w", err) } // Set the attachment blurhash. @@ -326,7 +327,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // Attempt to remove existing thumbnail at storage path (might be broken / out-of-date) if err := p.mgr.state.Storage.Delete(ctx, p.media.Thumbnail.Path); err != nil { - return fmt.Errorf("error removing thumbnail from storage: %v", err) + return gtserror.Newf("error removing thumbnail from storage: %v", err) } } @@ -338,7 +339,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { // Stream-encode the JPEG thumbnail image into storage. sz, err := p.mgr.state.Storage.PutStream(ctx, p.media.Thumbnail.Path, enc) if err != nil { - return fmt.Errorf("error stream-encoding thumbnail to storage: %w", err) + return gtserror.Newf("error stream-encoding thumbnail to storage: %w", err) } // Fill in remaining thumbnail now it's stored diff --git a/internal/media/prune.go b/internal/media/prune.go deleted file mode 100644 index 71c8e00ce..000000000 --- a/internal/media/prune.go +++ /dev/null @@ -1,373 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see <http://www.gnu.org/licenses/>. - -package media - -import ( - "context" - "errors" - "fmt" - "time" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/regexes" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -const ( - selectPruneLimit = 50 // Amount of media entries to select at a time from the db when pruning. - unusedLocalAttachmentDays = 3 // Number of days to keep local media in storage if not attached to a status. -) - -// PruneAll runs all of the below pruning/uncacheing functions, and then cleans up any resulting -// empty directories from the storage driver. It can be called as a shortcut for calling the below -// pruning functions one by one. -// -// If blocking is true, then any errors encountered during the prune will be combined + returned to -// the caller. If blocking is false, the prune is run in the background and errors are just logged -// instead. -func (m *Manager) PruneAll(ctx context.Context, mediaCacheRemoteDays int, blocking bool) error { - const dry = false - - f := func(innerCtx context.Context) error { - errs := gtserror.MultiError{} - - pruned, err := m.PruneUnusedLocal(innerCtx, dry) - if err != nil { - errs = append(errs, fmt.Sprintf("error pruning unused local media (%s)", err)) - } else { - log.Infof(ctx, "pruned %d unused local media", pruned) - } - - pruned, err = m.PruneUnusedRemote(innerCtx, dry) - if err != nil { - errs = append(errs, fmt.Sprintf("error pruning unused remote media: (%s)", err)) - } else { - log.Infof(ctx, "pruned %d unused remote media", pruned) - } - - pruned, err = m.UncacheRemote(innerCtx, mediaCacheRemoteDays, dry) - if err != nil { - errs = append(errs, fmt.Sprintf("error uncacheing remote media older than %d day(s): (%s)", mediaCacheRemoteDays, err)) - } else { - log.Infof(ctx, "uncached %d remote media older than %d day(s)", pruned, mediaCacheRemoteDays) - } - - pruned, err = m.PruneOrphaned(innerCtx, dry) - if err != nil { - errs = append(errs, fmt.Sprintf("error pruning orphaned media: (%s)", err)) - } else { - log.Infof(ctx, "pruned %d orphaned media", pruned) - } - - if err := m.state.Storage.Storage.Clean(innerCtx); err != nil { - errs = append(errs, fmt.Sprintf("error cleaning storage: (%s)", err)) - } else { - log.Info(ctx, "cleaned storage") - } - - return errs.Combine() - } - - if blocking { - return f(ctx) - } - - go func() { - if err := f(context.Background()); err != nil { - log.Error(ctx, err) - } - }() - - return nil -} - -// PruneUnusedRemote prunes unused/out of date headers and avatars cached on this instance. -// -// The returned int is the amount of media that was pruned by this function. -func (m *Manager) PruneUnusedRemote(ctx context.Context, dry bool) (int, error) { - var ( - totalPruned int - maxID string - attachments []*gtsmodel.MediaAttachment - err error - ) - - // We don't know in advance how many remote attachments will meet - // our criteria for being 'unused'. So a dry run in this case just - // means we iterate through as normal, but do nothing with each entry - // instead of removing it. Define this here so we don't do the 'if dry' - // check inside the loop a million times. - var f func(ctx context.Context, attachment *gtsmodel.MediaAttachment) error - if !dry { - f = m.deleteAttachment - } else { - f = func(_ context.Context, _ *gtsmodel.MediaAttachment) error { - return nil // noop - } - } - - for attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { - maxID = attachments[len(attachments)-1].ID // use the id of the last attachment in the slice as the next 'maxID' value - - for _, attachment := range attachments { - // Retrieve owning account if possible. - var account *gtsmodel.Account - if accountID := attachment.AccountID; accountID != "" { - account, err = m.state.DB.GetAccountByID(ctx, attachment.AccountID) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - // Only return on a real error. - return 0, fmt.Errorf("PruneUnusedRemote: error fetching account with id %s: %w", accountID, err) - } - } - - // Prune each attachment that meets one of the following criteria: - // - Has no owning account in the database. - // - Is a header but isn't the owning account's current header. - // - Is an avatar but isn't the owning account's current avatar. - if account == nil || - (*attachment.Header && attachment.ID != account.HeaderMediaAttachmentID) || - (*attachment.Avatar && attachment.ID != account.AvatarMediaAttachmentID) { - if err := f(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - } - - // Make sure we don't have a real error when we leave the loop. - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return totalPruned, err - } - - return totalPruned, nil -} - -// PruneOrphaned prunes files that exist in storage but which do not have a corresponding -// entry in the database. -// -// If dry is true, then nothing will be changed, only the amount that *would* be removed -// is returned to the caller. -func (m *Manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) { - // Emojis are stored under the instance account, so we - // need the ID of the instance account for the next part. - instanceAccount, err := m.state.DB.GetInstanceAccount(ctx, "") - if err != nil { - return 0, fmt.Errorf("PruneOrphaned: error getting instance account: %w", err) - } - - instanceAccountID := instanceAccount.ID - - var orphanedKeys []string - - // Keys in storage will look like the following format: - // `[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[MEDIA_ID].[EXTENSION]` - // We can filter out keys we're not interested in by matching through a regex. - if err := m.state.Storage.WalkKeys(ctx, func(ctx context.Context, key string) error { - if !regexes.FilePath.MatchString(key) { - // This is not our expected key format. - return nil - } - - // Check whether this storage entry is orphaned. - orphaned, err := m.orphaned(ctx, key, instanceAccountID) - if err != nil { - return fmt.Errorf("error checking orphaned status: %w", err) - } - - if orphaned { - // Add this orphaned entry to list of keys. - orphanedKeys = append(orphanedKeys, key) - } - - return nil - }); err != nil { - return 0, fmt.Errorf("PruneOrphaned: error walking keys: %w", err) - } - - totalPruned := len(orphanedKeys) - - if dry { - // Dry run: don't remove anything. - return totalPruned, nil - } - - // This is not a drill! We have to delete stuff! - return m.removeFiles(ctx, orphanedKeys...) -} - -func (m *Manager) orphaned(ctx context.Context, key string, instanceAccountID string) (bool, error) { - pathParts := regexes.FilePath.FindStringSubmatch(key) - if len(pathParts) != 6 { - // This doesn't match our expectations so - // it wasn't created by gts; ignore it. - return false, nil - } - - var ( - mediaType = pathParts[2] - mediaID = pathParts[4] - orphaned = false - ) - - // Look for keys in storage that we don't have an attachment for. - switch Type(mediaType) { - case TypeAttachment, TypeHeader, TypeAvatar: - if _, err := m.state.DB.GetAttachmentByID(ctx, mediaID); err != nil { - if !errors.Is(err, db.ErrNoEntries) { - return false, fmt.Errorf("error calling GetAttachmentByID: %w", err) - } - orphaned = true - } - case TypeEmoji: - // Look using the static URL for the emoji. Emoji images can change, so - // the MEDIA_ID part of the key for emojis will not necessarily correspond - // to the file that's currently being used as the emoji image. - staticURL := uris.GenerateURIForAttachment(instanceAccountID, string(TypeEmoji), string(SizeStatic), mediaID, mimePng) - if _, err := m.state.DB.GetEmojiByStaticURL(ctx, staticURL); err != nil { - if !errors.Is(err, db.ErrNoEntries) { - return false, fmt.Errorf("error calling GetEmojiByStaticURL: %w", err) - } - orphaned = true - } - } - - return orphaned, nil -} - -// UncacheRemote uncaches all remote media attachments older than the given amount of days. -// -// In this context, uncacheing means deleting media files from storage and marking the attachment -// as cached=false in the database. -// -// If 'dry' is true, then only a dry run will be performed: nothing will actually be changed. -// -// The returned int is the amount of media that was/would be uncached by this function. -func (m *Manager) UncacheRemote(ctx context.Context, olderThanDays int, dry bool) (int, error) { - if olderThanDays < 0 { - return 0, nil - } - - olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays)) - - if dry { - // Dry run, just count eligible entries without removing them. - return m.state.DB.CountRemoteOlderThan(ctx, olderThan) - } - - var ( - totalPruned int - attachments []*gtsmodel.MediaAttachment - err error - ) - - for attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { - olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value - - for _, attachment := range attachments { - if err := m.uncacheAttachment(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - - // Make sure we don't have a real error when we leave the loop. - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return totalPruned, err - } - - return totalPruned, nil -} - -// PruneUnusedLocal prunes unused media attachments that were uploaded by -// a user on this instance, but never actually attached to a status, or attached but -// later detached. -// -// The returned int is the amount of media that was pruned by this function. -func (m *Manager) PruneUnusedLocal(ctx context.Context, dry bool) (int, error) { - olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(unusedLocalAttachmentDays)) - - if dry { - // Dry run, just count eligible entries without removing them. - return m.state.DB.CountLocalUnattachedOlderThan(ctx, olderThan) - } - - var ( - totalPruned int - attachments []*gtsmodel.MediaAttachment - err error - ) - - for attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit) { - olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value - - for _, attachment := range attachments { - if err := m.deleteAttachment(ctx, attachment); err != nil { - return totalPruned, err - } - totalPruned++ - } - } - - // Make sure we don't have a real error when we leave the loop. - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return totalPruned, err - } - - return totalPruned, nil -} - -/* - Handy little helpers -*/ - -func (m *Manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { - if _, err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil { - return err - } - - // Delete attachment completely. - return m.state.DB.DeleteAttachment(ctx, attachment.ID) -} - -func (m *Manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { - if _, err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil { - return err - } - - // Update attachment to reflect that we no longer have it cached. - attachment.Cached = func() *bool { i := false; return &i }() - return m.state.DB.UpdateAttachment(ctx, attachment, "cached") -} - -func (m *Manager) removeFiles(ctx context.Context, keys ...string) (int, error) { - errs := make(gtserror.MultiError, 0, len(keys)) - - for _, key := range keys { - if err := m.state.Storage.Delete(ctx, key); err != nil && !errors.Is(err, storage.ErrNotFound) { - errs = append(errs, "storage error removing "+key+": "+err.Error()) - } - } - - return len(keys) - len(errs), errs.Combine() -} diff --git a/internal/media/prune_test.go b/internal/media/prune_test.go deleted file mode 100644 index 375ce0c06..000000000 --- a/internal/media/prune_test.go +++ /dev/null @@ -1,357 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see <http://www.gnu.org/licenses/>. - -package media_test - -import ( - "bytes" - "context" - "io" - "os" - "testing" - - "codeberg.org/gruf/go-store/v2/storage" - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" - gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type PruneTestSuite struct { - MediaStandardTestSuite -} - -func (suite *PruneTestSuite) TestPruneOrphanedDry() { - // add a big orphan panda to store - b, err := os.ReadFile("./test/big-panda.gif") - if err != nil { - suite.FailNow(err.Error()) - } - - pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { - suite.FailNow(err.Error()) - } - - // dry run should show up 1 orphaned panda - totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // panda should still be in storage - hasKey, err := suite.storage.Has(context.Background(), pandaPath) - suite.NoError(err) - suite.True(hasKey) -} - -func (suite *PruneTestSuite) TestPruneOrphanedMoist() { - // add a big orphan panda to store - b, err := os.ReadFile("./test/big-panda.gif") - if err != nil { - suite.FailNow(err.Error()) - } - - pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { - suite.FailNow(err.Error()) - } - - // should show up 1 orphaned panda - totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // panda should no longer be in storage - hasKey, err := suite.storage.Has(context.Background(), pandaPath) - suite.NoError(err) - suite.False(hasKey) -} - -func (suite *PruneTestSuite) TestPruneUnusedLocal() { - testAttachment := suite.testAttachments["local_account_1_unattached_1"] - suite.True(*testAttachment.Cached) - - totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) - suite.NoError(err) - suite.Equal(1, totalPruned) - - _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func (suite *PruneTestSuite) TestPruneUnusedLocalDry() { - testAttachment := suite.testAttachments["local_account_1_unattached_1"] - suite.True(*testAttachment.Cached) - - totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true) - suite.NoError(err) - suite.Equal(1, totalPruned) - - _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) - suite.NoError(err) -} - -func (suite *PruneTestSuite) TestPruneRemoteTwice() { - totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) - suite.NoError(err) - suite.Equal(1, totalPruned) - - // final prune should prune nothing, since the first prune already happened - totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false) - suite.NoError(err) - suite.Equal(0, totalPrunedAgain) -} - -func (suite *PruneTestSuite) TestPruneOneNonExistent() { - ctx := context.Background() - testAttachment := suite.testAttachments["local_account_1_unattached_1"] - - // Delete this attachment cached on disk - media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) - suite.NoError(err) - suite.True(*media.Cached) - err = suite.storage.Delete(ctx, media.File.Path) - suite.NoError(err) - - // Now attempt to prune for item with db entry no file - totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false) - suite.NoError(err) - suite.Equal(1, totalPruned) -} - -func (suite *PruneTestSuite) TestPruneUnusedRemote() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] - zorkOldHeader := suite.testAttachments["local_account_1_avatar"] - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // attachments should no longer be in the db - _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) - suite.ErrorIs(err, db.ErrNoEntries) - _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func (suite *PruneTestSuite) TestPruneUnusedRemoteTwice() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // final prune should prune nothing, since the first prune already happened - totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false) - suite.NoError(err) - suite.Equal(0, totalPruned) -} - -func (suite *PruneTestSuite) TestPruneUnusedRemoteMultipleAccounts() { - ctx := context.Background() - - // start by clearing zork's avatar + header - zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] - zorkOldHeader := suite.testAttachments["local_account_1_avatar"] - zork := suite.testAccounts["local_account_1"] - zork.AvatarMediaAttachmentID = "" - zork.HeaderMediaAttachmentID = "" - if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { - panic(err) - } - - // set zork's unused header as belonging to turtle - turtle := suite.testAccounts["local_account_1"] - zorkOldHeader.AccountID = turtle.ID - if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil { - panic(err) - } - - totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) - suite.NoError(err) - suite.Equal(2, totalPruned) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // attachments should no longer be in the db - _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) - suite.ErrorIs(err, db.ErrNoEntries) - _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) - suite.ErrorIs(err, db.ErrNoEntries) -} - -func (suite *PruneTestSuite) TestUncacheRemote() { - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - suite.True(*testStatusAttachment.Cached) - - testHeader := suite.testAttachments["remote_account_3_header"] - suite.True(*testHeader.Cached) - - totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) - suite.NoError(err) - suite.Equal(2, totalUncached) - - uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) - suite.NoError(err) - suite.False(*uncachedAttachment.Cached) - - uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) - suite.NoError(err) - suite.False(*uncachedAttachment.Cached) -} - -func (suite *PruneTestSuite) TestUncacheRemoteDry() { - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - suite.True(*testStatusAttachment.Cached) - - testHeader := suite.testAttachments["remote_account_3_header"] - suite.True(*testHeader.Cached) - - totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, true) - suite.NoError(err) - suite.Equal(2, totalUncached) - - uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) - suite.NoError(err) - suite.True(*uncachedAttachment.Cached) - - uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) - suite.NoError(err) - suite.True(*uncachedAttachment.Cached) -} - -func (suite *PruneTestSuite) TestUncacheRemoteTwice() { - totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) - suite.NoError(err) - suite.Equal(2, totalUncached) - - // final uncache should uncache nothing, since the first uncache already happened - totalUncachedAgain, err := suite.manager.UncacheRemote(context.Background(), 1, false) - suite.NoError(err) - suite.Equal(0, totalUncachedAgain) -} - -func (suite *PruneTestSuite) TestUncacheAndRecache() { - ctx := context.Background() - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - testHeader := suite.testAttachments["remote_account_3_header"] - - totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) - suite.NoError(err) - suite.Equal(2, totalUncached) - - // media should no longer be stored - _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testHeader.File.Path) - suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) - suite.ErrorIs(err, storage.ErrNotFound) - - // now recache the image.... - data := func(_ context.Context) (io.ReadCloser, int64, error) { - // load bytes from a test image - b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") - if err != nil { - panic(err) - } - return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil - } - - for _, original := range []*gtsmodel.MediaAttachment{ - testStatusAttachment, - testHeader, - } { - processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, original.ID) - suite.NoError(err) - - // synchronously load the recached attachment - recachedAttachment, err := processingRecache.LoadAttachment(ctx) - suite.NoError(err) - suite.NotNil(recachedAttachment) - - // recachedAttachment should be basically the same as the old attachment - suite.True(*recachedAttachment.Cached) - suite.Equal(original.ID, recachedAttachment.ID) - suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place - suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail - suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same - - // recached files should be back in storage - _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) - suite.NoError(err) - _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) - suite.NoError(err) - } -} - -func (suite *PruneTestSuite) TestUncacheOneNonExistent() { - ctx := context.Background() - testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - - // Delete this attachment cached on disk - media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) - suite.NoError(err) - suite.True(*media.Cached) - err = suite.storage.Delete(ctx, media.File.Path) - suite.NoError(err) - - // Now attempt to uncache remote for item with db entry no file - totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) - suite.NoError(err) - suite.Equal(2, totalUncached) -} - -func TestPruneOrphanedTestSuite(t *testing.T) { - suite.Run(t, &PruneTestSuite{}) -} diff --git a/internal/media/refetch.go b/internal/media/refetch.go index 80dfe4f60..03f0fbf34 100644 --- a/internal/media/refetch.go +++ b/internal/media/refetch.go @@ -52,7 +52,7 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM // page through emojis 20 at a time, looking for those with missing images for { // Fetch next block of emojis from database - emojis, err := m.state.DB.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) + emojis, err := m.state.DB.GetEmojisBy(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) if err != nil { if !errors.Is(err, db.ErrNoEntries) { // an actual error has occurred diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 9b243e06d..0fa24452b 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -18,6 +18,7 @@ package admin import ( + "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/state" @@ -27,6 +28,7 @@ import ( type Processor struct { state *state.State + cleaner *cleaner.Cleaner tc typeutils.TypeConverter mediaManager *media.Manager transportController transport.Controller @@ -37,6 +39,7 @@ type Processor struct { func New(state *state.State, tc typeutils.TypeConverter, mediaManager *media.Manager, transportController transport.Controller, emailSender email.Sender) Processor { return Processor{ state: state, + cleaner: cleaner.New(state), tc: tc, mediaManager: mediaManager, transportController: transportController, diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 3a7868eb1..96b0bef07 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -109,7 +109,7 @@ func (p *Processor) EmojisGet( return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } - emojis, err := p.state.DB.GetEmojis(ctx, domain, includeDisabled, includeEnabled, shortcode, maxShortcodeDomain, minShortcodeDomain, limit) + emojis, err := p.state.DB.GetEmojisBy(ctx, domain, includeDisabled, includeEnabled, shortcode, maxShortcodeDomain, minShortcodeDomain, limit) if err != nil && !errors.Is(err, db.ErrNoEntries) { err := fmt.Errorf("EmojisGet: db error: %s", err) return nil, gtserror.NewErrorInternalError(err) @@ -385,13 +385,13 @@ func (p *Processor) emojiUpdateDisable(ctx context.Context, emoji *gtsmodel.Emoj emojiDisabled := true emoji.Disabled = &emojiDisabled - updatedEmoji, err := p.state.DB.UpdateEmoji(ctx, emoji, "disabled") + err := p.state.DB.UpdateEmoji(ctx, emoji, "disabled") if err != nil { err = fmt.Errorf("emojiUpdateDisable: error updating emoji %s: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) } - adminEmoji, err := p.tc.EmojiToAdminAPIEmoji(ctx, updatedEmoji) + adminEmoji, err := p.tc.EmojiToAdminAPIEmoji(ctx, emoji) if err != nil { err = fmt.Errorf("emojiUpdateDisable: error converting updated emoji %s to admin emoji: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) @@ -407,8 +407,6 @@ func (p *Processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji return nil, gtserror.NewErrorBadRequest(err, err.Error()) } - var updatedEmoji *gtsmodel.Emoji - // keep existing categoryID unless a new one is defined var ( updatedCategoryID = emoji.CategoryID @@ -442,7 +440,7 @@ func (p *Processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji } var err error - updatedEmoji, err = p.state.DB.UpdateEmoji(ctx, emoji, columns...) + err = p.state.DB.UpdateEmoji(ctx, emoji, columns...) if err != nil { err = fmt.Errorf("emojiUpdateModify: error updating emoji %s: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) @@ -467,14 +465,14 @@ func (p *Processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji return nil, gtserror.NewErrorInternalError(err) } - updatedEmoji, err = processingEmoji.LoadEmoji(ctx) + emoji, err = processingEmoji.LoadEmoji(ctx) if err != nil { err = fmt.Errorf("emojiUpdateModify: error loading processed emoji %s: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) } } - adminEmoji, err := p.tc.EmojiToAdminAPIEmoji(ctx, updatedEmoji) + adminEmoji, err := p.tc.EmojiToAdminAPIEmoji(ctx, emoji) if err != nil { err = fmt.Errorf("emojiUpdateModify: error converting updated emoji %s to admin emoji: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/admin/media.go b/internal/processing/admin/media.go index b8af183da..a457487b8 100644 --- a/internal/processing/admin/media.go +++ b/internal/processing/admin/media.go @@ -47,17 +47,19 @@ func (p *Processor) MediaRefetch(ctx context.Context, requestingAccount *gtsmode return nil } -// MediaPrune triggers a non-blocking prune of remote media, local unused media, etc. +// MediaPrune triggers a non-blocking prune of unused media, orphaned, uncaching remote and fixing cache states. func (p *Processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { if mediaRemoteCacheDays < 0 { err := fmt.Errorf("MediaPrune: invalid value for mediaRemoteCacheDays prune: value was %d, cannot be less than 0", mediaRemoteCacheDays) return gtserror.NewErrorBadRequest(err, err.Error()) } - if err := p.mediaManager.PruneAll(ctx, mediaRemoteCacheDays, false); err != nil { - err = fmt.Errorf("MediaPrune: %w", err) - return gtserror.NewErrorInternalError(err) - } + // Start background task performing all media cleanup tasks. + go func() { + ctx := context.Background() + p.cleaner.Media().All(ctx, mediaRemoteCacheDays) + p.cleaner.Emoji().All(ctx) + }() return nil } diff --git a/internal/processing/search/get.go b/internal/processing/search/get.go index 936e8acfa..aaade8908 100644 --- a/internal/processing/search/get.go +++ b/internal/processing/search/get.go @@ -26,11 +26,9 @@ import ( "strings" "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/ap" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -62,7 +60,6 @@ func (p *Processor) Get( account *gtsmodel.Account, req *apimodel.SearchRequest, ) (*apimodel.SearchResult, gtserror.WithCode) { - var ( maxID = req.MaxID minID = req.MinID @@ -127,7 +124,7 @@ func (p *Processor) Get( // accounts, since this is all namestring search can return. if includeAccounts(queryType) { // Copy query to avoid altering original. - var queryC = query + queryC := query // If query looks vaguely like an email address, ie. it doesn't // start with '@' but it has '@' in it somewhere, it's probably @@ -284,12 +281,7 @@ func (p *Processor) accountsByNamestring( if err != nil { // Check for semi-expected error types. // On one of these, we can continue. - var ( - errNotRetrievable = new(*dereferencing.ErrNotRetrievable) // Item can't be dereferenced. - errWrongType = new(*ap.ErrWrongType) // Item was dereferenced, but wasn't an account. - ) - - if !errors.As(err, errNotRetrievable) && !errors.As(err, errWrongType) { + if !gtserror.Unretrievable(err) && !gtserror.WrongType(err) { err = gtserror.Newf("error looking up %s as account: %w", query, err) return false, gtserror.NewErrorInternalError(err) } @@ -331,11 +323,10 @@ func (p *Processor) accountByUsernameDomain( if err != nil { err = gtserror.Newf("error checking domain block: %w", err) return nil, gtserror.NewErrorInternalError(err) - } - - if blocked { + } else if blocked { // Don't search on blocked domain. - return nil, dereferencing.NewErrNotRetrievable(err) + err = gtserror.New("domain blocked") + return nil, gtserror.SetUnretrievable(err) } } @@ -365,7 +356,7 @@ func (p *Processor) accountByUsernameDomain( } err = fmt.Errorf("account %s could not be retrieved locally and we cannot resolve", usernameDomain) - return nil, dereferencing.NewErrNotRetrievable(err) + return nil, gtserror.SetUnretrievable(err) } // byURI looks for account(s) or a status with the given URI @@ -419,12 +410,7 @@ func (p *Processor) byURI( if err != nil { // Check for semi-expected error types. // On one of these, we can continue. - var ( - errNotRetrievable = new(*dereferencing.ErrNotRetrievable) // Item can't be dereferenced. - errWrongType = new(*ap.ErrWrongType) // Item was dereferenced, but wasn't an account. - ) - - if !errors.As(err, errNotRetrievable) && !errors.As(err, errWrongType) { + if !gtserror.Unretrievable(err) && !gtserror.WrongType(err) { err = gtserror.Newf("error looking up %s as account: %w", uri, err) return false, gtserror.NewErrorInternalError(err) } @@ -443,12 +429,7 @@ func (p *Processor) byURI( if err != nil { // Check for semi-expected error types. // On one of these, we can continue. - var ( - errNotRetrievable = new(*dereferencing.ErrNotRetrievable) // Item can't be dereferenced. - errWrongType = new(*ap.ErrWrongType) // Item was dereferenced, but wasn't a status. - ) - - if !errors.As(err, errNotRetrievable) && !errors.As(err, errWrongType) { + if !gtserror.Unretrievable(err) && !gtserror.WrongType(err) { err = gtserror.Newf("error looking up %s as status: %w", uri, err) return false, gtserror.NewErrorInternalError(err) } @@ -519,7 +500,7 @@ func (p *Processor) accountByURI( } err = fmt.Errorf("account %s could not be retrieved locally and we cannot resolve", uriStr) - return nil, dereferencing.NewErrNotRetrievable(err) + return nil, gtserror.SetUnretrievable(err) } // statusByURI looks for one status with the given URI. @@ -575,7 +556,7 @@ func (p *Processor) statusByURI( } err = fmt.Errorf("status %s could not be retrieved locally and we cannot resolve", uriStr) - return nil, dereferencing.NewErrNotRetrievable(err) + return nil, gtserror.SetUnretrievable(err) } // byText searches in the database for accounts and/or diff --git a/internal/processing/search/lookup.go b/internal/processing/search/lookup.go index 0f2a4191b..d50183221 100644 --- a/internal/processing/search/lookup.go +++ b/internal/processing/search/lookup.go @@ -23,10 +23,8 @@ import ( "fmt" "strings" - errorsv2 "codeberg.org/gruf/go-errors/v2" "codeberg.org/gruf/go-kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -82,7 +80,7 @@ func (p *Processor) Lookup( false, // never resolve! ) if err != nil { - if errorsv2.Assignable(err, (*dereferencing.ErrNotRetrievable)(nil)) { + if gtserror.Unretrievable(err) { // ErrNotRetrievable is fine, just wrap it in // a 404 to indicate we couldn't find anything. err := fmt.Errorf("%s not found", query) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index f131b4292..ea8184881 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -46,9 +46,11 @@ type PresignedURL struct { Expiry time.Time // link expires at this time } -// ErrAlreadyExists is a ptr to underlying storage.ErrAlreadyExists, -// to put the related errors in the same package as our storage wrapper. -var ErrAlreadyExists = storage.ErrAlreadyExists +var ( + // Ptrs to underlying storage library errors. + ErrAlreadyExists = storage.ErrAlreadyExists + ErrNotFound = storage.ErrNotFound +) // Driver wraps a kv.KVStore to also provide S3 presigned GET URLs. type Driver struct { |