diff options
Diffstat (limited to 'internal/cleaner/emoji.go')
| -rw-r--r-- | internal/cleaner/emoji.go | 317 |
1 files changed, 282 insertions, 35 deletions
diff --git a/internal/cleaner/emoji.go b/internal/cleaner/emoji.go index 35e579171..d2baec7e8 100644 --- a/internal/cleaner/emoji.go +++ b/internal/cleaner/emoji.go @@ -20,6 +20,7 @@ package cleaner import ( "context" "errors" + "time" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" @@ -36,22 +37,26 @@ type Emoji struct { // All will execute all cleaner.Emoji utilities synchronously, including output logging. // Context will be checked for `gtscontext.DryRun()` in order to actually perform the action. -func (e *Emoji) All(ctx context.Context) { - e.LogPruneMissing(ctx) +func (e *Emoji) All(ctx context.Context, maxRemoteDays int) { + t := time.Now().Add(-24 * time.Hour * time.Duration(maxRemoteDays)) + e.LogUncacheRemote(ctx, t) e.LogFixBroken(ctx) + e.LogPruneUnused(ctx) + e.LogFixCacheStates(ctx) + _ = e.state.Storage.Storage.Clean(ctx) } -// LogPruneMissing performs emoji.PruneMissing(...), logging the start and outcome. -func (e *Emoji) LogPruneMissing(ctx context.Context) { - log.Info(ctx, "start") - if n, err := e.PruneMissing(ctx); err != nil { +// LogUncacheRemote performs Emoji.UncacheRemote(...), logging the start and outcome. +func (e *Emoji) LogUncacheRemote(ctx context.Context, olderThan time.Time) { + log.Infof(ctx, "start older than: %s", olderThan.Format(time.Stamp)) + if n, err := e.UncacheRemote(ctx, olderThan); err != nil { log.Error(ctx, err) } else { - log.Infof(ctx, "pruned: %d", n) + log.Infof(ctx, "uncached: %d", n) } } -// LogFixBroken performs emoji.FixBroken(...), logging the start and outcome. +// LogFixBroken performs Emoji.FixBroken(...), logging the start and outcome. func (e *Emoji) LogFixBroken(ctx context.Context) { log.Info(ctx, "start") if n, err := e.FixBroken(ctx); err != nil { @@ -61,10 +66,78 @@ func (e *Emoji) LogFixBroken(ctx context.Context) { } } -// PruneMissing will delete emoji with missing files from the database and storage driver. -// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function -// should be updated to match media.FixCacheStat() if we ever support emoji uncaching. -func (e *Emoji) PruneMissing(ctx context.Context) (int, error) { +// LogPruneUnused performs Emoji.PruneUnused(...), logging the start and outcome. +func (e *Emoji) LogPruneUnused(ctx context.Context) { + log.Info(ctx, "start") + if n, err := e.PruneUnused(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "pruned: %d", n) + } +} + +// LogFixCacheStates performs Emoji.FixCacheStates(...), logging the start and outcome. +func (e *Emoji) LogFixCacheStates(ctx context.Context) { + log.Info(ctx, "start") + if n, err := e.FixCacheStates(ctx); err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "fixed: %d", n) + } +} + +// UncacheRemote will uncache all remote emoji older than given input time. Context +// will be checked for `gtscontext.DryRun()` in order to actually perform the action. +func (e *Emoji) UncacheRemote(ctx context.Context, olderThan time.Time) (int, error) { + var total int + + // Drop time by a minute to improve search, + // (i.e. make it olderThan inclusive search). + olderThan = olderThan.Add(-time.Minute) + + // Store recent time. + mostRecent := olderThan + + for { + // Fetch the next batch of cached emojis older than last-set time. + emojis, err := e.state.DB.GetCachedEmojisOlderThan(ctx, olderThan, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting remote emoji: %w", err) + } + + if len(emojis) == 0 { + // reached end. + break + } + + // Use last created-at as the next 'olderThan' value. + olderThan = emojis[len(emojis)-1].CreatedAt + + for _, emoji := range emojis { + // Check / uncache each remote emoji. + uncached, err := e.uncacheRemote(ctx, + mostRecent, + emoji, + ) + if err != nil { + return total, err + } + + if uncached { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +// FixBroken will check all emojis for valid related models (e.g. category). +// Broken media will be automatically updated to remove now-missing models. +// Context will be checked for `gtscontext.DryRun()` to perform the action. +func (e *Emoji) FixBroken(ctx context.Context) (int, error) { var ( total int maxID string @@ -86,8 +159,8 @@ func (e *Emoji) PruneMissing(ctx context.Context) (int, error) { maxID = emojis[len(emojis)-1].ID for _, emoji := range emojis { - // Check / fix missing emoji media. - fixed, err := e.pruneMissing(ctx, emoji) + // Check / fix missing broken emoji. + fixed, err := e.fixBroken(ctx, emoji) if err != nil { return total, err } @@ -103,10 +176,10 @@ func (e *Emoji) PruneMissing(ctx context.Context) (int, error) { return total, nil } -// FixBroken will check all emojis for valid related models (e.g. category). -// Broken media will be automatically updated to remove now-missing models. -// Context will be checked for `gtscontext.DryRun()` to perform the action. -func (e *Emoji) FixBroken(ctx context.Context) (int, error) { +// PruneUnused will delete all unused emoji media from the database and storage driver. +// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function +// should be updated to match media.FixCacheStat() if we ever support emoji uncaching. +func (e *Emoji) PruneUnused(ctx context.Context) (int, error) { var ( total int maxID string @@ -114,9 +187,9 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) { for { // Fetch the next batch of emoji media up to next ID. - emojis, err := e.state.DB.GetEmojis(ctx, maxID, selectLimit) + emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit) if err != nil && !errors.Is(err, db.ErrNoEntries) { - return total, gtserror.Newf("error getting emojis: %w", err) + return total, gtserror.Newf("error getting remote emojis: %w", err) } if len(emojis) == 0 { @@ -128,8 +201,50 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) { maxID = emojis[len(emojis)-1].ID for _, emoji := range emojis { - // Check / fix missing broken emoji. - fixed, err := e.fixBroken(ctx, emoji) + // Check / prune unused emoji media. + fixed, err := e.pruneUnused(ctx, emoji) + if err != nil { + return total, err + } + + if fixed { + // Update + // count. + total++ + } + } + } + + return total, nil +} + +// FixCacheStatus will check all emoji for up-to-date cache status (i.e. in storage driver). +// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function +// should be updated to match media.FixCacheStat() if we ever support emoji uncaching. +func (e *Emoji) FixCacheStates(ctx context.Context) (int, error) { + var ( + total int + maxID string + ) + + for { + // Fetch the next batch of emoji media up to next ID. + emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return total, gtserror.Newf("error getting remote emojis: %w", err) + } + + if len(emojis) == 0 { + // reached end. + break + } + + // Use last as the next 'maxID' value. + maxID = emojis[len(emojis)-1].ID + + for _, emoji := range emojis { + // Check / fix required emoji cache states. + fixed, err := e.fixCacheState(ctx, emoji) if err != nil { return total, err } @@ -145,22 +260,113 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) { return total, nil } -func (e *Emoji) pruneMissing(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { - return e.checkFiles(ctx, func() error { - // Emoji missing files, delete it. - // NOTE: if we ever support uncaching - // of emojis, change to e.uncache(). - // In that case we should also rename - // this function to match the media - // equivalent -> fixCacheState(). - log.WithContext(ctx). - WithField("emoji", emoji.ID). - Debug("deleting due to missing emoji") - return e.delete(ctx, emoji) - }, +func (e *Emoji) pruneUnused(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { + // Start a log entry for emoji. + l := log.WithContext(ctx). + WithField("emoji", emoji.ID) + + // Load any related accounts using this emoji. + accounts, err := e.getRelatedAccounts(ctx, emoji) + if err != nil { + return false, err + } else if len(accounts) > 0 { + l.Debug("skipping as account emoji in use") + return false, nil + } + + // Load any related statuses using this emoji. + statuses, err := e.getRelatedStatuses(ctx, emoji) + if err != nil { + return false, err + } else if len(statuses) > 0 { + l.Debug("skipping as status emoji in use") + return false, nil + } + + // Check not recently created, give it some time to be "used" again. + if time.Now().Add(-24 * time.Hour * 7).Before(emoji.CreatedAt) { + l.Debug("skipping due to recently created") + return false, nil + } + + // Emoji totally unused, delete it. + l.Debug("deleting unused emoji") + return true, e.delete(ctx, emoji) +} + +func (e *Emoji) fixCacheState(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { + // Start a log entry for emoji. + l := log.WithContext(ctx). + WithField("emoji", emoji.ID) + + // Check whether files exist. + exist, err := e.haveFiles(ctx, emoji.ImageStaticPath, emoji.ImagePath, ) + if err != nil { + return false, err + } + + switch { + case *emoji.Cached && !exist: + // Mark as uncached if expected files don't exist. + l.Debug("cached=true exists=false => marking uncached") + return true, e.uncache(ctx, emoji) + + case !*emoji.Cached && exist: + // Remove files if we don't expect them to exist. + l.Debug("cached=false exists=true => removing files") + _, err := e.removeFiles(ctx, + emoji.ImageStaticPath, + emoji.ImagePath, + ) + return true, err + + default: + return false, nil + } +} + +func (e *Emoji) uncacheRemote(ctx context.Context, after time.Time, emoji *gtsmodel.Emoji) (bool, error) { + if !*emoji.Cached { + // Already uncached. + return false, nil + } + + // Start a log entry for emoji. + l := log.WithContext(ctx). + WithField("emoji", emoji.ID) + + // Load any related accounts using this emoji. + accounts, err := e.getRelatedAccounts(ctx, emoji) + if err != nil { + return false, err + } + + for _, account := range accounts { + if account.FetchedAt.After(after) { + l.Debug("skipping due to recently fetched account") + return false, nil + } + } + + // Load any related statuses using this emoji. + statuses, err := e.getRelatedStatuses(ctx, emoji) + if err != nil { + return false, err + } + + for _, status := range statuses { + if status.FetchedAt.After(after) { + l.Debug("skipping due to recently fetched status") + return false, nil + } + } + + // This emoji is too old, uncache it. + l.Debug("uncaching old remote emoji") + return true, e.uncache(ctx, emoji) } func (e *Emoji) fixBroken(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { @@ -214,6 +420,47 @@ func (e *Emoji) getRelatedCategory(ctx context.Context, emoji *gtsmodel.Emoji) ( return category, false, nil } +func (e *Emoji) getRelatedAccounts(ctx context.Context, emoji *gtsmodel.Emoji) ([]*gtsmodel.Account, error) { + accounts, err := e.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID) + if err != nil { + return nil, gtserror.Newf("error fetching accounts using emoji %s: %w", emoji.ID, err) + } + return accounts, nil +} + +func (e *Emoji) getRelatedStatuses(ctx context.Context, emoji *gtsmodel.Emoji) ([]*gtsmodel.Status, error) { + statuses, err := e.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID) + if err != nil { + return nil, gtserror.Newf("error fetching statuses using emoji %s: %w", emoji.ID, err) + } + return statuses, nil +} + +func (e *Emoji) uncache(ctx context.Context, emoji *gtsmodel.Emoji) error { + if gtscontext.DryRun(ctx) { + // Dry run, do nothing. + return nil + } + + // Remove emoji and static. + _, err := e.removeFiles(ctx, + emoji.ImagePath, + emoji.ImageStaticPath, + ) + if err != nil { + return gtserror.Newf("error removing emoji files: %w", err) + } + + // Update emoji to reflect that we no longer have it cached. + log.Debugf(ctx, "marking emoji as uncached: %s", emoji.ID) + emoji.Cached = func() *bool { i := false; return &i }() + if err := e.state.DB.UpdateEmoji(ctx, emoji, "cached"); err != nil { + return gtserror.Newf("error updating emoji: %w", err) + } + + return nil +} + func (e *Emoji) delete(ctx context.Context, emoji *gtsmodel.Emoji) error { if gtscontext.DryRun(ctx) { // Dry run, do nothing. |