[feature/performance] support uncaching remote emoji + scheduled cleanup functions (#1987)

5 files changed, 802 insertions, 63 deletions

diff --git a/internal/cleaner/cleaner.go b/internal/cleaner/cleaner.go
index ee1e4785f..70497c10e 100644
--- a/internal/cleaner/cleaner.go
+++ b/internal/cleaner/cleaner.go
@@ -61,19 +61,19 @@ func (c *Cleaner) Media() *Media {
 	return &c.media
 }
 
-// checkFiles checks for each of the provided files, and calls onMissing() if any of them are missing. Returns true if missing.
-func (c *Cleaner) checkFiles(ctx context.Context, onMissing func() error, files ...string) (bool, error) {
+// haveFiles returns whether all of the provided files exist within current storage.
+func (c *Cleaner) haveFiles(ctx context.Context, files ...string) (bool, error) {
 	for _, file := range files {
 		// Check whether each file exists in storage.
 		have, err := c.state.Storage.Has(ctx, file)
 		if err != nil {
 			return false, gtserror.Newf("error checking storage for %s: %w", file, err)
 		} else if !have {
-			// Missing files, perform hook.
-			return true, onMissing()
+			// Missing file(s).
+			return false, nil
 		}
 	}
-	return false, nil
+	return true, nil
 }
 
 // removeFiles removes the provided files, returning the number of them returned.
@@ -129,7 +129,7 @@ func scheduleJobs(c *Cleaner) {
 	c.state.Workers.Scheduler.Schedule(sched.NewJob(func(start time.Time) {
 		log.Info(nil, "starting media clean")
 		c.Media().All(doneCtx, config.GetMediaRemoteCacheDays())
-		c.Emoji().All(doneCtx)
+		c.Emoji().All(doneCtx, config.GetMediaRemoteCacheDays())
 		log.Infof(nil, "finished media clean after %s", time.Since(start))
 	}).EveryAt(midnight, day))
 }
diff --git a/internal/cleaner/cleaner_test.go b/internal/cleaner/cleaner_test.go
new file mode 100644
index 000000000..d23dac504
--- /dev/null
+++ b/internal/cleaner/cleaner_test.go
@@ -0,0 +1,80 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cleaner_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type CleanerTestSuite struct {
+	state   state.State
+	cleaner *cleaner.Cleaner
+	emojis  map[string]*gtsmodel.Emoji
+	suite.Suite
+}
+
+func TestCleanerTestSuite(t *testing.T) {
+	suite.Run(t, &CleanerTestSuite{})
+}
+
+func (suite *CleanerTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+}
+
+func (suite *CleanerTestSuite) SetupTest() {
+	// Initialize gts caches.
+	suite.state.Caches.Init()
+
+	// Ensure scheduler started (even if unused).
+	suite.state.Workers.Scheduler.Start(nil)
+
+	// Initialize test database.
+	_ = testrig.NewTestDB(&suite.state)
+	testrig.StandardDBSetup(suite.state.DB, nil)
+
+	// Initialize test storage (in-memory).
+	suite.state.Storage = testrig.NewInMemoryStorage()
+
+	// Initialize test cleaner instance.
+	suite.cleaner = cleaner.New(&suite.state)
+
+	// Allocate new test model emojis.
+	suite.emojis = testrig.NewTestEmojis()
+}
+
+func (suite *CleanerTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.state.DB)
+}
+
+// mapvals extracts a slice of values from the values contained within the map.
+func mapvals[Key comparable, Val any](m map[Key]Val) []Val {
+	var i int
+	vals := make([]Val, len(m))
+	for _, val := range m {
+		vals[i] = val
+		i++
+	}
+	return vals
+}
diff --git a/internal/cleaner/emoji.go b/internal/cleaner/emoji.go
index 35e579171..d2baec7e8 100644
--- a/internal/cleaner/emoji.go
+++ b/internal/cleaner/emoji.go
@@ -20,6 +20,7 @@ package cleaner
 import (
 	"context"
 	"errors"
+	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
@@ -36,22 +37,26 @@ type Emoji struct {
 
 // All will execute all cleaner.Emoji utilities synchronously, including output logging.
 // Context will be checked for `gtscontext.DryRun()` in order to actually perform the action.
-func (e *Emoji) All(ctx context.Context) {
-	e.LogPruneMissing(ctx)
+func (e *Emoji) All(ctx context.Context, maxRemoteDays int) {
+	t := time.Now().Add(-24 * time.Hour * time.Duration(maxRemoteDays))
+	e.LogUncacheRemote(ctx, t)
 	e.LogFixBroken(ctx)
+	e.LogPruneUnused(ctx)
+	e.LogFixCacheStates(ctx)
+	_ = e.state.Storage.Storage.Clean(ctx)
 }
 
-// LogPruneMissing performs emoji.PruneMissing(...), logging the start and outcome.
-func (e *Emoji) LogPruneMissing(ctx context.Context) {
-	log.Info(ctx, "start")
-	if n, err := e.PruneMissing(ctx); err != nil {
+// LogUncacheRemote performs Emoji.UncacheRemote(...), logging the start and outcome.
+func (e *Emoji) LogUncacheRemote(ctx context.Context, olderThan time.Time) {
+	log.Infof(ctx, "start older than: %s", olderThan.Format(time.Stamp))
+	if n, err := e.UncacheRemote(ctx, olderThan); err != nil {
 		log.Error(ctx, err)
 	} else {
-		log.Infof(ctx, "pruned: %d", n)
+		log.Infof(ctx, "uncached: %d", n)
 	}
 }
 
-// LogFixBroken performs emoji.FixBroken(...), logging the start and outcome.
+// LogFixBroken performs Emoji.FixBroken(...), logging the start and outcome.
 func (e *Emoji) LogFixBroken(ctx context.Context) {
 	log.Info(ctx, "start")
 	if n, err := e.FixBroken(ctx); err != nil {
@@ -61,10 +66,78 @@ func (e *Emoji) LogFixBroken(ctx context.Context) {
 	}
 }
 
-// PruneMissing will delete emoji with missing files from the database and storage driver.
-// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function
-// should be updated to match media.FixCacheStat() if we ever support emoji uncaching.
-func (e *Emoji) PruneMissing(ctx context.Context) (int, error) {
+// LogPruneUnused performs Emoji.PruneUnused(...), logging the start and outcome.
+func (e *Emoji) LogPruneUnused(ctx context.Context) {
+	log.Info(ctx, "start")
+	if n, err := e.PruneUnused(ctx); err != nil {
+		log.Error(ctx, err)
+	} else {
+		log.Infof(ctx, "pruned: %d", n)
+	}
+}
+
+// LogFixCacheStates performs Emoji.FixCacheStates(...), logging the start and outcome.
+func (e *Emoji) LogFixCacheStates(ctx context.Context) {
+	log.Info(ctx, "start")
+	if n, err := e.FixCacheStates(ctx); err != nil {
+		log.Error(ctx, err)
+	} else {
+		log.Infof(ctx, "fixed: %d", n)
+	}
+}
+
+// UncacheRemote will uncache all remote emoji older than given input time. Context
+// will be checked for `gtscontext.DryRun()` in order to actually perform the action.
+func (e *Emoji) UncacheRemote(ctx context.Context, olderThan time.Time) (int, error) {
+	var total int
+
+	// Drop time by a minute to improve search,
+	// (i.e. make it olderThan inclusive search).
+	olderThan = olderThan.Add(-time.Minute)
+
+	// Store recent time.
+	mostRecent := olderThan
+
+	for {
+		// Fetch the next batch of cached emojis older than last-set time.
+		emojis, err := e.state.DB.GetCachedEmojisOlderThan(ctx, olderThan, selectLimit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return total, gtserror.Newf("error getting remote emoji: %w", err)
+		}
+
+		if len(emojis) == 0 {
+			// reached end.
+			break
+		}
+
+		// Use last created-at as the next 'olderThan' value.
+		olderThan = emojis[len(emojis)-1].CreatedAt
+
+		for _, emoji := range emojis {
+			// Check / uncache each remote emoji.
+			uncached, err := e.uncacheRemote(ctx,
+				mostRecent,
+				emoji,
+			)
+			if err != nil {
+				return total, err
+			}
+
+			if uncached {
+				// Update
+				// count.
+				total++
+			}
+		}
+	}
+
+	return total, nil
+}
+
+// FixBroken will check all emojis for valid related models (e.g. category).
+// Broken media will be automatically updated to remove now-missing models.
+// Context will be checked for `gtscontext.DryRun()` to perform the action.
+func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
 	var (
 		total int
 		maxID string
@@ -86,8 +159,8 @@ func (e *Emoji) PruneMissing(ctx context.Context) (int, error) {
 		maxID = emojis[len(emojis)-1].ID
 
 		for _, emoji := range emojis {
-			// Check / fix missing emoji media.
-			fixed, err := e.pruneMissing(ctx, emoji)
+			// Check / fix missing broken emoji.
+			fixed, err := e.fixBroken(ctx, emoji)
 			if err != nil {
 				return total, err
 			}
@@ -103,10 +176,10 @@ func (e *Emoji) PruneMissing(ctx context.Context) (int, error) {
 	return total, nil
 }
 
-// FixBroken will check all emojis for valid related models (e.g. category).
-// Broken media will be automatically updated to remove now-missing models.
-// Context will be checked for `gtscontext.DryRun()` to perform the action.
-func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
+// PruneUnused will delete all unused emoji media from the database and storage driver.
+// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function
+// should be updated to match media.FixCacheStat() if we ever support emoji uncaching.
+func (e *Emoji) PruneUnused(ctx context.Context) (int, error) {
 	var (
 		total int
 		maxID string
@@ -114,9 +187,9 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
 
 	for {
 		// Fetch the next batch of emoji media up to next ID.
-		emojis, err := e.state.DB.GetEmojis(ctx, maxID, selectLimit)
+		emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit)
 		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			return total, gtserror.Newf("error getting emojis: %w", err)
+			return total, gtserror.Newf("error getting remote emojis: %w", err)
 		}
 
 		if len(emojis) == 0 {
@@ -128,8 +201,50 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
 		maxID = emojis[len(emojis)-1].ID
 
 		for _, emoji := range emojis {
-			// Check / fix missing broken emoji.
-			fixed, err := e.fixBroken(ctx, emoji)
+			// Check / prune unused emoji media.
+			fixed, err := e.pruneUnused(ctx, emoji)
+			if err != nil {
+				return total, err
+			}
+
+			if fixed {
+				// Update
+				// count.
+				total++
+			}
+		}
+	}
+
+	return total, nil
+}
+
+// FixCacheStatus will check all emoji for up-to-date cache status (i.e. in storage driver).
+// Context will be checked for `gtscontext.DryRun()` to perform the action. NOTE: this function
+// should be updated to match media.FixCacheStat() if we ever support emoji uncaching.
+func (e *Emoji) FixCacheStates(ctx context.Context) (int, error) {
+	var (
+		total int
+		maxID string
+	)
+
+	for {
+		// Fetch the next batch of emoji media up to next ID.
+		emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return total, gtserror.Newf("error getting remote emojis: %w", err)
+		}
+
+		if len(emojis) == 0 {
+			// reached end.
+			break
+		}
+
+		// Use last as the next 'maxID' value.
+		maxID = emojis[len(emojis)-1].ID
+
+		for _, emoji := range emojis {
+			// Check / fix required emoji cache states.
+			fixed, err := e.fixCacheState(ctx, emoji)
 			if err != nil {
 				return total, err
 			}
@@ -145,22 +260,113 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
 	return total, nil
 }
 
-func (e *Emoji) pruneMissing(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
-	return e.checkFiles(ctx, func() error {
-		// Emoji missing files, delete it.
-		// NOTE: if we ever support uncaching
-		// of emojis, change to e.uncache().
-		// In that case we should also rename
-		// this function to match the media
-		// equivalent -> fixCacheState().
-		log.WithContext(ctx).
-			WithField("emoji", emoji.ID).
-			Debug("deleting due to missing emoji")
-		return e.delete(ctx, emoji)
-	},
+func (e *Emoji) pruneUnused(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
+	// Start a log entry for emoji.
+	l := log.WithContext(ctx).
+		WithField("emoji", emoji.ID)
+
+	// Load any related accounts using this emoji.
+	accounts, err := e.getRelatedAccounts(ctx, emoji)
+	if err != nil {
+		return false, err
+	} else if len(accounts) > 0 {
+		l.Debug("skipping as account emoji in use")
+		return false, nil
+	}
+
+	// Load any related statuses using this emoji.
+	statuses, err := e.getRelatedStatuses(ctx, emoji)
+	if err != nil {
+		return false, err
+	} else if len(statuses) > 0 {
+		l.Debug("skipping as status emoji in use")
+		return false, nil
+	}
+
+	// Check not recently created, give it some time to be "used" again.
+	if time.Now().Add(-24 * time.Hour * 7).Before(emoji.CreatedAt) {
+		l.Debug("skipping due to recently created")
+		return false, nil
+	}
+
+	// Emoji totally unused, delete it.
+	l.Debug("deleting unused emoji")
+	return true, e.delete(ctx, emoji)
+}
+
+func (e *Emoji) fixCacheState(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
+	// Start a log entry for emoji.
+	l := log.WithContext(ctx).
+		WithField("emoji", emoji.ID)
+
+	// Check whether files exist.
+	exist, err := e.haveFiles(ctx,
 		emoji.ImageStaticPath,
 		emoji.ImagePath,
 	)
+	if err != nil {
+		return false, err
+	}
+
+	switch {
+	case *emoji.Cached && !exist:
+		// Mark as uncached if expected files don't exist.
+		l.Debug("cached=true exists=false => marking uncached")
+		return true, e.uncache(ctx, emoji)
+
+	case !*emoji.Cached && exist:
+		// Remove files if we don't expect them to exist.
+		l.Debug("cached=false exists=true => removing files")
+		_, err := e.removeFiles(ctx,
+			emoji.ImageStaticPath,
+			emoji.ImagePath,
+		)
+		return true, err
+
+	default:
+		return false, nil
+	}
+}
+
+func (e *Emoji) uncacheRemote(ctx context.Context, after time.Time, emoji *gtsmodel.Emoji) (bool, error) {
+	if !*emoji.Cached {
+		// Already uncached.
+		return false, nil
+	}
+
+	// Start a log entry for emoji.
+	l := log.WithContext(ctx).
+		WithField("emoji", emoji.ID)
+
+	// Load any related accounts using this emoji.
+	accounts, err := e.getRelatedAccounts(ctx, emoji)
+	if err != nil {
+		return false, err
+	}
+
+	for _, account := range accounts {
+		if account.FetchedAt.After(after) {
+			l.Debug("skipping due to recently fetched account")
+			return false, nil
+		}
+	}
+
+	// Load any related statuses using this emoji.
+	statuses, err := e.getRelatedStatuses(ctx, emoji)
+	if err != nil {
+		return false, err
+	}
+
+	for _, status := range statuses {
+		if status.FetchedAt.After(after) {
+			l.Debug("skipping due to recently fetched status")
+			return false, nil
+		}
+	}
+
+	// This emoji is too old, uncache it.
+	l.Debug("uncaching old remote emoji")
+	return true, e.uncache(ctx, emoji)
 }
 
 func (e *Emoji) fixBroken(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
@@ -214,6 +420,47 @@ func (e *Emoji) getRelatedCategory(ctx context.Context, emoji *gtsmodel.Emoji) (
 	return category, false, nil
 }
 
+func (e *Emoji) getRelatedAccounts(ctx context.Context, emoji *gtsmodel.Emoji) ([]*gtsmodel.Account, error) {
+	accounts, err := e.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return nil, gtserror.Newf("error fetching accounts using emoji %s: %w", emoji.ID, err)
+	}
+	return accounts, nil
+}
+
+func (e *Emoji) getRelatedStatuses(ctx context.Context, emoji *gtsmodel.Emoji) ([]*gtsmodel.Status, error) {
+	statuses, err := e.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return nil, gtserror.Newf("error fetching statuses using emoji %s: %w", emoji.ID, err)
+	}
+	return statuses, nil
+}
+
+func (e *Emoji) uncache(ctx context.Context, emoji *gtsmodel.Emoji) error {
+	if gtscontext.DryRun(ctx) {
+		// Dry run, do nothing.
+		return nil
+	}
+
+	// Remove emoji and static.
+	_, err := e.removeFiles(ctx,
+		emoji.ImagePath,
+		emoji.ImageStaticPath,
+	)
+	if err != nil {
+		return gtserror.Newf("error removing emoji files: %w", err)
+	}
+
+	// Update emoji to reflect that we no longer have it cached.
+	log.Debugf(ctx, "marking emoji as uncached: %s", emoji.ID)
+	emoji.Cached = func() *bool { i := false; return &i }()
+	if err := e.state.DB.UpdateEmoji(ctx, emoji, "cached"); err != nil {
+		return gtserror.Newf("error updating emoji: %w", err)
+	}
+
+	return nil
+}
+
 func (e *Emoji) delete(ctx context.Context, emoji *gtsmodel.Emoji) error {
 	if gtscontext.DryRun(ctx) {
 		// Dry run, do nothing.
diff --git a/internal/cleaner/emoji_test.go b/internal/cleaner/emoji_test.go
new file mode 100644
index 000000000..81fde6e48
--- /dev/null
+++ b/internal/cleaner/emoji_test.go
@@ -0,0 +1,402 @@
+package cleaner_test
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (suite *CleanerTestSuite) TestEmojiUncacheRemote() {
+	suite.testEmojiUncacheRemote(
+		context.Background(),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiUncacheRemoteDryRun() {
+	suite.testEmojiUncacheRemote(
+		gtscontext.SetDryRun(context.Background()),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiFixBroken() {
+	suite.testEmojiFixBroken(
+		context.Background(),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiFixBrokenDryRun() {
+	suite.testEmojiFixBroken(
+		gtscontext.SetDryRun(context.Background()),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiPruneUnused() {
+	suite.testEmojiPruneUnused(
+		context.Background(),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiPruneUnusedDryRun() {
+	suite.testEmojiPruneUnused(
+		gtscontext.SetDryRun(context.Background()),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiFixCacheStates() {
+	suite.testEmojiFixCacheStates(
+		context.Background(),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) TestEmojiFixCacheStatesDryRun() {
+	suite.testEmojiFixCacheStates(
+		gtscontext.SetDryRun(context.Background()),
+		mapvals(suite.emojis),
+	)
+}
+
+func (suite *CleanerTestSuite) testEmojiUncacheRemote(ctx context.Context, emojis []*gtsmodel.Emoji) {
+	var uncacheIDs []string
+
+	// Test state.
+	t := suite.T()
+
+	// Get max remote cache days to keep.
+	days := config.GetMediaRemoteCacheDays()
+	olderThan := time.Now().Add(-24 * time.Hour * time.Duration(days))
+
+	for _, emoji := range emojis {
+		// Check whether this emoji should be uncached.
+		ok, err := suite.shouldUncacheEmoji(ctx, emoji, olderThan)
+		if err != nil {
+			t.Fatalf("error checking whether emoji should be uncached: %v", err)
+		}
+
+		if ok {
+			// Mark this emoji ID as to be uncached.
+			uncacheIDs = append(uncacheIDs, emoji.ID)
+		}
+	}
+
+	// Attempt to uncache remote emojis.
+	found, err := suite.cleaner.Emoji().UncacheRemote(ctx, olderThan)
+	if err != nil {
+		t.Errorf("error uncaching remote emojis: %v", err)
+		return
+	}
+
+	// Check expected were uncached.
+	if found != len(uncacheIDs) {
+		t.Errorf("expected %d emojis to be uncached, %d were", len(uncacheIDs), found)
+		return
+	}
+
+	if gtscontext.DryRun(ctx) {
+		// nothing else to test.
+		return
+	}
+
+	for _, id := range uncacheIDs {
+		// Fetch the emoji by ID that should now be uncached.
+		emoji, err := suite.state.DB.GetEmojiByID(ctx, id)
+		if err != nil {
+			t.Fatalf("error fetching emoji from database: %v", err)
+		}
+
+		// Check cache state.
+		if *emoji.Cached {
+			t.Errorf("emoji %s@%s should have been uncached", emoji.Shortcode, emoji.Domain)
+		}
+
+		// Check that the emoji files in storage have been deleted.
+		if ok, err := suite.state.Storage.Has(ctx, emoji.ImagePath); err != nil {
+			t.Fatalf("error checking storage for emoji: %v", err)
+		} else if ok {
+			t.Errorf("emoji %s@%s image path should not exist", emoji.Shortcode, emoji.Domain)
+		} else if ok, err := suite.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil {
+			t.Fatalf("error checking storage for emoji: %v", err)
+		} else if ok {
+			t.Errorf("emoji %s@%s image static path should not exist", emoji.Shortcode, emoji.Domain)
+		}
+	}
+}
+
+func (suite *CleanerTestSuite) shouldUncacheEmoji(ctx context.Context, emoji *gtsmodel.Emoji, after time.Time) (bool, error) {
+	if emoji.ImageRemoteURL == "" {
+		// Local emojis are never uncached.
+		return false, nil
+	}
+
+	if emoji.Cached == nil || !*emoji.Cached {
+		// Emoji is already uncached.
+		return false, nil
+	}
+
+	// Get related accounts using this emoji (if any).
+	accounts, err := suite.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return false, err
+	}
+
+	// Check if accounts are recently updated.
+	for _, account := range accounts {
+		if account.FetchedAt.After(after) {
+			return false, nil
+		}
+	}
+
+	// Get related statuses using this emoji (if any).
+	statuses, err := suite.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return false, err
+	}
+
+	// Check if statuses are recently updated.
+	for _, status := range statuses {
+		if status.FetchedAt.After(after) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (suite *CleanerTestSuite) testEmojiFixBroken(ctx context.Context, emojis []*gtsmodel.Emoji) {
+	var fixIDs []string
+
+	// Test state.
+	t := suite.T()
+
+	for _, emoji := range emojis {
+		// Check whether this emoji should be fixed.
+		ok, err := suite.shouldFixBrokenEmoji(ctx, emoji)
+		if err != nil {
+			t.Fatalf("error checking whether emoji should be fixed: %v", err)
+		}
+
+		if ok {
+			// Mark this emoji ID as to be fixed.
+			fixIDs = append(fixIDs, emoji.ID)
+		}
+	}
+
+	// Attempt to fix broken emojis.
+	found, err := suite.cleaner.Emoji().FixBroken(ctx)
+	if err != nil {
+		t.Errorf("error fixing broken emojis: %v", err)
+		return
+	}
+
+	// Check expected were fixed.
+	if found != len(fixIDs) {
+		t.Errorf("expected %d emojis to be fixed, %d were", len(fixIDs), found)
+		return
+	}
+
+	if gtscontext.DryRun(ctx) {
+		// nothing else to test.
+		return
+	}
+
+	for _, id := range fixIDs {
+		// Fetch the emoji by ID that should now be fixed.
+		emoji, err := suite.state.DB.GetEmojiByID(ctx, id)
+		if err != nil {
+			t.Fatalf("error fetching emoji from database: %v", err)
+		}
+
+		// Ensure category was cleared.
+		if emoji.CategoryID != "" {
+			t.Errorf("emoji %s@%s should have empty category", emoji.Shortcode, emoji.Domain)
+		}
+	}
+}
+
+func (suite *CleanerTestSuite) shouldFixBrokenEmoji(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
+	if emoji.CategoryID == "" {
+		// no category issue.
+		return false, nil
+	}
+
+	// Get the related category for this emoji.
+	category, err := suite.state.DB.GetEmojiCategory(ctx, emoji.CategoryID)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		return false, nil
+	}
+
+	return (category == nil), nil
+}
+
+func (suite *CleanerTestSuite) testEmojiPruneUnused(ctx context.Context, emojis []*gtsmodel.Emoji) {
+	var pruneIDs []string
+
+	// Test state.
+	t := suite.T()
+
+	for _, emoji := range emojis {
+		// Check whether this emoji should be pruned.
+		ok, err := suite.shouldPruneEmoji(ctx, emoji)
+		if err != nil {
+			t.Fatalf("error checking whether emoji should be pruned: %v", err)
+		}
+
+		if ok {
+			// Mark this emoji ID as to be pruned.
+			pruneIDs = append(pruneIDs, emoji.ID)
+		}
+	}
+
+	// Attempt to prune emojis.
+	found, err := suite.cleaner.Emoji().PruneUnused(ctx)
+	if err != nil {
+		t.Errorf("error fixing broken emojis: %v", err)
+		return
+	}
+
+	// Check expected were pruned.
+	if found != len(pruneIDs) {
+		t.Errorf("expected %d emojis to be pruned, %d were", len(pruneIDs), found)
+		return
+	}
+
+	if gtscontext.DryRun(ctx) {
+		// nothing else to test.
+		return
+	}
+
+	for _, id := range pruneIDs {
+		// Fetch the emoji by ID that should now be pruned.
+		emoji, err := suite.state.DB.GetEmojiByID(ctx, id)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			t.Fatalf("error fetching emoji from database: %v", err)
+		}
+
+		// Ensure gone.
+		if emoji != nil {
+			t.Errorf("emoji %s@%s should have been pruned", emoji.Shortcode, emoji.Domain)
+		}
+	}
+}
+
+func (suite *CleanerTestSuite) shouldPruneEmoji(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
+	if emoji.ImageRemoteURL == "" {
+		// Local emojis are never pruned.
+		return false, nil
+	}
+
+	// Get related accounts using this emoji (if any).
+	accounts, err := suite.state.DB.GetAccountsUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return false, err
+	} else if len(accounts) > 0 {
+		return false, nil
+	}
+
+	// Get related statuses using this emoji (if any).
+	statuses, err := suite.state.DB.GetStatusesUsingEmoji(ctx, emoji.ID)
+	if err != nil {
+		return false, err
+	} else if len(statuses) > 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (suite *CleanerTestSuite) testEmojiFixCacheStates(ctx context.Context, emojis []*gtsmodel.Emoji) {
+	var fixIDs []string
+
+	// Test state.
+	t := suite.T()
+
+	for _, emoji := range emojis {
+		// Check whether this emoji should be fixed.
+		ok, err := suite.shouldFixEmojiCacheState(ctx, emoji)
+		if err != nil {
+			t.Fatalf("error checking whether emoji should be fixed: %v", err)
+		}
+
+		if ok {
+			// Mark this emoji ID as to be fixed.
+			fixIDs = append(fixIDs, emoji.ID)
+		}
+	}
+
+	// Attempt to fix broken emoji cache states.
+	found, err := suite.cleaner.Emoji().FixCacheStates(ctx)
+	if err != nil {
+		t.Errorf("error fixing broken emojis: %v", err)
+		return
+	}
+
+	// Check expected were fixed.
+	if found != len(fixIDs) {
+		t.Errorf("expected %d emojis to be fixed, %d were", len(fixIDs), found)
+		return
+	}
+
+	if gtscontext.DryRun(ctx) {
+		// nothing else to test.
+		return
+	}
+
+	for _, id := range fixIDs {
+		// Fetch the emoji by ID that should now be fixed.
+		emoji, err := suite.state.DB.GetEmojiByID(ctx, id)
+		if err != nil {
+			t.Fatalf("error fetching emoji from database: %v", err)
+		}
+
+		// Ensure emoji cache state has been fixed.
+		ok, err := suite.shouldFixEmojiCacheState(ctx, emoji)
+		if err != nil {
+			t.Fatalf("error checking whether emoji should be fixed: %v", err)
+		} else if ok {
+			t.Errorf("emoji %s@%s cache state should have been fixed", emoji.Shortcode, emoji.Domain)
+		}
+	}
+}
+
+func (suite *CleanerTestSuite) shouldFixEmojiCacheState(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
+	// Check whether emoji image path exists.
+	haveImage, err := suite.state.Storage.Has(ctx, emoji.ImagePath)
+	if err != nil {
+		return false, err
+	}
+
+	// Check whether emoji static path exists.
+	haveStatic, err := suite.state.Storage.Has(ctx, emoji.ImageStaticPath)
+	if err != nil {
+		return false, err
+	}
+
+	switch exists := (haveImage && haveStatic); {
+	case emoji.Cached != nil &&
+		*emoji.Cached && !exists:
+		// (cached can be nil in tests)
+		// Cached but missing files.
+		return true, nil
+
+	case emoji.Cached != nil &&
+		!*emoji.Cached && exists:
+		// (cached can be nil in tests)
+		// Uncached but unexpected files.
+		return true, nil
+
+	default:
+		// No cache state issue.
+		return false, nil
+	}
+}
diff --git a/internal/cleaner/media.go b/internal/cleaner/media.go
index 51a0aea6d..8b11a30bf 100644
--- a/internal/cleaner/media.go
+++ b/internal/cleaner/media.go
@@ -96,9 +96,9 @@ func (m *Media) PruneOrphaned(ctx context.Context) (int, error) {
 
 	// All media files in storage will have path fitting: {$account}/{$type}/{$size}/{$id}.{$ext}
 	if err := m.state.Storage.WalkKeys(ctx, func(ctx context.Context, path string) error {
+		// Check for our expected fileserver path format.
 		if !regexes.FilePath.MatchString(path) {
-			// This is not our expected media
-			// path format, skip this one.
+			log.Warn(ctx, "unexpected storage item: %s", path)
 			return nil
 		}
 
@@ -177,10 +177,10 @@ func (m *Media) UncacheRemote(ctx context.Context, olderThan time.Time) (int, er
 	mostRecent := olderThan
 
 	for {
-		// Fetch the next batch of attachments older than last-set time.
-		attachments, err := m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectLimit)
+		// Fetch the next batch of cached attachments older than last-set time.
+		attachments, err := m.state.DB.GetCachedAttachmentsOlderThan(ctx, olderThan, selectLimit)
 		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			return total, gtserror.Newf("error getting remote media: %w", err)
+			return total, gtserror.Newf("error getting remote attachments: %w", err)
 		}
 
 		if len(attachments) == 0 {
@@ -220,9 +220,9 @@ func (m *Media) FixCacheStates(ctx context.Context) (int, error) {
 
 	for {
 		// Fetch the next batch of media attachments up to next max ID.
-		attachments, err := m.state.DB.GetAttachments(ctx, maxID, selectLimit)
+		attachments, err := m.state.DB.GetRemoteAttachments(ctx, maxID, selectLimit)
 		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			return total, gtserror.Newf("error getting avatars / headers: %w", err)
+			return total, gtserror.Newf("error getting remote attachments: %w", err)
 		}
 
 		if len(attachments) == 0 {
@@ -323,7 +323,7 @@ func (m *Media) pruneUnused(ctx context.Context, media *gtsmodel.MediaAttachment
 	l := log.WithContext(ctx).
 		WithField("media", media.ID)
 
-		// Check whether we have the required account for media.
+	// Check whether we have the required account for media.
 	account, missing, err := m.getRelatedAccount(ctx, media)
 	if err != nil {
 		return false, err
@@ -367,14 +367,6 @@ func (m *Media) pruneUnused(ctx context.Context, media *gtsmodel.MediaAttachment
 }
 
 func (m *Media) fixCacheState(ctx context.Context, media *gtsmodel.MediaAttachment) (bool, error) {
-	if !*media.Cached {
-		// We ignore uncached media, a
-		// false negative is a much better
-		// situation than a false positive,
-		// re-cache will just overwrite it.
-		return false, nil
-	}
-
 	// Start a log entry for media.
 	l := log.WithContext(ctx).
 		WithField("media", media.ID)
@@ -397,15 +389,33 @@ func (m *Media) fixCacheState(ctx context.Context, media *gtsmodel.MediaAttachme
 		return false, nil
 	}
 
-	// So we know this a valid cached media entry.
-	// Check that we have the files on disk required....
-	return m.checkFiles(ctx, func() error {
-		l.Debug("uncaching due to missing media")
-		return m.uncache(ctx, media)
-	},
+	// Check whether files exist.
+	exist, err := m.haveFiles(ctx,
 		media.Thumbnail.Path,
 		media.File.Path,
 	)
+	if err != nil {
+		return false, err
+	}
+
+	switch {
+	case *media.Cached && !exist:
+		// Mark as uncached if expected files don't exist.
+		l.Debug("cached=true exists=false => uncaching")
+		return true, m.uncache(ctx, media)
+
+	case !*media.Cached && exist:
+		// Remove files if we don't expect them to exist.
+		l.Debug("cached=false exists=true => deleting")
+		_, err := m.removeFiles(ctx,
+			media.Thumbnail.Path,
+			media.File.Path,
+		)
+		return true, err
+
+	default:
+		return false, nil
+	}
 }
 
 func (m *Media) uncacheRemote(ctx context.Context, after time.Time, media *gtsmodel.MediaAttachment) (bool, error) {