[feature] Customizable media cleaner schedule (#2304)

12 files changed, 151 insertions, 60 deletions

diff --git a/internal/api/wellknown/webfinger/webfingerget_test.go b/internal/api/wellknown/webfinger/webfingerget_test.go
index fb450470f..6b3e2cc5d 100644
--- a/internal/api/wellknown/webfinger/webfingerget_test.go
+++ b/internal/api/wellknown/webfinger/webfingerget_test.go
@@ -33,6 +33,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -82,7 +83,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom
 	// to new host + account domain.
 	config.SetHost(host)
 	config.SetAccountDomain(accountDomain)
-	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(&suite.state), &suite.state, suite.emailSender)
+	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(&suite.state), &suite.state, suite.emailSender)
 	suite.webfingerModule = webfinger.New(suite.processor)
 
 	// Generate a new account for the
diff --git a/internal/cleaner/cleaner.go b/internal/cleaner/cleaner.go
index 31766bae6..1139a85bb 100644
--- a/internal/cleaner/cleaner.go
+++ b/internal/cleaner/cleaner.go
@@ -47,7 +47,6 @@ func New(state *state.State) *Cleaner {
 	c.state = state
 	c.emoji.Cleaner = c
 	c.media.Cleaner = c
-	scheduleJobs(c)
 	return c
 }
 
@@ -109,16 +108,46 @@ func (c *Cleaner) removeFiles(ctx context.Context, files ...string) (int, error)
 	return diff, nil
 }
 
-func scheduleJobs(c *Cleaner) {
-	const day = time.Hour * 24
+// ScheduleJobs schedules cleaning
+// jobs using configured parameters.
+//
+// Returns an error if `MediaCleanupFrom`
+// is not a valid format (hh:mm:ss).
+func (c *Cleaner) ScheduleJobs() error {
+	const hourMinute = "15:04"
+
+	var (
+		now            = time.Now()
+		cleanupEvery   = config.GetMediaCleanupEvery()
+		cleanupFromStr = config.GetMediaCleanupFrom()
+	)
+
+	// Parse cleanupFromStr as hh:mm.
+	// Resulting time will be on 1 Jan year zero.
+	cleanupFrom, err := time.Parse(hourMinute, cleanupFromStr)
+	if err != nil {
+		return gtserror.Newf(
+			"error parsing '%s' in time format 'hh:mm': %w",
+			cleanupFromStr, err,
+		)
+	}
 
-	// Calculate closest midnight.
-	now := time.Now()
-	midnight := now.Round(day)
+	// Time travel from
+	// year zero, groovy.
+	firstCleanupAt := time.Date(
+		now.Year(),
+		now.Month(),
+		now.Day(),
+		cleanupFrom.Hour(),
+		cleanupFrom.Minute(),
+		0,
+		0,
+		now.Location(),
+	)
 
-	if midnight.Before(now) {
-		// since <= 11:59am rounds down.
-		midnight = midnight.Add(day)
+	// Ensure first cleanup is in the future.
+	for firstCleanupAt.Before(now) {
+		firstCleanupAt = firstCleanupAt.Add(cleanupEvery)
 	}
 
 	// Get ctx associated with scheduler run state.
@@ -129,11 +158,18 @@ func scheduleJobs(c *Cleaner) {
 	// jobs restartable if we want to implement reloads in
 	// the future that make call to Workers.Stop() -> Workers.Start().
 
-	// Schedule the cleaning tasks to execute every day at midnight.
+	log.Infof(nil,
+		"scheduling media clean to run every %s, starting from %s; next clean will run at %s",
+		cleanupEvery, cleanupFromStr, firstCleanupAt,
+	)
+
+	// Schedule the cleaning tasks to execute according to given schedule.
 	c.state.Workers.Scheduler.Schedule(sched.NewJob(func(start time.Time) {
 		log.Info(nil, "starting media clean")
 		c.Media().All(doneCtx, config.GetMediaRemoteCacheDays())
 		c.Emoji().All(doneCtx, config.GetMediaRemoteCacheDays())
 		log.Infof(nil, "finished media clean after %s", time.Since(start))
-	}).EveryAt(midnight, day))
+	}).EveryAt(firstCleanupAt, cleanupEvery))
+
+	return nil
 }
diff --git a/internal/config/config.go b/internal/config/config.go
index a9fdef3c7..77e70185c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -97,6 +97,8 @@ type Configuration struct {
 	MediaRemoteCacheDays     int           `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
 	MediaEmojiLocalMaxSize   bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
 	MediaEmojiRemoteMaxSize  bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
+	MediaCleanupFrom         string        `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
+	MediaCleanupEvery        time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`
 
 	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
 	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 6ee52d162..0c2556e9d 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -76,6 +76,8 @@ var Defaults = Configuration{
 	MediaRemoteCacheDays:     7,
 	MediaEmojiLocalMaxSize:   50 * bytesize.KiB,
 	MediaEmojiRemoteMaxSize:  100 * bytesize.KiB,
+	MediaCleanupFrom:         "00:00",        // Midnight.
+	MediaCleanupEvery:        24 * time.Hour, // 1/day.
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",
diff --git a/internal/config/flags.go b/internal/config/flags.go
index 29e0726a6..b29d0fe04 100644
--- a/internal/config/flags.go
+++ b/internal/config/flags.go
@@ -103,6 +103,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
 		cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))
 		cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage"))
+		cmd.Flags().String(MediaCleanupFromFlag(), cfg.MediaCleanupFrom, fieldtag("MediaCleanupFrom", "usage"))
+		cmd.Flags().Duration(MediaCleanupEveryFlag(), cfg.MediaCleanupEvery, fieldtag("MediaCleanupEvery", "usage"))
 
 		// Storage
 		cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage"))
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 80687eb66..415035bea 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -1224,6 +1224,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem
 // SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
 func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
 
+// GetMediaCleanupFrom safely fetches the Configuration value for state's 'MediaCleanupFrom' field
+func (st *ConfigState) GetMediaCleanupFrom() (v string) {
+	st.mutex.RLock()
+	v = st.config.MediaCleanupFrom
+	st.mutex.RUnlock()
+	return
+}
+
+// SetMediaCleanupFrom safely sets the Configuration value for state's 'MediaCleanupFrom' field
+func (st *ConfigState) SetMediaCleanupFrom(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaCleanupFrom = v
+	st.reloadToViper()
+}
+
+// MediaCleanupFromFlag returns the flag name for the 'MediaCleanupFrom' field
+func MediaCleanupFromFlag() string { return "media-cleanup-from" }
+
+// GetMediaCleanupFrom safely fetches the value for global configuration 'MediaCleanupFrom' field
+func GetMediaCleanupFrom() string { return global.GetMediaCleanupFrom() }
+
+// SetMediaCleanupFrom safely sets the value for global configuration 'MediaCleanupFrom' field
+func SetMediaCleanupFrom(v string) { global.SetMediaCleanupFrom(v) }
+
+// GetMediaCleanupEvery safely fetches the Configuration value for state's 'MediaCleanupEvery' field
+func (st *ConfigState) GetMediaCleanupEvery() (v time.Duration) {
+	st.mutex.RLock()
+	v = st.config.MediaCleanupEvery
+	st.mutex.RUnlock()
+	return
+}
+
+// SetMediaCleanupEvery safely sets the Configuration value for state's 'MediaCleanupEvery' field
+func (st *ConfigState) SetMediaCleanupEvery(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaCleanupEvery = v
+	st.reloadToViper()
+}
+
+// MediaCleanupEveryFlag returns the flag name for the 'MediaCleanupEvery' field
+func MediaCleanupEveryFlag() string { return "media-cleanup-every" }
+
+// GetMediaCleanupEvery safely fetches the value for global configuration 'MediaCleanupEvery' field
+func GetMediaCleanupEvery() time.Duration { return global.GetMediaCleanupEvery() }
+
+// SetMediaCleanupEvery safely sets the value for global configuration 'MediaCleanupEvery' field
+func SetMediaCleanupEvery(v time.Duration) { global.SetMediaCleanupEvery(v) }
+
 // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
 func (st *ConfigState) GetStorageBackend() (v string) {
 	st.mutex.RLock()
diff --git a/internal/gotosocial/gotosocial.go b/internal/gotosocial/gotosocial.go
index 89d11c579..74cc09f65 100644
--- a/internal/gotosocial/gotosocial.go
+++ b/internal/gotosocial/gotosocial.go
@@ -20,62 +20,47 @@ package gotosocial
 import (
 	"context"
 
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/federation"
-	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 )
 
-// Server is the 'main' function of the gotosocial server, and the place where everything hangs together.
-// The logic of stopping and starting the entire server is contained here.
-type Server interface {
-	// Start starts up the gotosocial server. If something goes wrong
-	// while starting the server, then an error will be returned.
-	Start(context.Context) error
-	// Stop closes down the gotosocial server, first closing the router
-	// then the database. If something goes wrong while stopping, an
-	// error will be returned.
-	Stop(context.Context) error
+// Server represents a long-running
+// GoToSocial server instance.
+type Server struct {
+	db        db.DB
+	apiRouter router.Router
+	cleaner   *cleaner.Cleaner
 }
 
-// NewServer returns a new gotosocial server, initialized with the given configuration.
-// An error will be returned the caller if something goes wrong during initialization
-// eg., no db or storage connection, port for router already in use, etc.
+// NewServer returns a new
+// GoToSocial server instance.
 func NewServer(
 	db db.DB,
 	apiRouter router.Router,
-	federator *federation.Federator,
-	mediaManager *media.Manager,
-) (Server, error) {
-	return &gotosocial{
-		db:           db,
-		apiRouter:    apiRouter,
-		federator:    federator,
-		mediaManager: mediaManager,
-	}, nil
-}
-
-// gotosocial fulfils the gotosocial interface.
-type gotosocial struct {
-	db           db.DB
-	apiRouter    router.Router
-	federator    *federation.Federator
-	mediaManager *media.Manager
+	cleaner *cleaner.Cleaner,
+) *Server {
+	return &Server{
+		db:        db,
+		apiRouter: apiRouter,
+		cleaner:   cleaner,
+	}
 }
 
-// Start starts up the gotosocial server. If something goes wrong
-// while starting the server, then an error will be returned.
-func (gts *gotosocial) Start(ctx context.Context) error {
-	gts.apiRouter.Start()
-	return nil
+// Start starts up the GoToSocial server by starting the router,
+// then the cleaner. If something goes wrong while starting the
+// server, then an error will be returned.
+func (s *Server) Start(ctx context.Context) error {
+	s.apiRouter.Start()
+	return s.cleaner.ScheduleJobs()
 }
 
-// Stop closes down the gotosocial server, first closing the router,
-// then the media manager, then the database.
-// If something goes wrong while stopping, an error will be returned.
-func (gts *gotosocial) Stop(ctx context.Context) error {
-	if err := gts.apiRouter.Stop(ctx); err != nil {
+// Stop closes down the GoToSocial server, first closing the cleaner,
+// then the router, then the database. If something goes wrong while
+// stopping, an error will be returned.
+func (s *Server) Stop(ctx context.Context) error {
+	if err := s.apiRouter.Stop(ctx); err != nil {
 		return err
 	}
-	return gts.db.Close()
+	return s.db.Close()
 }
diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go
index 51429c11c..3093b3e36 100644
--- a/internal/processing/admin/admin.go
+++ b/internal/processing/admin/admin.go
@@ -45,10 +45,17 @@ func (p *Processor) Actions() *Actions {
 }
 
 // New returns a new admin processor.
-func New(state *state.State, converter *typeutils.Converter, mediaManager *media.Manager, transportController transport.Controller, emailSender email.Sender) Processor {
+func New(
+	state *state.State,
+	cleaner *cleaner.Cleaner,
+	converter *typeutils.Converter,
+	mediaManager *media.Manager,
+	transportController transport.Controller,
+	emailSender email.Sender,
+) Processor {
 	return Processor{
 		state:               state,
-		cleaner:             cleaner.New(state),
+		cleaner:             cleaner,
 		converter:           converter,
 		mediaManager:        mediaManager,
 		transportController: transportController,
diff --git a/internal/processing/admin/admin_test.go b/internal/processing/admin/admin_test.go
index a5a790763..614735ee1 100644
--- a/internal/processing/admin/admin_test.go
+++ b/internal/processing/admin/admin_test.go
@@ -19,6 +19,7 @@ package admin_test
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -105,6 +106,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
 
 	suite.processor = processing.NewProcessor(
+		cleaner.New(&suite.state),
 		suite.tc,
 		suite.federator,
 		suite.oauthServer,
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index 47f14a686..b571ff499 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -18,6 +18,7 @@
 package processing
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	mm "github.com/superseriousbusiness/gotosocial/internal/media"
@@ -126,6 +127,7 @@ func (p *Processor) Workers() *workers.Processor {
 
 // NewProcessor returns a new Processor.
 func NewProcessor(
+	cleaner *cleaner.Cleaner,
 	converter *typeutils.Converter,
 	federator *federation.Federator,
 	oauthServer oauth.Server,
@@ -156,7 +158,7 @@ func NewProcessor(
 	// Instantiate the rest of the sub
 	// processors + pin them to this struct.
 	processor.account = accountProcessor
-	processor.admin = admin.New(state, converter, mediaManager, federator.TransportController(), emailSender)
+	processor.admin = admin.New(state, cleaner, converter, mediaManager, federator.TransportController(), emailSender)
 	processor.fedi = fedi.New(state, converter, federator, filter)
 	processor.list = list.New(state, converter)
 	processor.markers = markers.New(state, converter)
diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
index 63d2c31fe..2e0baae96 100644
--- a/internal/processing/processor_test.go
+++ b/internal/processing/processor_test.go
@@ -21,6 +21,7 @@ import (
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -122,7 +123,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.emailSender = testrig.NewEmailSender("../../web/template/", nil)
 
-	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)
+	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)
 	suite.state.Workers.EnqueueClientAPI = suite.processor.Workers().EnqueueClientAPI
 	suite.state.Workers.EnqueueFediAPI = suite.processor.Workers().EnqueueFediAPI
 
diff --git a/internal/processing/workers/workers_test.go b/internal/processing/workers/workers_test.go
index 5712180f5..c97e9eeb8 100644
--- a/internal/processing/workers/workers_test.go
+++ b/internal/processing/workers/workers_test.go
@@ -21,6 +21,7 @@ import (
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -124,7 +125,7 @@ func (suite *WorkersTestSuite) SetupTest() {
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", nil)
 
-	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)
+	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)
 	suite.state.Workers.EnqueueClientAPI = suite.processor.Workers().EnqueueClientAPI
 	suite.state.Workers.EnqueueFediAPI = suite.processor.Workers().EnqueueFediAPI