diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/api/wellknown/webfinger/webfingerget_test.go | 3 | ||||
| -rw-r--r-- | internal/cleaner/cleaner.go | 58 | ||||
| -rw-r--r-- | internal/config/config.go | 2 | ||||
| -rw-r--r-- | internal/config/defaults.go | 2 | ||||
| -rw-r--r-- | internal/config/flags.go | 2 | ||||
| -rw-r--r-- | internal/config/helpers.gen.go | 50 | ||||
| -rw-r--r-- | internal/gotosocial/gotosocial.go | 71 | ||||
| -rw-r--r-- | internal/processing/admin/admin.go | 11 | ||||
| -rw-r--r-- | internal/processing/admin/admin_test.go | 2 | ||||
| -rw-r--r-- | internal/processing/processor.go | 4 | ||||
| -rw-r--r-- | internal/processing/processor_test.go | 3 | ||||
| -rw-r--r-- | internal/processing/workers/workers_test.go | 3 | 
12 files changed, 151 insertions, 60 deletions
diff --git a/internal/api/wellknown/webfinger/webfingerget_test.go b/internal/api/wellknown/webfinger/webfingerget_test.go index fb450470f..6b3e2cc5d 100644 --- a/internal/api/wellknown/webfinger/webfingerget_test.go +++ b/internal/api/wellknown/webfinger/webfingerget_test.go @@ -33,6 +33,7 @@ import (  	"github.com/superseriousbusiness/gotosocial/internal/ap"  	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"  	"github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger" +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/config"  	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"  	"github.com/superseriousbusiness/gotosocial/internal/processing" @@ -82,7 +83,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom  	// to new host + account domain.  	config.SetHost(host)  	config.SetAccountDomain(accountDomain) -	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(&suite.state), &suite.state, suite.emailSender) +	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(&suite.state), &suite.state, suite.emailSender)  	suite.webfingerModule = webfinger.New(suite.processor)  	// Generate a new account for the diff --git a/internal/cleaner/cleaner.go b/internal/cleaner/cleaner.go index 31766bae6..1139a85bb 100644 --- a/internal/cleaner/cleaner.go +++ b/internal/cleaner/cleaner.go @@ -47,7 +47,6 @@ func New(state *state.State) *Cleaner {  	c.state = state  	c.emoji.Cleaner = c  	c.media.Cleaner = c -	scheduleJobs(c)  	return c  } @@ -109,16 +108,46 @@ func (c *Cleaner) removeFiles(ctx context.Context, files ...string) (int, error)  	return diff, nil  } -func scheduleJobs(c *Cleaner) { -	const day = time.Hour * 24 +// ScheduleJobs schedules cleaning +// jobs using configured parameters. +// +// Returns an error if `MediaCleanupFrom` +// is not a valid format (hh:mm:ss). +func (c *Cleaner) ScheduleJobs() error { +	const hourMinute = "15:04" + +	var ( +		now            = time.Now() +		cleanupEvery   = config.GetMediaCleanupEvery() +		cleanupFromStr = config.GetMediaCleanupFrom() +	) + +	// Parse cleanupFromStr as hh:mm. +	// Resulting time will be on 1 Jan year zero. +	cleanupFrom, err := time.Parse(hourMinute, cleanupFromStr) +	if err != nil { +		return gtserror.Newf( +			"error parsing '%s' in time format 'hh:mm': %w", +			cleanupFromStr, err, +		) +	} -	// Calculate closest midnight. -	now := time.Now() -	midnight := now.Round(day) +	// Time travel from +	// year zero, groovy. +	firstCleanupAt := time.Date( +		now.Year(), +		now.Month(), +		now.Day(), +		cleanupFrom.Hour(), +		cleanupFrom.Minute(), +		0, +		0, +		now.Location(), +	) -	if midnight.Before(now) { -		// since <= 11:59am rounds down. -		midnight = midnight.Add(day) +	// Ensure first cleanup is in the future. +	for firstCleanupAt.Before(now) { +		firstCleanupAt = firstCleanupAt.Add(cleanupEvery)  	}  	// Get ctx associated with scheduler run state. @@ -129,11 +158,18 @@ func scheduleJobs(c *Cleaner) {  	// jobs restartable if we want to implement reloads in  	// the future that make call to Workers.Stop() -> Workers.Start(). -	// Schedule the cleaning tasks to execute every day at midnight. +	log.Infof(nil, +		"scheduling media clean to run every %s, starting from %s; next clean will run at %s", +		cleanupEvery, cleanupFromStr, firstCleanupAt, +	) + +	// Schedule the cleaning tasks to execute according to given schedule.  	c.state.Workers.Scheduler.Schedule(sched.NewJob(func(start time.Time) {  		log.Info(nil, "starting media clean")  		c.Media().All(doneCtx, config.GetMediaRemoteCacheDays())  		c.Emoji().All(doneCtx, config.GetMediaRemoteCacheDays())  		log.Infof(nil, "finished media clean after %s", time.Since(start)) -	}).EveryAt(midnight, day)) +	}).EveryAt(firstCleanupAt, cleanupEvery)) + +	return nil  } diff --git a/internal/config/config.go b/internal/config/config.go index a9fdef3c7..77e70185c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,6 +97,8 @@ type Configuration struct {  	MediaRemoteCacheDays     int           `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`  	MediaEmojiLocalMaxSize   bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`  	MediaEmojiRemoteMaxSize  bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."` +	MediaCleanupFrom         string        `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."` +	MediaCleanupEvery        time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`  	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`  	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 6ee52d162..0c2556e9d 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -76,6 +76,8 @@ var Defaults = Configuration{  	MediaRemoteCacheDays:     7,  	MediaEmojiLocalMaxSize:   50 * bytesize.KiB,  	MediaEmojiRemoteMaxSize:  100 * bytesize.KiB, +	MediaCleanupFrom:         "00:00",        // Midnight. +	MediaCleanupEvery:        24 * time.Hour, // 1/day.  	StorageBackend:       "local",  	StorageLocalBasePath: "/gotosocial/storage", diff --git a/internal/config/flags.go b/internal/config/flags.go index 29e0726a6..b29d0fe04 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -103,6 +103,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {  		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))  		cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))  		cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage")) +		cmd.Flags().String(MediaCleanupFromFlag(), cfg.MediaCleanupFrom, fieldtag("MediaCleanupFrom", "usage")) +		cmd.Flags().Duration(MediaCleanupEveryFlag(), cfg.MediaCleanupEvery, fieldtag("MediaCleanupEvery", "usage"))  		// Storage  		cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 80687eb66..415035bea 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1224,6 +1224,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem  // SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field  func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) } +// GetMediaCleanupFrom safely fetches the Configuration value for state's 'MediaCleanupFrom' field +func (st *ConfigState) GetMediaCleanupFrom() (v string) { +	st.mutex.RLock() +	v = st.config.MediaCleanupFrom +	st.mutex.RUnlock() +	return +} + +// SetMediaCleanupFrom safely sets the Configuration value for state's 'MediaCleanupFrom' field +func (st *ConfigState) SetMediaCleanupFrom(v string) { +	st.mutex.Lock() +	defer st.mutex.Unlock() +	st.config.MediaCleanupFrom = v +	st.reloadToViper() +} + +// MediaCleanupFromFlag returns the flag name for the 'MediaCleanupFrom' field +func MediaCleanupFromFlag() string { return "media-cleanup-from" } + +// GetMediaCleanupFrom safely fetches the value for global configuration 'MediaCleanupFrom' field +func GetMediaCleanupFrom() string { return global.GetMediaCleanupFrom() } + +// SetMediaCleanupFrom safely sets the value for global configuration 'MediaCleanupFrom' field +func SetMediaCleanupFrom(v string) { global.SetMediaCleanupFrom(v) } + +// GetMediaCleanupEvery safely fetches the Configuration value for state's 'MediaCleanupEvery' field +func (st *ConfigState) GetMediaCleanupEvery() (v time.Duration) { +	st.mutex.RLock() +	v = st.config.MediaCleanupEvery +	st.mutex.RUnlock() +	return +} + +// SetMediaCleanupEvery safely sets the Configuration value for state's 'MediaCleanupEvery' field +func (st *ConfigState) SetMediaCleanupEvery(v time.Duration) { +	st.mutex.Lock() +	defer st.mutex.Unlock() +	st.config.MediaCleanupEvery = v +	st.reloadToViper() +} + +// MediaCleanupEveryFlag returns the flag name for the 'MediaCleanupEvery' field +func MediaCleanupEveryFlag() string { return "media-cleanup-every" } + +// GetMediaCleanupEvery safely fetches the value for global configuration 'MediaCleanupEvery' field +func GetMediaCleanupEvery() time.Duration { return global.GetMediaCleanupEvery() } + +// SetMediaCleanupEvery safely sets the value for global configuration 'MediaCleanupEvery' field +func SetMediaCleanupEvery(v time.Duration) { global.SetMediaCleanupEvery(v) } +  // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field  func (st *ConfigState) GetStorageBackend() (v string) {  	st.mutex.RLock() diff --git a/internal/gotosocial/gotosocial.go b/internal/gotosocial/gotosocial.go index 89d11c579..74cc09f65 100644 --- a/internal/gotosocial/gotosocial.go +++ b/internal/gotosocial/gotosocial.go @@ -20,62 +20,47 @@ package gotosocial  import (  	"context" +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/db" -	"github.com/superseriousbusiness/gotosocial/internal/federation" -	"github.com/superseriousbusiness/gotosocial/internal/media"  	"github.com/superseriousbusiness/gotosocial/internal/router"  ) -// Server is the 'main' function of the gotosocial server, and the place where everything hangs together. -// The logic of stopping and starting the entire server is contained here. -type Server interface { -	// Start starts up the gotosocial server. If something goes wrong -	// while starting the server, then an error will be returned. -	Start(context.Context) error -	// Stop closes down the gotosocial server, first closing the router -	// then the database. If something goes wrong while stopping, an -	// error will be returned. -	Stop(context.Context) error +// Server represents a long-running +// GoToSocial server instance. +type Server struct { +	db        db.DB +	apiRouter router.Router +	cleaner   *cleaner.Cleaner  } -// NewServer returns a new gotosocial server, initialized with the given configuration. -// An error will be returned the caller if something goes wrong during initialization -// eg., no db or storage connection, port for router already in use, etc. +// NewServer returns a new +// GoToSocial server instance.  func NewServer(  	db db.DB,  	apiRouter router.Router, -	federator *federation.Federator, -	mediaManager *media.Manager, -) (Server, error) { -	return &gotosocial{ -		db:           db, -		apiRouter:    apiRouter, -		federator:    federator, -		mediaManager: mediaManager, -	}, nil -} - -// gotosocial fulfils the gotosocial interface. -type gotosocial struct { -	db           db.DB -	apiRouter    router.Router -	federator    *federation.Federator -	mediaManager *media.Manager +	cleaner *cleaner.Cleaner, +) *Server { +	return &Server{ +		db:        db, +		apiRouter: apiRouter, +		cleaner:   cleaner, +	}  } -// Start starts up the gotosocial server. If something goes wrong -// while starting the server, then an error will be returned. -func (gts *gotosocial) Start(ctx context.Context) error { -	gts.apiRouter.Start() -	return nil +// Start starts up the GoToSocial server by starting the router, +// then the cleaner. If something goes wrong while starting the +// server, then an error will be returned. +func (s *Server) Start(ctx context.Context) error { +	s.apiRouter.Start() +	return s.cleaner.ScheduleJobs()  } -// Stop closes down the gotosocial server, first closing the router, -// then the media manager, then the database. -// If something goes wrong while stopping, an error will be returned. -func (gts *gotosocial) Stop(ctx context.Context) error { -	if err := gts.apiRouter.Stop(ctx); err != nil { +// Stop closes down the GoToSocial server, first closing the cleaner, +// then the router, then the database. If something goes wrong while +// stopping, an error will be returned. +func (s *Server) Stop(ctx context.Context) error { +	if err := s.apiRouter.Stop(ctx); err != nil {  		return err  	} -	return gts.db.Close() +	return s.db.Close()  } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 51429c11c..3093b3e36 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -45,10 +45,17 @@ func (p *Processor) Actions() *Actions {  }  // New returns a new admin processor. -func New(state *state.State, converter *typeutils.Converter, mediaManager *media.Manager, transportController transport.Controller, emailSender email.Sender) Processor { +func New( +	state *state.State, +	cleaner *cleaner.Cleaner, +	converter *typeutils.Converter, +	mediaManager *media.Manager, +	transportController transport.Controller, +	emailSender email.Sender, +) Processor {  	return Processor{  		state:               state, -		cleaner:             cleaner.New(state), +		cleaner:             cleaner,  		converter:           converter,  		mediaManager:        mediaManager,  		transportController: transportController, diff --git a/internal/processing/admin/admin_test.go b/internal/processing/admin/admin_test.go index a5a790763..614735ee1 100644 --- a/internal/processing/admin/admin_test.go +++ b/internal/processing/admin/admin_test.go @@ -19,6 +19,7 @@ package admin_test  import (  	"github.com/stretchr/testify/suite" +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/db"  	"github.com/superseriousbusiness/gotosocial/internal/email"  	"github.com/superseriousbusiness/gotosocial/internal/federation" @@ -105,6 +106,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {  	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)  	suite.processor = processing.NewProcessor( +		cleaner.New(&suite.state),  		suite.tc,  		suite.federator,  		suite.oauthServer, diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 47f14a686..b571ff499 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -18,6 +18,7 @@  package processing  import ( +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/email"  	"github.com/superseriousbusiness/gotosocial/internal/federation"  	mm "github.com/superseriousbusiness/gotosocial/internal/media" @@ -126,6 +127,7 @@ func (p *Processor) Workers() *workers.Processor {  // NewProcessor returns a new Processor.  func NewProcessor( +	cleaner *cleaner.Cleaner,  	converter *typeutils.Converter,  	federator *federation.Federator,  	oauthServer oauth.Server, @@ -156,7 +158,7 @@ func NewProcessor(  	// Instantiate the rest of the sub  	// processors + pin them to this struct.  	processor.account = accountProcessor -	processor.admin = admin.New(state, converter, mediaManager, federator.TransportController(), emailSender) +	processor.admin = admin.New(state, cleaner, converter, mediaManager, federator.TransportController(), emailSender)  	processor.fedi = fedi.New(state, converter, federator, filter)  	processor.list = list.New(state, converter)  	processor.markers = markers.New(state, converter) diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index 63d2c31fe..2e0baae96 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -21,6 +21,7 @@ import (  	"context"  	"github.com/stretchr/testify/suite" +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/db"  	"github.com/superseriousbusiness/gotosocial/internal/email"  	"github.com/superseriousbusiness/gotosocial/internal/federation" @@ -122,7 +123,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {  	suite.oauthServer = testrig.NewTestOauthServer(suite.db)  	suite.emailSender = testrig.NewEmailSender("../../web/template/", nil) -	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender) +	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)  	suite.state.Workers.EnqueueClientAPI = suite.processor.Workers().EnqueueClientAPI  	suite.state.Workers.EnqueueFediAPI = suite.processor.Workers().EnqueueFediAPI diff --git a/internal/processing/workers/workers_test.go b/internal/processing/workers/workers_test.go index 5712180f5..c97e9eeb8 100644 --- a/internal/processing/workers/workers_test.go +++ b/internal/processing/workers/workers_test.go @@ -21,6 +21,7 @@ import (  	"context"  	"github.com/stretchr/testify/suite" +	"github.com/superseriousbusiness/gotosocial/internal/cleaner"  	"github.com/superseriousbusiness/gotosocial/internal/db"  	"github.com/superseriousbusiness/gotosocial/internal/email"  	"github.com/superseriousbusiness/gotosocial/internal/federation" @@ -124,7 +125,7 @@ func (suite *WorkersTestSuite) SetupTest() {  	suite.oauthServer = testrig.NewTestOauthServer(suite.db)  	suite.emailSender = testrig.NewEmailSender("../../../web/template/", nil) -	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender) +	suite.processor = processing.NewProcessor(cleaner.New(&suite.state), suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, &suite.state, suite.emailSender)  	suite.state.Workers.EnqueueClientAPI = suite.processor.Workers().EnqueueClientAPI  	suite.state.Workers.EnqueueFediAPI = suite.processor.Workers().EnqueueFediAPI  | 
