summaryrefslogtreecommitdiff
path: root/internal/media
diff options
context:
space:
mode:
Diffstat (limited to 'internal/media')
-rw-r--r--internal/media/manager.go122
-rw-r--r--internal/media/media_test.go2
-rw-r--r--internal/media/prunemeta.go87
-rw-r--r--internal/media/prunemeta_test.go131
-rw-r--r--internal/media/pruneremote.go21
-rw-r--r--internal/media/pruneremote_test.go10
6 files changed, 305 insertions, 68 deletions
diff --git a/internal/media/manager.go b/internal/media/manager.go
index 5b4a01021..60290e4ff 100644
--- a/internal/media/manager.go
+++ b/internal/media/manager.go
@@ -32,6 +32,9 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
)
+// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
+const selectPruneLimit = 20
+
// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs.
type Manager interface {
// ProcessMedia begins the process of decoding and storing the given data as an attachment.
@@ -66,10 +69,19 @@ type Manager interface {
ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error)
// RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote.
RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error)
- // PruneRemote prunes all remote media cached on this instance that's older than the given amount of days.
+
+ // PruneAllRemote prunes all remote media attachments cached on this instance which are older than the given amount of days.
// 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size),
// and setting 'cached' to false on the associated attachment.
- PruneRemote(ctx context.Context, olderThanDays int) (int, error)
+ //
+ // The returned int is the amount of media that was pruned by this function.
+ PruneAllRemote(ctx context.Context, olderThanDays int) (int, error)
+ // PruneAllMeta prunes unused meta media -- currently, this means unused avatars + headers, but can also be extended
+ // to include things like attachments that were uploaded on this server but left unused, etc.
+ //
+ // The returned int is the amount of media that was pruned by this function.
+ PruneAllMeta(ctx context.Context) (int, error)
+
// Stop stops the underlying worker pool of the manager. It should be called
// when closing GoToSocial in order to cleanly finish any in-progress jobs.
// It will block until workers are finished processing.
@@ -128,53 +140,8 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
return nil, err
}
- // start remote cache cleanup cronjob if configured
- cacheCleanupDays := viper.GetInt(config.Keys.MediaRemoteCacheDays)
- if cacheCleanupDays != 0 {
- // we need a way of cancelling running jobs if the media manager is told to stop
- pruneCtx, pruneCancel := context.WithCancel(context.Background())
-
- // create a new cron instance and add a function to it
- c := cron.New(cron.WithLogger(&logrusWrapper{}))
-
- pruneFunc := func() {
- begin := time.Now()
- pruned, err := m.PruneRemote(pruneCtx, cacheCleanupDays)
- if err != nil {
- logrus.Errorf("media manager: error pruning remote cache: %s", err)
- return
- }
- logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
- }
-
- // run every night
- entryID, err := c.AddFunc("@midnight", pruneFunc)
- if err != nil {
- pruneCancel()
- return nil, fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
- }
-
- // since we're running a cron job, we should define how the manager should stop them
- m.stopCronJobs = func() error {
- // try to stop any jobs gracefully by waiting til they're finished
- cronCtx := c.Stop()
-
- select {
- case <-cronCtx.Done():
- logrus.Infof("media manager: cron finished jobs and stopped gracefully")
- case <-time.After(1 * time.Minute):
- logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close")
- break
- }
-
- // whether the job is finished neatly or we had to wait a minute, cancel the context on the prune job
- pruneCancel()
- return nil
- }
-
- // now start all the cron stuff we've lined up
- c.Start()
- logrus.Infof("media manager: next scheduled remote cache cleanup is %q", c.Entry(entryID).Next)
+ if err := scheduleCleanupJobs(m); err != nil {
+ return nil, err
}
return m, nil
@@ -213,9 +180,7 @@ func (m *manager) Stop() error {
emojiErr := m.emojiWorker.Stop()
var cronErr error
-
if m.stopCronJobs != nil {
- // only set if cache prune age > 0
cronErr = m.stopCronJobs()
}
@@ -224,5 +189,60 @@ func (m *manager) Stop() error {
} else if emojiErr != nil {
return emojiErr
}
+
return cronErr
}
+
+func scheduleCleanupJobs(m *manager) error {
+ // create a new cron instance for scheduling cleanup jobs
+ c := cron.New(cron.WithLogger(&logrusWrapper{}))
+ pruneCtx, pruneCancel := context.WithCancel(context.Background())
+
+ if _, err := c.AddFunc("@midnight", func() {
+ begin := time.Now()
+ pruned, err := m.PruneAllMeta(pruneCtx)
+ if err != nil {
+ logrus.Errorf("media manager: error pruning meta: %s", err)
+ return
+ }
+ logrus.Infof("media manager: pruned %d meta entries in %s", pruned, time.Since(begin))
+ }); err != nil {
+ pruneCancel()
+ return fmt.Errorf("error starting media manager meta cleanup job: %s", err)
+ }
+
+ // start remote cache cleanup cronjob if configured
+ if mediaRemoteCacheDays := viper.GetInt(config.Keys.MediaRemoteCacheDays); mediaRemoteCacheDays > 0 {
+ if _, err := c.AddFunc("@midnight", func() {
+ begin := time.Now()
+ pruned, err := m.PruneAllRemote(pruneCtx, mediaRemoteCacheDays)
+ if err != nil {
+ logrus.Errorf("media manager: error pruning remote cache: %s", err)
+ return
+ }
+ logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
+ }); err != nil {
+ pruneCancel()
+ return fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
+ }
+ }
+
+ // try to stop any jobs gracefully by waiting til they're finished
+ m.stopCronJobs = func() error {
+ cronCtx := c.Stop()
+
+ select {
+ case <-cronCtx.Done():
+ logrus.Infof("media manager: cron finished jobs and stopped gracefully")
+ case <-time.After(1 * time.Minute):
+ logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close jobs")
+ break
+ }
+
+ pruneCancel()
+ return nil
+ }
+
+ c.Start()
+ return nil
+}
diff --git a/internal/media/media_test.go b/internal/media/media_test.go
index ee0fd8eea..1b5011801 100644
--- a/internal/media/media_test.go
+++ b/internal/media/media_test.go
@@ -34,6 +34,7 @@ type MediaStandardTestSuite struct {
storage *kv.KVStore
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
+ testAccounts map[string]*gtsmodel.Account
}
func (suite *MediaStandardTestSuite) SetupSuite() {
@@ -48,6 +49,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
testrig.StandardDBSetup(suite.db, nil)
suite.testAttachments = testrig.NewTestAttachments()
+ suite.testAccounts = testrig.NewTestAccounts()
suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)
}
diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go
new file mode 100644
index 000000000..aa838d2a4
--- /dev/null
+++ b/internal/media/prunemeta.go
@@ -0,0 +1,87 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+ "context"
+
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/sirupsen/logrus"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
+ var totalPruned int
+ var maxID string
+ var attachments []*gtsmodel.MediaAttachment
+ var err error
+
+ // select 20 attachments at a time and prune them
+ for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) {
+ // use the id of the last attachment in the slice as the next 'maxID' value
+ l := len(attachments)
+ logrus.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID)
+ maxID = attachments[l-1].ID
+
+ // prune each attachment that meets one of the following criteria:
+ // - has no owning account in the database
+ // - is a header but isn't the owning account's current header
+ // - is an avatar but isn't the owning account's current avatar
+ for _, attachment := range attachments {
+ if attachment.Account == nil ||
+ (attachment.Header && attachment.ID != attachment.Account.HeaderMediaAttachmentID) ||
+ (attachment.Avatar && attachment.ID != attachment.Account.AvatarMediaAttachmentID) {
+ if err := m.pruneOneAvatarOrHeader(ctx, attachment); err != nil {
+ return totalPruned, err
+ }
+ totalPruned++
+ }
+ }
+ }
+
+ // make sure we don't have a real error when we leave the loop
+ if err != nil && err != db.ErrNoEntries {
+ return totalPruned, err
+ }
+
+ logrus.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned)
+ return totalPruned, nil
+}
+
+func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+ if attachment.File.Path != "" {
+ // delete the full size attachment from storage
+ logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
+ if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ if attachment.Thumbnail.Path != "" {
+ // delete the thumbnail from storage
+ logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
+ if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ // delete the attachment entry completely
+ return m.db.DeleteByID(ctx, attachment.ID, &gtsmodel.MediaAttachment{})
+}
diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go
new file mode 100644
index 000000000..1358208a8
--- /dev/null
+++ b/internal/media/prunemeta_test.go
@@ -0,0 +1,131 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+ "context"
+ "testing"
+
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+type PruneMetaTestSuite struct {
+ MediaStandardTestSuite
+}
+
+func (suite *PruneMetaTestSuite) TestPruneMeta() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+ zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // media should no longer be stored
+ _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+
+ // attachments should no longer be in the db
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+}
+
+func (suite *PruneMetaTestSuite) TestPruneMetaTwice() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // final prune should prune nothing, since the first prune already happened
+ totalPruned, err = suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(0, totalPruned)
+}
+func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
+ ctx := context.Background()
+
+ // start by clearing zork's avatar + header
+ zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+ zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+ zork := suite.testAccounts["local_account_1"]
+ zork.AvatarMediaAttachmentID = ""
+ zork.HeaderMediaAttachmentID = ""
+ if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil {
+ panic(err)
+ }
+
+ // set zork's unused header as belonging to turtle
+ turtle := suite.testAccounts["local_account_1"]
+ zorkOldHeader.AccountID = turtle.ID
+ if err := suite.db.UpdateByPrimaryKey(ctx, zorkOldHeader); err != nil {
+ panic(err)
+ }
+
+ totalPruned, err := suite.manager.PruneAllMeta(ctx)
+ suite.NoError(err)
+ suite.Equal(2, totalPruned)
+
+ // media should no longer be stored
+ _, err = suite.storage.Get(zorkOldAvatar.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.File.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+ _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
+ suite.ErrorIs(err, storage.ErrNotFound)
+
+ // attachments should no longer be in the db
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+ _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+}
+
+func TestPruneMetaTestSuite(t *testing.T) {
+ suite.Run(t, &PruneMetaTestSuite{})
+}
diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go
index 372f7bbb9..f7b77d32e 100644
--- a/internal/media/pruneremote.go
+++ b/internal/media/pruneremote.go
@@ -29,10 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
-// amount of media attachments to select at a time from the db when pruning
-const selectPruneLimit = 20
-
-func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, error) {
+func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) {
var totalPruned int
// convert days into a duration string
@@ -40,23 +37,23 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro
// parse the duration string into a duration
olderThanHours, err := time.ParseDuration(olderThanHoursString)
if err != nil {
- return totalPruned, fmt.Errorf("PruneRemote: %d", err)
+ return totalPruned, fmt.Errorf("PruneAllRemote: %d", err)
}
// 'subtract' that from the time now to give our threshold
olderThan := time.Now().Add(-olderThanHours)
- logrus.Infof("PruneRemote: pruning media older than %s", olderThan)
+ logrus.Infof("PruneAllRemote: pruning media older than %s", olderThan)
// select 20 attachments at a time and prune them
for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
l := len(attachments)
- logrus.Tracef("PruneRemote: got %d attachments older than %s", l, olderThan)
+ logrus.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan)
olderThan = attachments[l-1].CreatedAt
// prune each attachment
for _, attachment := range attachments {
- if err := m.PruneOne(ctx, attachment); err != nil {
+ if err := m.pruneOneRemote(ctx, attachment); err != nil {
return totalPruned, err
}
totalPruned++
@@ -68,14 +65,14 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro
return totalPruned, err
}
- logrus.Infof("PruneRemote: finished pruning remote media: pruned %d entries", totalPruned)
+ logrus.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned)
return totalPruned, nil
}
-func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
if attachment.File.Path != "" {
// delete the full size attachment from storage
- logrus.Tracef("PruneOne: deleting %s", attachment.File.Path)
+ logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
@@ -84,7 +81,7 @@ func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttach
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
- logrus.Tracef("PruneOne: deleting %s", attachment.Thumbnail.Path)
+ logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go
index c9d040a6f..31c5128ff 100644
--- a/internal/media/pruneremote_test.go
+++ b/internal/media/pruneremote_test.go
@@ -37,7 +37,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
suite.True(testAttachment.Cached)
- totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
@@ -49,12 +49,12 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
}
func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
- totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
// final prune should prune nothing, since the first prune already happened
- totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
+ totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(0, totalPrunedAgain)
}
@@ -63,7 +63,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
ctx := context.Background()
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
- totalPruned, err := suite.manager.PruneRemote(ctx, 1)
+ totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
@@ -116,7 +116,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
suite.NoError(err)
// Now attempt to prune remote for item with db entry no file
- totalPruned, err := suite.manager.PruneRemote(ctx, 1)
+ totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
suite.NoError(err)
suite.Equal(2, totalPruned)
}
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-25 07:40:30 +0000