summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
Diffstat (limited to 'internal')
-rw-r--r--internal/db/bundb/media.go26
-rw-r--r--internal/db/bundb/media_test.go9
-rw-r--r--internal/db/media.go5
-rw-r--r--internal/media/manager.go26
-rw-r--r--internal/media/pruneremote.go10
-rw-r--r--internal/media/pruneunusedlocal.go86
-rw-r--r--internal/media/pruneunusedlocal_test.go75
-rw-r--r--internal/media/util.go17
-rw-r--r--internal/processing/admin/mediaprune.go9
9 files changed, 253 insertions, 10 deletions
diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go
index fc3280ddf..71433b901 100644
--- a/internal/db/bundb/media.go
+++ b/internal/db/bundb/media.go
@@ -98,3 +98,29 @@ func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit
return attachments, nil
}
+
+func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
+ attachments := []*gtsmodel.MediaAttachment{}
+
+ q := m.newMediaQ(&attachments).
+ Where("media_attachment.cached = true").
+ Where("media_attachment.avatar = false").
+ Where("media_attachment.header = false").
+ Where("media_attachment.created_at < ?", olderThan).
+ Where("media_attachment.remote_url IS NULL").
+ Where("media_attachment.status_id IS NULL")
+
+ if maxID != "" {
+ q = q.Where("media_attachment.id < ?", maxID)
+ }
+
+ if limit != 0 {
+ q = q.Limit(limit)
+ }
+
+ if err := q.Scan(ctx); err != nil {
+ return nil, m.conn.ProcessError(err)
+ }
+
+ return attachments, nil
+}
diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go
index f1809b3fb..d6a4981f8 100644
--- a/internal/db/bundb/media_test.go
+++ b/internal/db/bundb/media_test.go
@@ -24,6 +24,7 @@ import (
"time"
"github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/testrig"
)
type MediaTestSuite struct {
@@ -51,6 +52,14 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
suite.Len(attachments, 2)
}
+func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() {
+ ctx := context.Background()
+
+ attachments, err := suite.db.GetLocalUnattachedOlderThan(ctx, testrig.TimeMustParse("2090-06-04T13:12:00Z"), "", 10)
+ suite.NoError(err)
+ suite.Len(attachments, 1)
+}
+
func TestMediaTestSuite(t *testing.T) {
suite.Run(t, new(MediaTestSuite))
}
diff --git a/internal/db/media.go b/internal/db/media.go
index 636fc61f2..2f9ed79dc 100644
--- a/internal/db/media.go
+++ b/internal/db/media.go
@@ -38,4 +38,9 @@ type Media interface {
// GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers
// and avis may be in use or not; the caller should check this if it's important.
GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
+ // GetLocalUnattachedOlderThan fetches limit n local media attachments, older than the given time, which
+ // aren't header or avatars, and aren't attached to a status. In other words, attachments which were uploaded
+ // but never used for whatever reason, or attachments that were attached to a status which was subsequently
+ // deleted.
+ GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
}
diff --git a/internal/media/manager.go b/internal/media/manager.go
index 663f74123..aacf607cc 100644
--- a/internal/media/manager.go
+++ b/internal/media/manager.go
@@ -34,6 +34,10 @@ import (
// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
const selectPruneLimit = 20
+// UnusedLocalAttachmentCacheDays is the amount of days to keep local media in storage if it
+// is not attached to a status, or was never attached to a status.
+const UnusedLocalAttachmentCacheDays = 3
+
// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs.
type Manager interface {
// ProcessMedia begins the process of decoding and storing the given data as an attachment.
@@ -75,11 +79,16 @@ type Manager interface {
//
// The returned int is the amount of media that was pruned by this function.
PruneAllRemote(ctx context.Context, olderThanDays int) (int, error)
- // PruneAllMeta prunes unused meta media -- currently, this means unused avatars + headers, but can also be extended
- // to include things like attachments that were uploaded on this server but left unused, etc.
+ // PruneAllMeta prunes unused/out of date headers and avatars cached on this instance.
//
// The returned int is the amount of media that was pruned by this function.
PruneAllMeta(ctx context.Context) (int, error)
+ // PruneUnusedLocalAttachments prunes unused media attachments that were uploaded by
+ // a user on this instance, but never actually attached to a status, or attached but
+ // later detached.
+ //
+ // The returned int is the amount of media that was pruned by this function.
+ PruneUnusedLocalAttachments(ctx context.Context) (int, error)
// Stop stops the underlying worker pool of the manager. It should be called
// when closing GoToSocial in order to cleanly finish any in-progress jobs.
@@ -210,6 +219,19 @@ func scheduleCleanupJobs(m *manager) error {
return fmt.Errorf("error starting media manager meta cleanup job: %s", err)
}
+ if _, err := c.AddFunc("@midnight", func() {
+ begin := time.Now()
+ pruned, err := m.PruneUnusedLocalAttachments(pruneCtx)
+ if err != nil {
+ logrus.Errorf("media manager: error pruning unused local attachments: %s", err)
+ return
+ }
+ logrus.Infof("media manager: pruned %d unused local attachments in %s", pruned, time.Since(begin))
+ }); err != nil {
+ pruneCancel()
+ return fmt.Errorf("error starting media manager unused local attachments cleanup job: %s", err)
+ }
+
// start remote cache cleanup cronjob if configured
if mediaRemoteCacheDays := config.GetMediaRemoteCacheDays(); mediaRemoteCacheDays > 0 {
if _, err := c.AddFunc("@midnight", func() {
diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go
index f7b77d32e..a01995740 100644
--- a/internal/media/pruneremote.go
+++ b/internal/media/pruneremote.go
@@ -21,7 +21,6 @@ package media
import (
"context"
"fmt"
- "time"
"codeberg.org/gruf/go-store/storage"
"github.com/sirupsen/logrus"
@@ -32,15 +31,10 @@ import (
func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) {
var totalPruned int
- // convert days into a duration string
- olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
- // parse the duration string into a duration
- olderThanHours, err := time.ParseDuration(olderThanHoursString)
+ olderThan, err := parseOlderThan(olderThanDays)
if err != nil {
- return totalPruned, fmt.Errorf("PruneAllRemote: %d", err)
+ return totalPruned, fmt.Errorf("PruneAllRemote: error parsing olderThanDays %d: %s", olderThanDays, err)
}
- // 'subtract' that from the time now to give our threshold
- olderThan := time.Now().Add(-olderThanHours)
logrus.Infof("PruneAllRemote: pruning media older than %s", olderThan)
// select 20 attachments at a time and prune them
diff --git a/internal/media/pruneunusedlocal.go b/internal/media/pruneunusedlocal.go
new file mode 100644
index 000000000..0c464e857
--- /dev/null
+++ b/internal/media/pruneunusedlocal.go
@@ -0,0 +1,86 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+ "context"
+ "fmt"
+
+ "codeberg.org/gruf/go-store/storage"
+ "github.com/sirupsen/logrus"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error) {
+ var totalPruned int
+ var maxID string
+ var attachments []*gtsmodel.MediaAttachment
+ var err error
+
+ olderThan, err := parseOlderThan(UnusedLocalAttachmentCacheDays)
+ if err != nil {
+ return totalPruned, fmt.Errorf("PruneUnusedLocalAttachments: error parsing olderThanDays %d: %s", UnusedLocalAttachmentCacheDays, err)
+ }
+ logrus.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan)
+
+ // select 20 attachments at a time and prune them
+ for attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, maxID, selectPruneLimit) {
+ // use the id of the last attachment in the slice as the next 'maxID' value
+ l := len(attachments)
+ maxID = attachments[l-1].ID
+ logrus.Tracef("PruneUnusedLocalAttachments: got %d unused local attachments older than %s with maxID < %s", l, olderThan, maxID)
+
+ for _, attachment := range attachments {
+ if err := m.pruneOneLocal(ctx, attachment); err != nil {
+ return totalPruned, err
+ }
+ totalPruned++
+ }
+ }
+
+ // make sure we don't have a real error when we leave the loop
+ if err != nil && err != db.ErrNoEntries {
+ return totalPruned, err
+ }
+
+ logrus.Infof("PruneUnusedLocalAttachments: finished pruning: pruned %d entries", totalPruned)
+ return totalPruned, nil
+}
+
+func (m *manager) pruneOneLocal(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+ if attachment.File.Path != "" {
+ // delete the full size attachment from storage
+ logrus.Tracef("pruneOneLocal: deleting %s", attachment.File.Path)
+ if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ if attachment.Thumbnail.Path != "" {
+ // delete the thumbnail from storage
+ logrus.Tracef("pruneOneLocal: deleting %s", attachment.Thumbnail.Path)
+ if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+ return err
+ }
+ }
+
+ // delete the attachment completely
+ return m.db.DeleteByID(ctx, attachment.ID, attachment)
+}
diff --git a/internal/media/pruneunusedlocal_test.go b/internal/media/pruneunusedlocal_test.go
new file mode 100644
index 000000000..5f6bfbfba
--- /dev/null
+++ b/internal/media/pruneunusedlocal_test.go
@@ -0,0 +1,75 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+type PruneUnusedLocalTestSuite struct {
+ MediaStandardTestSuite
+}
+
+func (suite *PruneUnusedLocalTestSuite) TestPruneUnusedLocal() {
+ testAttachment := suite.testAttachments["local_account_1_unattached_1"]
+ suite.True(testAttachment.Cached)
+
+ totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background())
+ suite.NoError(err)
+ suite.Equal(1, totalPruned)
+
+ _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+ suite.ErrorIs(err, db.ErrNoEntries)
+}
+
+func (suite *PruneUnusedLocalTestSuite) TestPruneRemoteTwice() {
+ totalPruned, err := suite.manager.PruneUnusedLocalAttachments(context.Background())
+ suite.NoError(err)
+ suite.Equal(1, totalPruned)
+
+ // final prune should prune nothing, since the first prune already happened
+ totalPrunedAgain, err := suite.manager.PruneUnusedLocalAttachments(context.Background())
+ suite.NoError(err)
+ suite.Equal(0, totalPrunedAgain)
+}
+
+func (suite *PruneUnusedLocalTestSuite) TestPruneOneNonExistent() {
+ ctx := context.Background()
+ testAttachment := suite.testAttachments["local_account_1_unattached_1"]
+
+ // Delete this attachment cached on disk
+ media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
+ suite.NoError(err)
+ suite.True(media.Cached)
+ err = suite.storage.Delete(media.File.Path)
+ suite.NoError(err)
+
+ // Now attempt to prune for item with db entry no file
+ totalPruned, err := suite.manager.PruneUnusedLocalAttachments(ctx)
+ suite.NoError(err)
+ suite.Equal(1, totalPruned)
+}
+
+func TestPruneUnusedLocalTestSuite(t *testing.T) {
+ suite.Run(t, &PruneUnusedLocalTestSuite{})
+}
diff --git a/internal/media/util.go b/internal/media/util.go
index 6dfcede89..9d62619f5 100644
--- a/internal/media/util.go
+++ b/internal/media/util.go
@@ -21,6 +21,7 @@ package media
import (
"errors"
"fmt"
+ "time"
"github.com/h2non/filetype"
"github.com/sirupsen/logrus"
@@ -128,3 +129,19 @@ func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) {
func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) {
logrus.Error("media manager cron logger: ", err, msg, keysAndValues)
}
+
+func parseOlderThan(olderThanDays int) (time.Time, error) {
+ // convert days into a duration string
+ olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
+
+ // parse the duration string into a duration
+ olderThanHours, err := time.ParseDuration(olderThanHoursString)
+ if err != nil {
+ return time.Time{}, err
+ }
+
+ // 'subtract' that from the time now to give our threshold
+ olderThan := time.Now().Add(-olderThanHours)
+
+ return olderThan, nil
+}
diff --git a/internal/processing/admin/mediaprune.go b/internal/processing/admin/mediaprune.go
index 0e6abe028..1c3398b78 100644
--- a/internal/processing/admin/mediaprune.go
+++ b/internal/processing/admin/mediaprune.go
@@ -42,6 +42,15 @@ func (p *processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gt
}()
go func() {
+ pruned, err := p.mediaManager.PruneUnusedLocalAttachments(ctx)
+ if err != nil {
+ logrus.Errorf("MediaPrune: error pruning unused local cache: %s", err)
+ } else {
+ logrus.Infof("MediaPrune: pruned %d unused local cache entries", pruned)
+ }
+ }()
+
+ go func() {
pruned, err := p.mediaManager.PruneAllMeta(ctx)
if err != nil {
logrus.Errorf("MediaPrune: error pruning meta: %s", err)
generated by cgit v1.2.3 (git 2.46.0) at 2025-11-04 04:23:38 +0000