[bugfix/chore] oauth entropy fix + media cleanup tasks rewrite (#1853)

1 files changed, 427 insertions, 0 deletions

diff --git a/internal/cleaner/media_test.go b/internal/cleaner/media_test.go
new file mode 100644
index 000000000..824df2ca5
--- /dev/null
+++ b/internal/cleaner/media_test.go
@@ -0,0 +1,427 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cleaner_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/visibility"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type MediaTestSuite struct {
+	suite.Suite
+
+	db                  db.DB
+	storage             *storage.Driver
+	state               state.State
+	manager             *media.Manager
+	cleaner             *cleaner.Cleaner
+	transportController transport.Controller
+	testAttachments     map[string]*gtsmodel.MediaAttachment
+	testAccounts        map[string]*gtsmodel.Account
+	testEmojis          map[string]*gtsmodel.Emoji
+}
+
+func TestMediaTestSuite(t *testing.T) {
+	suite.Run(t, &MediaTestSuite{})
+}
+
+func (suite *MediaTestSuite) SetupTest() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+
+	suite.state.Caches.Init()
+	testrig.StartWorkers(&suite.state)
+
+	suite.db = testrig.NewTestDB(&suite.state)
+	suite.storage = testrig.NewInMemoryStorage()
+	suite.state.DB = suite.db
+	suite.state.Storage = suite.storage
+
+	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
+	testrig.StandardDBSetup(suite.db, nil)
+
+	testrig.StartTimelines(
+		&suite.state,
+		visibility.NewFilter(&suite.state),
+		testrig.NewTestTypeConverter(suite.db),
+	)
+
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testEmojis = testrig.NewTestEmojis()
+	suite.manager = testrig.NewTestMediaManager(&suite.state)
+	suite.cleaner = cleaner.New(&suite.state)
+	suite.transportController = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../testrig/media"))
+}
+
+func (suite *MediaTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+	testrig.StopWorkers(&suite.state)
+}
+
+// func (suite *MediaTestSuite) TestPruneOrphanedDry() {
+// 	// add a big orphan panda to store
+// 	b, err := os.ReadFile("../media/test/big-panda.gif")
+// 	if err != nil {
+// 		suite.FailNow(err.Error())
+// 	}
+
+// 	pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
+// 	if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil {
+// 		suite.FailNow(err.Error())
+// 	}
+
+// 	ctx := context.Background()
+
+// 	// dry run should show up 1 orphaned panda
+// 	totalPruned, err := suite.cleaner.Media().PruneOrphaned(gtscontext.SetDryRun(ctx))
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+
+// 	// panda should still be in storage
+// 	hasKey, err := suite.storage.Has(ctx, pandaPath)
+// 	suite.NoError(err)
+// 	suite.True(hasKey)
+// }
+
+// func (suite *MediaTestSuite) TestPruneOrphanedMoist() {
+// 	// i am not complicit in the moistness of this codebase :|
+
+// 	// add a big orphan panda to store
+// 	b, err := os.ReadFile("../media/test/big-panda.gif")
+// 	if err != nil {
+// 		suite.FailNow(err.Error())
+// 	}
+
+// 	pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
+// 	if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil {
+// 		suite.FailNow(err.Error())
+// 	}
+
+// 	ctx := context.Background()
+
+// 	// should show up 1 orphaned panda
+// 	totalPruned, err := suite.cleaner.Media().PruneOrphaned(ctx)
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+
+// 	// panda should no longer be in storage
+// 	hasKey, err := suite.storage.Has(ctx, pandaPath)
+// 	suite.NoError(err)
+// 	suite.False(hasKey)
+// }
+
+// func (suite *MediaTestSuite) TestPruneUnusedLocal() {
+// 	testAttachment := suite.testAttachments["local_account_1_unattached_1"]
+// 	suite.True(*testAttachment.Cached)
+
+// 	totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false)
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+
+// 	_, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+// 	suite.ErrorIs(err, db.ErrNoEntries)
+// }
+
+// func (suite *MediaTestSuite) TestPruneUnusedLocalDry() {
+// 	testAttachment := suite.testAttachments["local_account_1_unattached_1"]
+// 	suite.True(*testAttachment.Cached)
+
+// 	totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true)
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+
+// 	_, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+// 	suite.NoError(err)
+// }
+
+// func (suite *MediaTestSuite) TestPruneRemoteTwice() {
+// 	totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false)
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+
+// 	// final prune should prune nothing, since the first prune already happened
+// 	totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false)
+// 	suite.NoError(err)
+// 	suite.Equal(0, totalPrunedAgain)
+// }
+
+// func (suite *MediaTestSuite) TestPruneOneNonExistent() {
+// 	ctx := context.Background()
+// 	testAttachment := suite.testAttachments["local_account_1_unattached_1"]
+
+// 	// Delete this attachment cached on disk
+// 	media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
+// 	suite.NoError(err)
+// 	suite.True(*media.Cached)
+// 	err = suite.storage.Delete(ctx, media.File.Path)
+// 	suite.NoError(err)
+
+// 	// Now attempt to prune for item with db entry no file
+// 	totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false)
+// 	suite.NoError(err)
+// 	suite.Equal(1, totalPruned)
+// }
+
+// func (suite *MediaTestSuite) TestPruneUnusedRemote() {
+// 	ctx := context.Background()
+
+// 	// start by clearing zork's avatar + header
+// 	zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+// 	zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+// 	zork := suite.testAccounts["local_account_1"]
+// 	zork.AvatarMediaAttachmentID = ""
+// 	zork.HeaderMediaAttachmentID = ""
+// 	if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
+// 		panic(err)
+// 	}
+
+// 	totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
+// 	suite.NoError(err)
+// 	suite.Equal(2, totalPruned)
+
+// 	// media should no longer be stored
+// 	_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+
+// 	// attachments should no longer be in the db
+// 	_, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+// 	suite.ErrorIs(err, db.ErrNoEntries)
+// 	_, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+// 	suite.ErrorIs(err, db.ErrNoEntries)
+// }
+
+// func (suite *MediaTestSuite) TestPruneUnusedRemoteTwice() {
+// 	ctx := context.Background()
+
+// 	// start by clearing zork's avatar + header
+// 	zork := suite.testAccounts["local_account_1"]
+// 	zork.AvatarMediaAttachmentID = ""
+// 	zork.HeaderMediaAttachmentID = ""
+// 	if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
+// 		panic(err)
+// 	}
+
+// 	totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
+// 	suite.NoError(err)
+// 	suite.Equal(2, totalPruned)
+
+// 	// final prune should prune nothing, since the first prune already happened
+// 	totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false)
+// 	suite.NoError(err)
+// 	suite.Equal(0, totalPruned)
+// }
+
+// func (suite *MediaTestSuite) TestPruneUnusedRemoteMultipleAccounts() {
+// 	ctx := context.Background()
+
+// 	// start by clearing zork's avatar + header
+// 	zorkOldAvatar := suite.testAttachments["local_account_1_avatar"]
+// 	zorkOldHeader := suite.testAttachments["local_account_1_avatar"]
+// 	zork := suite.testAccounts["local_account_1"]
+// 	zork.AvatarMediaAttachmentID = ""
+// 	zork.HeaderMediaAttachmentID = ""
+// 	if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil {
+// 		panic(err)
+// 	}
+
+// 	// set zork's unused header as belonging to turtle
+// 	turtle := suite.testAccounts["local_account_1"]
+// 	zorkOldHeader.AccountID = turtle.ID
+// 	if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil {
+// 		panic(err)
+// 	}
+
+// 	totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false)
+// 	suite.NoError(err)
+// 	suite.Equal(2, totalPruned)
+
+// 	// media should no longer be stored
+// 	_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+// 	_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
+// 	suite.ErrorIs(err, storage.ErrNotFound)
+
+// 	// attachments should no longer be in the db
+// 	_, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID)
+// 	suite.ErrorIs(err, db.ErrNoEntries)
+// 	_, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID)
+// 	suite.ErrorIs(err, db.ErrNoEntries)
+// }
+
+func (suite *MediaTestSuite) TestUncacheRemote() {
+	ctx := context.Background()
+
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	suite.True(*testStatusAttachment.Cached)
+
+	testHeader := suite.testAttachments["remote_account_3_header"]
+	suite.True(*testHeader.Cached)
+
+	after := time.Now().Add(-24 * time.Hour)
+	totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after)
+	suite.NoError(err)
+	suite.Equal(2, totalUncached)
+
+	uncachedAttachment, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
+	suite.NoError(err)
+	suite.False(*uncachedAttachment.Cached)
+
+	uncachedAttachment, err = suite.db.GetAttachmentByID(ctx, testHeader.ID)
+	suite.NoError(err)
+	suite.False(*uncachedAttachment.Cached)
+}
+
+func (suite *MediaTestSuite) TestUncacheRemoteDry() {
+	ctx := context.Background()
+
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	suite.True(*testStatusAttachment.Cached)
+
+	testHeader := suite.testAttachments["remote_account_3_header"]
+	suite.True(*testHeader.Cached)
+
+	after := time.Now().Add(-24 * time.Hour)
+	totalUncached, err := suite.cleaner.Media().UncacheRemote(gtscontext.SetDryRun(ctx), after)
+	suite.NoError(err)
+	suite.Equal(2, totalUncached)
+
+	uncachedAttachment, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
+	suite.NoError(err)
+	suite.True(*uncachedAttachment.Cached)
+
+	uncachedAttachment, err = suite.db.GetAttachmentByID(ctx, testHeader.ID)
+	suite.NoError(err)
+	suite.True(*uncachedAttachment.Cached)
+}
+
+func (suite *MediaTestSuite) TestUncacheRemoteTwice() {
+	ctx := context.Background()
+	after := time.Now().Add(-24 * time.Hour)
+
+	totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after)
+	suite.NoError(err)
+	suite.Equal(2, totalUncached)
+
+	// final uncache should uncache nothing, since the first uncache already happened
+	totalUncachedAgain, err := suite.cleaner.Media().UncacheRemote(ctx, after)
+	suite.NoError(err)
+	suite.Equal(0, totalUncachedAgain)
+}
+
+func (suite *MediaTestSuite) TestUncacheAndRecache() {
+	ctx := context.Background()
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testHeader := suite.testAttachments["remote_account_3_header"]
+
+	after := time.Now().Add(-24 * time.Hour)
+	totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after)
+	suite.NoError(err)
+	suite.Equal(2, totalUncached)
+
+	// media should no longer be stored
+	_, err = suite.storage.Get(ctx, testStatusAttachment.File.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(ctx, testHeader.File.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+
+	// now recache the image....
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test image
+		b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	for _, original := range []*gtsmodel.MediaAttachment{
+		testStatusAttachment,
+		testHeader,
+	} {
+		processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, original.ID)
+		suite.NoError(err)
+
+		// synchronously load the recached attachment
+		recachedAttachment, err := processingRecache.LoadAttachment(ctx)
+		suite.NoError(err)
+		suite.NotNil(recachedAttachment)
+
+		// recachedAttachment should be basically the same as the old attachment
+		suite.True(*recachedAttachment.Cached)
+		suite.Equal(original.ID, recachedAttachment.ID)
+		suite.Equal(original.File.Path, recachedAttachment.File.Path)           // file should be stored in the same place
+		suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
+		suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta)       // and the filemeta should be the same
+
+		// recached files should be back in storage
+		_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
+		suite.NoError(err)
+		_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
+		suite.NoError(err)
+	}
+}
+
+func (suite *MediaTestSuite) TestUncacheOneNonExistent() {
+	ctx := context.Background()
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+
+	// Delete this attachment cached on disk
+	media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
+	suite.NoError(err)
+	suite.True(*media.Cached)
+	err = suite.storage.Delete(ctx, media.File.Path)
+	suite.NoError(err)
+
+	// Now attempt to uncache remote for item with db entry no file
+	after := time.Now().Add(-24 * time.Hour)
+	totalUncached, err := suite.cleaner.Media().UncacheRemote(ctx, after)
+	suite.NoError(err)
+	suite.Equal(2, totalUncached)
+}