[chore] media and emoji refactoring (#3000)

* start updating media manager interface ready for storing attachments / emoji right away

* store emoji and media as uncached immediately, then (re-)cache on Processing{}.Load()

* remove now unused media workers

* fix tests and issues

* fix another test!

* fix emoji activitypub uri setting behaviour, fix remainder of test compilation issues

* fix more tests

* fix (most of) remaining tests, add debouncing to repeatedly failing media / emojis

* whoops, rebase issue

* remove kim's whacky experiments

* do some reshuffling, ensure emoji uri gets set

* ensure marked as not cached on cleanup

* tweaks to media / emoji processing to handle context canceled better

* ensure newly fetched emojis actually get set in returned slice

* use different varnames to be a bit more obvious

* move emoji refresh rate limiting to dereferencer

* add exported dereferencer functions for remote media, use these for recaching in processor

* add check for nil attachment in updateAttachment()

* remove unused emoji and media fields + columns

* see previous commit

* fix old migrations expecting image_updated_at to exists (from copies of old models)

* remove freshness checking code (seems to be broken...)

* fix error arg causing nil ptr exception

* finish documentating functions with comments, slight tweaks to media / emoji deref error logic

* remove some extra unneeded boolean checking

* finish writing documentation (code comments) for exported media manager methods

* undo changes to migration snapshot gtsmodels, updated failing migration to have its own snapshot

* move doesColumnExist() to util.go in migrations package

1 files changed, 0 insertions, 24 deletions

diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index e3d31b7d2..3db8ef62f 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -739,13 +739,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    62529,
-				UpdatedAt:   TimeMustParse("2022-06-04T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    6872,
-				UpdatedAt:   TimeMustParse("2022-06-04T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg",
 				RemoteURL:   "",
 			},
@@ -788,13 +786,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif",
 				ContentType: "image/gif",
 				FileSize:    1109138,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    8803,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg",
 				RemoteURL:   "",
 			},
@@ -840,13 +836,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif",
 				ContentType: "video/mp4",
 				FileSize:    2273532,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    5272,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg",
 				RemoteURL:   "",
 			},
@@ -889,13 +883,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    27759,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    6177,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
 				RemoteURL:   "",
 			},
@@ -938,13 +930,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    457680,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    15374,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg",
 				RemoteURL:   "",
 			},
@@ -987,13 +977,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    517226,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    42308,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
 				RemoteURL:   "",
 			},
@@ -1036,13 +1024,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    19310,
-				UpdatedAt:   TimeMustParse("2021-09-20T12:40:37+02:00"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    19312,
-				UpdatedAt:   TimeMustParse("2021-09-20T12:40:37+02:00"),
 				URL:         "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
 				RemoteURL:   "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
 			},
@@ -1085,13 +1071,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    19310,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    20395,
-				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
 				URL:         "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
 				RemoteURL:   "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
 			},
@@ -1133,13 +1117,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7Y3C432WRSNS10EZM86SA5.jpg",
 				ContentType: "image/jpg",
 				FileSize:    5450054,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE7Y3C432WRSNS10EZM86SA5.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    50820,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 				URL:         "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE7Y3C432WRSNS10EZM86SA5.jpg",
 			},
 			Avatar: util.Ptr(false),
@@ -1163,13 +1145,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7ZFX9GKA5ZZVD4FACABSS9.svg",
 				ContentType: "image/svg",
 				FileSize:    147819,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE7ZFX9GKA5ZZVD4FACABSS9.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    0,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 				URL:         "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE7ZFX9GKA5ZZVD4FACABSS9.jpg",
 			},
 			Avatar: util.Ptr(false),
@@ -1193,13 +1173,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE88YG74PVAB81PX2XA9F3FG.mp3",
 				ContentType: "audio/mpeg",
 				FileSize:    147819,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
 				Path:        "01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE88YG74PVAB81PX2XA9F3FG.jpg",
 				ContentType: "image/jpeg",
 				FileSize:    0,
-				UpdatedAt:   TimeMustParse("2023-11-02T12:44:25+02:00"),
 				URL:         "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/small/01HE88YG74PVAB81PX2XA9F3FG.jpg",
 			},
 			Avatar: util.Ptr(false),
@@ -1228,7 +1206,6 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
 			ImageStaticContentType: "image/png",
 			ImageFileSize:          36702,
 			ImageStaticFileSize:    10413,
-			ImageUpdatedAt:         TimeMustParse("2021-09-20T12:40:37+02:00"),
 			Disabled:               util.Ptr(false),
 			URI:                    "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
 			VisibleInPicker:        util.Ptr(true),
@@ -1251,7 +1228,6 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
 			ImageStaticContentType: "image/png",
 			ImageFileSize:          10889,
 			ImageStaticFileSize:    10808,
-			ImageUpdatedAt:         TimeMustParse("2020-03-18T13:12:00+01:00"),
 			Disabled:               util.Ptr(false),
 			URI:                    "http://fossbros-anonymous.io/emoji/01GD5KP5CQEE1R3X43Y1EHS2CW",
 			VisibleInPicker:        util.Ptr(false),