[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

1 files changed, 7 insertions, 6 deletions

diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go
index c2f27d9c4..20cc6d3bf 100644
--- a/internal/gtsmodel/mediaattachment.go
+++ b/internal/gtsmodel/mediaattachment.go
@@ -32,17 +32,18 @@ type MediaAttachment struct {
 	URL               string           `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"`                           // Where can the attachment be retrieved on *this* server
 	RemoteURL         string           `validate:"required_without=URL,omitempty,url" bun:",nullzero"`                                 // Where can the attachment be retrieved on a remote server (empty for local media)
 	Type              FileType         `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"`                        // Type of file (image/gif/audio/video)
-	FileMeta          FileMeta         `validate:"required" bun:",nullzero,notnull"`                                                   // Metadata about the file
+	FileMeta          FileMeta         `validate:"required" bun:",embed:filemeta_,nullzero,notnull"`                                   // Metadata about the file
 	AccountID         string           `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"`                                 // To which account does this attachment belong
 	Account           *Account         `validate:"-" bun:"rel:has-one"`                                                                // Account corresponding to accountID
 	Description       string           `validate:"-" bun:""`                                                                           // Description of the attachment (for screenreaders)
 	ScheduledStatusID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                        // To which scheduled status does this attachment belong
 	Blurhash          string           `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
 	Processing        ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"`                                           // What is the processing status of this attachment
-	File              File             `validate:"required" bun:",notnull,nullzero"`                                                   // metadata for the whole file
-	Thumbnail         Thumbnail        `validate:"required" bun:",notnull,nullzero"`                                                   // small image thumbnail derived from a larger image, video, or audio file.
+	File              File             `validate:"required" bun:",embed:file_,notnull,nullzero"`                                       // metadata for the whole file
+	Thumbnail         Thumbnail        `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"`                                  // small image thumbnail derived from a larger image, video, or audio file.
 	Avatar            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as an avatar?
 	Header            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as a header?
+	Cached            bool             `validate:"-" bun:",notnull"`                                                                   // Is this attachment currently cached by our instance?
 }
 
 // File refers to the metadata for the whole file
@@ -88,9 +89,9 @@ const (
 
 // FileMeta describes metadata about the actual contents of the file.
 type FileMeta struct {
-	Original Original `validate:"required"`
-	Small    Small
-	Focus    Focus
+	Original Original `validate:"required" bun:"embed:original_"`
+	Small    Small    `bun:"embed:small_"`
+	Focus    Focus    `bun:"embed:focus_"`
 }
 
 // Small can be used for a thumbnail of any media type