From 07727753b96d209406783e5e539725bcdafebdc7 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Mon, 7 Mar 2022 11:08:26 +0100 Subject: [feature] Clean up/uncache remote media (#407) * Add whereNotEmptyAndNotNull * Add GetRemoteOlderThanDays * Add GetRemoteOlderThanDays * Add PruneRemote to Manager interface * Start implementing PruneRemote * add new attachment + status to tests * fix up and test GetRemoteOlderThan * fix bad import * PruneRemote: return number pruned * add Cached column to mediaattachment * update + test pruneRemote * update mediaTest * use Cached column * upstep bun to latest version * embed structs in mediaAttachment * migrate mediaAttachment to new format * don't default cached to true * select only remote media * update db dependencies * step bun back to last working version * update pruneRemote to use Cached field * fix storage path of test attachments * add recache logic to manager * fix trimmed aspect ratio * test prune and recache * return errwithcode * tidy up different paths for emoji vs attachment * fix incorrect thumbnail type being stored * expose TransportController to media processor * implement tee-ing recached content * add thoughts of dog to test fedi attachments * test get remote files * add comment on PruneRemote * add postData cleanup to recache * test thumbnail fetching * add incredible diagram * go mod tidy * buffer pipes for recache streaming * test for client stops reading after 1kb * add media-remote-cache-days to config * add cron package * wrap logrus so it's available to cron * start and stop cron jobs gracefully --- internal/gtsmodel/mediaattachment.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'internal/gtsmodel') diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go index c2f27d9c4..20cc6d3bf 100644 --- a/internal/gtsmodel/mediaattachment.go +++ b/internal/gtsmodel/mediaattachment.go @@ -32,17 +32,18 @@ type MediaAttachment struct { URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on *this* server RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media) Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video) - FileMeta FileMeta `validate:"required" bun:",nullzero,notnull"` // Metadata about the file + FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders) ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment Processing ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"` // What is the processing status of this attachment - File File `validate:"required" bun:",notnull,nullzero"` // metadata for the whole file - Thumbnail Thumbnail `validate:"required" bun:",notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file. + File File `validate:"required" bun:",embed:file_,notnull,nullzero"` // metadata for the whole file + Thumbnail Thumbnail `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file. Avatar bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as an avatar? Header bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as a header? + Cached bool `validate:"-" bun:",notnull"` // Is this attachment currently cached by our instance? } // File refers to the metadata for the whole file @@ -88,9 +89,9 @@ const ( // FileMeta describes metadata about the actual contents of the file. type FileMeta struct { - Original Original `validate:"required"` - Small Small - Focus Focus + Original Original `validate:"required" bun:"embed:original_"` + Small Small `bun:"embed:small_"` + Focus Focus `bun:"embed:focus_"` } // Small can be used for a thumbnail of any media type -- cgit v1.3