[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

1 files changed, 23 insertions, 3 deletions

diff --git a/vendor/modernc.org/sqlite/lib/capi_netbsd_amd64.go b/vendor/modernc.org/sqlite/lib/capi_netbsd_amd64.go
index 1c9ec912c..d12ca2ff1 100644
--- a/vendor/modernc.org/sqlite/lib/capi_netbsd_amd64.go
+++ b/vendor/modernc.org/sqlite/lib/capi_netbsd_amd64.go
@@ -1,4 +1,4 @@
-// Code generated by 'ccgo -DSQLITE_PRIVATE= -export-defines  -export-enums  -export-externs X -export-fields F -export-typedefs  -pkgname sqlite3 -o lib/sqlite_netbsd_amd64.go -trace-translation-units testdata/sqlite-amalgamation-3360000/sqlite3.c -DNDEBUG -DHAVE_USLEEP -DLONGDOUBLE_TYPE=double -DSQLITE_CORE -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_GEOPOLY -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_RBU -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_SNAPSHOT -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_SOUNDEX -DSQLITE_THREADSAFE=1 -DSQLITE_OS_UNIX=1 -D__libc_cond_broadcast=pthread_cond_broadcast -D__libc_cond_destroy=pthread_cond_destroy -D__libc_cond_init=pthread_cond_init -D__libc_cond_signal=pthread_cond_signal -D__libc_cond_wait=pthread_cond_wait -D__libc_mutex_destroy=pthread_mutex_destroy -D__libc_mutex_init=pthread_mutex_init -D__libc_mutex_lock=pthread_mutex_lock -D__libc_mutex_trylock=pthread_mutex_trylock -D__libc_mutex_unlock=pthread_mutex_unlock -D__libc_mutexattr_destroy=pthread_mutexattr_destroy -D__libc_mutexattr_init=pthread_mutexattr_init -D__libc_mutexattr_settype=pthread_mutexattr_settype -D__libc_thr_yield=sched_yield', DO NOT EDIT.
+// Code generated by 'ccgo -DSQLITE_PRIVATE= -export-defines  -export-enums  -export-externs X -export-fields F -export-typedefs  -ignore-unsupported-alignment -pkgname sqlite3 -o lib/sqlite_netbsd_amd64.go -trace-translation-units testdata/sqlite-amalgamation-3370200/sqlite3.c -DNDEBUG -DHAVE_USLEEP -DLONGDOUBLE_TYPE=double -DSQLITE_CORE -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_GEOPOLY -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MATH_FUNCTIONS -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_RBU -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_SNAPSHOT -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_SOUNDEX -DSQLITE_THREADSAFE=1 -DSQLITE_OS_UNIX=1 -D__libc_cond_broadcast=pthread_cond_broadcast -D__libc_cond_destroy=pthread_cond_destroy -D__libc_cond_init=pthread_cond_init -D__libc_cond_signal=pthread_cond_signal -D__libc_cond_wait=pthread_cond_wait -D__libc_mutex_destroy=pthread_mutex_destroy -D__libc_mutex_init=pthread_mutex_init -D__libc_mutex_lock=pthread_mutex_lock -D__libc_mutex_trylock=pthread_mutex_trylock -D__libc_mutex_unlock=pthread_mutex_unlock -D__libc_mutexattr_destroy=pthread_mutexattr_destroy -D__libc_mutexattr_init=pthread_mutexattr_init -D__libc_mutexattr_settype=pthread_mutexattr_settype -D__libc_thr_yield=sched_yield', DO NOT EDIT.
 
 package sqlite3
 
@@ -98,6 +98,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3BtreeGetReserveNoMutex":                {},
 	"sqlite3BtreeIncrVacuum":                       {},
 	"sqlite3BtreeIncrblobCursor":                   {},
+	"sqlite3BtreeIndexMoveto":                      {},
 	"sqlite3BtreeInsert":                           {},
 	"sqlite3BtreeIntegerKey":                       {},
 	"sqlite3BtreeIntegrityCheck":                   {},
@@ -111,7 +112,6 @@ var CAPI = map[string]struct{}{
 	"sqlite3BtreeLockTable":                        {},
 	"sqlite3BtreeMaxPageCount":                     {},
 	"sqlite3BtreeMaxRecordSize":                    {},
-	"sqlite3BtreeMovetoUnpacked":                   {},
 	"sqlite3BtreeNewDb":                            {},
 	"sqlite3BtreeNext":                             {},
 	"sqlite3BtreeOffset":                           {},
@@ -136,6 +136,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3BtreeSetSpillSize":                     {},
 	"sqlite3BtreeSetVersion":                       {},
 	"sqlite3BtreeSharable":                         {},
+	"sqlite3BtreeTableMoveto":                      {},
 	"sqlite3BtreeTransferRow":                      {},
 	"sqlite3BtreeTripAllCursors":                   {},
 	"sqlite3BtreeTxnState":                         {},
@@ -157,8 +158,12 @@ var CAPI = map[string]struct{}{
 	"sqlite3CodeVerifyNamedSchema":                 {},
 	"sqlite3CodeVerifySchema":                      {},
 	"sqlite3CollapseDatabaseArray":                 {},
+	"sqlite3ColumnColl":                            {},
 	"sqlite3ColumnDefault":                         {},
+	"sqlite3ColumnExpr":                            {},
 	"sqlite3ColumnIndex":                           {},
+	"sqlite3ColumnSetColl":                         {},
+	"sqlite3ColumnSetExpr":                         {},
 	"sqlite3ColumnType":                            {},
 	"sqlite3ColumnsFromExprList":                   {},
 	"sqlite3CommitInternalChanges":                 {},
@@ -204,6 +209,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3DeleteTriggerStep":                     {},
 	"sqlite3Dequote":                               {},
 	"sqlite3DequoteExpr":                           {},
+	"sqlite3DequoteToken":                          {},
 	"sqlite3Detach":                                {},
 	"sqlite3DropIndex":                             {},
 	"sqlite3DropTable":                             {},
@@ -283,6 +289,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3ExprListSetName":                       {},
 	"sqlite3ExprListSetSortOrder":                  {},
 	"sqlite3ExprListSetSpan":                       {},
+	"sqlite3ExprListToValues":                      {},
 	"sqlite3ExprNNCollSeq":                         {},
 	"sqlite3ExprNeedsNoAffinityChange":             {},
 	"sqlite3ExprReferencesUpdatedColumn":           {},
@@ -311,6 +318,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3FixTriggerStep":                        {},
 	"sqlite3FkActions":                             {},
 	"sqlite3FkCheck":                               {},
+	"sqlite3FkClearTriggerCache":                   {},
 	"sqlite3FkDelete":                              {},
 	"sqlite3FkDropTable":                           {},
 	"sqlite3FkLocateIndex":                         {},
@@ -320,7 +328,6 @@ var CAPI = map[string]struct{}{
 	"sqlite3FreeIndex":                             {},
 	"sqlite3Fts5Init":                              {},
 	"sqlite3FunctionSearch":                        {},
-	"sqlite3FunctionUsesThisSrc":                   {},
 	"sqlite3GenerateColumnNames":                   {},
 	"sqlite3GenerateConstraintChecks":              {},
 	"sqlite3GenerateIndexKey":                      {},
@@ -365,6 +372,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3Insert":                                {},
 	"sqlite3InsertBuiltinFuncs":                    {},
 	"sqlite3Int64ToText":                           {},
+	"sqlite3IntFloatCompare":                       {},
 	"sqlite3InvokeBusyHandler":                     {},
 	"sqlite3IsBinary":                              {},
 	"sqlite3IsIdChar":                              {},
@@ -402,6 +410,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3MallocMutex":                           {},
 	"sqlite3MallocSize":                            {},
 	"sqlite3MallocZero":                            {},
+	"sqlite3MarkAllShadowTablesOf":                 {},
 	"sqlite3MatchEName":                            {},
 	"sqlite3MaterializeView":                       {},
 	"sqlite3MayAbort":                              {},
@@ -572,6 +581,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3PendingByte":                           {},
 	"sqlite3Pragma":                                {},
 	"sqlite3PragmaVtabRegister":                    {},
+	"sqlite3PreferredTableName":                    {},
 	"sqlite3PrimaryKeyIndex":                       {},
 	"sqlite3PrngRestoreState":                      {},
 	"sqlite3PrngSaveState":                         {},
@@ -581,6 +591,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3ReadSchema":                            {},
 	"sqlite3RealSameAsInt":                         {},
 	"sqlite3Realloc":                               {},
+	"sqlite3ReferencesSrcList":                     {},
 	"sqlite3RegisterBuiltinFunctions":              {},
 	"sqlite3RegisterDateTimeFunctions":             {},
 	"sqlite3RegisterLikeFunctions":                 {},
@@ -604,6 +615,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3ResolveSelfReference":                  {},
 	"sqlite3ResultIntReal":                         {},
 	"sqlite3ResultSetOfSelect":                     {},
+	"sqlite3ResultStrAccum":                        {},
 	"sqlite3RollbackAll":                           {},
 	"sqlite3RootPageMoved":                         {},
 	"sqlite3RowSetClear":                           {},
@@ -661,10 +673,15 @@ var CAPI = map[string]struct{}{
 	"sqlite3StatusHighwater":                       {},
 	"sqlite3StatusUp":                              {},
 	"sqlite3StatusValue":                           {},
+	"sqlite3StdType":                               {},
+	"sqlite3StdTypeAffinity":                       {},
+	"sqlite3StdTypeLen":                            {},
+	"sqlite3StdTypeMap":                            {},
 	"sqlite3StmtCurrentTime":                       {},
 	"sqlite3StorageColumnToTable":                  {},
 	"sqlite3StrAccumFinish":                        {},
 	"sqlite3StrAccumInit":                          {},
+	"sqlite3StrAccumSetError":                      {},
 	"sqlite3StrBINARY":                             {},
 	"sqlite3StrICmp":                               {},
 	"sqlite3StrIHash":                              {},
@@ -964,6 +981,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3_aggregate_context":                    {},
 	"sqlite3_aggregate_count":                      {},
 	"sqlite3_auto_extension":                       {},
+	"sqlite3_autovacuum_pages":                     {},
 	"sqlite3_backup_finish":                        {},
 	"sqlite3_backup_init":                          {},
 	"sqlite3_backup_pagecount":                     {},
@@ -995,6 +1013,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3_busy_timeout":                         {},
 	"sqlite3_cancel_auto_extension":                {},
 	"sqlite3_changes":                              {},
+	"sqlite3_changes64":                            {},
 	"sqlite3_clear_bindings":                       {},
 	"sqlite3_close":                                {},
 	"sqlite3_close_v2":                             {},
@@ -1195,6 +1214,7 @@ var CAPI = map[string]struct{}{
 	"sqlite3_thread_cleanup":                       {},
 	"sqlite3_threadsafe":                           {},
 	"sqlite3_total_changes":                        {},
+	"sqlite3_total_changes64":                      {},
 	"sqlite3_trace":                                {},
 	"sqlite3_trace_v2":                             {},
 	"sqlite3_transfer_bindings":                    {},