[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

1 files changed, 73 insertions, 0 deletions

diff --git a/vendor/modernc.org/libc/libc_freebsd_amd64.go b/vendor/modernc.org/libc/libc_freebsd_amd64.go
index 2868e3b84..48f7fd786 100644
--- a/vendor/modernc.org/libc/libc_freebsd_amd64.go
+++ b/vendor/modernc.org/libc/libc_freebsd_amd64.go
@@ -6,15 +6,22 @@ package libc // import "modernc.org/libc"
 
 import (
 	"strings"
+	"syscall"
 	"unsafe"
 
 	"golang.org/x/sys/unix"
 	"modernc.org/libc/fcntl"
+	"modernc.org/libc/fts"
 	"modernc.org/libc/sys/types"
 	"modernc.org/libc/time"
 	"modernc.org/libc/utime"
 )
 
+type (
+	long  = int64
+	ulong = uint64
+)
+
 // int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact);
 func Xsigaction(t *TLS, signum int32, act, oldact uintptr) int32 {
 	if _, _, err := unix.Syscall(unix.SYS_SIGACTION, uintptr(signum), act, oldact); err != 0 {
@@ -572,3 +579,69 @@ func Xgetrlimit64(t *TLS, resource int32, rlim uintptr) int32 {
 
 	return 0
 }
+
+func newFtsent(t *TLS, info int, path string, stat *unix.Stat_t, err syscall.Errno) (r *fts.FTSENT) {
+	var statp uintptr
+	if stat != nil {
+		statp = Xmalloc(t, types.Size_t(unsafe.Sizeof(unix.Stat_t{})))
+		if statp == 0 {
+			panic("OOM")
+		}
+
+		*(*unix.Stat_t)(unsafe.Pointer(statp)) = *stat
+	}
+	csp, errx := CString(path)
+	if errx != nil {
+		panic("OOM")
+	}
+
+	return &fts.FTSENT{
+		Ffts_info:    int32(info),
+		Ffts_path:    csp,
+		Ffts_pathlen: uint64(len(path)),
+		Ffts_statp:   statp,
+		Ffts_errno:   int32(err),
+	}
+}
+
+// DIR *opendir(const char *name);
+func Xopendir(t *TLS, name uintptr) uintptr {
+	p := Xmalloc(t, uint64(unsafe.Sizeof(darwinDir{})))
+	if p == 0 {
+		panic("OOM")
+	}
+
+	fd := int(Xopen(t, name, fcntl.O_RDONLY|fcntl.O_DIRECTORY|fcntl.O_CLOEXEC, 0))
+	if fd < 0 {
+		if dmesgs {
+			dmesg("%v: FAIL %v", origin(1), (*darwinDir)(unsafe.Pointer(p)).fd)
+		}
+		Xfree(t, p)
+		return 0
+	}
+
+	if dmesgs {
+		dmesg("%v: ok", origin(1))
+	}
+	(*darwinDir)(unsafe.Pointer(p)).fd = fd
+	(*darwinDir)(unsafe.Pointer(p)).h = 0
+	(*darwinDir)(unsafe.Pointer(p)).l = 0
+	(*darwinDir)(unsafe.Pointer(p)).eof = false
+	return p
+}
+
+// int chflags(const char *path, u_int flags);
+func Xchflags(t *TLS, path uintptr, flags uint64) int32 {
+	if err := unix.Chflags(GoString(path), int(flags)); err != nil {
+		if dmesgs {
+			dmesg("%v: %v FAIL", origin(1), err)
+		}
+		t.setErrno(err)
+		return -1
+	}
+
+	if dmesgs {
+		dmesg("%v: ok", origin(1))
+	}
+	return 0
+}