[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

1 files changed, 52 insertions, 72 deletions

diff --git a/vendor/modernc.org/sqlite/Makefile b/vendor/modernc.org/sqlite/Makefile
index c330b6724..4439781c8 100644
--- a/vendor/modernc.org/sqlite/Makefile
+++ b/vendor/modernc.org/sqlite/Makefile
@@ -42,6 +42,8 @@ build_all_targets:
 	GOOS=darwin GOARCH=arm64 go test -c -o /dev/null
 	GOOS=freebsd GOARCH=amd64 go build -v ./...
 	GOOS=freebsd GOARCH=amd64 go test -c -o /dev/null
+	GOOS=freebsd GOARCH=386 go build -v ./...
+	GOOS=freebsd GOARCH=386 go test -c -o /dev/null
 	GOOS=linux GOARCH=386 go build -v ./...
 	GOOS=linux GOARCH=386 go test -c -o /dev/null
 	GOOS=linux GOARCH=amd64 go build -v ./...
@@ -54,67 +56,91 @@ build_all_targets:
 	GOOS=linux GOARCH=s390x go test -c -o /dev/null
 	GOOS=netbsd GOARCH=amd64 go build -v ./...
 	GOOS=netbsd GOARCH=amd64 go test -c -o /dev/null
+	GOOS=openbsd GOARCH=amd64 go build -v ./...
+	GOOS=openbsd GOARCH=amd64 go test -c -o /dev/null
 	GOOS=windows GOARCH=386 go build -v ./...
 	GOOS=windows GOARCH=386 go test -c -o /dev/null
 	GOOS=windows GOARCH=amd64 go build -v ./...
 	GOOS=windows GOARCH=amd64 go test -c -o /dev/null
 	echo done
 
+# 3900x
+windows_amd64:
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=x86_64-w64-mingw32-cpp TARGET_GOOS=windows TARGET_GOARCH=amd64 go generate 2>&1 | tee log-generate
+	GOOS=windows GOARCH=amd64 go build -v ./...
+
+# 3900x
+windows_386:
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=i686-w64-mingw32-cpp TARGET_GOOS=windows TARGET_GOARCH=386 go generate 2>&1 | tee log-generate
+	GOOS=windows GOARCH=386 go build -v ./...
+
+# 3900x/qemu
 darwin_amd64:
 	@echo "Should be executed only on darwin/amd64."
 	go generate 2>&1 | tee log-generate
 	go build -v ./...
 
+# 3900x/qemu
+netbsd_amd64:
+	@echo "Should be executed only on netbsd/amd64."
+	go generate 2>&1 | tee log-generate
+	go build -v ./...
+
+# darwin-m1
 darwin_arm64:
 	@echo "Should be executed only on darwin/arm64."
 	go generate 2>&1 | tee log-generate
 	go build -v ./...
 
+# 3900x/VBox
 freebsd_amd64:
 	@echo "Should be executed only on freebsd/amd64."
 	go generate 2>&1 | tee log-generate
 	go build -v ./...
 
-netbsd_amd64:
-	@echo "Should be executed only on netbsd/amd64."
+# 3900x/qemu
+freebsd_386:
+	@echo "Should be executed only on freebsd/386."
 	go generate 2>&1 | tee log-generate
 	go build -v ./...
 
+# 3900x
 linux_amd64:
-	TARGET_GOOS=linux TARGET_GOARCH=amd64 go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-amd64
-	GOOS=linux GOARCH=amd64 go build -v ./...
+	@echo "Should be executed only on linux/amd64."
+	go generate 2>&1 | tee log-generate
+	go build -v ./...
 
+# 3900x
 linux_386:
-	CCGO_CPP=i686-linux-gnu-cpp TARGET_GOARCH=386 TARGET_GOOS=linux go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-386
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=i686-linux-gnu-cpp TARGET_GOARCH=386 TARGET_GOOS=linux go generate 2>&1 | tee log-generate
 	GOOS=linux GOARCH=386 go build -v ./...
 
-linux_arm_on_linux_amd64:
-	CCGO_CPP=arm-linux-gnueabi-cpp TARGET_GOARCH=arm TARGET_GOOS=linux go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-arm
-	GOOS=linux GOARCH=arm go build -v ./...
-
-linux_arm_on_linux_arm:
-	go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-arm
+# 3900x
+linux_arm:
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=arm-linux-gnueabi-cpp TARGET_GOARCH=arm TARGET_GOOS=linux go generate 2>&1 | tee log-generate
 	GOOS=linux GOARCH=arm go build -v ./...
 
-linux_arm64_on_linux_amd64:
-	CCGO_CPP=aarch64-linux-gnu-cpp TARGET_GOARCH=arm64 TARGET_GOOS=linux go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-arm64
+# 3900x
+linux_arm64:
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=aarch64-linux-gnu-cpp TARGET_GOARCH=arm64 TARGET_GOOS=linux go generate 2>&1 | tee log-generate
 	GOOS=linux GOARCH=arm64 go build -v ./...
 
-linux_arm64_on_linux_arm64:
-	go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-arm
-	GOOS=linux GOARCH=arm go build -v ./...
-
+# 3900x
 linux_s390x:
-	CCGO_CPP=s390x-linux-gnu-cpp TARGET_GOARCH=s390x TARGET_GOOS=linux go generate 2>&1 | tee /tmp/log-generate-sqlite-linux-s390x
+	@echo "Should be executed only on linux/amd64."
+	CCGO_CPP=s390x-linux-gnu-cpp TARGET_GOARCH=s390x TARGET_GOOS=linux go generate 2>&1 | tee log-generate
 	GOOS=linux GOARCH=s390x go build -v ./...
 
-windows_amd64:
-	CCGO_CPP=x86_64-w64-mingw32-cpp TARGET_GOOS=windows TARGET_GOARCH=amd64 go generate 2>&1 | tee /tmp/log-generate-sqlite-windows-amd64
-	GOOS=windows GOARCH=amd64 go build -v ./...
-
-windows_386:
-	CCGO_CPP=i686-w64-mingw32-cpp TARGET_GOOS=windows TARGET_GOARCH=386 go generate 2>&1 | tee /tmp/log-generate-sqlite-windows-386
-	GOOS=windows GOARCH=386 go build -v ./...
+# 3900x/VBox
+openbsd_amd64:
+	@echo "Should be executed only on openbsd/amd64."
+	go generate 2>&1 | tee log-generate
+	go build -v ./...
 
 generate_all_targets_on_linux_amd64: linux_amd64 linux_386 linux_arm_on_linux_amd64 linux_arm64 linux_s390x windows_amd64 #TODO windows_386
 	gofmt -l -s -w .
@@ -126,40 +152,8 @@ tcl_test_wine:
 run_tcl_test_wine:
 	TCL_LIBRARY=Z:/home/jnml/src/modernc.org/tcl/assets wine testfixture.exe ./testdata/tcl/zipfile.test
 
-test:
-	go version | tee $(testlog)
-	uname -a | tee -a $(testlog)
-	go test -v -timeout 24h | tee -a $(testlog)
-	grep -ni fail $(testlog) | tee -a $(testlog) || true
-	LC_ALL=C date | tee -a $(testlog)
-	grep -ni --color=always fail $(testlog) || true
-
-test_darwin_amd64:
-	GOOS=darwin GOARCH=amd64 make test
-
-test_darwin_arm64:
-	GOOS=darwin GOARCH=arm64 make test
-
-test_linux_amd64:
-	GOOS=linux GOARCH=amd64 make test
-
-test_linux_386:
-	GOOS=linux GOARCH=386 make test
-
-test_linux_386_hosted:
-	GOOS=linux GOARCH=386 SQLITE_TEST_SUFFIX=-hosted-$(host) make test
-
-test_linux_arm:
-	GOOS=linux GOARCH=arm make test
-
-test_linux_arm64:
-	GOOS=linux GOARCH=arm64 make test
-
-test_linux_s390x:
-	GOOS=linux GOARCH=s390x make test
-
 extraquick:
-	go test -timeout 24h -v -run Tcl -suite extraquick -maxerror 1 2>&1 | tee log-extraquick
+	go test -timeout 24h -v -failfast -suite extraquick -maxerror 1 2>&1 | tee log-extraquick
 	date
 
 full:
@@ -217,17 +211,3 @@ todo:
 	@grep -nr $(grep) TODO * | grep -v $(ngrep) || true
 	@grep -nr $(grep) BUG * | grep -v $(ngrep) || true
 	@grep -nr $(grep) [^[:alpha:]]println * | grep -v $(ngrep) || true
-
-tcl:
-	cp log log-0
-	go test -run Tcl$$ 2>&1 -timeout 24h -trc | tee log
-	grep -c '\.\.\. \?Ok' log || true
-	grep -c '^!' log || true
-	# grep -c 'Error:' log || true
-
-tclshort:
-	cp log log-0
-	go test -run Tcl$$ -short 2>&1 -timeout 24h -trc | tee log
-	grep -c '\.\.\. \?Ok' log || true
-	grep -c '^!' log || true
-	# grep -c 'Error:' log || true