[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

2 files changed, 29 insertions, 25 deletions

diff --git a/vendor/github.com/uptrace/bun/schema/formatter.go b/vendor/github.com/uptrace/bun/schema/formatter.go
index dd1859acf..1fba1b59e 100644
--- a/vendor/github.com/uptrace/bun/schema/formatter.go
+++ b/vendor/github.com/uptrace/bun/schema/formatter.go
@@ -177,8 +177,6 @@ type NamedArgAppender interface {
 	AppendNamedArg(fmter Formatter, b []byte, name string) ([]byte, bool)
 }
 
-//------------------------------------------------------------------------------
-
 type namedArgList struct {
 	arg  NamedArgAppender
 	next *namedArgList
@@ -219,13 +217,13 @@ func (a *namedArg) AppendNamedArg(fmter Formatter, b []byte, name string) ([]byt
 
 //------------------------------------------------------------------------------
 
-var _ NamedArgAppender = (*structArgs)(nil)
-
 type structArgs struct {
 	table *Table
 	strct reflect.Value
 }
 
+var _ NamedArgAppender = (*structArgs)(nil)
+
 func newStructArgs(fmter Formatter, strct interface{}) (*structArgs, bool) {
 	v := reflect.ValueOf(strct)
 	if !v.IsValid() {
diff --git a/vendor/github.com/uptrace/bun/schema/table.go b/vendor/github.com/uptrace/bun/schema/table.go
index 823d4c69f..88b8d8e25 100644
--- a/vendor/github.com/uptrace/bun/schema/table.go
+++ b/vendor/github.com/uptrace/bun/schema/table.go
@@ -203,7 +203,7 @@ func (t *Table) fieldByGoName(name string) *Field {
 func (t *Table) initFields() {
 	t.Fields = make([]*Field, 0, t.Type.NumField())
 	t.FieldMap = make(map[string]*Field, t.Type.NumField())
-	t.addFields(t.Type, nil)
+	t.addFields(t.Type, "", nil)
 
 	if len(t.PKs) == 0 {
 		for _, name := range []string{"id", "uuid", "pk_" + t.ModelName} {
@@ -230,7 +230,7 @@ func (t *Table) initFields() {
 	}
 }
 
-func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
+func (t *Table) addFields(typ reflect.Type, prefix string, index []int) {
 	for i := 0; i < typ.NumField(); i++ {
 		f := typ.Field(i)
 		unexported := f.PkgPath != ""
@@ -242,10 +242,6 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			continue
 		}
 
-		// Make a copy so the slice is not shared between fields.
-		index := make([]int, len(baseIndex))
-		copy(index, baseIndex)
-
 		if f.Anonymous {
 			if f.Name == "BaseModel" && f.Type == baseModelType {
 				if len(index) == 0 {
@@ -258,7 +254,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			if fieldType.Kind() != reflect.Struct {
 				continue
 			}
-			t.addFields(fieldType, append(index, f.Index...))
+			t.addFields(fieldType, "", withIndex(index, f.Index))
 
 			tag := tagparser.Parse(f.Tag.Get("bun"))
 			if _, inherit := tag.Options["inherit"]; inherit {
@@ -274,7 +270,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			continue
 		}
 
-		if field := t.newField(f, index); field != nil {
+		if field := t.newField(f, prefix, index); field != nil {
 			t.addField(field)
 		}
 	}
@@ -315,10 +311,20 @@ func (t *Table) processBaseModelField(f reflect.StructField) {
 }
 
 //nolint
-func (t *Table) newField(f reflect.StructField, index []int) *Field {
-	sqlName := internal.Underscore(f.Name)
+func (t *Table) newField(f reflect.StructField, prefix string, index []int) *Field {
 	tag := tagparser.Parse(f.Tag.Get("bun"))
 
+	if prefix, ok := tag.Option("embed"); ok {
+		fieldType := indirectType(f.Type)
+		if fieldType.Kind() != reflect.Struct {
+			panic(fmt.Errorf("bun: embed %s.%s: got %s, wanted reflect.Struct",
+				t.TypeName, f.Name, fieldType.Kind()))
+		}
+		t.addFields(fieldType, prefix, withIndex(index, f.Index))
+		return nil
+	}
+
+	sqlName := internal.Underscore(f.Name)
 	if tag.Name != "" && tag.Name != sqlName {
 		if isKnownFieldOption(tag.Name) {
 			internal.Warn.Printf(
@@ -328,10 +334,10 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
 		}
 		sqlName = tag.Name
 	}
-
 	if s, ok := tag.Option("column"); ok {
 		sqlName = s
 	}
+	sqlName = prefix + sqlName
 
 	for name := range tag.Options {
 		if !isKnownFieldOption(name) {
@@ -339,7 +345,7 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
 		}
 	}
 
-	index = append(index, f.Index...)
+	index = withIndex(index, f.Index)
 	if field := t.fieldWithLock(sqlName); field != nil {
 		if indexEqual(field.Index, index) {
 			return field
@@ -795,7 +801,7 @@ func (t *Table) inlineFields(field *Field, seen map[reflect.Type]struct{}) {
 		f.GoName = field.GoName + "_" + f.GoName
 		f.Name = field.Name + "__" + f.Name
 		f.SQLName = t.quoteIdent(f.Name)
-		f.Index = appendNew(field.Index, f.Index...)
+		f.Index = withIndex(field.Index, f.Index)
 
 		t.fieldsMapMu.Lock()
 		if _, ok := t.FieldMap[f.Name]; !ok {
@@ -834,7 +840,7 @@ func (t *Table) AppendNamedArg(
 	fmter Formatter, b []byte, name string, strct reflect.Value,
 ) ([]byte, bool) {
 	if field, ok := t.FieldMap[name]; ok {
-		return fmter.appendArg(b, field.Value(strct).Interface()), true
+		return field.AppendValue(fmter, b, strct), true
 	}
 	return b, false
 }
@@ -853,13 +859,6 @@ func (t *Table) quoteIdent(s string) Safe {
 	return Safe(NewFormatter(t.dialect).AppendIdent(nil, s))
 }
 
-func appendNew(dst []int, src ...int) []int {
-	cp := make([]int, len(dst)+len(src))
-	copy(cp, dst)
-	copy(cp[len(dst):], src)
-	return cp
-}
-
 func isKnownTableOption(name string) bool {
 	switch name {
 	case "table", "alias", "select":
@@ -991,3 +990,10 @@ func softDeleteFieldUpdaterFallback(field *Field) func(fv reflect.Value, tm time
 		return field.ScanWithCheck(fv, tm)
 	}
 }
+
+func withIndex(a, b []int) []int {
+	dest := make([]int, 0, len(a)+len(b))
+	dest = append(dest, a...)
+	dest = append(dest, b...)
+	return dest
+}