[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

1 files changed, 15 insertions, 40 deletions

diff --git a/vendor/github.com/uptrace/bun/migrate/migrator.go b/vendor/github.com/uptrace/bun/migrate/migrator.go
index 1808d9db8..9c8d44173 100644
--- a/vendor/github.com/uptrace/bun/migrate/migrator.go
+++ b/vendor/github.com/uptrace/bun/migrate/migrator.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"path/filepath"
 	"regexp"
 	"time"
@@ -59,11 +58,16 @@ func (m *Migrator) DB() *bun.DB {
 
 // MigrationsWithStatus returns migrations with status in ascending order.
 func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, error) {
+	sorted, _, err := m.migrationsWithStatus(ctx)
+	return sorted, err
+}
+
+func (m *Migrator) migrationsWithStatus(ctx context.Context) (MigrationSlice, int64, error) {
 	sorted := m.migrations.Sorted()
 
 	applied, err := m.selectAppliedMigrations(ctx)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	appliedMap := migrationMap(applied)
@@ -76,7 +80,7 @@ func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, er
 		}
 	}
 
-	return sorted, nil
+	return sorted, applied.LastGroupID(), nil
 }
 
 func (m *Migrator) Init(ctx context.Context) error {
@@ -128,7 +132,7 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
 	}
 	defer m.Unlock(ctx) //nolint:errcheck
 
-	migrations, err := m.MigrationsWithStatus(ctx)
+	migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -139,21 +143,22 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
 	if len(group.Migrations) == 0 {
 		return group, nil
 	}
-	group.ID = migrations.LastGroupID() + 1
+	group.ID = lastGroupID + 1
 
 	for i := range group.Migrations {
 		migration := &group.Migrations[i]
 		migration.GroupID = group.ID
 
+		// Always mark migration as applied so the rollback has a chance to fix the database.
+		if err := m.MarkApplied(ctx, migration); err != nil {
+			return nil, err
+		}
+
 		if !cfg.nop && migration.Up != nil {
 			if err := migration.Up(ctx, m.db); err != nil {
 				return group, err
 			}
 		}
-
-		if err := m.MarkApplied(ctx, migration); err != nil {
-			return nil, err
-		}
 	}
 
 	return group, nil
@@ -195,36 +200,6 @@ func (m *Migrator) Rollback(ctx context.Context, opts ...MigrationOption) (*Migr
 	return lastGroup, nil
 }
 
-type MigrationStatus struct {
-	Migrations    MigrationSlice
-	NewMigrations MigrationSlice
-	LastGroup     *MigrationGroup
-}
-
-func (m *Migrator) Status(ctx context.Context) (*MigrationStatus, error) {
-	log.Printf(
-		"DEPRECATED: bun: replace Status(ctx) with " +
-			"MigrationsWithStatus(ctx)")
-
-	migrations, err := m.MigrationsWithStatus(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return &MigrationStatus{
-		Migrations:    migrations,
-		NewMigrations: migrations.Unapplied(),
-		LastGroup:     migrations.LastGroup(),
-	}, nil
-}
-
-func (m *Migrator) MarkCompleted(ctx context.Context) (*MigrationGroup, error) {
-	log.Printf(
-		"DEPRECATED: bun: replace MarkCompleted(ctx) with " +
-			"Migrate(ctx, migrate.WithNopMigration())")
-
-	return m.Migrate(ctx, WithNopMigration())
-}
-
 type goMigrationConfig struct {
 	packageName string
 }
@@ -320,7 +295,7 @@ func (m *Migrator) genMigrationName(name string) (string, error) {
 	return fmt.Sprintf("%s_%s", version, name), nil
 }
 
-// MarkApplied marks the migration as applied (applied).
+// MarkApplied marks the migration as applied (completed).
 func (m *Migrator) MarkApplied(ctx context.Context, migration *Migration) error {
 	_, err := m.db.NewInsert().Model(migration).
 		ModelTableExpr(m.table).