From 53180548083c0a100db2f703d5f5da047a9e0031 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Wed, 11 Jan 2023 11:13:13 +0000 Subject: [performance] media processing improvements (#1288) * media processor consolidation and reformatting, reduce amount of required syscalls Signed-off-by: kim * update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling Signed-off-by: kim * fix duration not being set, fix mp4 test expecting error Signed-off-by: kim * fix test expecting media files with different extension Signed-off-by: kim * remove unused code Signed-off-by: kim * fix expected storage paths in tests, update expected test thumbnails Signed-off-by: kim * remove dead code Signed-off-by: kim * fix cached presigned s3 url fetching Signed-off-by: kim * fix tests Signed-off-by: kim * fix test models Signed-off-by: kim * update media processing to use sync.Once{} for concurrency protection Signed-off-by: kim * shutup linter Signed-off-by: kim * fix passing in KVStore GetStream() as stream to PutStream() Signed-off-by: kim * fix unlocks of storage keys Signed-off-by: kim * whoops, return the error... Signed-off-by: kim * pour one out for tobi's code <3 Signed-off-by: kim * add back the byte slurping code Signed-off-by: kim * check for both ErrUnexpectedEOF and EOF Signed-off-by: kim * add back links to file format header information Signed-off-by: kim Signed-off-by: kim --- internal/storage/storage.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'internal/storage/storage.go') diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 48971f25c..6541a1fc5 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -26,12 +26,14 @@ import ( "path" "time" + "codeberg.org/gruf/go-bytesize" "codeberg.org/gruf/go-cache/v3/ttl" "codeberg.org/gruf/go-store/v2/kv" "codeberg.org/gruf/go-store/v2/storage" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/log" ) const ( @@ -63,9 +65,14 @@ func (d *Driver) URL(ctx context.Context, key string) *url.URL { return nil } - // access the cache member directly to avoid extending the TTL - if u, ok := d.PresignedCache.Cache.Get(key); ok { - return u.Value + // Check cache underlying cache map directly to + // avoid extending the TTL (which cache.Get() does). + d.PresignedCache.Lock() + e, ok := d.PresignedCache.Cache.Get(key) + d.PresignedCache.Unlock() + + if ok { + return e.Value } u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{ @@ -88,7 +95,6 @@ func AutoConfig() (*Driver, error) { default: return nil, fmt.Errorf("invalid storage backend: %s", backend) } - } func NewFileStorage() (*Driver, error) { @@ -102,12 +108,17 @@ func NewFileStorage() (*Driver, error) { // overwriting the lockfile if we store a file called 'store.lock'. // However, in this case it's OK because the keys are set by // GtS and not the user, so we know we're never going to overwrite it. - LockFile: path.Join(basePath, "store.lock"), + LockFile: path.Join(basePath, "store.lock"), + WriteBufSize: int(16 * bytesize.KiB), }) if err != nil { return nil, fmt.Errorf("error opening disk storage: %w", err) } + if err := disk.Clean(context.Background()); err != nil { + log.Errorf("error performing storage cleanup: %v", err) + } + return &Driver{ KVStore: kv.New(disk), Storage: disk, -- cgit v1.2.3