diff options
| author |  kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> | 2023-01-11 11:13:13 +0000 |
|---|---|---|
| committer |  GitHub <noreply@github.com> | 2023-01-11 12:13:13 +0100 |
| commit | 53180548083c0a100db2f703d5f5da047a9e0031 (patch) | |
| tree | a8eb1df9d03b37f907a747ae42cc8992d2ff9f52 /internal/storage/storage.go | |
| parent | [feature] Add local user and post count to nodeinfo responses (#1325) (diff) | |
| download | gotosocial-53180548083c0a100db2f703d5f5da047a9e0031.tar.xz | |
[performance] media processing improvements (#1288)
* media processor consolidation and reformatting, reduce amount of required syscalls
Signed-off-by: kim <grufwub@gmail.com>
* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling
Signed-off-by: kim <grufwub@gmail.com>
* fix duration not being set, fix mp4 test expecting error
Signed-off-by: kim <grufwub@gmail.com>
* fix test expecting media files with different extension
Signed-off-by: kim <grufwub@gmail.com>
* remove unused code
Signed-off-by: kim <grufwub@gmail.com>
* fix expected storage paths in tests, update expected test thumbnails
Signed-off-by: kim <grufwub@gmail.com>
* remove dead code
Signed-off-by: kim <grufwub@gmail.com>
* fix cached presigned s3 url fetching
Signed-off-by: kim <grufwub@gmail.com>
* fix tests
Signed-off-by: kim <grufwub@gmail.com>
* fix test models
Signed-off-by: kim <grufwub@gmail.com>
* update media processing to use sync.Once{} for concurrency protection
Signed-off-by: kim <grufwub@gmail.com>
* shutup linter
Signed-off-by: kim <grufwub@gmail.com>
* fix passing in KVStore GetStream() as stream to PutStream()
Signed-off-by: kim <grufwub@gmail.com>
* fix unlocks of storage keys
Signed-off-by: kim <grufwub@gmail.com>
* whoops, return the error...
Signed-off-by: kim <grufwub@gmail.com>
* pour one out for tobi's code <3
Signed-off-by: kim <grufwub@gmail.com>
* add back the byte slurping code
Signed-off-by: kim <grufwub@gmail.com>
* check for both ErrUnexpectedEOF and EOF
Signed-off-by: kim <grufwub@gmail.com>
* add back links to file format header information
Signed-off-by: kim <grufwub@gmail.com>
Signed-off-by: kim <grufwub@gmail.com>
Diffstat (limited to 'internal/storage/storage.go')
| -rw-r--r-- | internal/storage/storage.go | 21 |
1 files changed, 16 insertions, 5 deletions
diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 48971f25c..6541a1fc5 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -26,12 +26,14 @@ import ( "path" "time" + "codeberg.org/gruf/go-bytesize" "codeberg.org/gruf/go-cache/v3/ttl" "codeberg.org/gruf/go-store/v2/kv" "codeberg.org/gruf/go-store/v2/storage" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/log" ) const ( @@ -63,9 +65,14 @@ func (d *Driver) URL(ctx context.Context, key string) *url.URL { return nil } - // access the cache member directly to avoid extending the TTL - if u, ok := d.PresignedCache.Cache.Get(key); ok { - return u.Value + // Check cache underlying cache map directly to + // avoid extending the TTL (which cache.Get() does). + d.PresignedCache.Lock() + e, ok := d.PresignedCache.Cache.Get(key) + d.PresignedCache.Unlock() + + if ok { + return e.Value } u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{ @@ -88,7 +95,6 @@ func AutoConfig() (*Driver, error) { default: return nil, fmt.Errorf("invalid storage backend: %s", backend) } - } func NewFileStorage() (*Driver, error) { @@ -102,12 +108,17 @@ func NewFileStorage() (*Driver, error) { // overwriting the lockfile if we store a file called 'store.lock'. // However, in this case it's OK because the keys are set by // GtS and not the user, so we know we're never going to overwrite it. - LockFile: path.Join(basePath, "store.lock"), + LockFile: path.Join(basePath, "store.lock"), + WriteBufSize: int(16 * bytesize.KiB), }) if err != nil { return nil, fmt.Errorf("error opening disk storage: %w", err) } + if err := disk.Clean(context.Background()); err != nil { + log.Errorf("error performing storage cleanup: %v", err) + } + return &Driver{ KVStore: kv.New(disk), Storage: disk, |