diff options
| author |  kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> | 2022-11-22 13:28:55 +0000 |
|---|---|---|
| committer |  GitHub <noreply@github.com> | 2022-11-22 14:28:55 +0100 |
| commit | a898160b0c8591be9677c158fa32d6a78d679152 (patch) | |
| tree | da8eb8324ec12374200aa9e5d7d25c3296a6b4da /vendor/codeberg.org/gruf/go-store | |
| parent | [docs] Document non-buildx cross compilation for docker image (#1115) (diff) | |
| download | gotosocial-a898160b0c8591be9677c158fa32d6a78d679152.tar.xz | |
[chore] use kv.KVStore also for S3 storage (#1113)
* replace s3 storage implementation to also use kv.KVStore
Signed-off-by: kim <grufwub@gmail.com>
* pull in latest `go-store` fix
Signed-off-by: kim <grufwub@gmail.com>
* pull-in go-store v2.0.9 fixes, update s3 put chunk size to 5MiB
Signed-off-by: kim <grufwub@gmail.com>
Signed-off-by: kim <grufwub@gmail.com>
Diffstat (limited to 'vendor/codeberg.org/gruf/go-store')
| -rw-r--r-- | vendor/codeberg.org/gruf/go-store/v2/storage/s3.go | 25 |
1 files changed, 15 insertions, 10 deletions
diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go b/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go index baf2a1b3c..f8011114f 100644 --- a/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go +++ b/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go @@ -50,14 +50,17 @@ type S3Config struct { // getS3Config returns a valid S3Config for supplied ptr. func getS3Config(cfg *S3Config) S3Config { + const minChunkSz = 5 * 1024 * 1024 + // If nil, use default if cfg == nil { cfg = DefaultS3Config } - // Assume 0 chunk size == use default - if cfg.PutChunkSize <= 0 { - cfg.PutChunkSize = 4 * 1024 * 1024 + // Ensure a minimum compatible chunk size + if cfg.PutChunkSize <= minChunkSz { + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + cfg.PutChunkSize = minChunkSz } // Assume 0 list size == use default @@ -67,11 +70,13 @@ func getS3Config(cfg *S3Config) S3Config { // Return owned config copy return S3Config{ - CoreOpts: cfg.CoreOpts, - GetOpts: cfg.GetOpts, - PutOpts: cfg.PutOpts, - StatOpts: cfg.StatOpts, - RemoveOpts: cfg.RemoveOpts, + CoreOpts: cfg.CoreOpts, + GetOpts: cfg.GetOpts, + PutOpts: cfg.PutOpts, + PutChunkSize: cfg.PutChunkSize, + ListSize: cfg.ListSize, + StatOpts: cfg.StatOpts, + RemoveOpts: cfg.RemoveOpts, } } @@ -198,7 +203,7 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e } var ( - count int + count = 1 parts []minio.CompletePart chunk = make([]byte, st.config.PutChunkSize) rdr = bytes.NewReader(nil) @@ -243,7 +248,7 @@ loop: uploadID, count, rdr, - st.config.PutChunkSize, + int64(n), "", "", nil, |