diff options
| author | 2024-09-26 12:43:10 +0000 | |
|---|---|---|
| committer | 2024-09-26 14:43:10 +0200 | |
| commit | 53ee6aef0885b4055ef95bf4f20ee78fd381e333 (patch) | |
| tree | a181b038dc969482b34690603328dd6b756a42fb | |
| parent | [chore] reduce number admin process workers (#3354) (diff) | |
| download | gotosocial-53ee6aef0885b4055ef95bf4f20ee78fd381e333.tar.xz | |
[bugfix] s3 media uploaded without content-type (#3353)
* update go-storage dependency, for S3Storage manually call PutObject() so we can set content-type
* update calls to PutFile() to include the contentType
| -rw-r--r-- | go.mod | 4 | ||||
| -rw-r--r-- | go.sum | 8 | ||||
| -rw-r--r-- | internal/media/processingemoji.go | 16 | ||||
| -rw-r--r-- | internal/media/processingmedia.go | 16 | ||||
| -rw-r--r-- | internal/storage/storage.go | 42 | ||||
| -rw-r--r-- | vendor/codeberg.org/gruf/go-storage/s3/errors.go | 26 | ||||
| -rw-r--r-- | vendor/codeberg.org/gruf/go-storage/s3/s3.go | 159 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go | 49 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/api-put-object.go | 28 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go | 2 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/api.go | 4 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/functional_tests.go | 253 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/post-policy.go | 19 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/v7/retry.go | 5 | ||||
| -rw-r--r-- | vendor/modules.txt | 4 | 
15 files changed, 429 insertions, 206 deletions
@@ -21,7 +21,7 @@ require (  	codeberg.org/gruf/go-mutexes v1.5.1  	codeberg.org/gruf/go-runners v1.6.2  	codeberg.org/gruf/go-sched v1.2.3 -	codeberg.org/gruf/go-storage v0.1.2 +	codeberg.org/gruf/go-storage v0.2.0  	codeberg.org/gruf/go-structr v0.8.9  	codeberg.org/superseriousbusiness/exif-terminator v0.9.0  	github.com/DmitriyVTitov/size v1.5.0 @@ -42,7 +42,7 @@ require (  	github.com/k3a/html2text v1.2.1  	github.com/microcosm-cc/bluemonday v1.0.27  	github.com/miekg/dns v1.1.62 -	github.com/minio/minio-go/v7 v7.0.76 +	github.com/minio/minio-go/v7 v7.0.77  	github.com/mitchellh/mapstructure v1.5.0  	github.com/ncruces/go-sqlite3 v0.18.3  	github.com/oklog/ulid v1.3.1 @@ -76,8 +76,8 @@ codeberg.org/gruf/go-runners v1.6.2 h1:oQef9niahfHu/wch14xNxlRMP8i+ABXH1Cb9PzZ4o  codeberg.org/gruf/go-runners v1.6.2/go.mod h1:Tq5PrZ/m/rBXbLZz0u5if+yP3nG5Sf6S8O/GnyEePeQ=  codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk=  codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A= -codeberg.org/gruf/go-storage v0.1.2 h1:dIOVOKq1CJpRmuhbB8Zok3mmo8V6VV/nX5GLIm6hywA= -codeberg.org/gruf/go-storage v0.1.2/go.mod h1:LRDpFHqRJi0f+35c3ltBH2e/pGfwY5dGlNlgCJ/R1DA= +codeberg.org/gruf/go-storage v0.2.0 h1:mKj3Lx6AavEkuXXtxqPhdq+akW9YwrnP16yQBF7K5ZI= +codeberg.org/gruf/go-storage v0.2.0/go.mod h1:o3GzMDE5QNUaRnm/daUzFqvuAaC4utlgXDXYO79sWKU=  codeberg.org/gruf/go-structr v0.8.9 h1:OyiSspWYCeJOm356fFPd+bDRumPrard2VAUXAPqZiJ0=  codeberg.org/gruf/go-structr v0.8.9/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM=  codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go= @@ -419,8 +419,8 @@ github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=  github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=  github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=  github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng= -github.com/minio/minio-go/v7 v7.0.76/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= +github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw= +github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=  github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=  github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=  github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 89a1bcc91..18d7895fd 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -216,10 +216,18 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {  		"png",  	) +	// Get mimetype for the file container +	// type, falling back to generic data. +	p.emoji.ImageContentType = getMimeType(ext) + +	// Set the known emoji static content type. +	p.emoji.ImageStaticContentType = "image/png" +  	// Copy temporary file into storage at path.  	filesz, err := p.mgr.state.Storage.PutFile(ctx,  		p.emoji.ImagePath,  		temppath, +		p.emoji.ImageContentType,  	)  	if err != nil {  		return gtserror.Newf("error writing emoji to storage: %w", err) @@ -229,6 +237,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {  	staticsz, err := p.mgr.state.Storage.PutFile(ctx,  		p.emoji.ImageStaticPath,  		staticpath, +		p.emoji.ImageStaticContentType,  	)  	if err != nil {  		return gtserror.Newf("error writing static to storage: %w", err) @@ -256,13 +265,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {  		"png",  	) -	// Get mimetype for the file container -	// type, falling back to generic data. -	p.emoji.ImageContentType = getMimeType(ext) - -	// Set the known emoji static content type. -	p.emoji.ImageStaticContentType = "image/png" -  	// We can now consider this cached.  	p.emoji.Cached = util.Ptr(true) diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 78c6c61a9..acd326457 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -261,10 +261,15 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		ext,  	) +	// Get mimetype for the file container +	// type, falling back to generic data. +	p.media.File.ContentType = getMimeType(ext) +  	// Copy temporary file into storage at path.  	filesz, err := p.mgr.state.Storage.PutFile(ctx,  		p.media.File.Path,  		temppath, +		p.media.File.ContentType,  	)  	if err != nil {  		return gtserror.Newf("error writing media to storage: %w", err) @@ -286,10 +291,14 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  			thumbExt,  		) +		// Determine thumbnail content-type from thumb ext. +		p.media.Thumbnail.ContentType = getMimeType(thumbExt) +  		// Copy thumbnail file into storage at path.  		thumbsz, err := p.mgr.state.Storage.PutFile(ctx,  			p.media.Thumbnail.Path,  			thumbpath, +			p.media.Thumbnail.ContentType,  		)  		if err != nil {  			return gtserror.Newf("error writing thumb to storage: %w", err) @@ -298,9 +307,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		// Set final determined thumbnail size.  		p.media.Thumbnail.FileSize = int(thumbsz) -		// Determine thumbnail content-type from thumb ext. -		p.media.Thumbnail.ContentType = getMimeType(thumbExt) -  		// Generate a media attachment thumbnail URL.  		p.media.Thumbnail.URL = uris.URIForAttachment(  			p.media.AccountID, @@ -320,10 +326,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		ext,  	) -	// Get mimetype for the file container -	// type, falling back to generic data. -	p.media.File.ContentType = getMimeType(ext) -  	// We can now consider this cached.  	p.media.Cached = util.Ptr(true) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index f3cb814f1..dcef9cde4 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -97,23 +97,39 @@ func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error)  	return d.Storage.WriteBytes(ctx, key, value)  } -// PutStream writes the bytes from supplied reader at key in the storage -func (d *Driver) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) { -	return d.Storage.WriteStream(ctx, key, r) -} - -// PutFile moves the contents of file at path, to storage.Driver{} under given key. -func (d *Driver) PutFile(ctx context.Context, key string, filepath string) (int64, error) { +// PutFile moves the contents of file at path, to storage.Driver{} under given key (with content-type if supported). +func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) (int64, error) {  	// Open file at path for reading.  	file, err := os.Open(filepath)  	if err != nil {  		return 0, gtserror.Newf("error opening file %s: %w", filepath, err)  	} -	// Write the file data to storage under key. Note -	// that for disk.DiskStorage{} this should end up -	// being a highly optimized Linux sendfile syscall. -	sz, err := d.Storage.WriteStream(ctx, key, file) +	var sz int64 + +	switch d := d.Storage.(type) { +	case *s3.S3Storage: +		var info minio.UploadInfo + +		// For S3 storage, write the file but specifically pass in the +		// content-type as an extra option. This handles the case of media +		// being served via CDN redirect (where we don't handle content-type). +		info, err = d.PutObject(ctx, key, file, minio.PutObjectOptions{ +			ContentType: contentType, +		}) + +		// Get size from +		// uploaded info. +		sz = info.Size + +	default: +		// Write the file data to storage under key. Note +		// that for disk.DiskStorage{} this should end up +		// being a highly optimized Linux sendfile syscall. +		sz, err = d.WriteStream(ctx, key, file) +	} + +	// Wrap write error.  	if err != nil {  		err = gtserror.Newf("error writing file %s: %w", key, err)  	} @@ -305,11 +321,7 @@ func NewS3Storage() (*Driver, error) {  			Creds:  credentials.NewStaticV4(access, secret, ""),  			Secure: secure,  		}, -		GetOpts:      minio.GetObjectOptions{}, -		PutOpts:      minio.PutObjectOptions{},  		PutChunkSize: 5 * 1024 * 1024, // 5MiB -		StatOpts:     minio.StatObjectOptions{}, -		RemoveOpts:   minio.RemoveObjectOptions{},  		ListSize:     200,  	})  	if err != nil { diff --git a/vendor/codeberg.org/gruf/go-storage/s3/errors.go b/vendor/codeberg.org/gruf/go-storage/s3/errors.go index 2cbdd2e9d..1f4404469 100644 --- a/vendor/codeberg.org/gruf/go-storage/s3/errors.go +++ b/vendor/codeberg.org/gruf/go-storage/s3/errors.go @@ -3,35 +3,9 @@ package s3  import (  	"strings" -	"codeberg.org/gruf/go-storage" -	"codeberg.org/gruf/go-storage/internal"  	"github.com/minio/minio-go/v7"  ) -// transformS3Error transforms an error returned from S3Storage underlying -// minio.Core client, by wrapping where necessary with our own error types. -func transformS3Error(err error) error { -	// Cast this to a minio error response -	ersp, ok := err.(minio.ErrorResponse) -	if ok { -		switch ersp.Code { -		case "NoSuchKey": -			return internal.WrapErr(err, storage.ErrNotFound) -		case "Conflict": -			return internal.WrapErr(err, storage.ErrAlreadyExists) -		default: -			return err -		} -	} - -	// Check if error has an invalid object name prefix -	if strings.HasPrefix(err.Error(), "Object name ") { -		return internal.WrapErr(err, storage.ErrInvalidKey) -	} - -	return err -} -  func isNotFoundError(err error) bool {  	errRsp, ok := err.(minio.ErrorResponse)  	return ok && errRsp.Code == "NoSuchKey" diff --git a/vendor/codeberg.org/gruf/go-storage/s3/s3.go b/vendor/codeberg.org/gruf/go-storage/s3/s3.go index 0067d3e19..ad7686737 100644 --- a/vendor/codeberg.org/gruf/go-storage/s3/s3.go +++ b/vendor/codeberg.org/gruf/go-storage/s3/s3.go @@ -5,6 +5,7 @@ import (  	"context"  	"errors"  	"io" +	"net/http"  	"codeberg.org/gruf/go-storage"  	"codeberg.org/gruf/go-storage/internal" @@ -34,12 +35,7 @@ func DefaultConfig() Config {  // immutable default configuration.  var defaultConfig = Config{  	CoreOpts:     minio.Options{}, -	GetOpts:      minio.GetObjectOptions{}, -	PutOpts:      minio.PutObjectOptions{}, -	PutChunkOpts: minio.PutObjectPartOptions{},  	PutChunkSize: 4 * 1024 * 1024, // 4MiB -	StatOpts:     minio.StatObjectOptions{}, -	RemoveOpts:   minio.RemoveObjectOptions{},  	ListSize:     200,  } @@ -50,31 +46,11 @@ type Config struct {  	// passed during initialization.  	CoreOpts minio.Options -	// GetOpts are S3 client options -	// passed during .Read___() calls. -	GetOpts minio.GetObjectOptions - -	// PutOpts are S3 client options -	// passed during .Write___() calls. -	PutOpts minio.PutObjectOptions -  	// PutChunkSize is the chunk size (in bytes)  	// to use when sending a byte stream reader  	// of unknown size as a multi-part object.  	PutChunkSize int64 -	// PutChunkOpts are S3 client options -	// passed during chunked .Write___() calls. -	PutChunkOpts minio.PutObjectPartOptions - -	// StatOpts are S3 client options -	// passed during .Stat() calls. -	StatOpts minio.StatObjectOptions - -	// RemoveOpts are S3 client options -	// passed during .Remove() calls. -	RemoveOpts minio.RemoveObjectOptions -  	// ListSize determines how many items  	// to include in each list request, made  	// during calls to .WalkKeys(). @@ -103,12 +79,8 @@ func getS3Config(cfg *Config) Config {  	return Config{  		CoreOpts:     cfg.CoreOpts, -		GetOpts:      cfg.GetOpts, -		PutOpts:      cfg.PutOpts,  		PutChunkSize: cfg.PutChunkSize,  		ListSize:     cfg.ListSize, -		StatOpts:     cfg.StatOpts, -		RemoveOpts:   cfg.RemoveOpts,  	}  } @@ -183,36 +155,50 @@ func (st *S3Storage) ReadBytes(ctx context.Context, key string) ([]byte, error)  // ReadStream: implements Storage.ReadStream().  func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) { -	// Fetch object reader from S3 bucket -	rc, _, _, err := st.client.GetObject( +	rc, _, _, err := st.GetObject(ctx, key, minio.GetObjectOptions{}) +	return rc, err +} + +// GetObject wraps minio.Core{}.GetObject() to handle wrapping with our own storage library error types. +func (st *S3Storage) GetObject(ctx context.Context, key string, opts minio.GetObjectOptions) (io.ReadCloser, minio.ObjectInfo, http.Header, error) { + +	// Query bucket for object data and info. +	rc, info, hdr, err := st.client.GetObject(  		ctx,  		st.bucket,  		key, -		st.config.GetOpts, +		opts,  	)  	if err != nil {  		if isNotFoundError(err) {  			// Wrap not found errors as our not found type.  			err = internal.WrapErr(err, storage.ErrNotFound) -		} else if !isObjectNameError(err) { +		} else if isObjectNameError(err) {  			// Wrap object name errors as our invalid key type.  			err = internal.WrapErr(err, storage.ErrInvalidKey)  		} -		return nil, transformS3Error(err)  	} -	return rc, nil + +	return rc, info, hdr, err  }  // WriteBytes: implements Storage.WriteBytes().  func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { -	n, err := st.WriteStream(ctx, key, bytes.NewReader(value)) -	return int(n), err +	info, err := st.PutObject(ctx, key, bytes.NewReader(value), minio.PutObjectOptions{}) +	return int(info.Size), err  }  // WriteStream: implements Storage.WriteStream().  func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { +	info, err := st.PutObject(ctx, key, r, minio.PutObjectOptions{}) +	return info.Size, err +} + +// PutObject wraps minio.Core{}.PutObject() to handle wrapping with our own storage library error types, and in the case of an io.Reader +// that does not implement ReaderSize{}, it will instead handle upload by using minio.Core{}.NewMultipartUpload() in chunks of PutChunkSize. +func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opts minio.PutObjectOptions) (minio.UploadInfo, error) {  	if rs, ok := r.(ReaderSize); ok {  		// This reader supports providing us the size of  		// the encompassed data, allowing us to perform @@ -225,22 +211,21 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (  			rs.Size(),  			"",  			"", -			st.config.PutOpts, +			opts,  		)  		if err != nil {  			if isConflictError(err) {  				// Wrap conflict errors as our already exists type.  				err = internal.WrapErr(err, storage.ErrAlreadyExists) -			} else if !isObjectNameError(err) { +			} else if isObjectNameError(err) {  				// Wrap object name errors as our invalid key type.  				err = internal.WrapErr(err, storage.ErrInvalidKey)  			} -			return 0, err  		} -		return info.Size, nil +		return info, err  	}  	// Start a new multipart upload to get ID. @@ -248,24 +233,24 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (  		ctx,  		st.bucket,  		key, -		st.config.PutOpts, +		opts,  	)  	if err != nil {  		if isConflictError(err) {  			// Wrap conflict errors as our already exists type.  			err = internal.WrapErr(err, storage.ErrAlreadyExists) -		} else if !isObjectNameError(err) { +		} else if isObjectNameError(err) {  			// Wrap object name errors as our invalid key type.  			err = internal.WrapErr(err, storage.ErrInvalidKey)  		} -		return 0, transformS3Error(err) +		return minio.UploadInfo{}, err  	}  	var ( -		index = int(1) // parts index  		total = int64(0) +		index = int(1) // parts index  		parts []minio.CompletePart  		chunk = make([]byte, st.config.PutChunkSize)  		rbuf  = bytes.NewReader(nil) @@ -296,7 +281,7 @@ loop:  		// All other errors.  		default: -			return 0, err +			return minio.UploadInfo{}, err  		}  		// Reset byte reader. @@ -311,10 +296,13 @@ loop:  			index,  			rbuf,  			int64(n), -			st.config.PutChunkOpts, +			minio.PutObjectPartOptions{ +				SSE:                  opts.ServerSideEncryption, +				DisableContentSha256: opts.DisableContentSha256, +			},  		)  		if err != nil { -			return 0, err +			return minio.UploadInfo{}, err  		}  		// Append completed part to slice. @@ -327,101 +315,104 @@ loop:  			ChecksumSHA256: pt.ChecksumSHA256,  		}) +		// Update total. +		total += int64(n) +  		// Iterate.  		index++ - -		// Update total size. -		total += pt.Size  	}  	// Complete this multi-part upload operation -	_, err = st.client.CompleteMultipartUpload( +	info, err := st.client.CompleteMultipartUpload(  		ctx,  		st.bucket,  		key,  		uploadID,  		parts, -		st.config.PutOpts, +		opts,  	)  	if err != nil { -		return 0, err +		return minio.UploadInfo{}, err  	} -	return total, nil +	// Set correct size. +	info.Size = total +	return info, nil  }  // Stat: implements Storage.Stat().  func (st *S3Storage) Stat(ctx context.Context, key string) (*storage.Entry, error) { -	// Query object in S3 bucket. -	stat, err := st.client.StatObject( -		ctx, -		st.bucket, -		key, -		st.config.StatOpts, -	) +	info, err := st.StatObject(ctx, key, minio.StatObjectOptions{})  	if err != nil { - -		if isNotFoundError(err) { -			// Ignore err return -			// for not-found. -			err = nil -		} else if !isObjectNameError(err) { -			// Wrap object name errors as our invalid key type. -			err = internal.WrapErr(err, storage.ErrInvalidKey) +		if errors.Is(err, storage.ErrNotFound) { +			err = nil // mask not-found errors  		} -  		return nil, err  	} -  	return &storage.Entry{  		Key:  key, -		Size: stat.Size, +		Size: info.Size,  	}, nil  } -// Remove: implements Storage.Remove(). -func (st *S3Storage) Remove(ctx context.Context, key string) error { -	// Query object in S3 bucket. -	_, err := st.client.StatObject( +// StatObject wraps minio.Core{}.StatObject() to handle wrapping with our own storage library error types. +func (st *S3Storage) StatObject(ctx context.Context, key string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + +	// Query bucket for object info. +	info, err := st.client.StatObject(  		ctx,  		st.bucket,  		key, -		st.config.StatOpts, +		opts,  	)  	if err != nil {  		if isNotFoundError(err) {  			// Wrap not found errors as our not found type.  			err = internal.WrapErr(err, storage.ErrNotFound) -		} else if !isObjectNameError(err) { +		} else if isObjectNameError(err) {  			// Wrap object name errors as our invalid key type.  			err = internal.WrapErr(err, storage.ErrInvalidKey)  		} +	} + +	return info, err +} + +// Remove: implements Storage.Remove(). +func (st *S3Storage) Remove(ctx context.Context, key string) error { +	_, err := st.StatObject(ctx, key, minio.StatObjectOptions{}) +	if err != nil {  		return err  	} +	return st.RemoveObject(ctx, key, minio.RemoveObjectOptions{}) +} + +// RemoveObject wraps minio.Core{}.RemoveObject() to handle wrapping with our own storage library error types. +func (st *S3Storage) RemoveObject(ctx context.Context, key string, opts minio.RemoveObjectOptions) error {  	// Remove object from S3 bucket -	err = st.client.RemoveObject( +	err := st.client.RemoveObject(  		ctx,  		st.bucket,  		key, -		st.config.RemoveOpts, +		opts,  	) +  	if err != nil {  		if isNotFoundError(err) {  			// Wrap not found errors as our not found type.  			err = internal.WrapErr(err, storage.ErrNotFound) -		} else if !isObjectNameError(err) { +		} else if isObjectNameError(err) {  			// Wrap object name errors as our invalid key type.  			err = internal.WrapErr(err, storage.ErrInvalidKey)  		} -		return err  	} -	return nil +	return err  }  // WalkKeys: implements Storage.WalkKeys(). diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index 7f316564b..eef976c8c 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -108,7 +108,9 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN  	if err != nil {  		return UploadInfo{}, err  	} - +	if opts.Checksum.IsSet() { +		opts.AutoChecksum = opts.Checksum +	}  	withChecksum := c.trailingHeaderSupport  	if withChecksum {  		if opts.UserMetadata == nil { @@ -304,6 +306,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b  		return UploadInfo{}, err  	} +	if opts.Checksum.IsSet() { +		opts.AutoChecksum = opts.Checksum +		opts.SendContentMd5 = false +	} +  	if !opts.SendContentMd5 {  		if opts.UserMetadata == nil {  			opts.UserMetadata = make(map[string]string, 1) @@ -463,7 +470,10 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam  	if err = s3utils.CheckValidObjectName(objectName); err != nil {  		return UploadInfo{}, err  	} - +	if opts.Checksum.IsSet() { +		opts.SendContentMd5 = false +		opts.AutoChecksum = opts.Checksum +	}  	if !opts.SendContentMd5 {  		if opts.UserMetadata == nil {  			opts.UserMetadata = make(map[string]string, 1) @@ -555,7 +565,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam  		// Calculate md5sum.  		customHeader := make(http.Header)  		if !opts.SendContentMd5 { -			// Add CRC32C instead. +			// Add Checksum instead.  			crc.Reset()  			crc.Write(buf[:length])  			cSum := crc.Sum(nil) @@ -677,6 +687,9 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r  	if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 {  		return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'")  	} +	if opts.Checksum.IsSet() { +		opts.SendContentMd5 = false +	}  	var readSeeker io.Seeker  	if size > 0 { @@ -746,17 +759,6 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,  	// Set headers.  	customHeader := opts.Header() -	// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. -	addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) - -	if addCrc { -		// If user has added checksums, don't add them ourselves. -		for k := range opts.UserMetadata { -			if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { -				addCrc = false -			} -		} -	}  	// Populate request metadata.  	reqMetadata := requestMetadata{  		bucketName:       bucketName, @@ -768,10 +770,23 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,  		contentSHA256Hex: sha256Hex,  		streamSha256:     !opts.DisableContentSha256,  	} -	if addCrc { -		opts.AutoChecksum.SetDefault(ChecksumCRC32C) -		reqMetadata.addCrc = &opts.AutoChecksum +	// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. +	addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) +	if opts.Checksum.IsSet() { +		reqMetadata.addCrc = &opts.Checksum +	} else if addCrc { +		// If user has added checksums, don't add them ourselves. +		for k := range opts.UserMetadata { +			if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { +				addCrc = false +			} +		} +		if addCrc { +			opts.AutoChecksum.SetDefault(ChecksumCRC32C) +			reqMetadata.addCrc = &opts.AutoChecksum +		}  	} +  	if opts.Internal.SourceVersionID != "" {  		if opts.Internal.SourceVersionID != nullVersionID {  			if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index a792cfe39..d769648a7 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -94,6 +94,13 @@ type PutObjectOptions struct {  	// If none is specified CRC32C is used, since it is generally the fastest.  	AutoChecksum ChecksumType +	// Checksum will force a checksum of the specific type. +	// This requires that the client was created with "TrailingHeaders:true" option, +	// and that the destination server supports it. +	// Unavailable with V2 signatures & Google endpoints. +	// This will disable content MD5 checksums if set. +	Checksum ChecksumType +  	// ConcurrentStreamParts will create NumThreads buffers of PartSize bytes,  	// fill them serially and upload them in parallel.  	// This can be used for faster uploads on non-seekable or slow-to-seek input. @@ -240,7 +247,7 @@ func (opts PutObjectOptions) Header() (header http.Header) {  }  // validate() checks if the UserMetadata map has standard headers or and raises an error if so. -func (opts PutObjectOptions) validate() (err error) { +func (opts PutObjectOptions) validate(c *Client) (err error) {  	for k, v := range opts.UserMetadata {  		if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) {  			return errInvalidArgument(k + " unsupported user defined metadata name") @@ -255,6 +262,17 @@ func (opts PutObjectOptions) validate() (err error) {  	if opts.LegalHold != "" && !opts.LegalHold.IsValid() {  		return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status")  	} +	if opts.Checksum.IsSet() { +		switch { +		case !c.trailingHeaderSupport: +			return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled") +		case c.overrideSignerType.IsV2(): +			return errInvalidArgument("Checksum cannot be used with v2 signatures") +		case s3utils.IsGoogleEndpoint(*c.endpointURL): +			return errInvalidArgument("Checksum cannot be used with GCS endpoints") +		} +	} +  	return nil  } @@ -291,7 +309,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r  		return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")  	} -	err = opts.validate() +	err = opts.validate(c)  	if err != nil {  		return UploadInfo{}, err  	} @@ -333,7 +351,7 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str  		return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts)  	} -	if size < int64(partSize) || opts.DisableMultipart { +	if size <= int64(partSize) || opts.DisableMultipart {  		return c.putObject(ctx, bucketName, objectName, reader, size, opts)  	} @@ -362,6 +380,10 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam  		return UploadInfo{}, err  	} +	if opts.Checksum.IsSet() { +		opts.SendContentMd5 = false +		opts.AutoChecksum = opts.Checksum +	}  	if !opts.SendContentMd5 {  		if opts.UserMetadata == nil {  			opts.UserMetadata = make(map[string]string, 1) diff --git a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go index eb4da4147..6b6559bf7 100644 --- a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go +++ b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go @@ -107,7 +107,7 @@ type readSeekCloser interface {  // Total size should be < 5TB.  // This function blocks until 'objs' is closed and the content has been uploaded.  func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) { -	err = opts.Opts.validate() +	err = opts.Opts.validate(&c)  	if err != nil {  		return err  	} diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index be28e3fdf..1d6b66502 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -128,7 +128,7 @@ type Options struct {  // Global constants.  const (  	libraryName    = "minio-go" -	libraryVersion = "v7.0.76" +	libraryVersion = "v7.0.77"  )  // User Agent should always following the below style. @@ -661,7 +661,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ  		// Initiate the request.  		res, err = c.do(req)  		if err != nil { -			if isRequestErrorRetryable(err) { +			if isRequestErrorRetryable(ctx, err) {  				// Retry the request  				continue  			} diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index 8a908e3fd..780dc8997 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -83,7 +83,7 @@ func createHTTPTransport() (transport *http.Transport) {  		return nil  	} -	if mustParseBool(os.Getenv(skipCERTValidation)) { +	if mustParseBool(os.Getenv(enableHTTPS)) && mustParseBool(os.Getenv(skipCERTValidation)) {  		transport.TLSClientConfig.InsecureSkipVerify = true  	} @@ -2334,7 +2334,7 @@ func testPutObjectWithChecksums() {  }  // Test PutObject with custom checksums. -func testPutMultipartObjectWithChecksums() { +func testPutObjectWithTrailingChecksums() {  	// initialize logging params  	startTime := time.Now()  	testName := getFuncName() @@ -2342,7 +2342,7 @@ func testPutMultipartObjectWithChecksums() {  	args := map[string]interface{}{  		"bucketName": "",  		"objectName": "", -		"opts":       "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", +		"opts":       "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}",  	}  	if !isFullMode() { @@ -2356,9 +2356,201 @@ func testPutMultipartObjectWithChecksums() {  	// Instantiate new minio client object.  	c, err := minio.New(os.Getenv(serverEndpoint),  		&minio.Options{ -			Creds:     credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), -			Transport: createHTTPTransport(), -			Secure:    mustParseBool(os.Getenv(enableHTTPS)), +			Creds:           credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), +			Transport:       createHTTPTransport(), +			Secure:          mustParseBool(os.Getenv(enableHTTPS)), +			TrailingHeaders: true, +		}) +	if err != nil { +		logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) +		return +	} + +	// Enable tracing, write to stderr. +	// c.TraceOn(os.Stderr) + +	// Set user agent. +	c.SetAppInfo("MinIO-go-FunctionalTest", appVersion) + +	// Generate a new random bucket name. +	bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") +	args["bucketName"] = bucketName + +	// Make a new bucket. +	err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) +	if err != nil { +		logError(testName, function, args, startTime, "", "Make bucket failed", err) +		return +	} + +	defer cleanupBucket(bucketName, c) +	tests := []struct { +		cs minio.ChecksumType +	}{ +		{cs: minio.ChecksumCRC32C}, +		{cs: minio.ChecksumCRC32}, +		{cs: minio.ChecksumSHA1}, +		{cs: minio.ChecksumSHA256}, +	} + +	for _, test := range tests { +		function := "PutObject(bucketName, objectName, reader,size, opts)" +		bufSize := dataFileMap["datafile-10-kB"] + +		// Save the data +		objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") +		args["objectName"] = objectName + +		cmpChecksum := func(got, want string) { +			if want != got { +				logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) +				return +			} +		} + +		meta := map[string]string{} +		reader := getDataReader("datafile-10-kB") +		b, err := io.ReadAll(reader) +		if err != nil { +			logError(testName, function, args, startTime, "", "Read failed", err) +			return +		} +		h := test.cs.Hasher() +		h.Reset() + +		// Test with Wrong CRC. +		args["metadata"] = meta +		args["range"] = "false" +		args["checksum"] = test.cs.String() + +		resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ +			DisableMultipart:     true, +			DisableContentSha256: true, +			UserMetadata:         meta, +			Checksum:             test.cs, +		}) +		if err != nil { +			logError(testName, function, args, startTime, "", "PutObject failed", err) +			return +		} + +		h.Write(b) +		meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) + +		cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) +		cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) +		cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) +		cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + +		// Read the data back +		gopts := minio.GetObjectOptions{Checksum: true} + +		function = "GetObject(...)" +		r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) +		if err != nil { +			logError(testName, function, args, startTime, "", "GetObject failed", err) +			return +		} + +		st, err := r.Stat() +		if err != nil { +			logError(testName, function, args, startTime, "", "Stat failed", err) +			return +		} +		cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) +		cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) +		cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) +		cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + +		if st.Size != int64(bufSize) { +			logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) +			return +		} + +		if err := r.Close(); err != nil { +			logError(testName, function, args, startTime, "", "Object Close failed", err) +			return +		} +		if err := r.Close(); err == nil { +			logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) +			return +		} + +		function = "GetObject( Range...)" +		args["range"] = "true" +		err = gopts.SetRange(100, 1000) +		if err != nil { +			logError(testName, function, args, startTime, "", "SetRange failed", err) +			return +		} +		r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) +		if err != nil { +			logError(testName, function, args, startTime, "", "GetObject failed", err) +			return +		} + +		b, err = io.ReadAll(r) +		if err != nil { +			logError(testName, function, args, startTime, "", "Read failed", err) +			return +		} +		st, err = r.Stat() +		if err != nil { +			logError(testName, function, args, startTime, "", "Stat failed", err) +			return +		} + +		// Range requests should return empty checksums... +		cmpChecksum(st.ChecksumSHA256, "") +		cmpChecksum(st.ChecksumSHA1, "") +		cmpChecksum(st.ChecksumCRC32, "") +		cmpChecksum(st.ChecksumCRC32C, "") + +		function = "GetObjectAttributes(...)" +		s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) +		if err != nil { +			logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) +			return +		} +		cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"]) +		cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"]) +		cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"]) +		cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + +		delete(args, "range") +		delete(args, "metadata") +	} + +	logSuccess(testName, function, args, startTime) +} + +// Test PutObject with custom checksums. +func testPutMultipartObjectWithChecksums(trailing bool) { +	// initialize logging params +	startTime := time.Now() +	testName := getFuncName() +	function := "PutObject(bucketName, objectName, reader,size, opts)" +	args := map[string]interface{}{ +		"bucketName": "", +		"objectName": "", +		"opts":       fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: %v}", trailing), +	} + +	if !isFullMode() { +		logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") +		return +	} + +	// Seed random based on current time. +	rand.Seed(time.Now().Unix()) + +	// Instantiate new minio client object. +	c, err := minio.New(os.Getenv(serverEndpoint), +		&minio.Options{ +			Creds:           credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), +			Transport:       createHTTPTransport(), +			Secure:          mustParseBool(os.Getenv(enableHTTPS)), +			TrailingHeaders: trailing,  		})  	if err != nil {  		logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) @@ -2445,14 +2637,20 @@ func testPutMultipartObjectWithChecksums() {  		h.Reset()  		want := hashMultiPart(b, partSize, test.cs.Hasher()) +		var cs minio.ChecksumType +		rd := io.Reader(io.NopCloser(bytes.NewReader(b))) +		if trailing { +			cs = test.cs +			rd = bytes.NewReader(b) +		}  		// Set correct CRC. - -		resp, err := c.PutObject(context.Background(), bucketName, objectName, io.NopCloser(bytes.NewReader(b)), int64(bufSize), minio.PutObjectOptions{ +		resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{  			DisableContentSha256: true,  			DisableMultipart:     false,  			UserMetadata:         nil,  			PartSize:             partSize,  			AutoChecksum:         test.cs, +			Checksum:             cs,  		})  		if err != nil {  			logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -2982,6 +3180,7 @@ func testGetObjectAttributes() {  		testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{  			ContentType:    v.ContentType,  			SendContentMd5: v.SendContentMd5, +			Checksum:       minio.ChecksumCRC32C,  		})  		if err != nil {  			logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -3063,7 +3262,7 @@ func testGetObjectAttributes() {  		test: objectAttributesTestOptions{  			TestFileName:    "file1",  			StorageClass:    "STANDARD", -			HasFullChecksum: false, +			HasFullChecksum: true,  		},  	} @@ -3152,9 +3351,10 @@ func testGetObjectAttributesSSECEncryption() {  	info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{  		ContentType:          "content/custom", -		SendContentMd5:       true, +		SendContentMd5:       false,  		ServerSideEncryption: sse,  		PartSize:             uint64(bufSize) / 2, +		Checksum:             minio.ChecksumCRC32C,  	})  	if err != nil {  		logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -3174,9 +3374,9 @@ func testGetObjectAttributesSSECEncryption() {  		ETag:             info.ETag,  		NumberOfParts:    2,  		ObjectSize:       int(info.Size), -		HasFullChecksum:  false, +		HasFullChecksum:  true,  		HasParts:         true, -		HasPartChecksums: false, +		HasPartChecksums: true,  	})  	if err != nil {  		logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err) @@ -5594,18 +5794,12 @@ func testPresignedPostPolicy() {  	}  	writer.Close() -	transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) -	if err != nil { -		logError(testName, function, args, startTime, "", "DefaultTransport failed", err) -		return -	} -  	httpClient := &http.Client{  		// Setting a sensible time out of 30secs to wait for response  		// headers. Request is pro-actively canceled after 30secs  		// with no response.  		Timeout:   30 * time.Second, -		Transport: transport, +		Transport: createHTTPTransport(),  	}  	args["url"] = presignedPostPolicyURL.String() @@ -7519,7 +7713,7 @@ func testFunctional() {  		return  	} -	transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) +	transport := createHTTPTransport()  	if err != nil {  		logError(testName, function, args, startTime, "", "DefaultTransport failed", err)  		return @@ -12450,18 +12644,12 @@ func testFunctionalV2() {  		return  	} -	transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) -	if err != nil { -		logError(testName, function, args, startTime, "", "DefaultTransport failed", err) -		return -	} -  	httpClient := &http.Client{  		// Setting a sensible time out of 30secs to wait for response  		// headers. Request is pro-actively canceled after 30secs  		// with no response.  		Timeout:   30 * time.Second, -		Transport: transport, +		Transport: createHTTPTransport(),  	}  	req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) @@ -13556,14 +13744,9 @@ func testCors() {  	bucketURL := c.EndpointURL().String() + "/" + bucketName + "/"  	objectURL := bucketURL + objectName -	transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) -	if err != nil { -		logError(testName, function, args, startTime, "", "DefaultTransport failed", err) -		return -	}  	httpClient := &http.Client{  		Timeout:   30 * time.Second, -		Transport: transport, +		Transport: createHTTPTransport(),  	}  	errStrAccessForbidden := `<Error><Code>AccessForbidden</Code><Message>CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted` @@ -14757,7 +14940,9 @@ func main() {  		testCompose10KSourcesV2()  		testUserMetadataCopyingV2()  		testPutObjectWithChecksums() -		testPutMultipartObjectWithChecksums() +		testPutObjectWithTrailingChecksums() +		testPutMultipartObjectWithChecksums(false) +		testPutMultipartObjectWithChecksums(true)  		testPutObject0ByteV2()  		testPutObjectNoLengthV2()  		testPutObjectsUnknownV2() diff --git a/vendor/github.com/minio/minio-go/v7/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go index 3f023704a..19687e027 100644 --- a/vendor/github.com/minio/minio-go/v7/post-policy.go +++ b/vendor/github.com/minio/minio-go/v7/post-policy.go @@ -301,6 +301,25 @@ func (p *PostPolicy) SetUserMetadata(key, value string) error {  	return nil  } +// SetUserMetadataStartsWith - Set how an user metadata should starts with. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserMetadataStartsWith(key, value string) error { +	if strings.TrimSpace(key) == "" || key == "" { +		return errInvalidArgument("Key is empty") +	} +	headerName := fmt.Sprintf("x-amz-meta-%s", key) +	policyCond := policyCondition{ +		matchType: "starts-with", +		condition: fmt.Sprintf("$%s", headerName), +		value:     value, +	} +	if err := p.addNewPolicy(policyCond); err != nil { +		return err +	} +	p.formData[headerName] = value +	return nil +} +  // SetChecksum sets the checksum of the request.  func (p *PostPolicy) SetChecksum(c Checksum) {  	if c.IsSet() { diff --git a/vendor/github.com/minio/minio-go/v7/retry.go b/vendor/github.com/minio/minio-go/v7/retry.go index 5ddcad897..d15eb5901 100644 --- a/vendor/github.com/minio/minio-go/v7/retry.go +++ b/vendor/github.com/minio/minio-go/v7/retry.go @@ -129,9 +129,10 @@ func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {  }  // For now, all http Do() requests are retriable except some well defined errors -func isRequestErrorRetryable(err error) bool { +func isRequestErrorRetryable(ctx context.Context, err error) bool {  	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { -		return false +		// Retry if internal timeout in the HTTP call. +		return ctx.Err() == nil  	}  	if ue, ok := err.(*url.Error); ok {  		e := ue.Unwrap() diff --git a/vendor/modules.txt b/vendor/modules.txt index 8bad9b85b..26cd7850d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -65,7 +65,7 @@ codeberg.org/gruf/go-runners  # codeberg.org/gruf/go-sched v1.2.3  ## explicit; go 1.19  codeberg.org/gruf/go-sched -# codeberg.org/gruf/go-storage v0.1.2 +# codeberg.org/gruf/go-storage v0.2.0  ## explicit; go 1.22  codeberg.org/gruf/go-storage  codeberg.org/gruf/go-storage/disk @@ -491,7 +491,7 @@ github.com/miekg/dns  # github.com/minio/md5-simd v1.1.2  ## explicit; go 1.14  github.com/minio/md5-simd -# github.com/minio/minio-go/v7 v7.0.76 +# github.com/minio/minio-go/v7 v7.0.77  ## explicit; go 1.21  github.com/minio/minio-go/v7  github.com/minio/minio-go/v7/pkg/cors  | 
