diff options
Diffstat (limited to 'vendor/github.com/minio/minio-go')
32 files changed, 407 insertions, 168 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/Makefile b/vendor/github.com/minio/minio-go/v7/Makefile index ace666705..40d57c130 100644 --- a/vendor/github.com/minio/minio-go/v7/Makefile +++ b/vendor/github.com/minio/minio-go/v7/Makefile @@ -16,6 +16,8 @@ lint: vet: @GO111MODULE=on go vet ./... + @echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest + ${GOPATH}/bin/staticcheck -tests=false -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-ST1023,-ST1005" test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go index 7e2199732..3f88d0777 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go @@ -21,7 +21,7 @@ import ( "bytes" "context" "encoding/xml" - "io/ioutil" + "io" "net/http" "net/url" @@ -143,5 +143,5 @@ func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]b } } - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) } diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go b/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go index e7edf9c97..dbb5259a8 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go @@ -18,7 +18,7 @@ package minio import ( "context" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -137,7 +137,7 @@ func (c *Client) getBucketPolicy(ctx context.Context, bucketName string) (string } } - bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) + bucketPolicyBuf, err := io.ReadAll(resp.Body) if err != nil { return "", err } diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go index c80488eee..d5895dfe0 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go @@ -22,7 +22,7 @@ import ( "context" "encoding/json" "encoding/xml" - "io/ioutil" + "io" "net/http" "net/url" "time" @@ -180,7 +180,7 @@ func (c *Client) GetBucketReplicationMetrics(ctx context.Context, bucketName str if resp.StatusCode != http.StatusOK { return s, httpRespToErrorResponse(resp, bucketName, "") } - respBytes, err := ioutil.ReadAll(resp.Body) + respBytes, err := io.ReadAll(resp.Body) if err != nil { return s, err } diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go b/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go index 1615f8f87..86d74298a 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go @@ -22,7 +22,6 @@ import ( "encoding/xml" "errors" "io" - "io/ioutil" "net/http" "net/url" @@ -58,7 +57,7 @@ func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags return nil, httpRespToErrorResponse(resp, bucketName, "") } - defer io.Copy(ioutil.Discard, resp.Body) + defer io.Copy(io.Discard, resp.Body) return tags.ParseBucketXML(resp.Body) } diff --git a/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/vendor/github.com/minio/minio-go/v7/api-compose-object.go index 835c8bd8a..1ba68c79e 100644 --- a/vendor/github.com/minio/minio-go/v7/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -516,7 +515,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. return UploadInfo{}, err } if dst.Progress != nil { - io.CopyN(ioutil.Discard, dst.Progress, end-start+1) + io.CopyN(io.Discard, dst.Progress, end-start+1) } objParts = append(objParts, complPart) partIndex++ @@ -525,7 +524,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. // 4. Make final complete-multipart request. uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID, - completeMultipartUpload{Parts: objParts}, PutObjectOptions{}) + completeMultipartUpload{Parts: objParts}, PutObjectOptions{ServerSideEncryption: dst.Encryption}) if err != nil { return UploadInfo{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/api-copy-object.go b/vendor/github.com/minio/minio-go/v7/api-copy-object.go index 1c0ad2be4..0c95d91ec 100644 --- a/vendor/github.com/minio/minio-go/v7/api-copy-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-copy-object.go @@ -20,7 +20,6 @@ package minio import ( "context" "io" - "io/ioutil" "net/http" ) @@ -54,7 +53,7 @@ func (c *Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySr // Update the progress properly after successful copy. if dst.Progress != nil { - io.Copy(ioutil.Discard, io.LimitReader(dst.Progress, dst.Size)) + io.Copy(io.Discard, io.LimitReader(dst.Progress, dst.Size)) } cpObjRes := copyObjectResult{} diff --git a/vendor/github.com/minio/minio-go/v7/api-error-response.go b/vendor/github.com/minio/minio-go/v7/api-error-response.go index 33ca39458..4ec0c87c2 100644 --- a/vendor/github.com/minio/minio-go/v7/api-error-response.go +++ b/vendor/github.com/minio/minio-go/v7/api-error-response.go @@ -22,7 +22,6 @@ import ( "encoding/xml" "fmt" "io" - "io/ioutil" "net/http" ) @@ -108,7 +107,7 @@ const ( func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) { // read the whole body (up to 1MB) const maxBodyLength = 1 << 20 - body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) + body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) if err != nil { return nil, err } @@ -253,26 +252,6 @@ func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) } } -// errInvalidBucketName - Invalid bucket name response. -func errInvalidBucketName(message string) error { - return ErrorResponse{ - StatusCode: http.StatusBadRequest, - Code: "InvalidBucketName", - Message: message, - RequestID: "minio", - } -} - -// errInvalidObjectName - Invalid object name response. -func errInvalidObjectName(message string) error { - return ErrorResponse{ - StatusCode: http.StatusNotFound, - Code: "NoSuchKey", - Message: message, - RequestID: "minio", - } -} - // errInvalidArgument - Invalid argument response. func errInvalidArgument(message string) error { return ErrorResponse{ diff --git a/vendor/github.com/minio/minio-go/v7/api-list.go b/vendor/github.com/minio/minio-go/v7/api-list.go index d216afb39..bfb2c1151 100644 --- a/vendor/github.com/minio/minio-go/v7/api-list.go +++ b/vendor/github.com/minio/minio-go/v7/api-list.go @@ -897,6 +897,8 @@ func (c *Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyM } // listObjectParts list all object parts recursively. +// +//lint:ignore U1000 Keep this around func (c *Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { // Part number marker for the next batch of request. var nextPartNumberMarker int diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go index 3c9a13ff2..18bdeb24c 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go @@ -26,7 +26,6 @@ import ( "fmt" "hash/crc32" "io" - "io/ioutil" "net/http" "net/url" "sort" @@ -201,7 +200,9 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - opts = PutObjectOptions{} + opts = PutObjectOptions{ + ServerSideEncryption: opts.ServerSideEncryption, + } if len(crcBytes) > 0 { // Add hash of hashes. crc.Reset() @@ -412,7 +413,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object // Read resp.Body into a []bytes to parse for Error response inside the body var b []byte - b, err = ioutil.ReadAll(resp.Body) + b, err = io.ReadAll(resp.Body) if err != nil { return UploadInfo{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index e3a14c59d..cdc7d3d38 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -28,6 +28,7 @@ import ( "net/url" "sort" "strings" + "sync" "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/s3utils" @@ -44,7 +45,9 @@ import ( func (c *Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { - if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { + if opts.ConcurrentStreamParts && opts.NumThreads > 1 { + info, err = c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts) + } else if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { @@ -266,6 +269,9 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) + opts = PutObjectOptions{ + ServerSideEncryption: opts.ServerSideEncryption, + } if withChecksum { // Add hash of hashes. crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) @@ -278,7 +284,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))} } - uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{}) + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } @@ -425,6 +431,211 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) + opts = PutObjectOptions{ + ServerSideEncryption: opts.ServerSideEncryption, + } + if len(crcBytes) > 0 { + // Add hash of hashes. + crc.Reset() + crc.Write(crcBytes) + opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))} + } + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) + if err != nil { + return UploadInfo{}, err + } + + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil +} + +// putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel. +// This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer. +func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string, + reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return UploadInfo{}, err + } + + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return UploadInfo{}, err + } + + if !opts.SendContentMd5 { + if opts.UserMetadata == nil { + opts.UserMetadata = make(map[string]string, 1) + } + opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C" + } + + // Cancel all when an error occurs. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) + if err != nil { + return UploadInfo{}, err + } + + // Initiates a new multipart request + uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) + if err != nil { + return UploadInfo{}, err + } + delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm") + + // Aborts the multipart upload if the function returns + // any error, since we do not resume we should purge + // the parts which have been uploaded to relinquish + // storage space. + defer func() { + if err != nil { + c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + } + }() + + // Create checksums + // CRC32C is ~50% faster on AMD64 @ 30GB/s + var crcBytes []byte + crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + md5Hash := c.md5Hasher() + defer md5Hash.Close() + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Create a buffer. + nBuffers := int64(opts.NumThreads) + bufs := make(chan []byte, nBuffers) + all := make([]byte, nBuffers*partSize) + for i := int64(0); i < nBuffers; i++ { + bufs <- all[i*partSize : i*partSize+partSize] + } + + var wg sync.WaitGroup + var mu sync.Mutex + errCh := make(chan error, opts.NumThreads) + + reader = newHook(reader, opts.Progress) + + // Part number always starts with '1'. + var partNumber int + for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { + // Proceed to upload the part. + var buf []byte + select { + case buf = <-bufs: + case err = <-errCh: + cancel() + wg.Wait() + return UploadInfo{}, err + } + + if int64(len(buf)) != partSize { + return UploadInfo{}, fmt.Errorf("read buffer < %d than expected partSize: %d", len(buf), partSize) + } + + length, rerr := readFull(reader, buf) + if rerr == io.EOF && partNumber > 1 { + // Done + break + } + + if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { + cancel() + wg.Wait() + return UploadInfo{}, rerr + } + + // Calculate md5sum. + customHeader := make(http.Header) + if !opts.SendContentMd5 { + // Add CRC32C instead. + crc.Reset() + crc.Write(buf[:length]) + cSum := crc.Sum(nil) + customHeader.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(cSum)) + crcBytes = append(crcBytes, cSum...) + } + + wg.Add(1) + go func(partNumber int) { + // Avoid declaring variables in the for loop + var md5Base64 string + + if opts.SendContentMd5 { + md5Hash.Reset() + md5Hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)) + } + + defer wg.Done() + p := uploadPartParams{ + bucketName: bucketName, + objectName: objectName, + uploadID: uploadID, + reader: bytes.NewReader(buf[:length]), + partNumber: partNumber, + md5Base64: md5Base64, + size: int64(length), + sse: opts.ServerSideEncryption, + streamSha256: !opts.DisableContentSha256, + customHeader: customHeader, + } + objPart, uerr := c.uploadPart(ctx, p) + if uerr != nil { + errCh <- uerr + } + + // Save successfully uploaded part metadata. + mu.Lock() + partsInfo[partNumber] = objPart + mu.Unlock() + + // Send buffer back so it can be reused. + bufs <- buf + }(partNumber) + + // Save successfully uploaded size. + totalUploadedSize += int64(length) + } + wg.Wait() + + // Collect any error + select { + case err = <-errCh: + return UploadInfo{}, err + default: + } + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + ChecksumCRC32: part.ChecksumCRC32, + ChecksumCRC32C: part.ChecksumCRC32C, + ChecksumSHA1: part.ChecksumSHA1, + ChecksumSHA256: part.ChecksumSHA256, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + opts = PutObjectOptions{} if len(crcBytes) > 0 { // Add hash of hashes. diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index 5735bee5e..2376ee874 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -87,7 +87,12 @@ type PutObjectOptions struct { SendContentMd5 bool DisableContentSha256 bool DisableMultipart bool - Internal AdvancedPutOptions + + // ConcurrentStreamParts will create NumThreads buffers of PartSize bytes, + // fill them serially and upload them in parallel. + // This can be used for faster uploads on non-seekable or slow-to-seek input. + ConcurrentStreamParts bool + Internal AdvancedPutOptions } // getNumThreads - gets the number of threads to be used in the multipart @@ -272,6 +277,9 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str if opts.DisableMultipart { return UploadInfo{}, errors.New("no length provided and multipart disabled") } + if opts.ConcurrentStreamParts && opts.NumThreads > 1 { + return c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts) + } return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } diff --git a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go index b7502e2d9..899bc6b73 100644 --- a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go +++ b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go @@ -24,7 +24,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "strings" "sync" @@ -107,7 +106,7 @@ func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts return nopReadSeekCloser{bytes.NewReader(b.Bytes())}, int64(b.Len()), nil } } else { - f, err := ioutil.TempFile("", "s3-putsnowballobjects-*") + f, err := os.CreateTemp("", "s3-putsnowballobjects-*") if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go index f0ba0d94b..827395ee1 100644 --- a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go @@ -316,8 +316,6 @@ type completeMultipartUploadResult struct { // CompletePart sub container lists individual part numbers and their // md5sum, part of completeMultipartUpload. type CompletePart struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` - // Part number identifies the part. PartNumber int ETag string diff --git a/vendor/github.com/minio/minio-go/v7/api-select.go b/vendor/github.com/minio/minio-go/v7/api-select.go index 5d47d7ec5..628d967ff 100644 --- a/vendor/github.com/minio/minio-go/v7/api-select.go +++ b/vendor/github.com/minio/minio-go/v7/api-select.go @@ -41,8 +41,8 @@ type CSVFileHeaderInfo string // Constants for file header info. const ( CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE" - CSVFileHeaderInfoIgnore = "IGNORE" - CSVFileHeaderInfoUse = "USE" + CSVFileHeaderInfoIgnore CSVFileHeaderInfo = "IGNORE" + CSVFileHeaderInfoUse CSVFileHeaderInfo = "USE" ) // SelectCompressionType - is the parameter for what type of compression is @@ -52,15 +52,15 @@ type SelectCompressionType string // Constants for compression types under select API. const ( SelectCompressionNONE SelectCompressionType = "NONE" - SelectCompressionGZIP = "GZIP" - SelectCompressionBZIP = "BZIP2" + SelectCompressionGZIP SelectCompressionType = "GZIP" + SelectCompressionBZIP SelectCompressionType = "BZIP2" // Non-standard compression schemes, supported by MinIO hosts: - SelectCompressionZSTD = "ZSTD" // Zstandard compression. - SelectCompressionLZ4 = "LZ4" // LZ4 Stream - SelectCompressionS2 = "S2" // S2 Stream - SelectCompressionSNAPPY = "SNAPPY" // Snappy stream + SelectCompressionZSTD SelectCompressionType = "ZSTD" // Zstandard compression. + SelectCompressionLZ4 SelectCompressionType = "LZ4" // LZ4 Stream + SelectCompressionS2 SelectCompressionType = "S2" // S2 Stream + SelectCompressionSNAPPY SelectCompressionType = "SNAPPY" // Snappy stream ) // CSVQuoteFields - is the parameter for how CSV fields are quoted. @@ -69,7 +69,7 @@ type CSVQuoteFields string // Constants for csv quote styles. const ( CSVQuoteFieldsAlways CSVQuoteFields = "Always" - CSVQuoteFieldsAsNeeded = "AsNeeded" + CSVQuoteFieldsAsNeeded CSVQuoteFields = "AsNeeded" ) // QueryExpressionType - is of what syntax the expression is, this should only @@ -87,7 +87,7 @@ type JSONType string // Constants for JSONTypes. const ( JSONDocumentType JSONType = "DOCUMENT" - JSONLinesType = "LINES" + JSONLinesType JSONType = "LINES" ) // ParquetInputOptions parquet input specific options @@ -378,8 +378,8 @@ type SelectObjectType string // Constants for input data types. const ( SelectObjectTypeCSV SelectObjectType = "CSV" - SelectObjectTypeJSON = "JSON" - SelectObjectTypeParquet = "Parquet" + SelectObjectTypeJSON SelectObjectType = "JSON" + SelectObjectTypeParquet SelectObjectType = "Parquet" ) // preludeInfo is used for keeping track of necessary information from the @@ -416,7 +416,7 @@ type messageType string const ( errorMsg messageType = "error" - commonMsg = "event" + commonMsg messageType = "event" ) // eventType represents the type of event. @@ -425,9 +425,9 @@ type eventType string // list of event-types returned by Select API. const ( endEvent eventType = "End" - recordsEvent = "Records" - progressEvent = "Progress" - statsEvent = "Stats" + recordsEvent eventType = "Records" + progressEvent eventType = "Progress" + statsEvent eventType = "Stats" ) // contentType represents content type of event. diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index 8e440eb56..99a03df95 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -25,7 +25,6 @@ import ( "fmt" "hash/crc32" "io" - "io/ioutil" "math/rand" "net" "net/http" @@ -119,7 +118,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.44" + libraryVersion = "v7.0.47" ) // User Agent should always following the below style. @@ -635,7 +634,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ } // Read the body to be saved later. - errBodyBytes, err := ioutil.ReadAll(res.Body) + errBodyBytes, err := io.ReadAll(res.Body) // res.Body should be closed closeResponse(res) if err != nil { @@ -644,14 +643,14 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ // Save the body. errBodySeeker := bytes.NewReader(errBodyBytes) - res.Body = ioutil.NopCloser(errBodySeeker) + res.Body = io.NopCloser(errBodySeeker) // For errors verify if its retryable otherwise fail quickly. errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)) // Save the body back again. errBodySeeker.Seek(0, 0) // Seek back to starting point. - res.Body = ioutil.NopCloser(errBodySeeker) + res.Body = io.NopCloser(errBodySeeker) // Bucket region if set in error response and the error // code dictates invalid region, we can retry the request @@ -814,7 +813,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request if metadata.contentLength == 0 { req.Body = nil } else { - req.Body = ioutil.NopCloser(metadata.contentBody) + req.Body = io.NopCloser(metadata.contentBody) } // Set incoming content-length. @@ -846,7 +845,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request // Additionally, we also look if the initialized client is secure, // if yes then we don't need to perform streaming signature. req = signer.StreamingSignV4(req, accessKeyID, - secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) + secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher()) default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index e86e142e5..23dd7e9b9 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -31,7 +31,6 @@ import ( "hash" "hash/crc32" "io" - "io/ioutil" "math/rand" "mime/multipart" "net/http" @@ -346,7 +345,7 @@ func getDataReader(fileName string) io.ReadCloser { if _, ok := dataFileCRC32[fileName]; !ok { dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size)) } - return ioutil.NopCloser(newRandomReader(size, size)) + return io.NopCloser(newRandomReader(size, size)) } reader, _ := os.Open(getMintDataDirFilePath(fileName)) if _, ok := dataFileCRC32[fileName]; !ok { @@ -989,7 +988,7 @@ func testGetObjectWithVersioning() { for _, testFile := range testFiles { r := getDataReader(testFile) - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return @@ -1131,7 +1130,7 @@ func testPutObjectWithVersioning() { var errs [n]error for i := 0; i < n; i++ { r := newRandomReader(int64((1<<20)*i+i), int64(i)) - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return @@ -1271,7 +1270,7 @@ func testCopyObjectWithVersioning() { testFiles := []string{"datafile-1-b", "datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return @@ -1304,7 +1303,7 @@ func testCopyObjectWithVersioning() { return } - oldestContent, err := ioutil.ReadAll(reader) + oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return @@ -1338,7 +1337,7 @@ func testCopyObjectWithVersioning() { } defer readerCopy.Close() - newestContent, err := ioutil.ReadAll(readerCopy) + newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return @@ -1408,7 +1407,7 @@ func testConcurrentCopyObjectWithVersioning() { testFiles := []string{"datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return @@ -1441,7 +1440,7 @@ func testConcurrentCopyObjectWithVersioning() { return } - oldestContent, err := ioutil.ReadAll(reader) + oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return @@ -1491,7 +1490,7 @@ func testConcurrentCopyObjectWithVersioning() { } defer readerCopy.Close() - newestContent, err := ioutil.ReadAll(readerCopy) + newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return @@ -1571,7 +1570,7 @@ func testComposeObjectWithVersioning() { for _, testFile := range testFiles { r := getDataReader(testFile) - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return @@ -1633,7 +1632,7 @@ func testComposeObjectWithVersioning() { } defer readerCopy.Close() - copyContentBytes, err := ioutil.ReadAll(readerCopy) + copyContentBytes, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err) return @@ -1733,12 +1732,39 @@ func testRemoveObjectWithVersioning() { logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) return } - - err = c.RemoveBucket(context.Background(), bucketName) + // test delete marker version id is non-null + _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "CleanupBucket failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + // create delete marker + err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "DeleteObject failed", err) return } + objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + idx := 0 + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + if idx == 0 { + if !info.IsDeleteMarker { + logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to have been created", err) + return + } + if info.VersionID == "" { + logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to be versioned", err) + return + } + } + idx++ + } + + defer cleanupBucket(bucketName, c) successLogger(testName, function, args, startTime).Info() } @@ -2461,7 +2487,7 @@ func testGetObjectSeekEnd() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -2982,7 +3008,7 @@ func testFPutObjectMultipart() { fileName := getMintDataDirFilePath("datafile-129-MB") if fileName == "" { // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -3091,7 +3117,7 @@ func testFPutObject() { fName := getMintDataDirFilePath("datafile-129-MB") if fName == "" { // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -3257,7 +3283,7 @@ func testFPutObjectContext() { fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest") + file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -3357,7 +3383,7 @@ func testFPutObjectContextV2() { fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest") + file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "Temp file creation failed", err) return @@ -3621,7 +3647,7 @@ func testGetObjectS3Zip() { logError(testName, function, args, startTime, "", "file.Open failed", err) return } - want, err := ioutil.ReadAll(zfr) + want, err := io.ReadAll(zfr) if err != nil { logError(testName, function, args, startTime, "", "fzip file read failed", err) return @@ -3638,7 +3664,7 @@ func testGetObjectS3Zip() { } return } - got, err := ioutil.ReadAll(r) + got, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -3722,7 +3748,7 @@ func testGetObjectReadSeekFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -3885,7 +3911,7 @@ func testGetObjectReadAtFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -4062,7 +4088,7 @@ func testGetObjectReadAtWhenEOFWasReached() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -4181,7 +4207,7 @@ func testPresignedPostPolicy() { metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -4245,7 +4271,7 @@ func testPresignedPostPolicy() { filePath := getMintDataDirFilePath("datafile-33-kB") if filePath == "" { // Make a temp file with 33 KB data. - file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest") + file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -4588,7 +4614,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -4770,7 +4796,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -4944,7 +4970,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -5127,7 +5153,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -6138,7 +6164,7 @@ func testFunctional() { return } - newReadBytes, err := ioutil.ReadAll(newReader) + newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -6269,7 +6295,7 @@ func testFunctional() { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) + newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return @@ -6312,7 +6338,7 @@ func testFunctional() { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) + newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return @@ -6372,7 +6398,7 @@ func testFunctional() { return } - newReadBytes, err = ioutil.ReadAll(newReader) + newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) return @@ -6428,7 +6454,7 @@ func testFunctional() { return } - newReadBytes, err = ioutil.ReadAll(newReader) + newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err) return @@ -6652,7 +6678,7 @@ func testPutObjectUploadSeekedObject() { } args["fileToUpload"] = fileName } else { - tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") + tempfile, err = os.CreateTemp("", "minio-go-upload-test-") if err != nil { logError(testName, function, args, startTime, "", "TempFile create failed", err) return @@ -6916,7 +6942,7 @@ func testFPutObjectV2() { defer cleanupBucket(bucketName, c) // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -7145,7 +7171,7 @@ func testGetObjectReadSeekFunctionalV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -7299,7 +7325,7 @@ func testGetObjectReadAtFunctionalV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - buf, err := ioutil.ReadAll(reader) + buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -7837,7 +7863,7 @@ func testEncryptedEmptyObject() { } defer reader.Close() - decBytes, err := ioutil.ReadAll(reader) + decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) return @@ -7915,7 +7941,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, return } - decBytes, err := ioutil.ReadAll(reader) + decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -7955,7 +7981,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, return } - decBytes, err = ioutil.ReadAll(reader) + decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -7994,7 +8020,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } defer reader.Close() - decBytes, err = ioutil.ReadAll(reader) + decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -11040,7 +11066,7 @@ func testFunctionalV2() { return } - newReadBytes, err := ioutil.ReadAll(newReader) + newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -11146,7 +11172,7 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) + newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -11185,7 +11211,7 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) + newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return @@ -11239,7 +11265,7 @@ func testFunctionalV2() { return } - newReadBytes, err = ioutil.ReadAll(newReader) + newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err) return @@ -11553,7 +11579,7 @@ func testGetObjectRanges() { } for _, test := range tests { wantRC := getDataReader("datafile-129-MB") - io.CopyN(ioutil.Discard, wantRC, test.start) + io.CopyN(io.Discard, wantRC, test.start) want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1)) opts := minio.GetObjectOptions{} opts.SetRange(test.start, test.end) diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go index e964b5217..1c73d1008 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go @@ -24,7 +24,6 @@ import ( "encoding/xml" "errors" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -139,7 +138,7 @@ func closeResponse(resp *http.Response) { // Without this closing connection would disallow re-using // the same connection for future uses. // - http://stackoverflow.com/a/17961593/4465767 - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } } @@ -191,7 +190,7 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume defer closeResponse(resp) if resp.StatusCode != http.StatusOK { var errResp ErrorResponse - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleResponse{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go index f4b027a41..07a9c2f09 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go @@ -22,7 +22,6 @@ import ( "encoding/xml" "fmt" "io" - "io/ioutil" ) // ErrorResponse - Is the typed error returned. @@ -88,7 +87,7 @@ func xmlDecoder(body io.Reader, v interface{}) error { func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) { // read the whole body (up to 1MB) const maxBodyLength = 1 << 20 - body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) + body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) if err != nil { return nil, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go index 56437edb2..e4c0f6717 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go @@ -18,7 +18,6 @@ package credentials import ( - "io/ioutil" "os" "path/filepath" "runtime" @@ -114,6 +113,7 @@ type hostConfig struct { type config struct { Version string `json:"version"` Hosts map[string]hostConfig `json:"hosts"` + Aliases map[string]hostConfig `json:"aliases"` } // loadAliass loads from the file pointed to by shared credentials filename for alias. @@ -123,12 +123,17 @@ func loadAlias(filename, alias string) (hostConfig, error) { cfg := &config{} json := jsoniter.ConfigCompatibleWithStandardLibrary - configBytes, err := ioutil.ReadFile(filename) + configBytes, err := os.ReadFile(filename) if err != nil { return hostConfig{}, err } if err = json.Unmarshal(configBytes, cfg); err != nil { return hostConfig{}, err } + + if cfg.Version == "10" { + return cfg.Aliases[alias], nil + } + return cfg.Hosts[alias], nil } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go index 14369cf10..e641639c9 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go @@ -22,7 +22,7 @@ import ( "context" "errors" "fmt" - "io/ioutil" + "io" "net" "net/http" "net/url" @@ -106,7 +106,7 @@ func (m *IAM) Retrieve() (Value, error) { Client: m.Client, STSEndpoint: endpoint, GetWebIDTokenExpiry: func() (*WebIdentityToken, error) { - token, err := ioutil.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) + token, err := os.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) if err != nil { return nil, err } @@ -268,7 +268,7 @@ func fetchIMDSToken(client *http.Client, endpoint string) (string, error) { return "", err } defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return "", err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go index 34598bd8e..9e92c1e0f 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go @@ -22,7 +22,7 @@ import ( "encoding/xml" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -138,7 +138,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index 25b45ecb0..ec5f3f097 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -21,7 +21,7 @@ import ( "bytes" "encoding/xml" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -156,7 +156,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return value, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go index c7ac4db3b..a0ad09046 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go @@ -21,7 +21,6 @@ import ( "encoding/xml" "errors" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -152,7 +151,7 @@ func (i *STSCertificateIdentity) Retrieve() (Value, error) { } if resp.StatusCode != http.StatusOK { var errResp ErrorResponse - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return Value{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index 50f5f1ce6..2e2af50b4 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -22,7 +22,7 @@ import ( "encoding/xml" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strconv" @@ -155,7 +155,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go index 931ca5bc2..fd034fdc8 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go @@ -34,19 +34,19 @@ type EventType string // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations const ( ObjectCreatedAll EventType = "s3:ObjectCreated:*" - ObjectCreatedPut = "s3:ObjectCreated:Put" - ObjectCreatedPost = "s3:ObjectCreated:Post" - ObjectCreatedCopy = "s3:ObjectCreated:Copy" - ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" - ObjectAccessedGet = "s3:ObjectAccessed:Get" - ObjectAccessedHead = "s3:ObjectAccessed:Head" - ObjectAccessedAll = "s3:ObjectAccessed:*" - ObjectRemovedAll = "s3:ObjectRemoved:*" - ObjectRemovedDelete = "s3:ObjectRemoved:Delete" - ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" - ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" - BucketCreatedAll = "s3:BucketCreated:*" - BucketRemovedAll = "s3:BucketRemoved:*" + ObjectCreatedPut EventType = "s3:ObjectCreated:Put" + ObjectCreatedPost EventType = "s3:ObjectCreated:Post" + ObjectCreatedCopy EventType = "s3:ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload" + ObjectAccessedGet EventType = "s3:ObjectAccessed:Get" + ObjectAccessedHead EventType = "s3:ObjectAccessed:Head" + ObjectAccessedAll EventType = "s3:ObjectAccessed:*" + ObjectRemovedAll EventType = "s3:ObjectRemoved:*" + ObjectRemovedDelete EventType = "s3:ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated EventType = "s3:ObjectRemoved:DeleteMarkerCreated" + ObjectReducedRedundancyLostObject EventType = "s3:ReducedRedundancyLostObject" + BucketCreatedAll EventType = "s3:BucketCreated:*" + BucketRemovedAll EventType = "s3:BucketRemoved:*" ) // FilterRule - child of S3Key, a tag in the notification xml which diff --git a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go index 97abf8df8..645fe18cb 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go @@ -700,6 +700,10 @@ type TargetMetrics struct { PendingCount uint64 `json:"pendingReplicationCount"` // Total number of failed operations including metadata updates FailedCount uint64 `json:"failedReplicationCount"` + // Bandwidth limit in bytes/sec for this target + BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"` + // Current bandwidth used in bytes/sec for this target + CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"` } // Metrics represents inline replication metrics for a bucket. diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go index 5838b9de9..77540e2d8 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go @@ -21,7 +21,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "net/http" "strconv" "strings" @@ -132,7 +131,7 @@ func StreamingUnsignedV4(req *http.Request, sessionToken string, dataLen int64, prepareUSStreamingRequest(req, sessionToken, dataLen, reqTime) if req.Body == nil { - req.Body = ioutil.NopCloser(bytes.NewReader([]byte(""))) + req.Body = io.NopCloser(bytes.NewReader([]byte(""))) } stReader := &StreamingUSReader{ diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go index 49e999b01..1c2f1dc9d 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go @@ -22,11 +22,12 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "net/http" "strconv" "strings" "time" + + md5simd "github.com/minio/md5-simd" ) // Reference for constants used below - @@ -91,14 +92,14 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { // buildChunkStringToSign - returns the string to sign given chunk data // and previous signature. -func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { +func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingPayloadHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, emptySHA256, - hex.EncodeToString(sum256(chunkData)), + chunkChecksum, } return strings.Join(stringToSignParts, "\n") @@ -106,13 +107,13 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [ // buildTrailerChunkStringToSign - returns the string to sign given chunk data // and previous signature. -func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { +func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingTrailerHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, - hex.EncodeToString(sum256(chunkData)), + chunkChecksum, } return strings.Join(stringToSignParts, "\n") @@ -149,21 +150,21 @@ func buildChunkHeader(chunkLen int64, signature string) []byte { } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. -func buildChunkSignature(chunkData []byte, reqTime time.Time, region, +func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildChunkStringToSign(reqTime, region, - previousSignature, chunkData) + previousSignature, chunkCheckSum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. -func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region, +func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region, - previousSignature, chunkData) + previousSignature, chunkChecksum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } @@ -203,12 +204,17 @@ type StreamingReader struct { totalChunks int lastChunkSize int trailer http.Header + sh256 md5simd.Hasher } // signChunk - signs a chunk read from s.baseReader of chunkLen size. func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) { // Compute chunk signature for next header - signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime, + s.sh256.Reset() + s.sh256.Write(s.chunkBuf[:chunkLen]) + chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil)) + + signature := buildChunkSignature(chunckChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation @@ -240,8 +246,11 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) { s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...) } + s.sh256.Reset() + s.sh256.Write(s.chunkBuf) + chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil)) // Compute chunk signature - signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime, + signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation @@ -274,13 +283,13 @@ func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { // StreamingSignV4 - provides chunked upload signatureV4 support by // implementing io.Reader. func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, - region string, dataLen int64, reqTime time.Time, + region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher, ) *http.Request { // Set headers needed for streaming signature. prepareStreamingRequest(req, sessionToken, dataLen, reqTime) if req.Body == nil { - req.Body = ioutil.NopCloser(bytes.NewReader([]byte(""))) + req.Body = io.NopCloser(bytes.NewReader([]byte(""))) } stReader := &StreamingReader{ @@ -295,6 +304,7 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok chunkNum: 1, totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, lastChunkSize: int(dataLen % payloadChunkSize), + sh256: sh256, } if len(req.Trailer) > 0 { stReader.trailer = req.Trailer @@ -385,5 +395,9 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { // Close - this method makes underlying io.ReadCloser's Close method available. func (s *StreamingReader) Close() error { + if s.sh256 != nil { + s.sh256.Close() + s.sh256 = nil + } return s.baseReadCloser.Close() } diff --git a/vendor/github.com/minio/minio-go/v7/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go index 4b3df1912..31b340dcf 100644 --- a/vendor/github.com/minio/minio-go/v7/post-policy.go +++ b/vendor/github.com/minio/minio-go/v7/post-policy.go @@ -25,7 +25,7 @@ import ( ) // expirationDateFormat date format for expiration key in json policy. -const expirationDateFormat = "2006-01-02T15:04:05.999Z" +const expirationDateFormat = "2006-01-02T15:04:05.000Z" // policyCondition explanation: // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html diff --git a/vendor/github.com/minio/minio-go/v7/transport.go b/vendor/github.com/minio/minio-go/v7/transport.go index a88477b73..1bff66462 100644 --- a/vendor/github.com/minio/minio-go/v7/transport.go +++ b/vendor/github.com/minio/minio-go/v7/transport.go @@ -23,7 +23,6 @@ package minio import ( "crypto/tls" "crypto/x509" - "io/ioutil" "net" "net/http" "os" @@ -73,7 +72,7 @@ var DefaultTransport = func(secure bool) (*http.Transport, error) { } if f := os.Getenv("SSL_CERT_FILE"); f != "" { rootCAs := mustGetSystemCertPool() - data, err := ioutil.ReadFile(f) + data, err := os.ReadFile(f) if err == nil { rootCAs.AppendCertsFromPEM(data) } diff --git a/vendor/github.com/minio/minio-go/v7/utils.go b/vendor/github.com/minio/minio-go/v7/utils.go index a8a45b1a8..7ce8ec5ca 100644 --- a/vendor/github.com/minio/minio-go/v7/utils.go +++ b/vendor/github.com/minio/minio-go/v7/utils.go @@ -28,7 +28,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "math/rand" "net" "net/http" @@ -142,7 +141,7 @@ func closeResponse(resp *http.Response) { // Without this closing connection would disallow re-using // the same connection for future uses. // - http://stackoverflow.com/a/17961593/4465767 - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } } |