diff options
Diffstat (limited to 'vendor/github.com/minio')
18 files changed, 808 insertions, 86 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go index 3f88d0777..fec5cece5 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/url" + "time" "github.com/minio/minio-go/v7/pkg/lifecycle" "github.com/minio/minio-go/v7/pkg/s3utils" @@ -102,29 +103,36 @@ func (c *Client) removeBucketLifecycle(ctx context.Context, bucketName string) e // GetBucketLifecycle fetch bucket lifecycle configuration func (c *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) { + lc, _, err := c.GetBucketLifecycleWithInfo(ctx, bucketName) + return lc, err +} + +// GetBucketLifecycleWithInfo fetch bucket lifecycle configuration along with when it was last updated +func (c *Client) GetBucketLifecycleWithInfo(ctx context.Context, bucketName string) (*lifecycle.Configuration, time.Time, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return nil, err + return nil, time.Time{}, err } - bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName) + bucketLifecycle, updatedAt, err := c.getBucketLifecycle(ctx, bucketName) if err != nil { - return nil, err + return nil, time.Time{}, err } config := lifecycle.NewConfiguration() if err = xml.Unmarshal(bucketLifecycle, config); err != nil { - return nil, err + return nil, time.Time{}, err } - return config, nil + return config, updatedAt, nil } // Request server for current bucket lifecycle. -func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, error) { +func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, time.Time, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("lifecycle", "") + urlValues.Set("withUpdatedAt", "true") // Execute GET on bucket to get lifecycle. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ @@ -134,14 +142,28 @@ func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]b defer closeResponse(resp) if err != nil { - return nil, err + return nil, time.Time{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return nil, httpRespToErrorResponse(resp, bucketName, "") + return nil, time.Time{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + lcBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, time.Time{}, err + } + + const minIOLifecycleCfgUpdatedAt = "X-Minio-LifecycleConfig-UpdatedAt" + var updatedAt time.Time + if timeStr := resp.Header.Get(minIOLifecycleCfgUpdatedAt); timeStr != "" { + updatedAt, err = time.Parse(iso8601DateFormat, timeStr) + if err != nil { + return nil, time.Time{}, err } } - return io.ReadAll(resp.Body) + return lcBytes, updatedAt, nil } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go b/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go index f355d422a..9016ec4b4 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go @@ -27,11 +27,12 @@ import ( "strconv" "strings" "time" + + "github.com/minio/minio-go/v7/pkg/encrypt" ) -// PutObjectFanOutRequest this is the request structure sent -// to the server to fan-out the stream to multiple objects. -type PutObjectFanOutRequest struct { +// PutObjectFanOutEntry is per object entry fan-out metadata +type PutObjectFanOutEntry struct { Key string `json:"key"` UserMetadata map[string]string `json:"metadata,omitempty"` UserTags map[string]string `json:"tags,omitempty"` @@ -44,9 +45,17 @@ type PutObjectFanOutRequest struct { RetainUntilDate *time.Time `json:"retainUntil,omitempty"` } +// PutObjectFanOutRequest this is the request structure sent +// to the server to fan-out the stream to multiple objects. +type PutObjectFanOutRequest struct { + Entries []PutObjectFanOutEntry + Checksum Checksum + SSE encrypt.ServerSide +} + // PutObjectFanOutResponse this is the response structure sent // by the server upon success or failure for each object -// fan-out keys. Additionally this response carries ETag, +// fan-out keys. Additionally, this response carries ETag, // VersionID and LastModified for each object fan-out. type PutObjectFanOutResponse struct { Key string `json:"key"` @@ -60,8 +69,8 @@ type PutObjectFanOutResponse struct { // stream multiple objects are written, defined via a list of PutObjectFanOutRequests. Each entry // in PutObjectFanOutRequest carries an object keyname and its relevant metadata if any. `Key` is // mandatory, rest of the other options in PutObjectFanOutRequest are optional. -func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Reader, fanOutReq ...PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) { - if len(fanOutReq) == 0 { +func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, fanOutData io.Reader, fanOutReq PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) { + if len(fanOutReq.Entries) == 0 { return nil, errInvalidArgument("fan out requests cannot be empty") } @@ -72,6 +81,12 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea // Expires in 15 minutes. policy.SetExpires(time.Now().UTC().Add(15 * time.Minute)) + // Set encryption headers if any. + policy.SetEncryption(fanOutReq.SSE) + + // Set checksum headers if any. + policy.SetChecksum(fanOutReq.Checksum) + url, formData, err := c.PresignedPostPolicy(ctx, policy) if err != nil { return nil, err @@ -87,7 +102,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea var b strings.Builder enc := json.NewEncoder(&b) - for _, req := range fanOutReq { + for _, req := range fanOutReq.Entries { if req.Key == "" { w.Close() return nil, errors.New("PutObjectFanOutRequest.Key is mandatory and cannot be empty") @@ -120,7 +135,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea return } - if _, err = io.Copy(mw, body); err != nil { + if _, err = io.Copy(mw, fanOutData); err != nil { return } }() @@ -136,7 +151,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea } dec := json.NewDecoder(resp.Body) - fanOutResp := make([]PutObjectFanOutResponse, 0, len(fanOutReq)) + fanOutResp := make([]PutObjectFanOutResponse, 0, len(fanOutReq.Entries)) for dec.More() { var m PutObjectFanOutResponse if err = dec.Decode(&m); err != nil { diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index 55b3f38e6..9182d4eac 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -193,7 +193,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN } sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) - var trailer = make(http.Header, 1) + trailer := make(http.Header, 1) if withChecksum { crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(crc.Sum(nil))) @@ -203,7 +203,8 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN } // Proceed to upload the part. - p := uploadPartParams{bucketName: bucketName, + p := uploadPartParams{ + bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: sectionReader, @@ -244,7 +245,6 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN return UploadInfo{}, ctx.Err() case uploadRes := <-uploadedPartsCh: if uploadRes.Error != nil { - return UploadInfo{}, uploadRes.Error } @@ -452,7 +452,8 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b // putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel. // This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer. func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string, - reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { + reader io.Reader, opts PutObjectOptions, +) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err @@ -741,6 +742,17 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string, // Set headers. customHeader := opts.Header() + // Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. + addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) + + if addCrc { + // If user has added checksums, don't add them ourselves. + for k := range opts.UserMetadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { + addCrc = false + } + } + } // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, @@ -751,6 +763,7 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentMD5Base64: md5Base64, contentSHA256Hex: sha256Hex, streamSha256: !opts.DisableContentSha256, + addCrc: addCrc, } if opts.Internal.SourceVersionID != "" { if opts.Internal.SourceVersionID != nullVersionID { diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index f5334560b..0546b1ac0 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -124,7 +124,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.53" + libraryVersion = "v7.0.55" ) // User Agent should always following the below style. diff --git a/vendor/github.com/minio/minio-go/v7/checksum.go b/vendor/github.com/minio/minio-go/v7/checksum.go new file mode 100644 index 000000000..a1f6f434f --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/checksum.go @@ -0,0 +1,210 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2023 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "hash" + "hash/crc32" + "io" + "math/bits" +) + +// ChecksumType contains information about the checksum type. +type ChecksumType uint32 + +const ( + + // ChecksumSHA256 indicates a SHA256 checksum. + ChecksumSHA256 ChecksumType = 1 << iota + // ChecksumSHA1 indicates a SHA-1 checksum. + ChecksumSHA1 + // ChecksumCRC32 indicates a CRC32 checksum with IEEE table. + ChecksumCRC32 + // ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table. + ChecksumCRC32C + + // Keep after all valid checksums + checksumLast + + // checksumMask is a mask for valid checksum types. + checksumMask = checksumLast - 1 + + // ChecksumNone indicates no checksum. + ChecksumNone ChecksumType = 0 + + amzChecksumAlgo = "x-amz-checksum-algorithm" + amzChecksumCRC32 = "x-amz-checksum-crc32" + amzChecksumCRC32C = "x-amz-checksum-crc32c" + amzChecksumSHA1 = "x-amz-checksum-sha1" + amzChecksumSHA256 = "x-amz-checksum-sha256" +) + +// Is returns if c is all of t. +func (c ChecksumType) Is(t ChecksumType) bool { + return c&t == t +} + +// Key returns the header key. +// returns empty string if invalid or none. +func (c ChecksumType) Key() string { + switch c & checksumMask { + case ChecksumCRC32: + return amzChecksumCRC32 + case ChecksumCRC32C: + return amzChecksumCRC32C + case ChecksumSHA1: + return amzChecksumSHA1 + case ChecksumSHA256: + return amzChecksumSHA256 + } + return "" +} + +// RawByteLen returns the size of the un-encoded checksum. +func (c ChecksumType) RawByteLen() int { + switch c & checksumMask { + case ChecksumCRC32, ChecksumCRC32C: + return 4 + case ChecksumSHA1: + return sha1.Size + case ChecksumSHA256: + return sha256.Size + } + return 0 +} + +// Hasher returns a hasher corresponding to the checksum type. +// Returns nil if no checksum. +func (c ChecksumType) Hasher() hash.Hash { + switch c & checksumMask { + case ChecksumCRC32: + return crc32.NewIEEE() + case ChecksumCRC32C: + return crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case ChecksumSHA1: + return sha1.New() + case ChecksumSHA256: + return sha256.New() + } + return nil +} + +// IsSet returns whether the type is valid and known. +func (c ChecksumType) IsSet() bool { + return bits.OnesCount32(uint32(c)) == 1 +} + +// String returns the type as a string. +// CRC32, CRC32C, SHA1, and SHA256 for valid values. +// Empty string for unset and "<invalid>" if not valid. +func (c ChecksumType) String() string { + switch c & checksumMask { + case ChecksumCRC32: + return "CRC32" + case ChecksumCRC32C: + return "CRC32C" + case ChecksumSHA1: + return "SHA1" + case ChecksumSHA256: + return "SHA256" + case ChecksumNone: + return "" + } + return "<invalid>" +} + +// ChecksumReader reads all of r and returns a checksum of type c. +// Returns any error that may have occurred while reading. +func (c ChecksumType) ChecksumReader(r io.Reader) (Checksum, error) { + h := c.Hasher() + if h == nil { + return Checksum{}, nil + } + _, err := io.Copy(h, r) + if err != nil { + return Checksum{}, err + } + return NewChecksum(c, h.Sum(nil)), nil +} + +// ChecksumBytes returns a checksum of the content b with type c. +func (c ChecksumType) ChecksumBytes(b []byte) Checksum { + h := c.Hasher() + if h == nil { + return Checksum{} + } + n, err := h.Write(b) + if err != nil || n != len(b) { + // Shouldn't happen with these checksummers. + return Checksum{} + } + return NewChecksum(c, h.Sum(nil)) +} + +// Checksum is a type and encoded value. +type Checksum struct { + Type ChecksumType + r []byte +} + +// NewChecksum sets the checksum to the value of b, +// which is the raw hash output. +// If the length of c does not match t.RawByteLen, +// a checksum with ChecksumNone is returned. +func NewChecksum(t ChecksumType, b []byte) Checksum { + if t.IsSet() && len(b) == t.RawByteLen() { + return Checksum{Type: t, r: b} + } + return Checksum{} +} + +// NewChecksumString sets the checksum to the value of s, +// which is the base 64 encoded raw hash output. +// If the length of c does not match t.RawByteLen, it is not added. +func NewChecksumString(t ChecksumType, s string) Checksum { + b, _ := base64.StdEncoding.DecodeString(s) + if t.IsSet() && len(b) == t.RawByteLen() { + return Checksum{Type: t, r: b} + } + return Checksum{} +} + +// IsSet returns whether the checksum is valid and known. +func (c Checksum) IsSet() bool { + return c.Type.IsSet() && len(c.r) == c.Type.RawByteLen() +} + +// Encoded returns the encoded value. +// Returns the empty string if not set or valid. +func (c Checksum) Encoded() string { + if !c.IsSet() { + return "" + } + return base64.StdEncoding.EncodeToString(c.r) +} + +// Raw returns the raw checksum value if set. +func (c Checksum) Raw() []byte { + if !c.IsSet() { + return nil + } + return c.r +} diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index d7eb30322..ed1b7340e 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -2312,7 +2312,7 @@ func testPutMultipartObjectWithChecksums() { cmpChecksum := func(got, want string) { if want != got { - //logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) + // logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) fmt.Printf("want %s, got %s\n", want, got) return } @@ -2387,6 +2387,369 @@ func testPutMultipartObjectWithChecksums() { successLogger(testName, function, args, startTime).Info() } +// Test PutObject with trailing checksums. +func testTrailingChecksums() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", + } + + if !isFullMode() { + ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() + return + } + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + TrailingHeaders: true, + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) string { + r := bytes.NewReader(b) + tmp := make([]byte, partSize) + parts := 0 + var all []byte + for { + n, err := io.ReadFull(r, tmp) + if err != nil && err != io.ErrUnexpectedEOF { + logError(testName, function, args, startTime, "", "Calc crc failed", err) + } + if n == 0 { + break + } + parts++ + hasher.Reset() + hasher.Write(tmp[:n]) + all = append(all, hasher.Sum(nil)...) + if err != nil { + break + } + } + hasher.Reset() + hasher.Write(all) + return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts) + } + defer cleanupBucket(bucketName, c) + tests := []struct { + header string + hasher hash.Hash + + // Checksum values + ChecksumCRC32 string + ChecksumCRC32C string + ChecksumSHA1 string + ChecksumSHA256 string + PO minio.PutObjectOptions + }{ + // Currently there is no way to override the checksum type. + {header: "x-amz-checksum-crc32c", + hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), + ChecksumCRC32C: "set", + PO: minio.PutObjectOptions{ + DisableContentSha256: true, + DisableMultipart: false, + UserMetadata: nil, + PartSize: 5 << 20, + }, + }, + {header: "x-amz-checksum-crc32c", + hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), + ChecksumCRC32C: "set", + PO: minio.PutObjectOptions{ + DisableContentSha256: true, + DisableMultipart: false, + UserMetadata: nil, + PartSize: 6_645_654, // Rather arbitrary size + }, + }, + {header: "x-amz-checksum-crc32c", + hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), + ChecksumCRC32C: "set", + PO: minio.PutObjectOptions{ + DisableContentSha256: false, + DisableMultipart: false, + UserMetadata: nil, + PartSize: 5 << 20, + }, + }, + {header: "x-amz-checksum-crc32c", + hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), + ChecksumCRC32C: "set", + PO: minio.PutObjectOptions{ + DisableContentSha256: false, + DisableMultipart: false, + UserMetadata: nil, + PartSize: 6_645_654, // Rather arbitrary size + }, + }, + } + + for _, test := range tests { + bufSize := dataFileMap["datafile-11-MB"] + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + cmpChecksum := func(got, want string) { + if want != got { + logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %q, got %q", want, got)) + return + } + } + + reader := getDataReader("datafile-11-MB") + b, err := io.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + reader.Close() + h := test.hasher + h.Reset() + test.ChecksumCRC32C = hashMultiPart(b, int(test.PO.PartSize), test.hasher) + + // Set correct CRC. + c.TraceOn(os.Stdout) + resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), test.PO) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + c.TraceOff() + cmpChecksum(resp.ChecksumSHA256, test.ChecksumSHA256) + cmpChecksum(resp.ChecksumSHA1, test.ChecksumSHA1) + cmpChecksum(resp.ChecksumCRC32, test.ChecksumCRC32) + cmpChecksum(resp.ChecksumCRC32C, test.ChecksumCRC32C) + + // Read the data back + gopts := minio.GetObjectOptions{Checksum: true} + gopts.PartNumber = 2 + + // We cannot use StatObject, since it ignores partnumber. + r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + io.Copy(io.Discard, r) + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + // Test part 2 checksum... + h.Reset() + p2 := b[test.PO.PartSize:] + if len(p2) > int(test.PO.PartSize) { + p2 = p2[:test.PO.PartSize] + } + h.Write(p2) + got := base64.StdEncoding.EncodeToString(h.Sum(nil)) + if test.ChecksumSHA256 != "" { + cmpChecksum(st.ChecksumSHA256, got) + } + if test.ChecksumSHA1 != "" { + cmpChecksum(st.ChecksumSHA1, got) + } + if test.ChecksumCRC32 != "" { + cmpChecksum(st.ChecksumCRC32, got) + } + if test.ChecksumCRC32C != "" { + cmpChecksum(st.ChecksumCRC32C, got) + } + + delete(args, "metadata") + } +} + +// Test PutObject with custom checksums. +func testPutObjectWithAutomaticChecksums() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", + } + + if !isFullMode() { + ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + TrailingHeaders: true, + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + defer cleanupBucket(bucketName, c) + tests := []struct { + header string + hasher hash.Hash + + // Checksum values + ChecksumCRC32 string + ChecksumCRC32C string + ChecksumSHA1 string + ChecksumSHA256 string + }{ + // Built-in will only add crc32c, when no MD5 nor SHA256. + {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, + } + + // Enable tracing, write to stderr. + c.TraceOn(os.Stderr) + defer c.TraceOff() + + for i, test := range tests { + bufSize := dataFileMap["datafile-10-kB"] + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + cmpChecksum := func(got, want string) { + if want != got { + logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) + return + } + } + + meta := map[string]string{} + reader := getDataReader("datafile-10-kB") + b, err := io.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + + h := test.hasher + h.Reset() + h.Write(b) + meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) + args["metadata"] = meta + + resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + UserMetadata: nil, + DisableContentSha256: true, + SendContentMd5: false, + }) + if err == nil { + if i == 0 && resp.ChecksumCRC32C == "" { + ignoredLog(testName, function, args, startTime, "Checksums does not appear to be supported by backend").Info() + return + } + } else { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + // Usually this will be the same as above, since we skip automatic checksum when SHA256 content is sent. + // When/if we add a checksum control to PutObjectOptions this will make more sense. + resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + UserMetadata: nil, + DisableContentSha256: false, + SendContentMd5: false, + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + // The checksum will not be enabled on HTTP, since it uses SHA256 blocks. + if mustParseBool(os.Getenv(enableHTTPS)) { + cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + } + + // Set SHA256 header manually + sh256 := sha256.Sum256(b) + meta = map[string]string{"x-amz-checksum-sha256": base64.StdEncoding.EncodeToString(sh256[:])} + args["metadata"] = meta + resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + UserMetadata: meta, + DisableContentSha256: true, + SendContentMd5: false, + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + delete(args, "metadata") + } + + successLogger(testName, function, args, startTime).Info() +} + // Test PutObject using a large data to trigger multipart readat func testPutObjectWithMetadata() { // initialize logging params @@ -12576,6 +12939,8 @@ func main() { testRemoveObjectWithVersioning() testRemoveObjectsWithVersioning() testObjectTaggingWithVersioning() + testTrailingChecksums() + testPutObjectWithAutomaticChecksums() // SSE-C tests will only work over TLS connection. if tls { diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go index 34914490c..ffd251451 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go @@ -289,7 +289,7 @@ func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } - req.TransferEncoding = []string{"aws-chunked"} + req.Header.Set("Content-Encoding", "aws-chunked") req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10)) } diff --git a/vendor/github.com/minio/minio-go/v7/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go index 31b340dcf..0191909bd 100644 --- a/vendor/github.com/minio/minio-go/v7/post-policy.go +++ b/vendor/github.com/minio/minio-go/v7/post-policy.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,11 @@ package minio import ( "encoding/base64" "fmt" + "net/http" "strings" "time" + + "github.com/minio/minio-go/v7/pkg/encrypt" ) // expirationDateFormat date format for expiration key in json policy. @@ -258,6 +261,26 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error { return nil } +// SetChecksum sets the checksum of the request. +func (p *PostPolicy) SetChecksum(c Checksum) { + if c.IsSet() { + p.formData[amzChecksumAlgo] = c.Type.String() + p.formData[c.Type.Key()] = c.Encoded() + } +} + +// SetEncryption - sets encryption headers for POST API +func (p *PostPolicy) SetEncryption(sse encrypt.ServerSide) { + if sse == nil { + return + } + h := http.Header{} + sse.Marshal(h) + for k, v := range h { + p.formData[k] = v[0] + } +} + // SetUserData - Set user data as a key/value couple. // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserData(key string, value string) error { diff --git a/vendor/github.com/minio/sha256-simd/cpuid_other.go b/vendor/github.com/minio/sha256-simd/cpuid_other.go index cd9fbf2d9..97af6a195 100644 --- a/vendor/github.com/minio/sha256-simd/cpuid_other.go +++ b/vendor/github.com/minio/sha256-simd/cpuid_other.go @@ -23,6 +23,11 @@ import ( "github.com/klauspost/cpuid/v2" ) +var ( + hasIntelSha = runtime.GOARCH == "amd64" && cpuid.CPU.Supports(cpuid.SHA, cpuid.SSSE3, cpuid.SSE4) + hasAvx512 = cpuid.CPU.Supports(cpuid.AVX512F, cpuid.AVX512DQ, cpuid.AVX512BW, cpuid.AVX512VL) +) + func hasArmSha2() bool { if cpuid.CPU.Has(cpuid.SHA2) { return true @@ -42,5 +47,4 @@ func hasArmSha2() bool { return false } return bytes.Contains(cpuInfo, []byte(sha256Feature)) - } diff --git a/vendor/github.com/minio/sha256-simd/sha256.go b/vendor/github.com/minio/sha256-simd/sha256.go index b137ead9f..f146bbdb5 100644 --- a/vendor/github.com/minio/sha256-simd/sha256.go +++ b/vendor/github.com/minio/sha256-simd/sha256.go @@ -19,10 +19,8 @@ package sha256 import ( "crypto/sha256" "encoding/binary" + "errors" "hash" - "runtime" - - "github.com/klauspost/cpuid/v2" ) // Size - The size of a SHA256 checksum in bytes. @@ -68,42 +66,34 @@ func (d *digest) Reset() { type blockfuncType int const ( - blockfuncGeneric blockfuncType = iota - blockfuncSha blockfuncType = iota - blockfuncArm blockfuncType = iota + blockfuncStdlib blockfuncType = iota + blockfuncIntelSha + blockfuncArmSha2 + blockfuncForceGeneric = -1 ) var blockfunc blockfuncType func init() { - blockfunc = blockfuncGeneric switch { - case hasSHAExtensions(): - blockfunc = blockfuncSha + case hasIntelSha: + blockfunc = blockfuncIntelSha case hasArmSha2(): - blockfunc = blockfuncArm - default: - blockfunc = blockfuncGeneric + blockfunc = blockfuncArmSha2 } } -var avx512 = cpuid.CPU.Supports(cpuid.AVX512F, cpuid.AVX512DQ, cpuid.AVX512BW, cpuid.AVX512VL) - -// hasSHAExtensions return whether the cpu supports SHA extensions. -func hasSHAExtensions() bool { - return cpuid.CPU.Supports(cpuid.SHA, cpuid.SSSE3, cpuid.SSE4) && runtime.GOARCH == "amd64" -} - // New returns a new hash.Hash computing the SHA256 checksum. func New() hash.Hash { - if blockfunc != blockfuncGeneric { - d := new(digest) - d.Reset() - return d + if blockfunc == blockfuncStdlib { + // Fallback to the standard golang implementation + // if no features were found. + return sha256.New() } - // Fallback to the standard golang implementation - // if no features were found. - return sha256.New() + + d := new(digest) + d.Reset() + return d } // Sum256 - single caller sha256 helper @@ -272,11 +262,11 @@ func (d *digest) checkSum() (digest [Size]byte) { } func block(dig *digest, p []byte) { - if blockfunc == blockfuncSha { - blockShaGo(dig, p) - } else if blockfunc == blockfuncArm { - blockArmGo(dig, p) - } else if blockfunc == blockfuncGeneric { + if blockfunc == blockfuncIntelSha { + blockIntelShaGo(dig, p) + } else if blockfunc == blockfuncArmSha2 { + blockArmSha2Go(dig, p) + } else { blockGeneric(dig, p) } } @@ -397,3 +387,82 @@ var _K = []uint32{ 0xbef9a3f7, 0xc67178f2, } + +const ( + magic256 = "sha\x03" + marshaledSize = len(magic256) + 8*4 + chunk + 8 +) + +func (d *digest) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize) + b = append(b, magic256...) + b = appendUint32(b, d.h[0]) + b = appendUint32(b, d.h[1]) + b = appendUint32(b, d.h[2]) + b = appendUint32(b, d.h[3]) + b = appendUint32(b, d.h[4]) + b = appendUint32(b, d.h[5]) + b = appendUint32(b, d.h[6]) + b = appendUint32(b, d.h[7]) + b = append(b, d.x[:d.nx]...) + b = b[:len(b)+len(d.x)-d.nx] // already zero + b = appendUint64(b, d.len) + return b, nil +} + +func (d *digest) UnmarshalBinary(b []byte) error { + if len(b) < len(magic256) || string(b[:len(magic256)]) != magic256 { + return errors.New("crypto/sha256: invalid hash state identifier") + } + if len(b) != marshaledSize { + return errors.New("crypto/sha256: invalid hash state size") + } + b = b[len(magic256):] + b, d.h[0] = consumeUint32(b) + b, d.h[1] = consumeUint32(b) + b, d.h[2] = consumeUint32(b) + b, d.h[3] = consumeUint32(b) + b, d.h[4] = consumeUint32(b) + b, d.h[5] = consumeUint32(b) + b, d.h[6] = consumeUint32(b) + b, d.h[7] = consumeUint32(b) + b = b[copy(d.x[:], b):] + b, d.len = consumeUint64(b) + d.nx = int(d.len % chunk) + return nil +} + +func appendUint32(b []byte, v uint32) []byte { + return append(b, + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v), + ) +} + +func appendUint64(b []byte, v uint64) []byte { + return append(b, + byte(v>>56), + byte(v>>48), + byte(v>>40), + byte(v>>32), + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v), + ) +} + +func consumeUint64(b []byte) ([]byte, uint64) { + _ = b[7] + x := uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 + return b[8:], x +} + +func consumeUint32(b []byte) ([]byte, uint32) { + _ = b[3] + x := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 + return b[4:], x +} diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go index b7d7c1637..4b9473a4e 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.go @@ -1,4 +1,5 @@ -//+build !noasm,!appengine,gc +//go:build !noasm && !appengine && gc +// +build !noasm,!appengine,gc /* * Minio Cloud Storage, (C) 2017 Minio, Inc. diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.s b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.s index 275bcacbc..cca534e46 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.s +++ b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.s @@ -1,4 +1,4 @@ -//+build !noasm,!appengine +//+build !noasm,!appengine,gc TEXT ·sha256X16Avx512(SB), 7, $0 MOVQ digests+0(FP), DI diff --git a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go b/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go deleted file mode 100644 index bef949419..000000000 --- a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.go +++ /dev/null @@ -1,6 +0,0 @@ -//+build !noasm,!appengine,gc - -package sha256 - -//go:noescape -func blockSha(h *[8]uint32, message []uint8) diff --git a/vendor/github.com/minio/sha256-simd/sha256block_amd64.go b/vendor/github.com/minio/sha256-simd/sha256block_amd64.go index 0c48d45f8..e536f54e1 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_amd64.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_amd64.go @@ -1,4 +1,5 @@ -//+build !noasm,!appengine,gc +//go:build !noasm && !appengine && gc +// +build !noasm,!appengine,gc /* * Minio Cloud Storage, (C) 2016 Minio, Inc. @@ -18,10 +19,13 @@ package sha256 -func blockArmGo(dig *digest, p []byte) { - panic("blockArmGo called unexpectedly") +func blockArmSha2Go(dig *digest, p []byte) { + panic("blockArmSha2Go called unexpectedly") } -func blockShaGo(dig *digest, p []byte) { - blockSha(&dig.h, p) +//go:noescape +func blockIntelSha(h *[8]uint32, message []uint8) + +func blockIntelShaGo(dig *digest, p []byte) { + blockIntelSha(&dig.h, p) } diff --git a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.s b/vendor/github.com/minio/sha256-simd/sha256block_amd64.s index 909fc0ef8..c98a1d8f0 100644 --- a/vendor/github.com/minio/sha256-simd/sha256blockSha_amd64.s +++ b/vendor/github.com/minio/sha256-simd/sha256block_amd64.s @@ -1,4 +1,4 @@ -//+build !noasm,!appengine +//+build !noasm,!appengine,gc // SHA intrinsic version of SHA256 @@ -106,7 +106,7 @@ GLOBL SHUF_MASK<>(SB), RODATA|NOPTR, $16 // X13 saved hash state // CDGH // X15 data shuffle mask (constant) -TEXT ·blockSha(SB), NOSPLIT, $0-32 +TEXT ·blockIntelSha(SB), NOSPLIT, $0-32 MOVQ h+0(FP), DX MOVQ message_base+8(FP), SI MOVQ message_len+16(FP), DI diff --git a/vendor/github.com/minio/sha256-simd/sha256block_arm64.go b/vendor/github.com/minio/sha256-simd/sha256block_arm64.go index 58ccf6eb5..d4369e24a 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_arm64.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_arm64.go @@ -1,4 +1,5 @@ -//+build !noasm,!appengine,gc +//go:build !noasm && !appengine && gc +// +build !noasm,!appengine,gc /* * Minio Cloud Storage, (C) 2016 Minio, Inc. @@ -18,18 +19,18 @@ package sha256 -func blockShaGo(dig *digest, p []byte) { - panic("blockShaGoc called unexpectedly") +func blockIntelShaGo(dig *digest, p []byte) { + panic("blockIntelShaGo called unexpectedly") } //go:noescape -func blockArm(h []uint32, message []uint8) +func blockArmSha2(h []uint32, message []uint8) -func blockArmGo(dig *digest, p []byte) { +func blockArmSha2Go(dig *digest, p []byte) { h := []uint32{dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7]} - blockArm(h[:], p[:]) + blockArmSha2(h[:], p[:]) dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7] diff --git a/vendor/github.com/minio/sha256-simd/sha256block_arm64.s b/vendor/github.com/minio/sha256-simd/sha256block_arm64.s index c6ddb3717..7ab88b163 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_arm64.s +++ b/vendor/github.com/minio/sha256-simd/sha256block_arm64.s @@ -1,4 +1,4 @@ -//+build !noasm,!appengine +//+build !noasm,!appengine,gc // ARM64 version of SHA256 @@ -25,7 +25,7 @@ // their Plan9 equivalents // -TEXT ·blockArm(SB), 7, $0 +TEXT ·blockArmSha2(SB), 7, $0 MOVD h+0(FP), R0 MOVD message+24(FP), R1 MOVD message_len+32(FP), R2 // length of message diff --git a/vendor/github.com/minio/sha256-simd/sha256block_other.go b/vendor/github.com/minio/sha256-simd/sha256block_other.go index ec586c060..94d7eb0b4 100644 --- a/vendor/github.com/minio/sha256-simd/sha256block_other.go +++ b/vendor/github.com/minio/sha256-simd/sha256block_other.go @@ -1,4 +1,5 @@ -//+build appengine noasm !amd64,!arm64 !gc +//go:build appengine || noasm || (!amd64 && !arm64) || !gc +// +build appengine noasm !amd64,!arm64 !gc /* * Minio Cloud Storage, (C) 2019 Minio, Inc. @@ -18,11 +19,11 @@ package sha256 -func blockShaGo(dig *digest, p []byte) { - panic("blockShaGo called unexpectedly") +func blockIntelShaGo(dig *digest, p []byte) { + panic("blockIntelShaGo called unexpectedly") } -func blockArmGo(dig *digest, p []byte) { - panic("blockArmGo called unexpectedly") +func blockArmSha2Go(dig *digest, p []byte) { + panic("blockArmSha2Go called unexpectedly") } |