summaryrefslogtreecommitdiff
path: root/vendor/github.com/minio/minio-go
diff options
context:
space:
mode:
authorLibravatar dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>2023-05-29 13:47:11 +0100
committerLibravatar GitHub <noreply@github.com>2023-05-29 13:47:11 +0100
commit9ed96bc57083b4261a9e6571d86ec94b1e771e40 (patch)
tree87b6521816ed4d1242f47a731895cd5a8c6cabc6 /vendor/github.com/minio/minio-go
parent[bugfix/chore] Inbox post updates (#1821) (diff)
downloadgotosocial-9ed96bc57083b4261a9e6571d86ec94b1e771e40.tar.xz
[chore]: Bump github.com/minio/minio-go/v7 from 7.0.53 to 7.0.55 (#1844)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Diffstat (limited to 'vendor/github.com/minio/minio-go')
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go40
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go33
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go21
-rw-r--r--vendor/github.com/minio/minio-go/v7/api.go2
-rw-r--r--vendor/github.com/minio/minio-go/v7/checksum.go210
-rw-r--r--vendor/github.com/minio/minio-go/v7/functional_tests.go367
-rw-r--r--vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go2
-rw-r--r--vendor/github.com/minio/minio-go/v7/post-policy.go25
8 files changed, 674 insertions, 26 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go
index 3f88d0777..fec5cece5 100644
--- a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go
+++ b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go
@@ -24,6 +24,7 @@ import (
"io"
"net/http"
"net/url"
+ "time"
"github.com/minio/minio-go/v7/pkg/lifecycle"
"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -102,29 +103,36 @@ func (c *Client) removeBucketLifecycle(ctx context.Context, bucketName string) e
// GetBucketLifecycle fetch bucket lifecycle configuration
func (c *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) {
+ lc, _, err := c.GetBucketLifecycleWithInfo(ctx, bucketName)
+ return lc, err
+}
+
+// GetBucketLifecycleWithInfo fetch bucket lifecycle configuration along with when it was last updated
+func (c *Client) GetBucketLifecycleWithInfo(ctx context.Context, bucketName string) (*lifecycle.Configuration, time.Time, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
- return nil, err
+ return nil, time.Time{}, err
}
- bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName)
+ bucketLifecycle, updatedAt, err := c.getBucketLifecycle(ctx, bucketName)
if err != nil {
- return nil, err
+ return nil, time.Time{}, err
}
config := lifecycle.NewConfiguration()
if err = xml.Unmarshal(bucketLifecycle, config); err != nil {
- return nil, err
+ return nil, time.Time{}, err
}
- return config, nil
+ return config, updatedAt, nil
}
// Request server for current bucket lifecycle.
-func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, error) {
+func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, time.Time, error) {
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("lifecycle", "")
+ urlValues.Set("withUpdatedAt", "true")
// Execute GET on bucket to get lifecycle.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
@@ -134,14 +142,28 @@ func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]b
defer closeResponse(resp)
if err != nil {
- return nil, err
+ return nil, time.Time{}, err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
- return nil, httpRespToErrorResponse(resp, bucketName, "")
+ return nil, time.Time{}, httpRespToErrorResponse(resp, bucketName, "")
+ }
+ }
+
+ lcBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, time.Time{}, err
+ }
+
+ const minIOLifecycleCfgUpdatedAt = "X-Minio-LifecycleConfig-UpdatedAt"
+ var updatedAt time.Time
+ if timeStr := resp.Header.Get(minIOLifecycleCfgUpdatedAt); timeStr != "" {
+ updatedAt, err = time.Parse(iso8601DateFormat, timeStr)
+ if err != nil {
+ return nil, time.Time{}, err
}
}
- return io.ReadAll(resp.Body)
+ return lcBytes, updatedAt, nil
}
diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go b/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go
index f355d422a..9016ec4b4 100644
--- a/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go
+++ b/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go
@@ -27,11 +27,12 @@ import (
"strconv"
"strings"
"time"
+
+ "github.com/minio/minio-go/v7/pkg/encrypt"
)
-// PutObjectFanOutRequest this is the request structure sent
-// to the server to fan-out the stream to multiple objects.
-type PutObjectFanOutRequest struct {
+// PutObjectFanOutEntry is per object entry fan-out metadata
+type PutObjectFanOutEntry struct {
Key string `json:"key"`
UserMetadata map[string]string `json:"metadata,omitempty"`
UserTags map[string]string `json:"tags,omitempty"`
@@ -44,9 +45,17 @@ type PutObjectFanOutRequest struct {
RetainUntilDate *time.Time `json:"retainUntil,omitempty"`
}
+// PutObjectFanOutRequest this is the request structure sent
+// to the server to fan-out the stream to multiple objects.
+type PutObjectFanOutRequest struct {
+ Entries []PutObjectFanOutEntry
+ Checksum Checksum
+ SSE encrypt.ServerSide
+}
+
// PutObjectFanOutResponse this is the response structure sent
// by the server upon success or failure for each object
-// fan-out keys. Additionally this response carries ETag,
+// fan-out keys. Additionally, this response carries ETag,
// VersionID and LastModified for each object fan-out.
type PutObjectFanOutResponse struct {
Key string `json:"key"`
@@ -60,8 +69,8 @@ type PutObjectFanOutResponse struct {
// stream multiple objects are written, defined via a list of PutObjectFanOutRequests. Each entry
// in PutObjectFanOutRequest carries an object keyname and its relevant metadata if any. `Key` is
// mandatory, rest of the other options in PutObjectFanOutRequest are optional.
-func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Reader, fanOutReq ...PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) {
- if len(fanOutReq) == 0 {
+func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, fanOutData io.Reader, fanOutReq PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) {
+ if len(fanOutReq.Entries) == 0 {
return nil, errInvalidArgument("fan out requests cannot be empty")
}
@@ -72,6 +81,12 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea
// Expires in 15 minutes.
policy.SetExpires(time.Now().UTC().Add(15 * time.Minute))
+ // Set encryption headers if any.
+ policy.SetEncryption(fanOutReq.SSE)
+
+ // Set checksum headers if any.
+ policy.SetChecksum(fanOutReq.Checksum)
+
url, formData, err := c.PresignedPostPolicy(ctx, policy)
if err != nil {
return nil, err
@@ -87,7 +102,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea
var b strings.Builder
enc := json.NewEncoder(&b)
- for _, req := range fanOutReq {
+ for _, req := range fanOutReq.Entries {
if req.Key == "" {
w.Close()
return nil, errors.New("PutObjectFanOutRequest.Key is mandatory and cannot be empty")
@@ -120,7 +135,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea
return
}
- if _, err = io.Copy(mw, body); err != nil {
+ if _, err = io.Copy(mw, fanOutData); err != nil {
return
}
}()
@@ -136,7 +151,7 @@ func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, body io.Rea
}
dec := json.NewDecoder(resp.Body)
- fanOutResp := make([]PutObjectFanOutResponse, 0, len(fanOutReq))
+ fanOutResp := make([]PutObjectFanOutResponse, 0, len(fanOutReq.Entries))
for dec.More() {
var m PutObjectFanOutResponse
if err = dec.Decode(&m); err != nil {
diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
index 55b3f38e6..9182d4eac 100644
--- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
+++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
@@ -193,7 +193,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
}
sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress)
- var trailer = make(http.Header, 1)
+ trailer := make(http.Header, 1)
if withChecksum {
crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(crc.Sum(nil)))
@@ -203,7 +203,8 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
}
// Proceed to upload the part.
- p := uploadPartParams{bucketName: bucketName,
+ p := uploadPartParams{
+ bucketName: bucketName,
objectName: objectName,
uploadID: uploadID,
reader: sectionReader,
@@ -244,7 +245,6 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
return UploadInfo{}, ctx.Err()
case uploadRes := <-uploadedPartsCh:
if uploadRes.Error != nil {
-
return UploadInfo{}, uploadRes.Error
}
@@ -452,7 +452,8 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
// putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel.
// This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer.
func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string,
- reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) {
+ reader io.Reader, opts PutObjectOptions,
+) (info UploadInfo, err error) {
// Input validation.
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
return UploadInfo{}, err
@@ -741,6 +742,17 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
// Set headers.
customHeader := opts.Header()
+ // Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks.
+ addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure)
+
+ if addCrc {
+ // If user has added checksums, don't add them ourselves.
+ for k := range opts.UserMetadata {
+ if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
+ addCrc = false
+ }
+ }
+ }
// Populate request metadata.
reqMetadata := requestMetadata{
bucketName: bucketName,
@@ -751,6 +763,7 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
contentMD5Base64: md5Base64,
contentSHA256Hex: sha256Hex,
streamSha256: !opts.DisableContentSha256,
+ addCrc: addCrc,
}
if opts.Internal.SourceVersionID != "" {
if opts.Internal.SourceVersionID != nullVersionID {
diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go
index f5334560b..0546b1ac0 100644
--- a/vendor/github.com/minio/minio-go/v7/api.go
+++ b/vendor/github.com/minio/minio-go/v7/api.go
@@ -124,7 +124,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
- libraryVersion = "v7.0.53"
+ libraryVersion = "v7.0.55"
)
// User Agent should always following the below style.
diff --git a/vendor/github.com/minio/minio-go/v7/checksum.go b/vendor/github.com/minio/minio-go/v7/checksum.go
new file mode 100644
index 000000000..a1f6f434f
--- /dev/null
+++ b/vendor/github.com/minio/minio-go/v7/checksum.go
@@ -0,0 +1,210 @@
+/*
+ * MinIO Go Library for Amazon S3 Compatible Cloud Storage
+ * Copyright 2015-2023 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package minio
+
+import (
+ "crypto/sha1"
+ "crypto/sha256"
+ "encoding/base64"
+ "hash"
+ "hash/crc32"
+ "io"
+ "math/bits"
+)
+
+// ChecksumType contains information about the checksum type.
+type ChecksumType uint32
+
+const (
+
+ // ChecksumSHA256 indicates a SHA256 checksum.
+ ChecksumSHA256 ChecksumType = 1 << iota
+ // ChecksumSHA1 indicates a SHA-1 checksum.
+ ChecksumSHA1
+ // ChecksumCRC32 indicates a CRC32 checksum with IEEE table.
+ ChecksumCRC32
+ // ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table.
+ ChecksumCRC32C
+
+ // Keep after all valid checksums
+ checksumLast
+
+ // checksumMask is a mask for valid checksum types.
+ checksumMask = checksumLast - 1
+
+ // ChecksumNone indicates no checksum.
+ ChecksumNone ChecksumType = 0
+
+ amzChecksumAlgo = "x-amz-checksum-algorithm"
+ amzChecksumCRC32 = "x-amz-checksum-crc32"
+ amzChecksumCRC32C = "x-amz-checksum-crc32c"
+ amzChecksumSHA1 = "x-amz-checksum-sha1"
+ amzChecksumSHA256 = "x-amz-checksum-sha256"
+)
+
+// Is returns if c is all of t.
+func (c ChecksumType) Is(t ChecksumType) bool {
+ return c&t == t
+}
+
+// Key returns the header key.
+// returns empty string if invalid or none.
+func (c ChecksumType) Key() string {
+ switch c & checksumMask {
+ case ChecksumCRC32:
+ return amzChecksumCRC32
+ case ChecksumCRC32C:
+ return amzChecksumCRC32C
+ case ChecksumSHA1:
+ return amzChecksumSHA1
+ case ChecksumSHA256:
+ return amzChecksumSHA256
+ }
+ return ""
+}
+
+// RawByteLen returns the size of the un-encoded checksum.
+func (c ChecksumType) RawByteLen() int {
+ switch c & checksumMask {
+ case ChecksumCRC32, ChecksumCRC32C:
+ return 4
+ case ChecksumSHA1:
+ return sha1.Size
+ case ChecksumSHA256:
+ return sha256.Size
+ }
+ return 0
+}
+
+// Hasher returns a hasher corresponding to the checksum type.
+// Returns nil if no checksum.
+func (c ChecksumType) Hasher() hash.Hash {
+ switch c & checksumMask {
+ case ChecksumCRC32:
+ return crc32.NewIEEE()
+ case ChecksumCRC32C:
+ return crc32.New(crc32.MakeTable(crc32.Castagnoli))
+ case ChecksumSHA1:
+ return sha1.New()
+ case ChecksumSHA256:
+ return sha256.New()
+ }
+ return nil
+}
+
+// IsSet returns whether the type is valid and known.
+func (c ChecksumType) IsSet() bool {
+ return bits.OnesCount32(uint32(c)) == 1
+}
+
+// String returns the type as a string.
+// CRC32, CRC32C, SHA1, and SHA256 for valid values.
+// Empty string for unset and "<invalid>" if not valid.
+func (c ChecksumType) String() string {
+ switch c & checksumMask {
+ case ChecksumCRC32:
+ return "CRC32"
+ case ChecksumCRC32C:
+ return "CRC32C"
+ case ChecksumSHA1:
+ return "SHA1"
+ case ChecksumSHA256:
+ return "SHA256"
+ case ChecksumNone:
+ return ""
+ }
+ return "<invalid>"
+}
+
+// ChecksumReader reads all of r and returns a checksum of type c.
+// Returns any error that may have occurred while reading.
+func (c ChecksumType) ChecksumReader(r io.Reader) (Checksum, error) {
+ h := c.Hasher()
+ if h == nil {
+ return Checksum{}, nil
+ }
+ _, err := io.Copy(h, r)
+ if err != nil {
+ return Checksum{}, err
+ }
+ return NewChecksum(c, h.Sum(nil)), nil
+}
+
+// ChecksumBytes returns a checksum of the content b with type c.
+func (c ChecksumType) ChecksumBytes(b []byte) Checksum {
+ h := c.Hasher()
+ if h == nil {
+ return Checksum{}
+ }
+ n, err := h.Write(b)
+ if err != nil || n != len(b) {
+ // Shouldn't happen with these checksummers.
+ return Checksum{}
+ }
+ return NewChecksum(c, h.Sum(nil))
+}
+
+// Checksum is a type and encoded value.
+type Checksum struct {
+ Type ChecksumType
+ r []byte
+}
+
+// NewChecksum sets the checksum to the value of b,
+// which is the raw hash output.
+// If the length of c does not match t.RawByteLen,
+// a checksum with ChecksumNone is returned.
+func NewChecksum(t ChecksumType, b []byte) Checksum {
+ if t.IsSet() && len(b) == t.RawByteLen() {
+ return Checksum{Type: t, r: b}
+ }
+ return Checksum{}
+}
+
+// NewChecksumString sets the checksum to the value of s,
+// which is the base 64 encoded raw hash output.
+// If the length of c does not match t.RawByteLen, it is not added.
+func NewChecksumString(t ChecksumType, s string) Checksum {
+ b, _ := base64.StdEncoding.DecodeString(s)
+ if t.IsSet() && len(b) == t.RawByteLen() {
+ return Checksum{Type: t, r: b}
+ }
+ return Checksum{}
+}
+
+// IsSet returns whether the checksum is valid and known.
+func (c Checksum) IsSet() bool {
+ return c.Type.IsSet() && len(c.r) == c.Type.RawByteLen()
+}
+
+// Encoded returns the encoded value.
+// Returns the empty string if not set or valid.
+func (c Checksum) Encoded() string {
+ if !c.IsSet() {
+ return ""
+ }
+ return base64.StdEncoding.EncodeToString(c.r)
+}
+
+// Raw returns the raw checksum value if set.
+func (c Checksum) Raw() []byte {
+ if !c.IsSet() {
+ return nil
+ }
+ return c.r
+}
diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go
index d7eb30322..ed1b7340e 100644
--- a/vendor/github.com/minio/minio-go/v7/functional_tests.go
+++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go
@@ -2312,7 +2312,7 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum := func(got, want string) {
if want != got {
- //logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got))
+ // logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got))
fmt.Printf("want %s, got %s\n", want, got)
return
}
@@ -2387,6 +2387,369 @@ func testPutMultipartObjectWithChecksums() {
successLogger(testName, function, args, startTime).Info()
}
+// Test PutObject with trailing checksums.
+func testTrailingChecksums() {
+ // initialize logging params
+ startTime := time.Now()
+ testName := getFuncName()
+ function := "PutObject(bucketName, objectName, reader,size, opts)"
+ args := map[string]interface{}{
+ "bucketName": "",
+ "objectName": "",
+ "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
+ }
+
+ if !isFullMode() {
+ ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info()
+ return
+ }
+
+ // Instantiate new minio client object.
+ c, err := minio.New(os.Getenv(serverEndpoint),
+ &minio.Options{
+ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
+ Secure: mustParseBool(os.Getenv(enableHTTPS)),
+ TrailingHeaders: true,
+ })
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
+ return
+ }
+
+ // Enable tracing, write to stderr.
+ // c.TraceOn(os.Stderr)
+
+ // Set user agent.
+ c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
+
+ // Generate a new random bucket name.
+ bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+ args["bucketName"] = bucketName
+
+ // Make a new bucket.
+ err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Make bucket failed", err)
+ return
+ }
+
+ hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) string {
+ r := bytes.NewReader(b)
+ tmp := make([]byte, partSize)
+ parts := 0
+ var all []byte
+ for {
+ n, err := io.ReadFull(r, tmp)
+ if err != nil && err != io.ErrUnexpectedEOF {
+ logError(testName, function, args, startTime, "", "Calc crc failed", err)
+ }
+ if n == 0 {
+ break
+ }
+ parts++
+ hasher.Reset()
+ hasher.Write(tmp[:n])
+ all = append(all, hasher.Sum(nil)...)
+ if err != nil {
+ break
+ }
+ }
+ hasher.Reset()
+ hasher.Write(all)
+ return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts)
+ }
+ defer cleanupBucket(bucketName, c)
+ tests := []struct {
+ header string
+ hasher hash.Hash
+
+ // Checksum values
+ ChecksumCRC32 string
+ ChecksumCRC32C string
+ ChecksumSHA1 string
+ ChecksumSHA256 string
+ PO minio.PutObjectOptions
+ }{
+ // Currently there is no way to override the checksum type.
+ {header: "x-amz-checksum-crc32c",
+ hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
+ ChecksumCRC32C: "set",
+ PO: minio.PutObjectOptions{
+ DisableContentSha256: true,
+ DisableMultipart: false,
+ UserMetadata: nil,
+ PartSize: 5 << 20,
+ },
+ },
+ {header: "x-amz-checksum-crc32c",
+ hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
+ ChecksumCRC32C: "set",
+ PO: minio.PutObjectOptions{
+ DisableContentSha256: true,
+ DisableMultipart: false,
+ UserMetadata: nil,
+ PartSize: 6_645_654, // Rather arbitrary size
+ },
+ },
+ {header: "x-amz-checksum-crc32c",
+ hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
+ ChecksumCRC32C: "set",
+ PO: minio.PutObjectOptions{
+ DisableContentSha256: false,
+ DisableMultipart: false,
+ UserMetadata: nil,
+ PartSize: 5 << 20,
+ },
+ },
+ {header: "x-amz-checksum-crc32c",
+ hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
+ ChecksumCRC32C: "set",
+ PO: minio.PutObjectOptions{
+ DisableContentSha256: false,
+ DisableMultipart: false,
+ UserMetadata: nil,
+ PartSize: 6_645_654, // Rather arbitrary size
+ },
+ },
+ }
+
+ for _, test := range tests {
+ bufSize := dataFileMap["datafile-11-MB"]
+
+ // Save the data
+ objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+ args["objectName"] = objectName
+
+ cmpChecksum := func(got, want string) {
+ if want != got {
+ logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %q, got %q", want, got))
+ return
+ }
+ }
+
+ reader := getDataReader("datafile-11-MB")
+ b, err := io.ReadAll(reader)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Read failed", err)
+ return
+ }
+ reader.Close()
+ h := test.hasher
+ h.Reset()
+ test.ChecksumCRC32C = hashMultiPart(b, int(test.PO.PartSize), test.hasher)
+
+ // Set correct CRC.
+ c.TraceOn(os.Stdout)
+ resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), test.PO)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ c.TraceOff()
+ cmpChecksum(resp.ChecksumSHA256, test.ChecksumSHA256)
+ cmpChecksum(resp.ChecksumSHA1, test.ChecksumSHA1)
+ cmpChecksum(resp.ChecksumCRC32, test.ChecksumCRC32)
+ cmpChecksum(resp.ChecksumCRC32C, test.ChecksumCRC32C)
+
+ // Read the data back
+ gopts := minio.GetObjectOptions{Checksum: true}
+ gopts.PartNumber = 2
+
+ // We cannot use StatObject, since it ignores partnumber.
+ r, err := c.GetObject(context.Background(), bucketName, objectName, gopts)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "GetObject failed", err)
+ return
+ }
+ io.Copy(io.Discard, r)
+ st, err := r.Stat()
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Stat failed", err)
+ return
+ }
+
+ // Test part 2 checksum...
+ h.Reset()
+ p2 := b[test.PO.PartSize:]
+ if len(p2) > int(test.PO.PartSize) {
+ p2 = p2[:test.PO.PartSize]
+ }
+ h.Write(p2)
+ got := base64.StdEncoding.EncodeToString(h.Sum(nil))
+ if test.ChecksumSHA256 != "" {
+ cmpChecksum(st.ChecksumSHA256, got)
+ }
+ if test.ChecksumSHA1 != "" {
+ cmpChecksum(st.ChecksumSHA1, got)
+ }
+ if test.ChecksumCRC32 != "" {
+ cmpChecksum(st.ChecksumCRC32, got)
+ }
+ if test.ChecksumCRC32C != "" {
+ cmpChecksum(st.ChecksumCRC32C, got)
+ }
+
+ delete(args, "metadata")
+ }
+}
+
+// Test PutObject with custom checksums.
+func testPutObjectWithAutomaticChecksums() {
+ // initialize logging params
+ startTime := time.Now()
+ testName := getFuncName()
+ function := "PutObject(bucketName, objectName, reader,size, opts)"
+ args := map[string]interface{}{
+ "bucketName": "",
+ "objectName": "",
+ "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
+ }
+
+ if !isFullMode() {
+ ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info()
+ return
+ }
+
+ // Seed random based on current time.
+ rand.Seed(time.Now().Unix())
+
+ // Instantiate new minio client object.
+ c, err := minio.New(os.Getenv(serverEndpoint),
+ &minio.Options{
+ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
+ Secure: mustParseBool(os.Getenv(enableHTTPS)),
+ TrailingHeaders: true,
+ })
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
+ return
+ }
+
+ // Set user agent.
+ c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
+
+ // Generate a new random bucket name.
+ bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+ args["bucketName"] = bucketName
+
+ // Make a new bucket.
+ err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Make bucket failed", err)
+ return
+ }
+
+ defer cleanupBucket(bucketName, c)
+ tests := []struct {
+ header string
+ hasher hash.Hash
+
+ // Checksum values
+ ChecksumCRC32 string
+ ChecksumCRC32C string
+ ChecksumSHA1 string
+ ChecksumSHA256 string
+ }{
+ // Built-in will only add crc32c, when no MD5 nor SHA256.
+ {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
+ }
+
+ // Enable tracing, write to stderr.
+ c.TraceOn(os.Stderr)
+ defer c.TraceOff()
+
+ for i, test := range tests {
+ bufSize := dataFileMap["datafile-10-kB"]
+
+ // Save the data
+ objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+ args["objectName"] = objectName
+
+ cmpChecksum := func(got, want string) {
+ if want != got {
+ logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got))
+ return
+ }
+ }
+
+ meta := map[string]string{}
+ reader := getDataReader("datafile-10-kB")
+ b, err := io.ReadAll(reader)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Read failed", err)
+ return
+ }
+
+ h := test.hasher
+ h.Reset()
+ h.Write(b)
+ meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil))
+ args["metadata"] = meta
+
+ resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{
+ DisableMultipart: true,
+ UserMetadata: nil,
+ DisableContentSha256: true,
+ SendContentMd5: false,
+ })
+ if err == nil {
+ if i == 0 && resp.ChecksumCRC32C == "" {
+ ignoredLog(testName, function, args, startTime, "Checksums does not appear to be supported by backend").Info()
+ return
+ }
+ } else {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"])
+ cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
+ cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
+ cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
+
+ // Usually this will be the same as above, since we skip automatic checksum when SHA256 content is sent.
+ // When/if we add a checksum control to PutObjectOptions this will make more sense.
+ resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{
+ DisableMultipart: true,
+ UserMetadata: nil,
+ DisableContentSha256: false,
+ SendContentMd5: false,
+ })
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ // The checksum will not be enabled on HTTP, since it uses SHA256 blocks.
+ if mustParseBool(os.Getenv(enableHTTPS)) {
+ cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"])
+ cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
+ cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
+ cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
+ }
+
+ // Set SHA256 header manually
+ sh256 := sha256.Sum256(b)
+ meta = map[string]string{"x-amz-checksum-sha256": base64.StdEncoding.EncodeToString(sh256[:])}
+ args["metadata"] = meta
+ resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{
+ DisableMultipart: true,
+ UserMetadata: meta,
+ DisableContentSha256: true,
+ SendContentMd5: false,
+ })
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"])
+ cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
+ cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
+ cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
+ delete(args, "metadata")
+ }
+
+ successLogger(testName, function, args, startTime).Info()
+}
+
// Test PutObject using a large data to trigger multipart readat
func testPutObjectWithMetadata() {
// initialize logging params
@@ -12576,6 +12939,8 @@ func main() {
testRemoveObjectWithVersioning()
testRemoveObjectsWithVersioning()
testObjectTaggingWithVersioning()
+ testTrailingChecksums()
+ testPutObjectWithAutomaticChecksums()
// SSE-C tests will only work over TLS connection.
if tls {
diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go
index 34914490c..ffd251451 100644
--- a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go
+++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go
@@ -289,7 +289,7 @@ func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati
req.Header.Add("X-Amz-Trailer", strings.ToLower(k))
}
- req.TransferEncoding = []string{"aws-chunked"}
+ req.Header.Set("Content-Encoding", "aws-chunked")
req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10))
}
diff --git a/vendor/github.com/minio/minio-go/v7/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go
index 31b340dcf..0191909bd 100644
--- a/vendor/github.com/minio/minio-go/v7/post-policy.go
+++ b/vendor/github.com/minio/minio-go/v7/post-policy.go
@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2015-2017 MinIO, Inc.
+ * Copyright 2015-2023 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,8 +20,11 @@ package minio
import (
"encoding/base64"
"fmt"
+ "net/http"
"strings"
"time"
+
+ "github.com/minio/minio-go/v7/pkg/encrypt"
)
// expirationDateFormat date format for expiration key in json policy.
@@ -258,6 +261,26 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error {
return nil
}
+// SetChecksum sets the checksum of the request.
+func (p *PostPolicy) SetChecksum(c Checksum) {
+ if c.IsSet() {
+ p.formData[amzChecksumAlgo] = c.Type.String()
+ p.formData[c.Type.Key()] = c.Encoded()
+ }
+}
+
+// SetEncryption - sets encryption headers for POST API
+func (p *PostPolicy) SetEncryption(sse encrypt.ServerSide) {
+ if sse == nil {
+ return
+ }
+ h := http.Header{}
+ sse.Marshal(h)
+ for k, v := range h {
+ p.formData[k] = v[0]
+ }
+}
+
// SetUserData - Set user data as a key/value couple.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserData(key string, value string) error {