summaryrefslogtreecommitdiff
path: root/vendor/github.com/minio/minio-go/v7/functional_tests.go
diff options
context:
space:
mode:
authorLibravatar kim <grufwub@gmail.com>2025-11-05 21:58:38 +0100
committerLibravatar tobi <tobi.smethurst@protonmail.com>2025-11-17 14:12:41 +0100
commitd0c551acb5ab79efafd325f5b19c76a3de835356 (patch)
treeba9462cdc5081015fbd6bf5d98d9f39334d13207 /vendor/github.com/minio/minio-go/v7/functional_tests.go
parent[performance] when transforming media, perform read operations of large files... (diff)
downloadgotosocial-d0c551acb5ab79efafd325f5b19c76a3de835356.tar.xz
[chore] update dependencies (#4542)
- github.com/minio/minio-go/v7: v7.0.95 -> v7.0.97 - github.com/ncruces/go-sqlite3: v0.29.1 -> v0.30.0 - github.com/tdewolff/minify/v2: v2.24.5 -> v2.24.6 - codeberg.org/gruf/go-mmap: fixes build for BSD platforms Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4542 Co-authored-by: kim <grufwub@gmail.com> Co-committed-by: kim <grufwub@gmail.com>
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/functional_tests.go')
-rw-r--r--vendor/github.com/minio/minio-go/v7/functional_tests.go560
1 files changed, 502 insertions, 58 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go
index 3ade9a6af..4f8f9dd8c 100644
--- a/vendor/github.com/minio/minio-go/v7/functional_tests.go
+++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go
@@ -88,7 +88,7 @@ func createHTTPTransport() (transport *http.Transport) {
transport.TLSClientConfig.InsecureSkipVerify = true
}
- return
+ return transport
}
var readFull = func(r io.Reader, buf []byte) (n int, err error) {
@@ -123,7 +123,7 @@ var readFull = func(r io.Reader, buf []byte) (n int, err error) {
} else if n > 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
- return
+ return n, err
}
func baseLogger(testName, function string, args map[string]interface{}, startTime time.Time) *slog.Logger {
@@ -282,7 +282,7 @@ var mintDataDir = os.Getenv("MINT_DATA_DIR")
func getMintDataDirFilePath(filename string) (fp string) {
if mintDataDir == "" {
- return
+ return fp
}
return filepath.Join(mintDataDir, filename)
}
@@ -2075,6 +2075,9 @@ func testPutObjectWithChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
+ if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
+ logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
+ }
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@@ -2095,6 +2098,9 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
+ if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
+ logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
+ }
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@@ -2435,7 +2441,10 @@ func testPutMultipartObjectWithChecksums() {
reader.Close()
h := test.cs.Hasher()
h.Reset()
- want := hashMultiPart(b, partSize, test.cs)
+ // wantChksm might be the full object checksum or the multipart checksum, depending on the test.cs type.
+ wantChksm := hashMultiPart(b, partSize, test.cs)
+ // wantFullObjectChksm is always the full object checksum that is returned after CopyObject.
+ wantFullObjectChksm := hashMultiPart(b, len(b), test.cs)
rd := bytes.NewReader(b)
cs := test.cs
@@ -2456,15 +2465,15 @@ func testPutMultipartObjectWithChecksums() {
switch test.cs.Base() {
case minio.ChecksumCRC32C:
- cmpChecksum(resp.ChecksumCRC32C, want)
+ cmpChecksum(resp.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
- cmpChecksum(resp.ChecksumCRC32, want)
+ cmpChecksum(resp.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
- cmpChecksum(resp.ChecksumSHA1, want)
+ cmpChecksum(resp.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
- cmpChecksum(resp.ChecksumSHA256, want)
+ cmpChecksum(resp.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
- cmpChecksum(resp.ChecksumCRC64NVME, want)
+ cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
}
args["section"] = "HeadObject"
@@ -2475,15 +2484,51 @@ func testPutMultipartObjectWithChecksums() {
}
switch test.cs.Base() {
case minio.ChecksumCRC32C:
- cmpChecksum(st.ChecksumCRC32C, want)
+ cmpChecksum(st.ChecksumCRC32C, wantChksm)
+ case minio.ChecksumCRC32:
+ cmpChecksum(st.ChecksumCRC32, wantChksm)
+ case minio.ChecksumSHA1:
+ cmpChecksum(st.ChecksumSHA1, wantChksm)
+ case minio.ChecksumSHA256:
+ cmpChecksum(st.ChecksumSHA256, wantChksm)
+ case minio.ChecksumCRC64NVME:
+ cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
+ }
+
+ // Use the CopyObject API to make a copy, in the case it was a composite checksum,
+ // it will change because the copy is no longer a multipart object. S3 returns the checksum
+ // of the full object when HeadObject is called on the copy.
+ args["section"] = "CopyObject"
+ objectCopyName := objectName + "-copy"
+ _, err = c.CopyObject(context.Background(), minio.CopyDestOptions{
+ Bucket: bucketName,
+ Object: objectCopyName,
+ }, minio.CopySrcOptions{
+ Bucket: bucketName,
+ Object: objectName,
+ })
+ if err != nil {
+ logError(testName, function, args, startTime, "", "CopyObject failed", err)
+ return
+ }
+
+ args["section"] = "HeadObject-Copy"
+ st, err = c.StatObject(context.Background(), bucketName, objectCopyName, minio.StatObjectOptions{Checksum: true})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "StatObject failed", err)
+ return
+ }
+ switch test.cs.Base() {
+ case minio.ChecksumCRC32C:
+ cmpChecksum(st.ChecksumCRC32C, wantFullObjectChksm)
case minio.ChecksumCRC32:
- cmpChecksum(st.ChecksumCRC32, want)
+ cmpChecksum(st.ChecksumCRC32, wantFullObjectChksm)
case minio.ChecksumSHA1:
- cmpChecksum(st.ChecksumSHA1, want)
+ cmpChecksum(st.ChecksumSHA1, wantFullObjectChksm)
case minio.ChecksumSHA256:
- cmpChecksum(st.ChecksumSHA256, want)
+ cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
case minio.ChecksumCRC64NVME:
- cmpChecksum(st.ChecksumCRC64NVME, want)
+ cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
}
args["section"] = "GetObjectAttributes"
@@ -2493,19 +2538,19 @@ func testPutMultipartObjectWithChecksums() {
return
}
- if strings.ContainsRune(want, '-') {
- want = want[:strings.IndexByte(want, '-')]
+ if strings.ContainsRune(wantChksm, '-') {
+ wantChksm = wantChksm[:strings.IndexByte(wantChksm, '-')]
}
switch test.cs {
// Full Object CRC does not return anything with GetObjectAttributes
case minio.ChecksumCRC32C:
- cmpChecksum(s.Checksum.ChecksumCRC32C, want)
+ cmpChecksum(s.Checksum.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
- cmpChecksum(s.Checksum.ChecksumCRC32, want)
+ cmpChecksum(s.Checksum.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
- cmpChecksum(s.Checksum.ChecksumSHA1, want)
+ cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
- cmpChecksum(s.Checksum.ChecksumSHA256, want)
+ cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
}
// Read the data back
@@ -2529,22 +2574,22 @@ func testPutMultipartObjectWithChecksums() {
// Test part 2 checksum...
h.Reset()
h.Write(b[partSize : 2*partSize])
- want = base64.StdEncoding.EncodeToString(h.Sum(nil))
+ wantChksm = base64.StdEncoding.EncodeToString(h.Sum(nil))
switch test.cs {
// Full Object CRC does not return any part CRC for whatever reason.
case minio.ChecksumCRC32C:
- cmpChecksum(st.ChecksumCRC32C, want)
+ cmpChecksum(st.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
- cmpChecksum(st.ChecksumCRC32, want)
+ cmpChecksum(st.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
- cmpChecksum(st.ChecksumSHA1, want)
+ cmpChecksum(st.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
- cmpChecksum(st.ChecksumSHA256, want)
+ cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
// AWS doesn't return part checksum, but may in the future.
if st.ChecksumCRC64NVME != "" {
- cmpChecksum(st.ChecksumCRC64NVME, want)
+ cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
}
@@ -3321,7 +3366,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if opts.VersionID != "" {
if OA.VersionID != opts.VersionID {
err = fmt.Errorf("Expected versionId %s but got versionId %s", opts.VersionID, OA.VersionID)
- return
+ return err
}
}
@@ -3348,12 +3393,12 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if test.HasPartChecksums {
if partsMissingChecksum {
err = fmt.Errorf("One or all parts were missing a checksum")
- return
+ return err
}
} else {
if foundPartChecksum {
err = fmt.Errorf("Did not expect ObjectParts to have checksums but found one")
- return
+ return err
}
}
@@ -3365,52 +3410,52 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if test.HasFullChecksum {
if !hasFullObjectChecksum {
err = fmt.Errorf("Full object checksum not found")
- return
+ return err
}
} else {
if hasFullObjectChecksum {
err = fmt.Errorf("Did not expect a full object checksum but we got one")
- return
+ return err
}
}
if OA.ETag != test.ETag {
err = fmt.Errorf("Etags do not match, got %s but expected %s", OA.ETag, test.ETag)
- return
+ return err
}
if test.HasParts {
if len(OA.ObjectParts.Parts) < 1 {
err = fmt.Errorf("Was expecting ObjectParts but none were present")
- return
+ return err
}
}
if OA.StorageClass == "" {
err = fmt.Errorf("Was expecting a StorageClass but got none")
- return
+ return err
}
if OA.ObjectSize != test.ObjectSize {
err = fmt.Errorf("Was expecting a ObjectSize but got none")
- return
+ return err
}
if test.HasParts {
if opts.MaxParts == 0 {
if len(OA.ObjectParts.Parts) != OA.ObjectParts.PartsCount {
err = fmt.Errorf("expected %s parts but got %d", OA.ObjectParts.PartsCount, len(OA.ObjectParts.Parts))
- return
+ return err
}
} else if (opts.MaxParts + opts.PartNumberMarker) > OA.ObjectParts.PartsCount {
if len(OA.ObjectParts.Parts) != (OA.ObjectParts.PartsCount - opts.PartNumberMarker) {
err = fmt.Errorf("expected %d parts but got %d", (OA.ObjectParts.PartsCount - opts.PartNumberMarker), len(OA.ObjectParts.Parts))
- return
+ return err
}
} else if opts.MaxParts != 0 {
if opts.MaxParts != len(OA.ObjectParts.Parts) {
err = fmt.Errorf("expected %d parts but got %d", opts.MaxParts, len(OA.ObjectParts.Parts))
- return
+ return err
}
}
}
@@ -3418,18 +3463,18 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if OA.ObjectParts.NextPartNumberMarker == OA.ObjectParts.PartsCount {
if OA.ObjectParts.IsTruncated {
err = fmt.Errorf("Expected ObjectParts to NOT be truncated, but it was")
- return
+ return err
}
}
if OA.ObjectParts.NextPartNumberMarker != OA.ObjectParts.PartsCount {
if !OA.ObjectParts.IsTruncated {
err = fmt.Errorf("Expected ObjectParts to be truncated, but it was NOT")
- return
+ return err
}
}
- return
+ return err
}
// Test PutObject using a large data to trigger multipart readat
@@ -3645,6 +3690,93 @@ func testPutObjectStreaming() {
logSuccess(testName, function, args, startTime)
}
+// Test PutObject with preconditions on non-existent objects
+func testPutObjectPreconditionOnNonExistent() {
+ startTime := time.Now()
+ testName := getFuncName()
+ function := "PutObject(bucketName, objectName, reader, size, opts) with preconditions"
+ args := map[string]interface{}{
+ "bucketName": "",
+ "objectName": "",
+ "opts": "minio.PutObjectOptions{SetMatchETag/SetMatchETagExcept}",
+ }
+
+ c, err := NewClient(ClientConfig{})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
+ return
+ }
+
+ bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+ args["bucketName"] = bucketName
+
+ err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MakeBucket failed", err)
+ return
+ }
+
+ defer cleanupBucket(bucketName, c)
+
+ // Test 1: PutObject with SetMatchETag on non-existent object should fail
+ objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object-")
+ args["objectName"] = objectName
+
+ data := bytes.NewReader([]byte("test data"))
+
+ opts := minio.PutObjectOptions{}
+ opts.SetMatchETag("some-etag")
+
+ _, err = c.PutObject(context.Background(), bucketName, objectName, data, int64(data.Len()), opts)
+ if err == nil {
+ logError(testName, function, args, startTime, "", "PutObject with SetMatchETag on non-existent object should have failed", nil)
+ return
+ }
+
+ errResp := minio.ToErrorResponse(err)
+ if errResp.Code != "NoSuchKey" {
+ logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects), got %s", errResp.Code), err)
+ return
+ }
+
+ // Test 2: PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should succeed
+ objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object2-")
+ args["objectName"] = objectName2
+
+ data2 := bytes.NewReader([]byte("test data 2"))
+ opts2 := minio.PutObjectOptions{}
+ opts2.SetMatchETagExcept("some-etag")
+
+ _, err = c.PutObject(context.Background(), bucketName, objectName2, data2, int64(data2.Len()), opts2)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should have succeeded", err)
+ return
+ }
+ // Test 3: CompleteMultipartUpload with preconditions on non-existent object should fail
+ objectName3 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-multipart-")
+ args["objectName"] = objectName3
+
+ data3 := bytes.Repeat([]byte("a"), 5*1024*1024+1)
+ reader3 := bytes.NewReader(data3)
+
+ opts3 := minio.PutObjectOptions{}
+ opts3.SetMatchETag("non-existent-etag")
+
+ _, err = c.PutObject(context.Background(), bucketName, objectName3, reader3, int64(len(data3)), opts3)
+ if err == nil {
+ logError(testName, function, args, startTime, "", "CompleteMultipartUpload with SetMatchETag on non-existent object should have failed", nil)
+ return
+ }
+
+ errResp = minio.ToErrorResponse(err)
+ if errResp.Code != "NoSuchKey" {
+ logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects) for multipart, got %s", errResp.Code), err)
+ return
+ }
+
+ logSuccess(testName, function, args, startTime)
+}
+
// Test get object seeker from the end, using whence set to '2'.
func testGetObjectSeekEnd() {
// initialize logging params
@@ -8653,6 +8785,7 @@ func testCopyObjectV2() {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
+
// Check the various fields of source object against destination object.
objInfo, err = r.Stat()
if err != nil {
@@ -8691,6 +8824,260 @@ func testCopyObjectV2() {
logSuccess(testName, function, args, startTime)
}
+// Tests copy object with various checksum scenarios, tries to not repeat CopyObjectV2 test and
+// instead just focus on Checksum.
+func testCopyObjectWithChecksums() {
+ startTime := time.Now()
+ testName := getFuncName()
+ function := "CopyObjectWithChecksums(destination, source)"
+ args := map[string]interface{}{}
+
+ c, err := NewClient(ClientConfig{CredsV2: true})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
+ return
+ }
+
+ // Generate a new random bucket name.
+ bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+
+ // Make a new bucket in 'us-east-1' (source bucket).
+ err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MakeBucket failed", err)
+ return
+ }
+ defer cleanupBucket(bucketName, c)
+
+ // Make a new bucket in 'us-east-1' (destination bucket).
+ err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MakeBucket failed", err)
+ return
+ }
+ defer cleanupBucket(bucketName+"-copy", c)
+
+ // Generate 33K of data.
+ bufSize := dataFileMap["datafile-33-kB"]
+ reader := getDataReader("datafile-33-kB")
+ defer reader.Close()
+
+ // PutObject to upload the object to the bucket, this object will have a Crc64NVME checksum applied
+ // by default since nothing was explicitly specified.
+ objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+ _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ // GetObject to obtain the eTag
+ r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "GetObject failed", err)
+ return
+ }
+ objInfo, err := r.Stat()
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Stat failed", err)
+ return
+ }
+ r.Close()
+
+ // Copy source options
+ src := minio.CopySrcOptions{
+ Bucket: bucketName,
+ Object: objectName,
+ MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
+ MatchETag: objInfo.ETag,
+ }
+
+ tests := []struct {
+ csType minio.ChecksumType
+ cs wantChecksums
+ }{
+ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
+ {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}},
+ {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}},
+ {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}},
+ {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}},
+ // In S3, all copied objects without checksums and specified destination checksum algorithms
+ // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case.
+ {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
+ }
+
+ for _, test := range tests {
+ args := map[string]interface{}{}
+ args["srcOpts"] = src
+ args["section"] = "setup"
+ args["checksum"] = test.csType.String()
+
+ // Copy destination options
+ bucketCopyName := bucketName + "-copy"
+ objectCopyName := objectName + "-copy-" + test.csType.String()
+ dst := minio.CopyDestOptions{
+ Bucket: bucketCopyName,
+ Object: objectCopyName,
+ ReplaceMetadata: true,
+ }
+ if test.csType != minio.ChecksumNone {
+ // Request the server-side checksum on the copy.
+ // ChecksumNone is a flag to leave off the header
+ dst.ChecksumType = test.csType
+ }
+ args["destOpts"] = dst
+
+ // Perform the Copy
+ args["section"] = "CopyObject"
+ _, err = c.CopyObject(context.Background(), dst, src)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "CopyObject failed", err)
+ return
+ }
+
+ // Checksum verification
+ args["section"] = "HeadObject"
+ st, err := c.StatObject(context.Background(), bucketCopyName, objectCopyName, minio.StatObjectOptions{Checksum: true})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "StatObject failed", err)
+ return
+ }
+ if st.ChecksumMode != "FULL_OBJECT" {
+ logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil)
+ return
+ }
+ err = cmpChecksum(st, test.cs)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Checksum mismatch", err)
+ return
+ }
+
+ logSuccess(testName, function, args, startTime)
+ }
+}
+
+// Tests replacing an object with CopyObject and a new Checksum type
+func testReplaceObjectWithChecksums() {
+ startTime := time.Now()
+ testName := getFuncName()
+ function := "CopyObjectWithChecksums(destination, source)"
+ args := map[string]interface{}{}
+
+ c, err := NewClient(ClientConfig{CredsV2: true})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
+ return
+ }
+
+ // Generate a new random bucket name.
+ bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+
+ // Make a new bucket in 'us-east-1' (source bucket).
+ err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "MakeBucket failed", err)
+ return
+ }
+ defer cleanupBucket(bucketName, c)
+
+ tests := []struct {
+ csType minio.ChecksumType
+ cs wantChecksums
+ }{
+ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
+ {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}},
+ {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}},
+ {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}},
+ {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}},
+ // In S3, all copied objects without checksums and specified destination checksum algorithms
+ // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case.
+ {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
+ }
+
+ for _, test := range tests {
+ args := map[string]interface{}{}
+ args["section"] = "setup"
+ args["destOpts"] = ""
+ args["checksum"] = test.csType.String()
+
+ bufSize := dataFileMap["datafile-33-kB"]
+ reader := getDataReader("datafile-33-kB")
+ defer reader.Close()
+
+ // PutObject to upload the object to the bucket
+ objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+ _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "PutObject failed", err)
+ return
+ }
+ // GetObject to obtain the eTag
+ r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "GetObject failed", err)
+ return
+ }
+ objInfo, err := r.Stat()
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Stat failed", err)
+ return
+ }
+ r.Close()
+
+ // Copy source options
+ src := minio.CopySrcOptions{
+ Bucket: bucketName,
+ Object: objectName,
+ MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
+ MatchETag: objInfo.ETag,
+ }
+
+ // Copy destination options, overwrite the existing object
+ dst := minio.CopyDestOptions{
+ Bucket: bucketName,
+ Object: objectName,
+ // S3 requires that we send some new metadata otherwise it complains that the
+ // CopyObject is illegal.
+ UserMetadata: map[string]string{
+ "TestMeta": objectName + "-meta-" + test.csType.String(),
+ },
+ ReplaceMetadata: true,
+ }
+ if test.csType != minio.ChecksumNone {
+ // Request the server-side checksum on the copy.
+ // ChecksumNone is a flag to leave off the header
+ dst.ChecksumType = test.csType
+ }
+ args["destOpts"] = dst
+
+ // Perform the Copy
+ args["section"] = "CopyObject"
+ _, err = c.CopyObject(context.Background(), dst, src)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "CopyObject failed", err)
+ return
+ }
+
+ // Checksum verification
+ args["section"] = "HeadObject"
+ st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "StatObject failed", err)
+ return
+ }
+ if st.ChecksumMode != "FULL_OBJECT" {
+ logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil)
+ return
+ }
+ err = cmpChecksum(st, test.cs)
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Checksum mismatch", err)
+ return
+ }
+
+ logSuccess(testName, function, args, startTime)
+ }
+}
+
func testComposeObjectErrorCasesWrapper(c *minio.Client) {
// initialize logging params
startTime := time.Now()
@@ -8976,6 +9363,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
testName := getFuncNameLoc(2)
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
+ args["testName"] = testName
var srcEncryption, dstEncryption encrypt.ServerSide
// Make a new bucket in 'us-east-1' (source bucket).
@@ -8990,8 +9378,19 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
// 1. create an sse-c encrypted object to copy by uploading
const srcSize = 1024 * 1024
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
+
+ // Calculate the CRC32C checksum for the object
+ meta := map[string]string{}
+ h := minio.ChecksumCRC32C.Hasher()
+ h.Reset()
+ h.Write(buf)
+ meta[minio.ChecksumCRC32C.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil))
+
_, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
ServerSideEncryption: sseSrc,
+ DisableMultipart: true,
+ DisableContentSha256: true,
+ UserMetadata: meta,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject call failed", err)
@@ -9028,7 +9427,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// 3. get copied object and check if content is equal
coreClient := minio.Core{Client: c}
- reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption})
+ reader, oi, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption, Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9045,6 +9444,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
reader.Close()
+ err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Checksum mismatch on dstObject", err)
+ return
+ }
+
// Test key rotation for source object in-place.
var newSSE encrypt.ServerSide
if sseSrc != nil && sseSrc.Type() == encrypt.SSEC {
@@ -9068,7 +9473,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// Get copied object and check if content is equal
- reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE})
+ reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE, Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9085,6 +9490,13 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
reader.Close()
+ err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
+ if err != nil {
+ fmt.Printf("srcObject objectInfo: %+v\n", oi)
+ logError(testName, function, args, startTime, "", "Checksum mismatch on srcObject for in-place", err)
+ return
+ }
+
// Test in-place decryption.
dst = minio.CopyDestOptions{
Bucket: bucketName,
@@ -9106,7 +9518,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// Get copied decrypted object and check if content is equal
- reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{})
+ reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9123,6 +9535,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
return
}
+ err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
+ if err != nil {
+ logError(testName, function, args, startTime, "", "Checksum mismatch for decrypted object", err)
+ return
+ }
+
logSuccess(testName, function, args, startTime)
}
@@ -9134,7 +9552,7 @@ func testUnencryptedToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9154,7 +9572,7 @@ func testUnencryptedToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9175,7 +9593,7 @@ func testUnencryptedToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9195,7 +9613,7 @@ func testEncryptedSSECToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9216,7 +9634,7 @@ func testEncryptedSSECToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9237,7 +9655,7 @@ func testEncryptedSSECToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9258,7 +9676,7 @@ func testEncryptedSSES3ToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9279,7 +9697,7 @@ func testEncryptedSSES3ToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9300,7 +9718,7 @@ func testEncryptedSSES3ToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{})
+ c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9321,7 +9739,7 @@ func testEncryptedCopyObjectV2() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
- c, err := NewClient(ClientConfig{CredsV2: true})
+ c, err := NewClient(ClientConfig{CredsV2: true, TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -10080,7 +10498,7 @@ func testUnencryptedToSSECCopyObjectPart() {
function := "CopyObjectPart(destination, source)"
args := map[string]interface{}{}
- client, err := NewClient(ClientConfig{})
+ client, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
return
@@ -11080,7 +11498,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) {
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
- return
+ return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -11256,7 +11674,7 @@ func testStorageClassMetadataPutObject() {
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
- return
+ return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -11377,7 +11795,7 @@ func testStorageClassMetadataCopyObject() {
args["object"] = object
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
- return
+ return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -14195,6 +14613,29 @@ func mustParseBool(str string) bool {
return b
}
+// wantChecksums is a map of expected checksums for an object.
+type wantChecksums map[minio.ChecksumType]string
+
+// cmpChecksum compares the checksums of an object against expected values.
+func cmpChecksum(oi minio.ObjectInfo, chksums wantChecksums) error {
+ if oi.ChecksumCRC64NVME != chksums[minio.ChecksumCRC64NVME] {
+ return fmt.Errorf("Checksum mismatch for CRC64NVME, want: %s, got: %s", chksums[minio.ChecksumCRC64NVME], oi.ChecksumCRC64NVME)
+ }
+ if oi.ChecksumCRC32C != chksums[minio.ChecksumCRC32C] {
+ return fmt.Errorf("Checksum mismatch for CRC32C, want: %s, got: %s", chksums[minio.ChecksumCRC32C], oi.ChecksumCRC32C)
+ }
+ if oi.ChecksumCRC32 != chksums[minio.ChecksumCRC32] {
+ return fmt.Errorf("Checksum mismatch for CRC32, want: %s, got: %s", chksums[minio.ChecksumCRC32], oi.ChecksumCRC32)
+ }
+ if oi.ChecksumSHA1 != chksums[minio.ChecksumSHA1] {
+ return fmt.Errorf("Checksum mismatch for SHA1, want: %s, got: %s", chksums[minio.ChecksumSHA1], oi.ChecksumSHA1)
+ }
+ if oi.ChecksumSHA256 != chksums[minio.ChecksumSHA256] {
+ return fmt.Errorf("Checksum mismatch for SHA256, want: %s, got: %s", chksums[minio.ChecksumSHA256], oi.ChecksumSHA256)
+ }
+ return nil
+}
+
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(
os.Stdout,
@@ -14219,6 +14660,8 @@ func main() {
// execute tests
if isFullMode() {
+ testCopyObjectWithChecksums()
+ testReplaceObjectWithChecksums()
testCorsSetGetDelete()
testCors()
testListMultipartUpload()
@@ -14253,6 +14696,7 @@ func main() {
testPutObjectWithMetadata()
testPutObjectReadAt()
testPutObjectStreaming()
+ testPutObjectPreconditionOnNonExistent()
testGetObjectSeekEnd()
testGetObjectClosedTwice()
testGetObjectS3Zip()