summaryrefslogtreecommitdiff
path: root/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
diff options
context:
space:
mode:
authorLibravatar dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>2022-11-07 11:20:43 +0100
committerLibravatar GitHub <noreply@github.com>2022-11-07 11:20:43 +0100
commit459a5c8d967366b6b9f1d9815bf252c1107ae35e (patch)
tree348997f4d2eceb367a86a7853f1465a7ab7c538b /vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
parent[chore] Bump github.com/spf13/cobra from 1.5.0 to 1.6.1 (#982) (diff)
downloadgotosocial-459a5c8d967366b6b9f1d9815bf252c1107ae35e.tar.xz
[chore] Bump github.com/minio/minio-go/v7 from 7.0.37 to 7.0.43 (#983)
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.37 to 7.0.43. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](https://github.com/minio/minio-go/compare/v7.0.37...v7.0.43) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go')
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go92
1 files changed, 75 insertions, 17 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
index 464bde7f3..e3a14c59d 100644
--- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
+++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
@@ -107,11 +107,19 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
return UploadInfo{}, err
}
+ withChecksum := c.trailingHeaderSupport
+ if withChecksum {
+ if opts.UserMetadata == nil {
+ opts.UserMetadata = make(map[string]string, 1)
+ }
+ opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C"
+ }
// Initiate a new multipart upload.
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
if err != nil {
return UploadInfo{}, err
}
+ delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
// Aborts the multipart upload in progress, if the
// function returns any error, since we do not resume
@@ -177,14 +185,33 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
// As a special case if partNumber is lastPartNumber, we
// calculate the offset based on the last part size.
if uploadReq.PartNum == lastPartNumber {
- readOffset = (size - lastPartSize)
+ readOffset = size - lastPartSize
partSize = lastPartSize
}
sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress)
+ var trailer = make(http.Header, 1)
+ if withChecksum {
+ crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+ trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(crc.Sum(nil)))
+ sectionReader = newHashReaderWrapper(sectionReader, crc, func(hash []byte) {
+ trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(hash))
+ })
+ }
// Proceed to upload the part.
- objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption, !opts.DisableContentSha256, nil)
+ p := uploadPartParams{bucketName: bucketName,
+ objectName: objectName,
+ uploadID: uploadID,
+ reader: sectionReader,
+ partNumber: uploadReq.PartNum,
+ size: partSize,
+ sse: opts.ServerSideEncryption,
+ streamSha256: !opts.DisableContentSha256,
+ sha256Hex: "",
+ trailer: trailer,
+ }
+ objPart, err := c.uploadPart(ctx, p)
if err != nil {
uploadedPartsCh <- uploadedPartRes{
Error: err,
@@ -221,8 +248,12 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
// Update the totalUploadedSize.
totalUploadedSize += uploadRes.Size
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
- ETag: uploadRes.Part.ETag,
- PartNumber: uploadRes.Part.PartNumber,
+ ETag: uploadRes.Part.ETag,
+ PartNumber: uploadRes.Part.PartNumber,
+ ChecksumCRC32: uploadRes.Part.ChecksumCRC32,
+ ChecksumCRC32C: uploadRes.Part.ChecksumCRC32C,
+ ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
+ ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
})
}
}
@@ -235,6 +266,18 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
// Sort all completed parts.
sort.Sort(completedParts(complMultipartUpload.Parts))
+ if withChecksum {
+ // Add hash of hashes.
+ crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+ for _, part := range complMultipartUpload.Parts {
+ cs, err := base64.StdEncoding.DecodeString(part.ChecksumCRC32C)
+ if err == nil {
+ crc.Write(cs)
+ }
+ }
+ opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
+ }
+
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
@@ -339,8 +382,8 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
// Update progress reader appropriately to the latest offset
// as we read from the source.
hooked := newHook(bytes.NewReader(buf[:length]), opts.Progress)
-
- objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, hooked, partNumber, md5Base64, "", partSize, opts.ServerSideEncryption, !opts.DisableContentSha256, customHeader)
+ p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: hooked, partNumber: partNumber, md5Base64: md5Base64, size: partSize, sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
+ objPart, uerr := c.uploadPart(ctx, p)
if uerr != nil {
return UploadInfo{}, uerr
}
@@ -419,6 +462,7 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'")
}
+ var readSeeker io.Seeker
if size > 0 {
if isReadAt(reader) && !isObject(reader) {
seeker, ok := reader.(io.Seeker)
@@ -428,35 +472,49 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
return UploadInfo{}, errInvalidArgument(err.Error())
}
reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size)
+ readSeeker = reader.(io.Seeker)
}
}
}
var md5Base64 string
if opts.SendContentMd5 {
- // Create a buffer.
- buf := make([]byte, size)
+ // Calculate md5sum.
+ hash := c.md5Hasher()
- length, rErr := readFull(reader, buf)
- if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF {
- return UploadInfo{}, rErr
+ if readSeeker != nil {
+ if _, err := io.Copy(hash, reader); err != nil {
+ return UploadInfo{}, err
+ }
+ // Seek back to beginning of io.NewSectionReader's offset.
+ _, err = readSeeker.Seek(0, io.SeekStart)
+ if err != nil {
+ return UploadInfo{}, errInvalidArgument(err.Error())
+ }
+ } else {
+ // Create a buffer.
+ buf := make([]byte, size)
+
+ length, err := readFull(reader, buf)
+ if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+ return UploadInfo{}, err
+ }
+
+ hash.Write(buf[:length])
+ reader = bytes.NewReader(buf[:length])
}
- // Calculate md5sum.
- hash := c.md5Hasher()
- hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
- reader = bytes.NewReader(buf[:length])
hash.Close()
}
// Update progress reader appropriately to the latest offset as we
// read from the source.
- readSeeker := newHook(reader, opts.Progress)
+ progressReader := newHook(reader, opts.Progress)
// This function does not calculate sha256 and md5sum for payload.
// Execute put object.
- return c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts)
+ return c.putObjectDo(ctx, bucketName, objectName, progressReader, md5Base64, "", size, opts)
}
// putObjectDo - executes the put object http operation.