diff options
| author | 6543 <6543@obermui.de> | 2020-11-29 21:54:08 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-11-29 15:54:08 -0500 |
| commit | c4deb97ed105e923b922fae0ad8cc49327a13852 (patch) | |
| tree | 18d31ca19b5ade0219a020405647f06cd5b445bc /vendor/github.com/minio/minio-go | |
| parent | e8a6c425ecc649ba60885a49729d0fba9bba5e3e (diff) | |
| download | gitea-c4deb97ed105e923b922fae0ad8cc49327a13852.tar.gz gitea-c4deb97ed105e923b922fae0ad8cc49327a13852.zip | |
[Vendor] Update Batch 2020-11 (#13746)
* github.com/alecthomas/chroma v0.8.1 -> v0.8.2
Changelog: https://github.com/alecthomas/chroma/releases/tag/v0.8.2
* github.com/blevesearch/bleve v1.0.12 -> v1.0.13
Changelog: https://github.com/blevesearch/bleve/releases/tag/v1.0.13
* github.com/editorconfig/editorconfig-core-go v2.3.8 -> v2.3.9
Changelog: https://github.com/editorconfig/editorconfig-core-go/releases/tag/v2.3.9
* github.com/klauspost/compress v1.11.2 -> v1.11.3
Changelog: https://github.com/klauspost/compress/releases/tag/v1.11.3
* github.com/minio/minio-go v7.0.5 -> v7.0.6
Changelog: https://github.com/minio/minio-go/releases/tag/v7.0.6
Co-authored-by: Lauris BH <lauris@nix.lv>
Diffstat (limited to 'vendor/github.com/minio/minio-go')
29 files changed, 418 insertions, 129 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/Makefile b/vendor/github.com/minio/minio-go/v7/Makefile index d9b433443f..a6f0b9bcfc 100644 --- a/vendor/github.com/minio/minio-go/v7/Makefile +++ b/vendor/github.com/minio/minio-go/v7/Makefile @@ -1,4 +1,5 @@ GOPATH := $(shell go env GOPATH) +TMPDIR := $(shell mktemp -d) all: checks @@ -20,7 +21,7 @@ test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... examples: - @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done + @$(foreach v,$(wildcard examples/s3/*), go build -o ${TMPDIR}/$(basename $(v)) $(v) || exit 1;) functional-test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go diff --git a/vendor/github.com/minio/minio-go/v7/README.md b/vendor/github.com/minio/minio-go/v7/README.md index d289f615af..4fec6ff89f 100644 --- a/vendor/github.com/minio/minio-go/v7/README.md +++ b/vendor/github.com/minio/minio-go/v7/README.md @@ -19,7 +19,6 @@ MinIO client requires the following four parameters specified to connect to an A | endpoint | URL to object storage service. | | _minio.Options_ | All the options such as credentials, custom transport etc. | - ```go package main @@ -116,6 +115,7 @@ func main() { ### Run FileUploader ```sh +export GO111MODULE=on go run file-uploader.go 2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 @@ -135,7 +135,6 @@ The full API Reference is available here. * [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) * [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) * [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) * [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API Reference : Bucket policy Operations @@ -173,7 +172,6 @@ The full API Reference is available here. ### API Reference : Client custom settings * [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) * [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) * [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) diff --git a/vendor/github.com/minio/minio-go/v7/README_zh_CN.md b/vendor/github.com/minio/minio-go/v7/README_zh_CN.md index 0911b0905a..64e7934119 100644 --- a/vendor/github.com/minio/minio-go/v7/README_zh_CN.md +++ b/vendor/github.com/minio/minio-go/v7/README_zh_CN.md @@ -38,8 +38,10 @@ MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 package main import ( - "github.com/minio/minio-go/v7" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -49,7 +51,10 @@ func main() { useSSL := true // 初使化 minio client对象。 - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -68,18 +73,25 @@ func main() { package main import ( - "github.com/minio/minio-go/v7" + "context" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { + ctx := context.Background() endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true - // 初使化minio client对象。 - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + // 初使化 minio client对象。 + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -88,17 +100,18 @@ func main() { bucketName := "mymusic" location := "us-east-1" - err = minioClient.MakeBucket(bucketName, location) + err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { // 检查存储桶是否已经存在。 - exists, err := minioClient.BucketExists(bucketName) - if err == nil && exists { + exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) + if errBucketExists == nil && exists { log.Printf("We already own %s\n", bucketName) } else { log.Fatalln(err) } + } else { + log.Printf("Successfully created %s\n", bucketName) } - log.Printf("Successfully created %s\n", bucketName) // 上传一个zip文件。 objectName := "golden-oldies.zip" @@ -106,7 +119,7 @@ func main() { contentType := "application/zip" // 使用FPutObject上传一个zip文件。 - n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType}) + n, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType}) if err != nil { log.Fatalln(err) } @@ -135,7 +148,6 @@ mc ls play/mymusic/ * [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) * [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) * [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) * [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API文档 : 存储桶策略 @@ -146,7 +158,8 @@ mc ls play/mymusic/ * [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) * [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) * [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) -* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) +* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO 扩展) +* [`ListenNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenNotification) (MinIO 扩展) ### API文档 : 操作文件对象 * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) @@ -161,10 +174,7 @@ mc ls play/mymusic/ * [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject) * [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects) * [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload) - -### API文档: 操作加密对象 -* [`GetEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#GetEncryptedObject) -* [`PutEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#PutEncryptedObject) +* [`SelectObjectContent`](https://docs.min.io/docs/golang-client-api-reference#SelectObjectContent) ### API文档 : Presigned操作 * [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject) @@ -174,7 +184,6 @@ mc ls play/mymusic/ ### API文档 : 客户端自定义设置 * [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) * [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) * [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) @@ -194,11 +203,26 @@ mc ls play/mymusic/ * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) +### 完整示例 : 存储桶生命周期 +* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) +* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) + +### 完整示例 : 存储桶加密 +* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go) +* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) +* [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go) + +### 完整示例 : 存储桶复制 +* [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go) +* [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go) +* [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go) + ### 完整示例 : 存储桶通知 * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) * [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO扩展) +* [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO 扩展) ### 完整示例 : 操作文件对象 * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go b/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go index 0889d43b0e..e3ceeb336d 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go @@ -83,6 +83,23 @@ type BucketVersioningConfiguration struct { MFADelete string `xml:"MfaDelete,omitempty"` } +// Various supported states +const ( + Enabled = "Enabled" + // Disabled State = "Disabled" only used by MFA Delete not supported yet. + Suspended = "Suspended" +) + +// Enabled returns true if bucket versioning is enabled +func (b BucketVersioningConfiguration) Enabled() bool { + return b.Status == Enabled +} + +// Suspended returns true if bucket versioning is suspended +func (b BucketVersioningConfiguration) Suspended() bool { + return b.Status == Suspended +} + // GetBucketVersioning gets the versioning configuration on // an existing bucket with a context to control cancellations and timeouts. func (c Client) GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { diff --git a/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/vendor/github.com/minio/minio-go/v7/api-compose-object.go index e107375583..e55f9f7b5b 100644 --- a/vendor/github.com/minio/minio-go/v7/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -28,6 +28,7 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -201,7 +202,7 @@ func (opts CopySrcOptions) validate() (err error) { // Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, - metadata map[string]string) (ObjectInfo, error) { + metadata map[string]string, dstOpts PutObjectOptions) (ObjectInfo, error) { // Build headers. headers := make(http.Header) @@ -210,16 +211,38 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck for k, v := range metadata { headers.Set(k, v) } + if !dstOpts.Internal.ReplicationStatus.Empty() { + headers.Set(amzBucketReplicationStatus, string(dstOpts.Internal.ReplicationStatus)) + } + if !dstOpts.Internal.SourceMTime.IsZero() { + headers.Set(minIOBucketSourceMTime, dstOpts.Internal.SourceMTime.Format(time.RFC3339)) + } + if dstOpts.Internal.SourceETag != "" { + headers.Set(minIOBucketSourceETag, dstOpts.Internal.SourceETag) + } + if len(dstOpts.UserTags) != 0 { + headers.Set(amzTaggingHeader, s3utils.TagEncode(dstOpts.UserTags)) + } + + reqMetadata := requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + } + if dstOpts.Internal.SourceVersionID != "" { + if _, err := uuid.Parse(dstOpts.Internal.SourceVersionID); err != nil { + return ObjectInfo{}, errInvalidArgument(err.Error()) + } + urlValues := make(url.Values) + urlValues.Set("versionId", dstOpts.Internal.SourceVersionID) + reqMetadata.queryValues = urlValues + } // Set the source header headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) // Send upload-part-copy request - resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ - bucketName: destBucket, - objectName: destObject, - customHeader: headers, - }) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err diff --git a/vendor/github.com/minio/minio-go/v7/api-get-options.go b/vendor/github.com/minio/minio-go/v7/api-get-options.go index 5f3ed36566..6b924fa0e0 100644 --- a/vendor/github.com/minio/minio-go/v7/api-get-options.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-options.go @@ -25,12 +25,19 @@ import ( "github.com/minio/minio-go/v7/pkg/encrypt" ) +//AdvancedGetOptions for internal use by MinIO server - not intended for client use. +type AdvancedGetOptions struct { + ReplicationDeleteMarker bool +} + // GetObjectOptions are used to specify additional headers or options // during GET requests. type GetObjectOptions struct { headers map[string]string ServerSideEncryption encrypt.ServerSide VersionID string + // To be not used by external applications + Internal AdvancedGetOptions } // StatObjectOptions are used to specify additional headers or options diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index cd29096323..f4694fc552 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -149,7 +149,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa var partsBuf = make([][]byte, opts.getNumThreads()) for i := range partsBuf { - partsBuf[i] = make([]byte, partSize) + partsBuf[i] = make([]byte, 0, partSize) } // Receive each part number from the channel allowing three parallel uploads. @@ -451,15 +451,12 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentMD5Base64: md5Base64, contentSHA256Hex: sha256Hex, } - if opts.ReplicationVersionID != "" { - if _, err := uuid.Parse(opts.ReplicationVersionID); err != nil { + if opts.Internal.SourceVersionID != "" { + if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { return UploadInfo{}, errInvalidArgument(err.Error()) } urlValues := make(url.Values) - urlValues.Set("versionId", opts.ReplicationVersionID) - if opts.ReplicationETag != "" { - urlValues.Set("etag", opts.ReplicationETag) - } + urlValues.Set("versionId", opts.Internal.SourceVersionID) reqMetadata.queryValues = urlValues } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index 14a2c16a0e..9428c7fee6 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -52,6 +52,15 @@ func (r ReplicationStatus) Empty() bool { return r == "" } +// AdvancedPutOptions for internal use - to be utilized by replication, ILM transition +// implementation on MinIO server +type AdvancedPutOptions struct { + SourceVersionID string + SourceETag string + ReplicationStatus ReplicationStatus + SourceMTime time.Time +} + // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { UserMetadata map[string]string @@ -72,10 +81,7 @@ type PutObjectOptions struct { LegalHold LegalHoldStatus SendContentMd5 bool DisableMultipart bool - ReplicationVersionID string - ReplicationETag string - ReplicationStatus ReplicationStatus - ReplicationMTime time.Time + Internal AdvancedPutOptions } // getNumThreads - gets the number of threads to be used in the multipart @@ -137,14 +143,14 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } - if !opts.ReplicationStatus.Empty() { - header.Set(amzBucketReplicationStatus, string(opts.ReplicationStatus)) + if !opts.Internal.ReplicationStatus.Empty() { + header.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus)) } - if !opts.ReplicationMTime.IsZero() { - header.Set(minIOBucketReplicationSourceMTime, opts.ReplicationMTime.Format(time.RFC3339)) + if !opts.Internal.SourceMTime.IsZero() { + header.Set(minIOBucketSourceMTime, opts.Internal.SourceMTime.Format(time.RFC3339)) } - if opts.ReplicationETag != "" { - header.Set(minIOBucketReplicationETag, opts.ReplicationETag) + if opts.Internal.SourceETag != "" { + header.Set(minIOBucketSourceETag, opts.Internal.SourceETag) } if len(opts.UserTags) != 0 { header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) diff --git a/vendor/github.com/minio/minio-go/v7/api-remove.go b/vendor/github.com/minio/minio-go/v7/api-remove.go index 6c2ab78020..920612c4fe 100644 --- a/vendor/github.com/minio/minio-go/v7/api-remove.go +++ b/vendor/github.com/minio/minio-go/v7/api-remove.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/url" + "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -58,10 +59,18 @@ func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { return nil } +// AdvancedRemoveOptions intended for internal use by replication +type AdvancedRemoveOptions struct { + ReplicationDeleteMarker bool + ReplicationStatus ReplicationStatus + ReplicationMTime time.Time +} + // RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { GovernanceBypass bool VersionID string + Internal AdvancedRemoveOptions } // RemoveObject removes an object from a bucket. @@ -74,6 +83,11 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, return err } + return c.removeObject(ctx, bucketName, objectName, opts) +} + +func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { + // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -89,6 +103,15 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, // Set the bypass goverenance retention header headers.Set(amzBypassGovernance, "true") } + if opts.Internal.ReplicationDeleteMarker { + headers.Set(minIOBucketReplicationDeleteMarker, "true") + } + if !opts.Internal.ReplicationMTime.IsZero() { + headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339)) + } + if !opts.Internal.ReplicationStatus.Empty() { + headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus)) + } // Execute DELETE on objectName. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, @@ -147,8 +170,14 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, err // Fill deletion that returned an error. for _, obj := range rmResult.UnDeletedObjects { + // Version does not exist is not an error ignore and continue. + switch obj.Code { + case "InvalidArgument", "NoSuchVersion": + continue + } errorCh <- RemoveObjectError{ ObjectName: obj.Key, + VersionID: obj.VersionID, Err: ErrorResponse{ Code: obj.Code, Message: obj.Message, @@ -189,6 +218,26 @@ func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh return errorCh } +// Return true if the character is within the allowed characters in an XML 1.0 document +// The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets +func validXMLChar(r rune) (ok bool) { + return r == 0x09 || + r == 0x0A || + r == 0x0D || + r >= 0x20 && r <= 0xD7FF || + r >= 0xE000 && r <= 0xFFFD || + r >= 0x10000 && r <= 0x10FFFF +} + +func hasInvalidXMLChar(str string) bool { + for _, s := range str { + if !validXMLChar(s) { + return true + } + } + return false +} + // Generate and call MultiDelete S3 requests based on entries received from objectsCh func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { maxEntries := 1000 @@ -209,6 +258,27 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh // Try to gather 1000 entries for object := range objectsCh { + if hasInvalidXMLChar(object.Key) { + // Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document. + err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ + VersionID: object.VersionID, + GovernanceBypass: opts.GovernanceBypass, + }) + if err != nil { + // Version does not exist is not an error ignore and continue. + switch ToErrorResponse(err).Code { + case "InvalidArgument", "NoSuchVersion": + continue + } + errorCh <- RemoveObjectError{ + ObjectName: object.Key, + VersionID: object.VersionID, + Err: err, + } + } + continue + } + batch = append(batch, object) if count++; count >= maxEntries { break diff --git a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go index ac34457455..37ed97b725 100644 --- a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go @@ -89,6 +89,8 @@ type Version struct { } // ListVersionsResult is an element in the list object versions response +// and has a special Unmarshaler because we need to preserver the order +// of <Version> and <DeleteMarker> in ListVersionsResult.Versions slice type ListVersionsResult struct { Versions []Version @@ -125,8 +127,7 @@ func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement switch tagName { case "Name", "Prefix", "Delimiter", "EncodingType", - "KeyMarker", "VersionIdMarker", - "NextKeyMarker", "NextVersionIdMarker": + "KeyMarker", "NextKeyMarker": var s string if err = d.DecodeElement(&s, &se); err != nil { return err @@ -135,6 +136,20 @@ func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement if v.IsValid() { v.SetString(s) } + case "VersionIdMarker": + // VersionIdMarker is a special case because of 'Id' instead of 'ID' in field name + var s string + if err = d.DecodeElement(&s, &se); err != nil { + return err + } + l.VersionIDMarker = s + case "NextVersionIdMarker": + // NextVersionIdMarker is a special case because of 'Id' instead of 'ID' in field name + var s string + if err = d.DecodeElement(&s, &se); err != nil { + return err + } + l.NextVersionIDMarker = s case "IsTruncated": // bool var b bool if err = d.DecodeElement(&b, &se); err != nil { @@ -325,9 +340,10 @@ type deletedObject struct { // nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response type nonDeletedObject struct { - Key string - Code string - Message string + Key string + Code string + Message string + VersionID string `xml:"VersionId"` } // deletedMultiObjects container for MultiObjects Delete XML request diff --git a/vendor/github.com/minio/minio-go/v7/api-stat.go b/vendor/github.com/minio/minio-go/v7/api-stat.go index ea9c30970a..aa81cc4377 100644 --- a/vendor/github.com/minio/minio-go/v7/api-stat.go +++ b/vendor/github.com/minio/minio-go/v7/api-stat.go @@ -78,27 +78,48 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o if err := s3utils.CheckValidObjectName(objectName); err != nil { return ObjectInfo{}, err } + headers := opts.Header() + if opts.Internal.ReplicationDeleteMarker { + headers.Set(minIOBucketReplicationDeleteMarker, "true") + } urlValues := make(url.Values) if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } - // Execute HEAD on objectName. resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, - customHeader: opts.Header(), + customHeader: headers, }) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err } + deleteMarker := resp.Header.Get(amzDeleteMarker) == "true" + if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + if resp.StatusCode == http.StatusBadRequest && opts.VersionID != "" && deleteMarker { + errResp := ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "MethodNotAllowed", + Message: "The specified method is not allowed against this resource.", + BucketName: bucketName, + Key: objectName, + } + return ObjectInfo{ + VersionID: resp.Header.Get(amzVersionID), + IsDeleteMarker: deleteMarker, + }, errResp + } + return ObjectInfo{ + VersionID: resp.Header.Get(amzVersionID), + IsDeleteMarker: deleteMarker, + }, httpRespToErrorResponse(resp, bucketName, objectName) } } diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index e36c796603..69ed02cfe0 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.5" + libraryVersion = "v7.0.6" ) // User Agent should always following the below style. diff --git a/vendor/github.com/minio/minio-go/v7/bucket-cache.go b/vendor/github.com/minio/minio-go/v7/bucket-cache.go index 7d485a6b1c..156150f622 100644 --- a/vendor/github.com/minio/minio-go/v7/bucket-cache.go +++ b/vendor/github.com/minio/minio-go/v7/bucket-cache.go @@ -97,7 +97,7 @@ func (c Client) getBucketLocation(ctx context.Context, bucketName string) (strin } // Initialize a new request. - req, err := c.getBucketLocationRequest(bucketName) + req, err := c.getBucketLocationRequest(ctx, bucketName) if err != nil { return "", err } @@ -169,7 +169,7 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck } // getBucketLocationRequest - Wrapper creates a new getBucketLocation request. -func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) { +func (c Client) getBucketLocationRequest(ctx context.Context, bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("location", "") @@ -198,7 +198,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } // Get a new HTTP request for the method. - req, err := http.NewRequest(http.MethodGet, urlStr, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) if err != nil { return nil, err } diff --git a/vendor/github.com/minio/minio-go/v7/constants.go b/vendor/github.com/minio/minio-go/v7/constants.go index 6a1b8d3dda..4f45ce5895 100644 --- a/vendor/github.com/minio/minio-go/v7/constants.go +++ b/vendor/github.com/minio/minio-go/v7/constants.go @@ -70,6 +70,7 @@ const ( amzTaggingCount = "X-Amz-Tagging-Count" amzExpiration = "X-Amz-Expiration" amzReplicationStatus = "X-Amz-Replication-Status" + amzDeleteMarker = "X-Amz-Delete-Marker" // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" @@ -81,7 +82,9 @@ const ( // Replication status amzBucketReplicationStatus = "X-Amz-Replication-Status" - // Minio specific Replication extension - minIOBucketReplicationSourceMTime = "X-Minio-Source-Mtime" - minIOBucketReplicationETag = "X-Minio-Source-Etag" + // Minio specific Replication/lifecycle transition extension + minIOBucketSourceMTime = "X-Minio-Source-Mtime" + + minIOBucketSourceETag = "X-Minio-Source-Etag" + minIOBucketReplicationDeleteMarker = "X-Minio-Source-DeleteMarker" ) diff --git a/vendor/github.com/minio/minio-go/v7/core.go b/vendor/github.com/minio/minio-go/v7/core.go index 954e1a1bb5..f8daebf61d 100644 --- a/vendor/github.com/minio/minio-go/v7/core.go +++ b/vendor/github.com/minio/minio-go/v7/core.go @@ -56,8 +56,8 @@ func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, } // CopyObject - copies an object from source object to destination object on server side. -func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) +func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string, dstOpts PutObjectOptions) (ObjectInfo, error) { + return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata, dstOpts) } // CopyObjectPart - creates a part in a multipart upload by copying (a @@ -71,7 +71,8 @@ func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBuck // PutObject - Upload object. Uploads using single PUT call. func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (UploadInfo, error) { - return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) + hookReader := newHook(data, opts.Progress) + return c.putObjectDo(ctx, bucket, object, hookReader, md5Base64, sha256Hex, size, opts) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. diff --git a/vendor/github.com/minio/minio-go/v7/go.mod b/vendor/github.com/minio/minio-go/v7/go.mod index 448b4bbf17..34bbb6fdd6 100644 --- a/vendor/github.com/minio/minio-go/v7/go.mod +++ b/vendor/github.com/minio/minio-go/v7/go.mod @@ -3,6 +3,8 @@ module github.com/minio/minio-go/v7 go 1.12 require ( + github.com/cheggaaa/pb v1.0.29 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/json-iterator/go v1.1.10 github.com/klauspost/cpuid v1.3.1 // indirect @@ -13,6 +15,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/rs/xid v1.2.1 + github.com/sirupsen/logrus v1.7.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/stretchr/testify v1.4.0 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 diff --git a/vendor/github.com/minio/minio-go/v7/go.sum b/vendor/github.com/minio/minio-go/v7/go.sum index 1858b6ff38..6a5c22f1a0 100644 --- a/vendor/github.com/minio/minio-go/v7/go.sum +++ b/vendor/github.com/minio/minio-go/v7/go.sum @@ -1,6 +1,12 @@ +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -19,6 +25,13 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -37,11 +50,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -55,8 +71,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go index cc88a9ab67..3b1b547b94 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go @@ -144,7 +144,7 @@ func closeResponse(resp *http.Response) { func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { v := url.Values{} v.Set("Action", "AssumeRole") - v.Set("Version", "2011-06-15") + v.Set("Version", STSVersion) if opts.RoleARN != "" { v.Set("RoleArn", opts.RoleARN) } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go index 1a48751b5d..608edc7fc4 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go @@ -22,6 +22,9 @@ import ( "time" ) +// STSVersion sts version string +const STSVersion = "2011-06-15" + // A Value is the AWS credentials value for individual credential fields. type Value struct { // AWS Access key ID diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go index ceeab84ddb..def48a037f 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go @@ -48,7 +48,7 @@ type IAM struct { Client *http.Client // Custom endpoint to fetch IAM role credentials. - endpoint string + Endpoint string } // IAM Roles for Amazon EC2 @@ -62,13 +62,12 @@ const ( // NewIAM returns a pointer to a new Credentials object wrapping the IAM. func NewIAM(endpoint string) *Credentials { - p := &IAM{ + return New(&IAM{ Client: &http.Client{ Transport: http.DefaultTransport, }, - endpoint: endpoint, - } - return New(p) + Endpoint: endpoint, + }) } // Retrieve retrieves credentials from the EC2 service. @@ -78,7 +77,7 @@ func (m *IAM) Retrieve() (Value, error) { var roleCreds ec2RoleCredRespBody var err error - endpoint := m.endpoint + endpoint := m.Endpoint switch { case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0: if len(endpoint) == 0 { @@ -90,11 +89,9 @@ func (m *IAM) Retrieve() (Value, error) { } creds := &STSWebIdentity{ - Client: m.Client, - stsEndpoint: endpoint, - roleARN: os.Getenv("AWS_ROLE_ARN"), - roleSessionName: os.Getenv("AWS_ROLE_SESSION_NAME"), - getWebIDTokenExpiry: func() (*WebIdentityToken, error) { + Client: m.Client, + STSEndpoint: endpoint, + GetWebIDTokenExpiry: func() (*WebIdentityToken, error) { token, err := ioutil.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) if err != nil { return nil, err @@ -102,6 +99,8 @@ func (m *IAM) Retrieve() (Value, error) { return &WebIdentityToken{Token: string(token)}, nil }, + roleARN: os.Getenv("AWS_ROLE_ARN"), + roleSessionName: os.Getenv("AWS_ROLE_SESSION_NAME"), } stsWebIdentityCreds, err := creds.Retrieve() @@ -121,7 +120,6 @@ func (m *IAM) Retrieve() (Value, error) { case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI")) > 0: if len(endpoint) == 0 { endpoint = os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") - var ok bool if ok, err = isLoopback(endpoint); !ok { if err == nil { diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go index e89d5d4d87..b79f920f58 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go @@ -73,7 +73,7 @@ type STSClientGrants struct { Client *http.Client // MinIO endpoint to fetch STS credentials. - stsEndpoint string + STSEndpoint string // getClientGrantsTokenExpiry function to retrieve tokens // from IDP This function should return two values one is @@ -81,7 +81,7 @@ type STSClientGrants struct { // and second return value is the expiry associated with // this token. This is a customer provided function and // is mandatory. - getClientGrantsTokenExpiry func() (*ClientGrantsToken, error) + GetClientGrantsTokenExpiry func() (*ClientGrantsToken, error) } // NewSTSClientGrants returns a pointer to a new @@ -97,8 +97,8 @@ func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (* Client: &http.Client{ Transport: http.DefaultTransport, }, - stsEndpoint: stsEndpoint, - getClientGrantsTokenExpiry: getClientGrantsTokenExpiry, + STSEndpoint: stsEndpoint, + GetClientGrantsTokenExpiry: getClientGrantsTokenExpiry, }), nil } @@ -114,7 +114,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, v.Set("Action", "AssumeRoleWithClientGrants") v.Set("Token", accessToken.Token) v.Set("DurationSeconds", fmt.Sprintf("%d", accessToken.Expiry)) - v.Set("Version", "2011-06-15") + v.Set("Version", STSVersion) u, err := url.Parse(endpoint) if err != nil { @@ -145,7 +145,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSClientGrants) Retrieve() (Value, error) { - a, err := getClientGrantsCredentials(m.Client, m.stsEndpoint, m.getClientGrantsTokenExpiry) + a, err := getClientGrantsCredentials(m.Client, m.STSEndpoint, m.GetClientGrantsTokenExpiry) if err != nil { return Value{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index abbf616413..bcb3c36a17 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -52,36 +52,41 @@ type LDAPIdentityResult struct { type LDAPIdentity struct { Expiry - stsEndpoint string + // Required http Client to use when connecting to MinIO STS service. + Client *http.Client - ldapUsername, ldapPassword string + // Exported STS endpoint to fetch STS credentials. + STSEndpoint string + + // LDAP username/password used to fetch LDAP STS credentials. + LDAPUsername, LDAPPassword string } // NewLDAPIdentity returns new credentials object that uses LDAP // Identity. func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string) (*Credentials, error) { return New(&LDAPIdentity{ - stsEndpoint: stsEndpoint, - ldapUsername: ldapUsername, - ldapPassword: ldapPassword, + Client: &http.Client{Transport: http.DefaultTransport}, + STSEndpoint: stsEndpoint, + LDAPUsername: ldapUsername, + LDAPPassword: ldapPassword, }), nil } // Retrieve gets the credential by calling the MinIO STS API for // LDAP on the configured stsEndpoint. func (k *LDAPIdentity) Retrieve() (value Value, err error) { - u, kerr := url.Parse(k.stsEndpoint) + u, kerr := url.Parse(k.STSEndpoint) if kerr != nil { err = kerr return } - clnt := &http.Client{Transport: http.DefaultTransport} v := url.Values{} v.Set("Action", "AssumeRoleWithLDAPIdentity") - v.Set("Version", "2011-06-15") - v.Set("LDAPUsername", k.ldapUsername) - v.Set("LDAPPassword", k.ldapPassword) + v.Set("Version", STSVersion) + v.Set("LDAPUsername", k.LDAPUsername) + v.Set("LDAPPassword", k.LDAPPassword) u.RawQuery = v.Encode() @@ -91,7 +96,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { return } - resp, kerr := clnt.Do(req) + resp, kerr := k.Client.Do(req) if kerr != nil { err = kerr return diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index 5a5f6405ee..161ffd36cd 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -66,16 +66,16 @@ type STSWebIdentity struct { // Required http Client to use when connecting to MinIO STS service. Client *http.Client - // MinIO endpoint to fetch STS credentials. - stsEndpoint string + // Exported STS endpoint to fetch STS credentials. + STSEndpoint string - // getWebIDTokenExpiry function which returns ID tokens - // from IDP. This function should return two values one - // is ID token which is a self contained ID token (JWT) + // Exported GetWebIDTokenExpiry function which returns ID + // tokens from IDP. This function should return two values + // one is ID token which is a self contained ID token (JWT) // and second return value is the expiry associated with // this token. // This is a customer provided function and is mandatory. - getWebIDTokenExpiry func() (*WebIdentityToken, error) + GetWebIDTokenExpiry func() (*WebIdentityToken, error) // roleARN is the Amazon Resource Name (ARN) of the role that the caller is // assuming. @@ -98,8 +98,8 @@ func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdent Client: &http.Client{ Transport: http.DefaultTransport, }, - stsEndpoint: stsEndpoint, - getWebIDTokenExpiry: getWebIDTokenExpiry, + STSEndpoint: stsEndpoint, + GetWebIDTokenExpiry: getWebIDTokenExpiry, }), nil } @@ -124,7 +124,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession if idToken.Expiry > 0 { v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) } - v.Set("Version", "2011-06-15") + v.Set("Version", STSVersion) u, err := url.Parse(endpoint) if err != nil { @@ -159,7 +159,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSWebIdentity) Retrieve() (Value, error) { - a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.roleARN, m.roleSessionName, m.getWebIDTokenExpiry) + a, err := getWebIdentityCredentials(m.Client, m.STSEndpoint, m.roleARN, m.roleSessionName, m.GetWebIDTokenExpiry) if err != nil { return Value{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go index 3493003f7d..216588f74d 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go @@ -146,7 +146,7 @@ type Filter struct { XMLName xml.Name `xml:"Filter" json:"-"` And And `xml:"And,omitempty" json:"And,omitempty"` Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` - Tag Tag `xml:"Tag,omitempty" json:"-"` + Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` } // MarshalXML - produces the xml representation of the Filter struct diff --git a/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go index 55c58cb3b8..05c79bb48b 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go @@ -43,6 +43,8 @@ const ( ObjectRemovedDelete = "s3:ObjectRemoved:Delete" ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" + BucketCreatedAll = "s3:BucketCreated:*" + BucketRemovedAll = "s3:BucketRemoved:*" ) // FilterRule - child of S3Key, a tag in the notification xml which diff --git a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go index fdd0afbc8d..9083816901 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go @@ -46,21 +46,23 @@ const ( // Options represents options to set a replication configuration rule type Options struct { - Op OptionType - ID string - Prefix string - RuleStatus string - Priority string - TagString string - StorageClass string - RoleArn string - DestBucket string - IsTagSet bool - IsSCSet bool + Op OptionType + ID string + Prefix string + RuleStatus string + Priority string + TagString string + StorageClass string + RoleArn string + DestBucket string + IsTagSet bool + IsSCSet bool + ReplicateDeletes string // replicate versioned deletes + ReplicateDeleteMarkers string // replicate soft deletes } // Tags returns a slice of tags for a rule -func (opts Options) Tags() []Tag { +func (opts Options) Tags() ([]Tag, error) { var tagList []Tag tagTokens := strings.Split(opts.TagString, "&") for _, tok := range tagTokens { @@ -68,12 +70,15 @@ func (opts Options) Tags() []Tag { break } kv := strings.SplitN(tok, "=", 2) + if len(kv) != 2 { + return []Tag{}, fmt.Errorf("Tags should be entered as comma separated k=v pairs") + } tagList = append(tagList, Tag{ Key: kv[0], Value: kv[1], }) } - return tagList + return tagList, nil } // Config - replication configuration specified in @@ -110,9 +115,12 @@ func (c *Config) AddRule(opts Options) error { return fmt.Errorf("Rule state should be either [enable|disable]") } - tags := opts.Tags() + tags, err := opts.Tags() + if err != nil { + return err + } andVal := And{ - Tags: opts.Tags(), + Tags: tags, } filter := Filter{Prefix: opts.Prefix} // only a single tag is set. @@ -152,6 +160,30 @@ func (c *Config) AddRule(opts Options) error { return fmt.Errorf("destination bucket needs to be in Arn format") } } + dmStatus := Disabled + if opts.ReplicateDeleteMarkers != "" { + switch opts.ReplicateDeleteMarkers { + case "enable": + dmStatus = Enabled + case "disable": + dmStatus = Disabled + default: + return fmt.Errorf("ReplicateDeleteMarkers should be either enable|disable") + } + } + + vDeleteStatus := Disabled + if opts.ReplicateDeletes != "" { + switch opts.ReplicateDeletes { + case "enable": + vDeleteStatus = Enabled + case "disable": + vDeleteStatus = Disabled + default: + return fmt.Errorf("ReplicateDeletes should be either enable|disable") + } + } + newRule := Rule{ ID: opts.ID, Priority: priority, @@ -161,7 +193,8 @@ func (c *Config) AddRule(opts Options) error { Bucket: destBucket, StorageClass: opts.StorageClass, }, - DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + DeleteMarkerReplication: DeleteMarkerReplication{Status: dmStatus}, + DeleteReplication: DeleteReplication{Status: vDeleteStatus}, } // validate rule after overlaying priority for pre-existing rule being disabled. @@ -211,8 +244,12 @@ func (c *Config) EditRule(opts Options) error { if len(newRule.Filter.And.Tags) != 0 { tags = newRule.Filter.And.Tags } + var err error if opts.IsTagSet { - tags = opts.Tags() + tags, err = opts.Tags() + if err != nil { + return err + } } andVal := And{ Tags: tags, @@ -244,6 +281,30 @@ func (c *Config) EditRule(opts Options) error { return fmt.Errorf("Rule state should be either [enable|disable]") } } + // set DeleteMarkerReplication rule status for edit option + if opts.ReplicateDeleteMarkers != "" { + switch opts.ReplicateDeleteMarkers { + case "enable": + newRule.DeleteMarkerReplication.Status = Enabled + case "disable": + newRule.DeleteMarkerReplication.Status = Disabled + default: + return fmt.Errorf("ReplicateDeleteMarkers state should be either [enable|disable]") + } + } + + // set DeleteReplication rule status for edit option. This is a MinIO specific + // option to replicate versioned deletes + if opts.ReplicateDeletes != "" { + switch opts.ReplicateDeletes { + case "enable": + newRule.DeleteReplication.Status = Enabled + case "disable": + newRule.DeleteReplication.Status = Disabled + default: + return fmt.Errorf("ReplicateDeletes state should be either [enable|disable]") + } + } if opts.IsSCSet { newRule.Destination.StorageClass = opts.StorageClass @@ -314,6 +375,7 @@ type Rule struct { Status Status `xml:"Status"` Priority int `xml:"Priority"` DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"` + DeleteReplication DeleteReplication `xml:"DeleteReplication"` Destination Destination `xml:"Destination"` Filter Filter `xml:"Filter" json:"Filter"` } @@ -470,7 +532,7 @@ type Destination struct { type And struct { XMLName xml.Name `xml:"And,omitempty" json:"-"` Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` - Tags []Tag `xml:"Tags,omitempty" json:"Tags,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` } // isEmpty returns true if Tags field is null @@ -496,3 +558,14 @@ type DeleteMarkerReplication struct { func (d DeleteMarkerReplication) IsEmpty() bool { return len(d.Status) == 0 } + +// DeleteReplication - whether versioned deletes are replicated - this +// is a MinIO specific extension +type DeleteReplication struct { + Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteReplication is not set +func (d DeleteReplication) IsEmpty() bool { + return len(d.Status) == 0 +} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go index 65f939253d..d818e8ad9c 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go @@ -286,15 +286,15 @@ func EncodePath(pathName string) string { if reservedObjectNames.MatchString(pathName) { return pathName } - var encodedPathname string + var encodedPathname strings.Builder for _, s := range pathName { if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) - encodedPathname = encodedPathname + string(s) + encodedPathname.WriteRune(s) continue } switch s { case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedPathname = encodedPathname + string(s) + encodedPathname.WriteRune(s) continue default: len := utf8.RuneLen(s) @@ -306,11 +306,11 @@ func EncodePath(pathName string) string { utf8.EncodeRune(u, s) for _, r := range u { hex := hex.EncodeToString([]byte{r}) - encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + encodedPathname.WriteString("%" + strings.ToUpper(hex)) } } } - return encodedPathname + return encodedPathname.String() } // We support '.' with bucket names but we fallback to using path diff --git a/vendor/github.com/minio/minio-go/v7/retry.go b/vendor/github.com/minio/minio-go/v7/retry.go index 6e2826fa65..598af29752 100644 --- a/vendor/github.com/minio/minio-go/v7/retry.go +++ b/vendor/github.com/minio/minio-go/v7/retry.go @@ -33,12 +33,12 @@ const MaxJitter = 1.0 const NoJitter = 0.0 // DefaultRetryUnit - default unit multiplicative per retry. -// defaults to 1 second. -const DefaultRetryUnit = time.Second +// defaults to 200 * time.Millisecond +var DefaultRetryUnit = 200 * time.Millisecond // DefaultRetryCap - Each retry attempt never waits no longer than // this maximum time duration. -const DefaultRetryCap = time.Second * 30 +var DefaultRetryCap = time.Second // newRetryTimer creates a timer with exponentially increasing // delays until the maximum retry attempts are reached. diff --git a/vendor/github.com/minio/minio-go/v7/utils.go b/vendor/github.com/minio/minio-go/v7/utils.go index f1b711d226..4bdf1a3c3b 100644 --- a/vendor/github.com/minio/minio-go/v7/utils.go +++ b/vendor/github.com/minio/minio-go/v7/utils.go @@ -297,6 +297,8 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn // extract lifecycle expiry date and rule ID expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration)) + deleteMarker := h.Get(amzDeleteMarker) == "true" + // Save object metadata info. return ObjectInfo{ ETag: etag, @@ -306,6 +308,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn ContentType: contentType, Expires: expiry, VersionID: h.Get(amzVersionID), + IsDeleteMarker: deleteMarker, ReplicationStatus: h.Get(amzReplicationStatus), Expiration: expTime, ExpirationRuleID: ruleID, |