[Vendor] Update Batch 2020-11 (#13746)

* github.com/alecthomas/chroma v0.8.1 -> v0.8.2

Changelog: https://github.com/alecthomas/chroma/releases/tag/v0.8.2

* github.com/blevesearch/bleve v1.0.12 -> v1.0.13

Changelog: https://github.com/blevesearch/bleve/releases/tag/v1.0.13

* github.com/editorconfig/editorconfig-core-go v2.3.8 -> v2.3.9

Changelog: https://github.com/editorconfig/editorconfig-core-go/releases/tag/v2.3.9

* github.com/klauspost/compress v1.11.2 -> v1.11.3

Changelog: https://github.com/klauspost/compress/releases/tag/v1.11.3

* github.com/minio/minio-go v7.0.5 -> v7.0.6

Changelog: https://github.com/minio/minio-go/releases/tag/v7.0.6

Co-authored-by: Lauris BH <lauris@nix.lv>
This commit is contained in:
6543 2020-11-29 21:54:08 +01:00 committed by GitHub
parent e8a6c425ec
commit c4deb97ed1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
68 changed files with 1315 additions and 525 deletions

View file

@ -1,4 +1,5 @@
GOPATH := $(shell go env GOPATH)
TMPDIR := $(shell mktemp -d)
all: checks
@ -20,7 +21,7 @@ test:
@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./...
examples:
@mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done
@$(foreach v,$(wildcard examples/s3/*), go build -o ${TMPDIR}/$(basename $(v)) $(v) || exit 1;)
functional-test:
@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go

View file

@ -19,7 +19,6 @@ MinIO client requires the following four parameters specified to connect to an A
| endpoint | URL to object storage service. |
| _minio.Options_ | All the options such as credentials, custom transport etc. |
```go
package main
@ -116,6 +115,7 @@ func main() {
### Run FileUploader
```sh
export GO111MODULE=on
go run file-uploader.go
2016/08/13 17:03:28 Successfully created mymusic
2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413
@ -135,7 +135,6 @@ The full API Reference is available here.
* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists)
* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket)
* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects)
* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2)
* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads)
### API Reference : Bucket policy Operations
@ -173,7 +172,6 @@ The full API Reference is available here.
### API Reference : Client custom settings
* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo)
* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport)
* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn)
* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff)

View file

@ -38,8 +38,10 @@ MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。
package main
import (
"github.com/minio/minio-go/v7"
"log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
func main() {
@ -49,7 +51,10 @@ func main() {
useSSL := true
// 初使化 minio client对象。
minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
})
if err != nil {
log.Fatalln(err)
}
@ -68,18 +73,25 @@ func main() {
package main
import (
"github.com/minio/minio-go/v7"
"context"
"log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
func main() {
ctx := context.Background()
endpoint := "play.min.io"
accessKeyID := "Q3AM3UQ867SPQQA43P2F"
secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
useSSL := true
// 初使化minio client对象。
minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
// 初使化 minio client对象。
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
})
if err != nil {
log.Fatalln(err)
}
@ -88,17 +100,18 @@ func main() {
bucketName := "mymusic"
location := "us-east-1"
err = minioClient.MakeBucket(bucketName, location)
err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
if err != nil {
// 检查存储桶是否已经存在。
exists, err := minioClient.BucketExists(bucketName)
if err == nil && exists {
exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
if errBucketExists == nil && exists {
log.Printf("We already own %s\n", bucketName)
} else {
log.Fatalln(err)
}
} else {
log.Printf("Successfully created %s\n", bucketName)
}
log.Printf("Successfully created %s\n", bucketName)
// 上传一个zip文件。
objectName := "golden-oldies.zip"
@ -106,7 +119,7 @@ func main() {
contentType := "application/zip"
// 使用FPutObject上传一个zip文件。
n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType})
n, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
log.Fatalln(err)
}
@ -135,7 +148,6 @@ mc ls play/mymusic/
* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists)
* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket)
* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects)
* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2)
* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads)
### API文档 : 存储桶策略
@ -146,7 +158,8 @@ mc ls play/mymusic/
* [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification)
* [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification)
* [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification)
* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension)
* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO 扩展)
* [`ListenNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenNotification) (MinIO 扩展)
### API文档 : 操作文件对象
* [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject)
@ -161,10 +174,7 @@ mc ls play/mymusic/
* [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject)
* [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects)
* [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload)
### API文档: 操作加密对象
* [`GetEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#GetEncryptedObject)
* [`PutEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#PutEncryptedObject)
* [`SelectObjectContent`](https://docs.min.io/docs/golang-client-api-reference#SelectObjectContent)
### API文档 : Presigned操作
* [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject)
@ -174,7 +184,6 @@ mc ls play/mymusic/
### API文档 : 客户端自定义设置
* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo)
* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport)
* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn)
* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff)
@ -194,11 +203,26 @@ mc ls play/mymusic/
* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go)
* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go)
### 完整示例 : 存储桶生命周期
* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
### 完整示例 : 存储桶加密
* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go)
* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go)
* [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go)
### 完整示例 : 存储桶复制
* [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go)
* [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go)
* [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go)
### 完整示例 : 存储桶通知
* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)
* [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go)
* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO扩展)
* [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO 扩展)
### 完整示例 : 操作文件对象
* [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go)

View file

@ -83,6 +83,23 @@ type BucketVersioningConfiguration struct {
MFADelete string `xml:"MfaDelete,omitempty"`
}
// Various supported states
const (
Enabled = "Enabled"
// Disabled State = "Disabled" only used by MFA Delete not supported yet.
Suspended = "Suspended"
)
// Enabled returns true if bucket versioning is enabled
func (b BucketVersioningConfiguration) Enabled() bool {
return b.Status == Enabled
}
// Suspended returns true if bucket versioning is suspended
func (b BucketVersioningConfiguration) Suspended() bool {
return b.Status == Suspended
}
// GetBucketVersioning gets the versioning configuration on
// an existing bucket with a context to control cancellations and timeouts.
func (c Client) GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) {

View file

@ -28,6 +28,7 @@ import (
"strings"
"time"
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
@ -201,7 +202,7 @@ func (opts CopySrcOptions) validate() (err error) {
// Low level implementation of CopyObject API, supports only upto 5GiB worth of copy.
func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string,
metadata map[string]string) (ObjectInfo, error) {
metadata map[string]string, dstOpts PutObjectOptions) (ObjectInfo, error) {
// Build headers.
headers := make(http.Header)
@ -210,16 +211,38 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck
for k, v := range metadata {
headers.Set(k, v)
}
if !dstOpts.Internal.ReplicationStatus.Empty() {
headers.Set(amzBucketReplicationStatus, string(dstOpts.Internal.ReplicationStatus))
}
if !dstOpts.Internal.SourceMTime.IsZero() {
headers.Set(minIOBucketSourceMTime, dstOpts.Internal.SourceMTime.Format(time.RFC3339))
}
if dstOpts.Internal.SourceETag != "" {
headers.Set(minIOBucketSourceETag, dstOpts.Internal.SourceETag)
}
if len(dstOpts.UserTags) != 0 {
headers.Set(amzTaggingHeader, s3utils.TagEncode(dstOpts.UserTags))
}
reqMetadata := requestMetadata{
bucketName: destBucket,
objectName: destObject,
customHeader: headers,
}
if dstOpts.Internal.SourceVersionID != "" {
if _, err := uuid.Parse(dstOpts.Internal.SourceVersionID); err != nil {
return ObjectInfo{}, errInvalidArgument(err.Error())
}
urlValues := make(url.Values)
urlValues.Set("versionId", dstOpts.Internal.SourceVersionID)
reqMetadata.queryValues = urlValues
}
// Set the source header
headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject))
// Send upload-part-copy request
resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{
bucketName: destBucket,
objectName: destObject,
customHeader: headers,
})
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return ObjectInfo{}, err

View file

@ -25,12 +25,19 @@ import (
"github.com/minio/minio-go/v7/pkg/encrypt"
)
//AdvancedGetOptions for internal use by MinIO server - not intended for client use.
type AdvancedGetOptions struct {
ReplicationDeleteMarker bool
}
// GetObjectOptions are used to specify additional headers or options
// during GET requests.
type GetObjectOptions struct {
headers map[string]string
ServerSideEncryption encrypt.ServerSide
VersionID string
// To be not used by external applications
Internal AdvancedGetOptions
}
// StatObjectOptions are used to specify additional headers or options

View file

@ -149,7 +149,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
var partsBuf = make([][]byte, opts.getNumThreads())
for i := range partsBuf {
partsBuf[i] = make([]byte, partSize)
partsBuf[i] = make([]byte, 0, partSize)
}
// Receive each part number from the channel allowing three parallel uploads.
@ -451,15 +451,12 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string,
contentMD5Base64: md5Base64,
contentSHA256Hex: sha256Hex,
}
if opts.ReplicationVersionID != "" {
if _, err := uuid.Parse(opts.ReplicationVersionID); err != nil {
if opts.Internal.SourceVersionID != "" {
if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil {
return UploadInfo{}, errInvalidArgument(err.Error())
}
urlValues := make(url.Values)
urlValues.Set("versionId", opts.ReplicationVersionID)
if opts.ReplicationETag != "" {
urlValues.Set("etag", opts.ReplicationETag)
}
urlValues.Set("versionId", opts.Internal.SourceVersionID)
reqMetadata.queryValues = urlValues
}

View file

@ -52,6 +52,15 @@ func (r ReplicationStatus) Empty() bool {
return r == ""
}
// AdvancedPutOptions for internal use - to be utilized by replication, ILM transition
// implementation on MinIO server
type AdvancedPutOptions struct {
SourceVersionID string
SourceETag string
ReplicationStatus ReplicationStatus
SourceMTime time.Time
}
// PutObjectOptions represents options specified by user for PutObject call
type PutObjectOptions struct {
UserMetadata map[string]string
@ -72,10 +81,7 @@ type PutObjectOptions struct {
LegalHold LegalHoldStatus
SendContentMd5 bool
DisableMultipart bool
ReplicationVersionID string
ReplicationETag string
ReplicationStatus ReplicationStatus
ReplicationMTime time.Time
Internal AdvancedPutOptions
}
// getNumThreads - gets the number of threads to be used in the multipart
@ -137,14 +143,14 @@ func (opts PutObjectOptions) Header() (header http.Header) {
header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation)
}
if !opts.ReplicationStatus.Empty() {
header.Set(amzBucketReplicationStatus, string(opts.ReplicationStatus))
if !opts.Internal.ReplicationStatus.Empty() {
header.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
}
if !opts.ReplicationMTime.IsZero() {
header.Set(minIOBucketReplicationSourceMTime, opts.ReplicationMTime.Format(time.RFC3339))
if !opts.Internal.SourceMTime.IsZero() {
header.Set(minIOBucketSourceMTime, opts.Internal.SourceMTime.Format(time.RFC3339))
}
if opts.ReplicationETag != "" {
header.Set(minIOBucketReplicationETag, opts.ReplicationETag)
if opts.Internal.SourceETag != "" {
header.Set(minIOBucketSourceETag, opts.Internal.SourceETag)
}
if len(opts.UserTags) != 0 {
header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags))

View file

@ -24,6 +24,7 @@ import (
"io"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
@ -58,10 +59,18 @@ func (c Client) RemoveBucket(ctx context.Context, bucketName string) error {
return nil
}
// AdvancedRemoveOptions intended for internal use by replication
type AdvancedRemoveOptions struct {
ReplicationDeleteMarker bool
ReplicationStatus ReplicationStatus
ReplicationMTime time.Time
}
// RemoveObjectOptions represents options specified by user for RemoveObject call
type RemoveObjectOptions struct {
GovernanceBypass bool
VersionID string
Internal AdvancedRemoveOptions
}
// RemoveObject removes an object from a bucket.
@ -74,6 +83,11 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string,
return err
}
return c.removeObject(ctx, bucketName, objectName, opts)
}
func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error {
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
@ -89,6 +103,15 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string,
// Set the bypass goverenance retention header
headers.Set(amzBypassGovernance, "true")
}
if opts.Internal.ReplicationDeleteMarker {
headers.Set(minIOBucketReplicationDeleteMarker, "true")
}
if !opts.Internal.ReplicationMTime.IsZero() {
headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339))
}
if !opts.Internal.ReplicationStatus.Empty() {
headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
}
// Execute DELETE on objectName.
resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
bucketName: bucketName,
@ -147,8 +170,14 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, err
// Fill deletion that returned an error.
for _, obj := range rmResult.UnDeletedObjects {
// Version does not exist is not an error ignore and continue.
switch obj.Code {
case "InvalidArgument", "NoSuchVersion":
continue
}
errorCh <- RemoveObjectError{
ObjectName: obj.Key,
VersionID: obj.VersionID,
Err: ErrorResponse{
Code: obj.Code,
Message: obj.Message,
@ -189,6 +218,26 @@ func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh
return errorCh
}
// Return true if the character is within the allowed characters in an XML 1.0 document
// The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets
func validXMLChar(r rune) (ok bool) {
return r == 0x09 ||
r == 0x0A ||
r == 0x0D ||
r >= 0x20 && r <= 0xD7FF ||
r >= 0xE000 && r <= 0xFFFD ||
r >= 0x10000 && r <= 0x10FFFF
}
func hasInvalidXMLChar(str string) bool {
for _, s := range str {
if !validXMLChar(s) {
return true
}
}
return false
}
// Generate and call MultiDelete S3 requests based on entries received from objectsCh
func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) {
maxEntries := 1000
@ -209,6 +258,27 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh
// Try to gather 1000 entries
for object := range objectsCh {
if hasInvalidXMLChar(object.Key) {
// Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document.
err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
VersionID: object.VersionID,
GovernanceBypass: opts.GovernanceBypass,
})
if err != nil {
// Version does not exist is not an error ignore and continue.
switch ToErrorResponse(err).Code {
case "InvalidArgument", "NoSuchVersion":
continue
}
errorCh <- RemoveObjectError{
ObjectName: object.Key,
VersionID: object.VersionID,
Err: err,
}
}
continue
}
batch = append(batch, object)
if count++; count >= maxEntries {
break

View file

@ -89,6 +89,8 @@ type Version struct {
}
// ListVersionsResult is an element in the list object versions response
// and has a special Unmarshaler because we need to preserver the order
// of <Version> and <DeleteMarker> in ListVersionsResult.Versions slice
type ListVersionsResult struct {
Versions []Version
@ -125,8 +127,7 @@ func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement
switch tagName {
case "Name", "Prefix",
"Delimiter", "EncodingType",
"KeyMarker", "VersionIdMarker",
"NextKeyMarker", "NextVersionIdMarker":
"KeyMarker", "NextKeyMarker":
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
@ -135,6 +136,20 @@ func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement
if v.IsValid() {
v.SetString(s)
}
case "VersionIdMarker":
// VersionIdMarker is a special case because of 'Id' instead of 'ID' in field name
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
l.VersionIDMarker = s
case "NextVersionIdMarker":
// NextVersionIdMarker is a special case because of 'Id' instead of 'ID' in field name
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
l.NextVersionIDMarker = s
case "IsTruncated": // bool
var b bool
if err = d.DecodeElement(&b, &se); err != nil {
@ -325,9 +340,10 @@ type deletedObject struct {
// nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response
type nonDeletedObject struct {
Key string
Code string
Message string
Key string
Code string
Message string
VersionID string `xml:"VersionId"`
}
// deletedMultiObjects container for MultiObjects Delete XML request

View file

@ -78,27 +78,48 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return ObjectInfo{}, err
}
headers := opts.Header()
if opts.Internal.ReplicationDeleteMarker {
headers.Set(minIOBucketReplicationDeleteMarker, "true")
}
urlValues := make(url.Values)
if opts.VersionID != "" {
urlValues.Set("versionId", opts.VersionID)
}
// Execute HEAD on objectName.
resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
customHeader: opts.Header(),
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
return ObjectInfo{}, err
}
deleteMarker := resp.Header.Get(amzDeleteMarker) == "true"
if resp != nil {
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
if resp.StatusCode == http.StatusBadRequest && opts.VersionID != "" && deleteMarker {
errResp := ErrorResponse{
StatusCode: resp.StatusCode,
Code: "MethodNotAllowed",
Message: "The specified method is not allowed against this resource.",
BucketName: bucketName,
Key: objectName,
}
return ObjectInfo{
VersionID: resp.Header.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
}, errResp
}
return ObjectInfo{
VersionID: resp.Header.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
}, httpRespToErrorResponse(resp, bucketName, objectName)
}
}

View file

@ -108,7 +108,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.5"
libraryVersion = "v7.0.6"
)
// User Agent should always following the below style.

View file

@ -97,7 +97,7 @@ func (c Client) getBucketLocation(ctx context.Context, bucketName string) (strin
}
// Initialize a new request.
req, err := c.getBucketLocationRequest(bucketName)
req, err := c.getBucketLocationRequest(ctx, bucketName)
if err != nil {
return "", err
}
@ -169,7 +169,7 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck
}
// getBucketLocationRequest - Wrapper creates a new getBucketLocation request.
func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) {
func (c Client) getBucketLocationRequest(ctx context.Context, bucketName string) (*http.Request, error) {
// Set location query.
urlValues := make(url.Values)
urlValues.Set("location", "")
@ -198,7 +198,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
}
// Get a new HTTP request for the method.
req, err := http.NewRequest(http.MethodGet, urlStr, nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
if err != nil {
return nil, err
}

View file

@ -70,6 +70,7 @@ const (
amzTaggingCount = "X-Amz-Tagging-Count"
amzExpiration = "X-Amz-Expiration"
amzReplicationStatus = "X-Amz-Replication-Status"
amzDeleteMarker = "X-Amz-Delete-Marker"
// Object legal hold header
amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold"
@ -81,7 +82,9 @@ const (
// Replication status
amzBucketReplicationStatus = "X-Amz-Replication-Status"
// Minio specific Replication extension
minIOBucketReplicationSourceMTime = "X-Minio-Source-Mtime"
minIOBucketReplicationETag = "X-Minio-Source-Etag"
// Minio specific Replication/lifecycle transition extension
minIOBucketSourceMTime = "X-Minio-Source-Mtime"
minIOBucketSourceETag = "X-Minio-Source-Etag"
minIOBucketReplicationDeleteMarker = "X-Minio-Source-DeleteMarker"
)

View file

@ -56,8 +56,8 @@ func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string,
}
// CopyObject - copies an object from source object to destination object on server side.
func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) {
return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata)
func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string, dstOpts PutObjectOptions) (ObjectInfo, error) {
return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata, dstOpts)
}
// CopyObjectPart - creates a part in a multipart upload by copying (a
@ -71,7 +71,8 @@ func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBuck
// PutObject - Upload object. Uploads using single PUT call.
func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (UploadInfo, error) {
return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts)
hookReader := newHook(data, opts.Progress)
return c.putObjectDo(ctx, bucket, object, hookReader, md5Base64, sha256Hex, size, opts)
}
// NewMultipartUpload - Initiates new multipart upload and returns the new uploadID.

View file

@ -3,6 +3,8 @@ module github.com/minio/minio-go/v7
go 1.12
require (
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/google/uuid v1.1.1
github.com/json-iterator/go v1.1.10
github.com/klauspost/cpuid v1.3.1 // indirect
@ -13,6 +15,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/rs/xid v1.2.1
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
github.com/stretchr/testify v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899

View file

@ -1,6 +1,12 @@
github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -19,6 +25,13 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
@ -37,11 +50,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
@ -55,8 +71,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View file

@ -144,7 +144,7 @@ func closeResponse(resp *http.Response) {
func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) {
v := url.Values{}
v.Set("Action", "AssumeRole")
v.Set("Version", "2011-06-15")
v.Set("Version", STSVersion)
if opts.RoleARN != "" {
v.Set("RoleArn", opts.RoleARN)
}

View file

@ -22,6 +22,9 @@ import (
"time"
)
// STSVersion sts version string
const STSVersion = "2011-06-15"
// A Value is the AWS credentials value for individual credential fields.
type Value struct {
// AWS Access key ID

View file

@ -48,7 +48,7 @@ type IAM struct {
Client *http.Client
// Custom endpoint to fetch IAM role credentials.
endpoint string
Endpoint string
}
// IAM Roles for Amazon EC2
@ -62,13 +62,12 @@ const (
// NewIAM returns a pointer to a new Credentials object wrapping the IAM.
func NewIAM(endpoint string) *Credentials {
p := &IAM{
return New(&IAM{
Client: &http.Client{
Transport: http.DefaultTransport,
},
endpoint: endpoint,
}
return New(p)
Endpoint: endpoint,
})
}
// Retrieve retrieves credentials from the EC2 service.
@ -78,7 +77,7 @@ func (m *IAM) Retrieve() (Value, error) {
var roleCreds ec2RoleCredRespBody
var err error
endpoint := m.endpoint
endpoint := m.Endpoint
switch {
case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0:
if len(endpoint) == 0 {
@ -90,11 +89,9 @@ func (m *IAM) Retrieve() (Value, error) {
}
creds := &STSWebIdentity{
Client: m.Client,
stsEndpoint: endpoint,
roleARN: os.Getenv("AWS_ROLE_ARN"),
roleSessionName: os.Getenv("AWS_ROLE_SESSION_NAME"),
getWebIDTokenExpiry: func() (*WebIdentityToken, error) {
Client: m.Client,
STSEndpoint: endpoint,
GetWebIDTokenExpiry: func() (*WebIdentityToken, error) {
token, err := ioutil.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE"))
if err != nil {
return nil, err
@ -102,6 +99,8 @@ func (m *IAM) Retrieve() (Value, error) {
return &WebIdentityToken{Token: string(token)}, nil
},
roleARN: os.Getenv("AWS_ROLE_ARN"),
roleSessionName: os.Getenv("AWS_ROLE_SESSION_NAME"),
}
stsWebIdentityCreds, err := creds.Retrieve()
@ -121,7 +120,6 @@ func (m *IAM) Retrieve() (Value, error) {
case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI")) > 0:
if len(endpoint) == 0 {
endpoint = os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI")
var ok bool
if ok, err = isLoopback(endpoint); !ok {
if err == nil {

View file

@ -73,7 +73,7 @@ type STSClientGrants struct {
Client *http.Client
// MinIO endpoint to fetch STS credentials.
stsEndpoint string
STSEndpoint string
// getClientGrantsTokenExpiry function to retrieve tokens
// from IDP This function should return two values one is
@ -81,7 +81,7 @@ type STSClientGrants struct {
// and second return value is the expiry associated with
// this token. This is a customer provided function and
// is mandatory.
getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)
GetClientGrantsTokenExpiry func() (*ClientGrantsToken, error)
}
// NewSTSClientGrants returns a pointer to a new
@ -97,8 +97,8 @@ func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (*
Client: &http.Client{
Transport: http.DefaultTransport,
},
stsEndpoint: stsEndpoint,
getClientGrantsTokenExpiry: getClientGrantsTokenExpiry,
STSEndpoint: stsEndpoint,
GetClientGrantsTokenExpiry: getClientGrantsTokenExpiry,
}), nil
}
@ -114,7 +114,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string,
v.Set("Action", "AssumeRoleWithClientGrants")
v.Set("Token", accessToken.Token)
v.Set("DurationSeconds", fmt.Sprintf("%d", accessToken.Expiry))
v.Set("Version", "2011-06-15")
v.Set("Version", STSVersion)
u, err := url.Parse(endpoint)
if err != nil {
@ -145,7 +145,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string,
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSClientGrants) Retrieve() (Value, error) {
a, err := getClientGrantsCredentials(m.Client, m.stsEndpoint, m.getClientGrantsTokenExpiry)
a, err := getClientGrantsCredentials(m.Client, m.STSEndpoint, m.GetClientGrantsTokenExpiry)
if err != nil {
return Value{}, err
}

View file

@ -52,36 +52,41 @@ type LDAPIdentityResult struct {
type LDAPIdentity struct {
Expiry
stsEndpoint string
// Required http Client to use when connecting to MinIO STS service.
Client *http.Client
ldapUsername, ldapPassword string
// Exported STS endpoint to fetch STS credentials.
STSEndpoint string
// LDAP username/password used to fetch LDAP STS credentials.
LDAPUsername, LDAPPassword string
}
// NewLDAPIdentity returns new credentials object that uses LDAP
// Identity.
func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string) (*Credentials, error) {
return New(&LDAPIdentity{
stsEndpoint: stsEndpoint,
ldapUsername: ldapUsername,
ldapPassword: ldapPassword,
Client: &http.Client{Transport: http.DefaultTransport},
STSEndpoint: stsEndpoint,
LDAPUsername: ldapUsername,
LDAPPassword: ldapPassword,
}), nil
}
// Retrieve gets the credential by calling the MinIO STS API for
// LDAP on the configured stsEndpoint.
func (k *LDAPIdentity) Retrieve() (value Value, err error) {
u, kerr := url.Parse(k.stsEndpoint)
u, kerr := url.Parse(k.STSEndpoint)
if kerr != nil {
err = kerr
return
}
clnt := &http.Client{Transport: http.DefaultTransport}
v := url.Values{}
v.Set("Action", "AssumeRoleWithLDAPIdentity")
v.Set("Version", "2011-06-15")
v.Set("LDAPUsername", k.ldapUsername)
v.Set("LDAPPassword", k.ldapPassword)
v.Set("Version", STSVersion)
v.Set("LDAPUsername", k.LDAPUsername)
v.Set("LDAPPassword", k.LDAPPassword)
u.RawQuery = v.Encode()
@ -91,7 +96,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) {
return
}
resp, kerr := clnt.Do(req)
resp, kerr := k.Client.Do(req)
if kerr != nil {
err = kerr
return

View file

@ -66,16 +66,16 @@ type STSWebIdentity struct {
// Required http Client to use when connecting to MinIO STS service.
Client *http.Client
// MinIO endpoint to fetch STS credentials.
stsEndpoint string
// Exported STS endpoint to fetch STS credentials.
STSEndpoint string
// getWebIDTokenExpiry function which returns ID tokens
// from IDP. This function should return two values one
// is ID token which is a self contained ID token (JWT)
// Exported GetWebIDTokenExpiry function which returns ID
// tokens from IDP. This function should return two values
// one is ID token which is a self contained ID token (JWT)
// and second return value is the expiry associated with
// this token.
// This is a customer provided function and is mandatory.
getWebIDTokenExpiry func() (*WebIdentityToken, error)
GetWebIDTokenExpiry func() (*WebIdentityToken, error)
// roleARN is the Amazon Resource Name (ARN) of the role that the caller is
// assuming.
@ -98,8 +98,8 @@ func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdent
Client: &http.Client{
Transport: http.DefaultTransport,
},
stsEndpoint: stsEndpoint,
getWebIDTokenExpiry: getWebIDTokenExpiry,
STSEndpoint: stsEndpoint,
GetWebIDTokenExpiry: getWebIDTokenExpiry,
}), nil
}
@ -124,7 +124,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
if idToken.Expiry > 0 {
v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry))
}
v.Set("Version", "2011-06-15")
v.Set("Version", STSVersion)
u, err := url.Parse(endpoint)
if err != nil {
@ -159,7 +159,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSWebIdentity) Retrieve() (Value, error) {
a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.roleARN, m.roleSessionName, m.getWebIDTokenExpiry)
a, err := getWebIdentityCredentials(m.Client, m.STSEndpoint, m.roleARN, m.roleSessionName, m.GetWebIDTokenExpiry)
if err != nil {
return Value{}, err
}

View file

@ -146,7 +146,7 @@ type Filter struct {
XMLName xml.Name `xml:"Filter" json:"-"`
And And `xml:"And,omitempty" json:"And,omitempty"`
Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"`
Tag Tag `xml:"Tag,omitempty" json:"-"`
Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"`
}
// MarshalXML - produces the xml representation of the Filter struct

View file

@ -43,6 +43,8 @@ const (
ObjectRemovedDelete = "s3:ObjectRemoved:Delete"
ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated"
ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject"
BucketCreatedAll = "s3:BucketCreated:*"
BucketRemovedAll = "s3:BucketRemoved:*"
)
// FilterRule - child of S3Key, a tag in the notification xml which

View file

@ -46,21 +46,23 @@ const (
// Options represents options to set a replication configuration rule
type Options struct {
Op OptionType
ID string
Prefix string
RuleStatus string
Priority string
TagString string
StorageClass string
RoleArn string
DestBucket string
IsTagSet bool
IsSCSet bool
Op OptionType
ID string
Prefix string
RuleStatus string
Priority string
TagString string
StorageClass string
RoleArn string
DestBucket string
IsTagSet bool
IsSCSet bool
ReplicateDeletes string // replicate versioned deletes
ReplicateDeleteMarkers string // replicate soft deletes
}
// Tags returns a slice of tags for a rule
func (opts Options) Tags() []Tag {
func (opts Options) Tags() ([]Tag, error) {
var tagList []Tag
tagTokens := strings.Split(opts.TagString, "&")
for _, tok := range tagTokens {
@ -68,12 +70,15 @@ func (opts Options) Tags() []Tag {
break
}
kv := strings.SplitN(tok, "=", 2)
if len(kv) != 2 {
return []Tag{}, fmt.Errorf("Tags should be entered as comma separated k=v pairs")
}
tagList = append(tagList, Tag{
Key: kv[0],
Value: kv[1],
})
}
return tagList
return tagList, nil
}
// Config - replication configuration specified in
@ -110,9 +115,12 @@ func (c *Config) AddRule(opts Options) error {
return fmt.Errorf("Rule state should be either [enable|disable]")
}
tags := opts.Tags()
tags, err := opts.Tags()
if err != nil {
return err
}
andVal := And{
Tags: opts.Tags(),
Tags: tags,
}
filter := Filter{Prefix: opts.Prefix}
// only a single tag is set.
@ -152,6 +160,30 @@ func (c *Config) AddRule(opts Options) error {
return fmt.Errorf("destination bucket needs to be in Arn format")
}
}
dmStatus := Disabled
if opts.ReplicateDeleteMarkers != "" {
switch opts.ReplicateDeleteMarkers {
case "enable":
dmStatus = Enabled
case "disable":
dmStatus = Disabled
default:
return fmt.Errorf("ReplicateDeleteMarkers should be either enable|disable")
}
}
vDeleteStatus := Disabled
if opts.ReplicateDeletes != "" {
switch opts.ReplicateDeletes {
case "enable":
vDeleteStatus = Enabled
case "disable":
vDeleteStatus = Disabled
default:
return fmt.Errorf("ReplicateDeletes should be either enable|disable")
}
}
newRule := Rule{
ID: opts.ID,
Priority: priority,
@ -161,7 +193,8 @@ func (c *Config) AddRule(opts Options) error {
Bucket: destBucket,
StorageClass: opts.StorageClass,
},
DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled},
DeleteMarkerReplication: DeleteMarkerReplication{Status: dmStatus},
DeleteReplication: DeleteReplication{Status: vDeleteStatus},
}
// validate rule after overlaying priority for pre-existing rule being disabled.
@ -211,8 +244,12 @@ func (c *Config) EditRule(opts Options) error {
if len(newRule.Filter.And.Tags) != 0 {
tags = newRule.Filter.And.Tags
}
var err error
if opts.IsTagSet {
tags = opts.Tags()
tags, err = opts.Tags()
if err != nil {
return err
}
}
andVal := And{
Tags: tags,
@ -244,6 +281,30 @@ func (c *Config) EditRule(opts Options) error {
return fmt.Errorf("Rule state should be either [enable|disable]")
}
}
// set DeleteMarkerReplication rule status for edit option
if opts.ReplicateDeleteMarkers != "" {
switch opts.ReplicateDeleteMarkers {
case "enable":
newRule.DeleteMarkerReplication.Status = Enabled
case "disable":
newRule.DeleteMarkerReplication.Status = Disabled
default:
return fmt.Errorf("ReplicateDeleteMarkers state should be either [enable|disable]")
}
}
// set DeleteReplication rule status for edit option. This is a MinIO specific
// option to replicate versioned deletes
if opts.ReplicateDeletes != "" {
switch opts.ReplicateDeletes {
case "enable":
newRule.DeleteReplication.Status = Enabled
case "disable":
newRule.DeleteReplication.Status = Disabled
default:
return fmt.Errorf("ReplicateDeletes state should be either [enable|disable]")
}
}
if opts.IsSCSet {
newRule.Destination.StorageClass = opts.StorageClass
@ -314,6 +375,7 @@ type Rule struct {
Status Status `xml:"Status"`
Priority int `xml:"Priority"`
DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"`
DeleteReplication DeleteReplication `xml:"DeleteReplication"`
Destination Destination `xml:"Destination"`
Filter Filter `xml:"Filter" json:"Filter"`
}
@ -470,7 +532,7 @@ type Destination struct {
type And struct {
XMLName xml.Name `xml:"And,omitempty" json:"-"`
Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"`
Tags []Tag `xml:"Tags,omitempty" json:"Tags,omitempty"`
Tags []Tag `xml:"Tag,omitempty" json:"Tag,omitempty"`
}
// isEmpty returns true if Tags field is null
@ -496,3 +558,14 @@ type DeleteMarkerReplication struct {
func (d DeleteMarkerReplication) IsEmpty() bool {
return len(d.Status) == 0
}
// DeleteReplication - whether versioned deletes are replicated - this
// is a MinIO specific extension
type DeleteReplication struct {
Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default
}
// IsEmpty returns true if DeleteReplication is not set
func (d DeleteReplication) IsEmpty() bool {
return len(d.Status) == 0
}

View file

@ -286,15 +286,15 @@ func EncodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
}
var encodedPathname string
var encodedPathname strings.Builder
for _, s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string(s)
encodedPathname.WriteRune(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string(s)
encodedPathname.WriteRune(s)
continue
default:
len := utf8.RuneLen(s)
@ -306,11 +306,11 @@ func EncodePath(pathName string) string {
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
encodedPathname.WriteString("%" + strings.ToUpper(hex))
}
}
}
return encodedPathname
return encodedPathname.String()
}
// We support '.' with bucket names but we fallback to using path

View file

@ -33,12 +33,12 @@ const MaxJitter = 1.0
const NoJitter = 0.0
// DefaultRetryUnit - default unit multiplicative per retry.
// defaults to 1 second.
const DefaultRetryUnit = time.Second
// defaults to 200 * time.Millisecond
var DefaultRetryUnit = 200 * time.Millisecond
// DefaultRetryCap - Each retry attempt never waits no longer than
// this maximum time duration.
const DefaultRetryCap = time.Second * 30
var DefaultRetryCap = time.Second
// newRetryTimer creates a timer with exponentially increasing
// delays until the maximum retry attempts are reached.

View file

@ -297,6 +297,8 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn
// extract lifecycle expiry date and rule ID
expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration))
deleteMarker := h.Get(amzDeleteMarker) == "true"
// Save object metadata info.
return ObjectInfo{
ETag: etag,
@ -306,6 +308,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn
ContentType: contentType,
Expires: expiry,
VersionID: h.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
ReplicationStatus: h.Get(amzReplicationStatus),
Expiration: expTime,
ExpirationRuleID: ruleID,