From f9f205ffe43656eee7c4dd04fc675754424a7f73 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 16 Mar 2022 15:41:31 -0700 Subject: [PATCH 1/5] Add PreSignRequest to stow container Signed-off-by: Haytham Abuelfutuh --- azure/container.go | 7 +++++ b2/container.go | 7 +++++ clientmethod_enumer.go | 68 ++++++++++++++++++++++++++++++++++++++++++ google/container.go | 21 ++++++++++++- local/container.go | 7 +++++ oracle/container.go | 7 +++++ s3/container.go | 25 ++++++++++++++++ s3/stow_test.go | 34 +++++++++++++++++++++ s3/v2signer.go | 2 +- stow.go | 24 +++++++++++++++ 10 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 clientmethod_enumer.go diff --git a/azure/container.go b/azure/container.go index 82a4c387..da308eb1 100644 --- a/azure/container.go +++ b/azure/container.go @@ -1,6 +1,8 @@ package azure import ( + "context" + "fmt" "io" "strings" "time" @@ -32,6 +34,11 @@ func (c *container) Name() string { return c.id } +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + func (c *container) Item(id string) (stow.Item, error) { blob := c.client.GetContainerReference(c.id).GetBlobReference(id) err := blob.GetProperties(nil) diff --git a/b2/container.go b/b2/container.go index 852258cb..33480130 100644 --- a/b2/container.go +++ b/b2/container.go @@ -1,6 +1,8 @@ package b2 import ( + "context" + "fmt" "io" "strings" "time" @@ -24,6 +26,11 @@ func (c *container) ID() string { return c.bucket.Name } +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + // Name returns the name of the bucket func (c *container) Name() string { return c.bucket.Name diff --git a/clientmethod_enumer.go b/clientmethod_enumer.go new file mode 100644 index 00000000..f966d591 --- /dev/null +++ b/clientmethod_enumer.go @@ -0,0 +1,68 @@ +// Code generated by "enumer --type=ClientMethod --trimprefix=ClientMethod -json"; DO NOT EDIT. + +// +package stow + +import ( + "encoding/json" + "fmt" +) + +const _ClientMethodName = "GetPut" + +var _ClientMethodIndex = [...]uint8{0, 3, 6} + +func (i ClientMethod) String() string { + if i < 0 || i >= ClientMethod(len(_ClientMethodIndex)-1) { + return fmt.Sprintf("ClientMethod(%d)", i) + } + return _ClientMethodName[_ClientMethodIndex[i]:_ClientMethodIndex[i+1]] +} + +var _ClientMethodValues = []ClientMethod{0, 1} + +var _ClientMethodNameToValueMap = map[string]ClientMethod{ + _ClientMethodName[0:3]: 0, + _ClientMethodName[3:6]: 1, +} + +// ClientMethodString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func ClientMethodString(s string) (ClientMethod, error) { + if val, ok := _ClientMethodNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to ClientMethod values", s) +} + +// ClientMethodValues returns all values of the enum +func ClientMethodValues() []ClientMethod { + return _ClientMethodValues +} + +// IsAClientMethod returns "true" if the value is listed in the enum definition. "false" otherwise +func (i ClientMethod) IsAClientMethod() bool { + for _, v := range _ClientMethodValues { + if i == v { + return true + } + } + return false +} + +// MarshalJSON implements the json.Marshaler interface for ClientMethod +func (i ClientMethod) MarshalJSON() ([]byte, error) { + return json.Marshal(i.String()) +} + +// UnmarshalJSON implements the json.Unmarshaler interface for ClientMethod +func (i *ClientMethod) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return fmt.Errorf("ClientMethod should be a string, got %s", data) + } + + var err error + *i, err = ClientMethodString(s) + return err +} diff --git a/google/container.go b/google/container.go index 3caec3b9..1e0d2650 100644 --- a/google/container.go +++ b/google/container.go @@ -3,6 +3,8 @@ package google import ( "context" "io" + "net/http" + "time" "cloud.google.com/go/storage" "github.com/pkg/errors" @@ -33,10 +35,27 @@ func (c *Container) Name() string { } // Bucket returns the google bucket attributes -func (c *Container) Bucket() *storage.BucketHandle{ +func (c *Container) Bucket() *storage.BucketHandle { return c.client.Bucket(c.name) } +func (c *Container) PreSignRequest(_ context.Context, clientMethod stow.ClientMethod, id string, + params stow.PresignRequestParams) (url string, err error) { + if len(params.HttpMethod) == 0 { + switch clientMethod { + case stow.ClientMethodGet: + params.HttpMethod = http.MethodGet + case stow.ClientMethodPut: + params.HttpMethod = http.MethodPut + } + } + + return storage.SignedURL(c.name, id, &storage.SignedURLOptions{ + Method: params.HttpMethod, + Expires: time.Now().Add(params.ExpiresIn), + }) +} + // Item returns a stow.Item instance of a container based on the // name of the container func (c *Container) Item(id string) (stow.Item, error) { diff --git a/local/container.go b/local/container.go index 6b11feef..b70da6c2 100644 --- a/local/container.go +++ b/local/container.go @@ -1,7 +1,9 @@ package local import ( + "context" "errors" + "fmt" "io" "net/url" "os" @@ -31,6 +33,11 @@ func (c *container) URL() *url.URL { } } +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + func (c *container) CreateItem(name string) (stow.Item, io.WriteCloser, error) { path := filepath.Join(c.path, filepath.FromSlash(name)) item := &item{ diff --git a/oracle/container.go b/oracle/container.go index 5c5aa41f..cd091744 100644 --- a/oracle/container.go +++ b/oracle/container.go @@ -1,6 +1,8 @@ package oracle import ( + "context" + "fmt" "io" "strings" @@ -16,6 +18,11 @@ type container struct { var _ stow.Container = (*container)(nil) +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + // ID returns a string value representing a unique container, in this case it's // the Container's name. func (c *container) ID() string { diff --git a/s3/container.go b/s3/container.go index c172ea02..1253ceb8 100644 --- a/s3/container.go +++ b/s3/container.go @@ -1,9 +1,12 @@ package s3 import ( + "context" "io" "strings" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" @@ -23,6 +26,28 @@ type container struct { customEndpoint string } +func (c *container) PreSignRequest(ctx context.Context, clientMethod stow.ClientMethod, id string, + params stow.PresignRequestParams) (url string, err error) { + + var req *request.Request + switch clientMethod { + case stow.ClientMethodGet: + req, _ = c.client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(c.name), + Key: aws.String(id), + }) + case stow.ClientMethodPut: + req, _ = c.client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(c.name), + Key: aws.String(id), + }) + } + + req.SetContext(ctx) + + return req.Presign(params.ExpiresIn) +} + // ID returns a string value which represents the name of the container. func (c *container) ID() string { return c.name diff --git a/s3/stow_test.go b/s3/stow_test.go index b4b28db0..5a20ef56 100644 --- a/s3/stow_test.go +++ b/s3/stow_test.go @@ -2,6 +2,7 @@ package s3 import ( + "context" "encoding/json" "fmt" "net/http" @@ -11,6 +12,9 @@ import ( "reflect" "strings" "testing" + "time" + + "github.com/stretchr/testify/assert" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -37,6 +41,36 @@ func TestStow(t *testing.T) { test.All(t, "s3", config) } +func TestPreSignedURL(t *testing.T) { + is := is.New(t) + accessKeyId := os.Getenv("S3ACCESSKEYID") + secretKey := os.Getenv("S3SECRETKEY") + region := os.Getenv("S3REGION") + + if accessKeyId == "" || secretKey == "" || region == "" { + t.Skip("skipping test because missing one or more of S3ACCESSKEYID S3SECRETKEY S3REGION") + } + + config := stow.ConfigMap{ + "access_key_id": accessKeyId, + "secret_key": secretKey, + "region": region, + } + + location, err := stow.Dial("s3", config) + is.NoErr(err) + + container, err := location.Container("flyte-demo") + ctx := context.Background() + res, err := container.PreSignRequest(ctx, stow.ClientMethodPut, "blah/bloh/fileon", stow.PresignRequestParams{ + ExpiresIn: time.Hour, + }) + + is.NoErr(err) + t.Log(res) + assert.NotEmpty(t, res) +} + func TestEtagCleanup(t *testing.T) { etagValue := "9c51403a2255f766891a1382288dece4" permutations := []string{ diff --git a/s3/v2signer.go b/s3/v2signer.go index 4b82b8d3..46a3dd54 100644 --- a/s3/v2signer.go +++ b/s3/v2signer.go @@ -231,4 +231,4 @@ const logSignInfoMsg = `DEBUG: Request Signature: func (v2 *signer) logSigningInfo() { msg := fmt.Sprintf(logSignInfoMsg, v2.stringToSign, v2.signature) v2.Logger.Log(msg) -} \ No newline at end of file +} diff --git a/stow.go b/stow.go index 2f02e2b5..c7bfc9a0 100644 --- a/stow.go +++ b/stow.go @@ -1,6 +1,7 @@ package stow import ( + "context" "errors" "io" "net/url" @@ -45,6 +46,20 @@ var ( NoPrefix = "" ) +// HttpMethod defines an alias type for string to represent http Methods. These are defined in RFC 7231 section 4.3. +type HttpMethod = string + +//go:generate enumer --type=ClientMethod --trimprefix=ClientMethod -json + +// ClientMethod defines common client methods across storage providers +type ClientMethod int + +const ( + ClientMethodGet ClientMethod = iota + ClientMethodPut + ClientMethodPost +) + // IsCursorEnd checks whether the cursor indicates there are no // more items or not. func IsCursorEnd(cursor string) bool { @@ -77,6 +92,12 @@ type Location interface { ItemByURL(url *url.URL) (Item, error) } +type PresignRequestParams struct { + ExpiresIn time.Duration + ExtraParams map[string]interface{} + HttpMethod HttpMethod +} + // Container represents a container. type Container interface { // ID gets a unique string describing this Container. @@ -100,6 +121,9 @@ type Container interface { // Put creates a new Item with the specified name, and contents // read from the reader. Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (Item, error) + + // PreSignRequest generates a pre-signed url for the given id (key after bucket/container) and a given clientMethod. + PreSignRequest(ctx context.Context, clientMethod ClientMethod, id string, params PresignRequestParams) (url string, err error) } // Item represents an item inside a Container. From 1190b6127dd95eb8716c0e36b9af6ae993eba92a Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 16 Mar 2022 17:05:30 -0700 Subject: [PATCH 2/5] Add stubs for sftp and swift Signed-off-by: Haytham Abuelfutuh --- sftp/container.go | 7 +++++++ swift/container.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/sftp/container.go b/sftp/container.go index f1b96224..50654341 100644 --- a/sftp/container.go +++ b/sftp/container.go @@ -1,7 +1,9 @@ package sftp import ( + "context" "errors" + "fmt" "io" "os" "path/filepath" @@ -25,6 +27,11 @@ func (c *container) Name() string { return c.name } +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + // Item returns a stow.Item instance of a container based on the name of the // container and the file. func (c *container) Item(id string) (stow.Item, error) { diff --git a/swift/container.go b/swift/container.go index ab766d57..deab7357 100644 --- a/swift/container.go +++ b/swift/container.go @@ -1,6 +1,8 @@ package swift import ( + "context" + "fmt" "io" "strings" @@ -25,6 +27,11 @@ func (c *container) Name() string { return c.id } +func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, + _ stow.PresignRequestParams) (url string, err error) { + return "", fmt.Errorf("unsupported") +} + func (c *container) Item(id string) (stow.Item, error) { return c.getItem(id) } From b6548d46208ae1387f2d9f58caf56a93295ad9d2 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 17 Mar 2022 16:21:24 -0700 Subject: [PATCH 3/5] cleanup Signed-off-by: Haytham Abuelfutuh --- s3/container.go | 3 +++ stow.go | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/s3/container.go b/s3/container.go index 1253ceb8..02c0e368 100644 --- a/s3/container.go +++ b/s3/container.go @@ -2,6 +2,7 @@ package s3 import ( "context" + "fmt" "io" "strings" @@ -41,6 +42,8 @@ func (c *container) PreSignRequest(ctx context.Context, clientMethod stow.Client Bucket: aws.String(c.name), Key: aws.String(id), }) + default: + return "", fmt.Errorf("unsupported client method [%v]", clientMethod.String()) } req.SetContext(ctx) diff --git a/stow.go b/stow.go index c7bfc9a0..6ecda237 100644 --- a/stow.go +++ b/stow.go @@ -57,7 +57,6 @@ type ClientMethod int const ( ClientMethodGet ClientMethod = iota ClientMethodPut - ClientMethodPost ) // IsCursorEnd checks whether the cursor indicates there are no From 57524be2132582dc24ce33acaf938e3065e5dbc5 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 22 Mar 2022 17:20:15 -0700 Subject: [PATCH 4/5] Implement Presigned URL for Azure Signed-off-by: Haytham Abuelfutuh --- azure/config.go | 37 ++++++++++++++++++++++------ azure/container.go | 34 +++++++++++++++++++++++--- azure/location.go | 10 ++++++-- go.mod | 11 ++++----- go.sum | 61 +++++++++++++++++++++++++++++++++------------- 5 files changed, 118 insertions(+), 35 deletions(-) diff --git a/azure/config.go b/azure/config.go index 7bf21f1a..d512c75e 100644 --- a/azure/config.go +++ b/azure/config.go @@ -4,6 +4,8 @@ import ( "errors" "net/url" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + az "github.com/Azure/azure-sdk-for-go/storage" "github.com/graymeta/stow" ) @@ -42,16 +44,30 @@ func init() { l := &location{ config: config, } - var err error - l.client, err = newBlobStorageClient(l.config) + + acc, key, err := getAccount(l.config) + if err != nil { + return nil, err + } + + l.account = acc + + l.client, err = newBlobStorageClient(acc, key) + if err != nil { + return nil, err + } + + l.sharedCreds, err = azblob.NewSharedKeyCredential(acc, key) if err != nil { return nil, err } + // test the connection _, _, err = l.Containers("", stow.CursorStart, 1) if err != nil { return nil, err } + return l, nil } kindfn := func(u *url.URL) bool { @@ -60,19 +76,26 @@ func init() { stow.Register(Kind, makefn, kindfn, validatefn) } -func newBlobStorageClient(cfg stow.Config) (*az.BlobStorageClient, error) { +func getAccount(cfg stow.Config) (account, key string, err error) { acc, ok := cfg.Config(ConfigAccount) if !ok { - return nil, errors.New("missing account id") + return "", "", errors.New("missing account id") } - key, ok := cfg.Config(ConfigKey) + + key, ok = cfg.Config(ConfigKey) if !ok { - return nil, errors.New("missing auth key") + return "", "", errors.New("missing auth key") } - basicClient, err := az.NewBasicClient(acc, key) + + return acc, key, nil +} + +func newBlobStorageClient(account, key string) (*az.BlobStorageClient, error) { + basicClient, err := az.NewBasicClient(account, key) if err != nil { return nil, errors.New("bad credentials") } + client := basicClient.GetBlobService() return &client, err } diff --git a/azure/container.go b/azure/container.go index da308eb1..ad269688 100644 --- a/azure/container.go +++ b/azure/container.go @@ -7,6 +7,7 @@ import ( "strings" "time" + azblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" az "github.com/Azure/azure-sdk-for-go/storage" "github.com/graymeta/stow" "github.com/pkg/errors" @@ -20,8 +21,10 @@ var timeFormat = "Mon, 2 Jan 2006 15:04:05 MST" type container struct { id string + account string properties az.ContainerProperties client *az.BlobStorageClient + creds *azblob.SharedKeyCredential } var _ stow.Container = (*container)(nil) @@ -34,9 +37,34 @@ func (c *container) Name() string { return c.id } -func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string, - _ stow.PresignRequestParams) (url string, err error) { - return "", fmt.Errorf("unsupported") +func (c *container) PreSignRequest(_ context.Context, method stow.ClientMethod, key string, + params stow.PresignRequestParams) (url string, err error) { + containerName := c.id + blobName := key + + permissions := azblob.BlobSASPermissions{} + switch method { + case stow.ClientMethodGet: + permissions.Read = true + case stow.ClientMethodPut: + permissions.Add = true + permissions.Write = true + } + + sasQueryParams, err := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: time.Now().Add(params.ExpiresIn), + ContainerName: containerName, + BlobName: blobName, + Permissions: permissions.String(), + }.NewSASQueryParameters(c.creds) + if err != nil { + return "", err + } + + // Create the SAS URL for the resource you wish to access, and append the SAS query parameters. + qp := sasQueryParams.Encode() + return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?%s", c.account, containerName, blobName, qp), nil } func (c *container) Item(id string) (stow.Item, error) { diff --git a/azure/location.go b/azure/location.go index c6dd50a0..829268f5 100644 --- a/azure/location.go +++ b/azure/location.go @@ -6,13 +6,17 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + az "github.com/Azure/azure-sdk-for-go/storage" "github.com/graymeta/stow" ) type location struct { - config stow.Config - client *az.BlobStorageClient + config stow.Config + account string + client *az.BlobStorageClient + sharedCreds *azblob.SharedKeyCredential } func (l *location) Close() error { @@ -56,6 +60,8 @@ func (l *location) Containers(prefix, cursor string, count int) ([]stow.Containe id: azureContainer.Name, properties: azureContainer.Properties, client: l.client, + creds: l.sharedCreds, + account: l.account, } } return containers, response.NextMarker, nil diff --git a/go.mod b/go.mod index a756f26b..3c0a9a31 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,13 @@ go 1.14 require ( cloud.google.com/go v0.38.0 - github.com/Azure/azure-sdk-for-go v32.5.0+incompatible + github.com/Azure/azure-sdk-for-go v62.3.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 github.com/Azure/go-autorest/autorest v0.9.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/aws/aws-sdk-go v1.23.4 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 - github.com/dnaeon/go-vcr v1.1.0 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect github.com/hashicorp/go-multierror v1.0.0 github.com/kr/fs v0.1.0 // indirect @@ -17,10 +18,8 @@ require ( github.com/pkg/errors v0.8.1 github.com/pkg/sftp v1.10.0 github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9 // indirect - github.com/satori/go.uuid v1.2.0 // indirect - github.com/stretchr/testify v1.4.0 - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect + github.com/stretchr/testify v1.7.0 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 google.golang.org/api v0.8.0 gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465 diff --git a/go.sum b/go.sum index 85bd08ca..28e84046 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -github.com/Azure/azure-sdk-for-go v32.5.0+incompatible h1:Hn/DsObfmw0M7dMGS/c0MlVrJuGFzHzOpBWL89acR68= -github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible h1:Ctfsn9UoA/BB4HMYQlbPPgNXdX0tZ4tmb85+KFb2+RE= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= @@ -27,12 +33,16 @@ github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -64,6 +74,11 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= @@ -75,16 +90,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9 h1:kyf9snWXHvQc+yxE9imhdI8YAm4oKeZISlaAR+x73zs= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -98,8 +111,10 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -112,12 +127,19 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -140,13 +162,18 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465 h1:DKgyTtKkmpZZesLue2fz/LxEhzBDUWg4N8u/BVRJqlA= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From e1ea0813d731f18747154884c328efd8886b0865 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 22 Mar 2022 20:35:04 -0700 Subject: [PATCH 5/5] Add Azure Unit Test Signed-off-by: Haytham Abuelfutuh --- azure/stow_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/azure/stow_test.go b/azure/stow_test.go index b09e98aa..739a5a21 100644 --- a/azure/stow_test.go +++ b/azure/stow_test.go @@ -1,11 +1,14 @@ package azure import ( + "context" "fmt" "os" "reflect" "testing" + "github.com/stretchr/testify/assert" + "github.com/cheekybits/is" "github.com/graymeta/stow" "github.com/graymeta/stow/test" @@ -78,3 +81,23 @@ func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { _, err := prepMetadata(m) is.Err(err) } + +func TestContainer_PreSignRequest(t *testing.T) { + if azureaccount == "" || azurekey == "" { + t.Skip("skipping test because missing either AZUREACCOUNT or AZUREKEY") + } + + l, err := stow.Dial(Kind, stow.ConfigMap{ + ConfigAccount: azureaccount, + ConfigKey: azurekey, + }) + + assert.NoError(t, err) + c, err := l.Container("test") + assert.NoError(t, err) + + u, err := c.PreSignRequest(context.Background(), stow.ClientMethodPut, "file.txt", stow.PresignRequestParams{}) + assert.NoError(t, err) + + assert.NotEmpty(t, u) +}