Merge pull request #1 from flyteorg/signed-url

azure/config.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"net/url"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+
 	az "github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/graymeta/stow"
 )
@@ -42,16 +44,30 @@ func init() {
 		l := &location{
 			config: config,
 		}
-		var err error
-		l.client, err = newBlobStorageClient(l.config)
+
+		acc, key, err := getAccount(l.config)
+		if err != nil {
+			return nil, err
+		}
+
+		l.account = acc
+
+		l.client, err = newBlobStorageClient(acc, key)
+		if err != nil {
+			return nil, err
+		}
+
+		l.sharedCreds, err = azblob.NewSharedKeyCredential(acc, key)
 		if err != nil {
 			return nil, err
 		}
+
 		// test the connection
 		_, _, err = l.Containers("", stow.CursorStart, 1)
 		if err != nil {
 			return nil, err
 		}
+
 		return l, nil
 	}
 	kindfn := func(u *url.URL) bool {
@@ -60,19 +76,26 @@ func init() {
 	stow.Register(Kind, makefn, kindfn, validatefn)
 }
 
-func newBlobStorageClient(cfg stow.Config) (*az.BlobStorageClient, error) {
+func getAccount(cfg stow.Config) (account, key string, err error) {
 	acc, ok := cfg.Config(ConfigAccount)
 	if !ok {
-		return nil, errors.New("missing account id")
+		return "", "", errors.New("missing account id")
 	}
-	key, ok := cfg.Config(ConfigKey)
+
+	key, ok = cfg.Config(ConfigKey)
 	if !ok {
-		return nil, errors.New("missing auth key")
+		return "", "", errors.New("missing auth key")
 	}
-	basicClient, err := az.NewBasicClient(acc, key)
+
+	return acc, key, nil
+}
+
+func newBlobStorageClient(account, key string) (*az.BlobStorageClient, error) {
+	basicClient, err := az.NewBasicClient(account, key)
 	if err != nil {
 		return nil, errors.New("bad credentials")
 	}
+
 	client := basicClient.GetBlobService()
 	return &client, err
 }

azure/container.go

@@ -1,10 +1,13 @@
 package azure
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 	"time"
 
+	azblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	az "github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/graymeta/stow"
 	"github.com/pkg/errors"
@@ -18,8 +21,10 @@ var timeFormat = "Mon, 2 Jan 2006 15:04:05 MST"
 
 type container struct {
 	id         string
+	account    string
 	properties az.ContainerProperties
 	client     *az.BlobStorageClient
+	creds      *azblob.SharedKeyCredential
 }
 
 var _ stow.Container = (*container)(nil)
@@ -32,6 +37,36 @@ func (c *container) Name() string {
 	return c.id
 }
 
+func (c *container) PreSignRequest(_ context.Context, method stow.ClientMethod, key string,
+	params stow.PresignRequestParams) (url string, err error) {
+	containerName := c.id
+	blobName := key
+
+	permissions := azblob.BlobSASPermissions{}
+	switch method {
+	case stow.ClientMethodGet:
+		permissions.Read = true
+	case stow.ClientMethodPut:
+		permissions.Add = true
+		permissions.Write = true
+	}
+
+	sasQueryParams, err := azblob.BlobSASSignatureValues{
+		Protocol:      azblob.SASProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(params.ExpiresIn),
+		ContainerName: containerName,
+		BlobName:      blobName,
+		Permissions:   permissions.String(),
+	}.NewSASQueryParameters(c.creds)
+	if err != nil {
+		return "", err
+	}
+
+	// Create the SAS URL for the resource you wish to access, and append the SAS query parameters.
+	qp := sasQueryParams.Encode()
+	return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?%s", c.account, containerName, blobName, qp), nil
+}
+
 func (c *container) Item(id string) (stow.Item, error) {
 	blob := c.client.GetContainerReference(c.id).GetBlobReference(id)
 	err := blob.GetProperties(nil)

azure/location.go

@@ -6,13 +6,17 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+
 	az "github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/graymeta/stow"
 )
 
 type location struct {
-	config stow.Config
-	client *az.BlobStorageClient
+	config      stow.Config
+	account     string
+	client      *az.BlobStorageClient
+	sharedCreds *azblob.SharedKeyCredential
 }
 
 func (l *location) Close() error {
@@ -56,6 +60,8 @@ func (l *location) Containers(prefix, cursor string, count int) ([]stow.Containe
 			id:         azureContainer.Name,
 			properties: azureContainer.Properties,
 			client:     l.client,
+			creds:      l.sharedCreds,
+			account:    l.account,
 		}
 	}
 	return containers, response.NextMarker, nil

azure/stow_test.go

@@ -1,11 +1,14 @@
 package azure
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/cheekybits/is"
 	"github.com/graymeta/stow"
 	"github.com/graymeta/stow/test"
@@ -78,3 +81,23 @@ func TestPrepMetadataFailureWithNonStringValues(t *testing.T) {
 	_, err := prepMetadata(m)
 	is.Err(err)
 }
+
+func TestContainer_PreSignRequest(t *testing.T) {
+	if azureaccount == "" || azurekey == "" {
+		t.Skip("skipping test because missing either AZUREACCOUNT or AZUREKEY")
+	}
+
+	l, err := stow.Dial(Kind, stow.ConfigMap{
+		ConfigAccount: azureaccount,
+		ConfigKey:     azurekey,
+	})
+
+	assert.NoError(t, err)
+	c, err := l.Container("test")
+	assert.NoError(t, err)
+
+	u, err := c.PreSignRequest(context.Background(), stow.ClientMethodPut, "file.txt", stow.PresignRequestParams{})
+	assert.NoError(t, err)
+
+	assert.NotEmpty(t, u)
+}

b2/container.go

@@ -1,6 +1,8 @@
 package b2
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 	"time"
@@ -24,6 +26,11 @@ func (c *container) ID() string {
 	return c.bucket.Name
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 // Name returns the name of the bucket
 func (c *container) Name() string {
 	return c.bucket.Name

go.mod

@@ -4,23 +4,22 @@ go 1.14
 
 require (
 	cloud.google.com/go v0.38.0
-	github.com/Azure/azure-sdk-for-go v32.5.0+incompatible
+	github.com/Azure/azure-sdk-for-go v62.3.0+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
 	github.com/Azure/go-autorest/autorest v0.9.0 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/aws/aws-sdk-go v1.23.4
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927
-	github.com/dnaeon/go-vcr v1.1.0 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/ncw/swift v1.0.49
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/sftp v1.10.0
 	github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9 // indirect
-	github.com/satori/go.uuid v1.2.0 // indirect
-	github.com/stretchr/testify v1.4.0
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.8.0
 	gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465