Add support for PreSigned URLs

azure/container.go

@@ -1,6 +1,8 @@
 package azure
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 	"time"
@@ -32,6 +34,11 @@ func (c *container) Name() string {
 	return c.id
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 func (c *container) Item(id string) (stow.Item, error) {
 	blob := c.client.GetContainerReference(c.id).GetBlobReference(id)
 	err := blob.GetProperties(nil)

b2/container.go

@@ -1,6 +1,8 @@
 package b2
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 	"time"
@@ -24,6 +26,11 @@ func (c *container) ID() string {
 	return c.bucket.Name
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 // Name returns the name of the bucket
 func (c *container) Name() string {
 	return c.bucket.Name

google/container.go

@@ -3,6 +3,8 @@ package google
 import (
 	"context"
 	"io"
+	"net/http"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/pkg/errors"
@@ -33,10 +35,27 @@ func (c *Container) Name() string {
 }
 
 // Bucket returns the google bucket attributes
-func (c *Container) Bucket() *storage.BucketHandle{
+func (c *Container) Bucket() *storage.BucketHandle {
 	return c.client.Bucket(c.name)
 }
 
+func (c *Container) PreSignRequest(_ context.Context, clientMethod stow.ClientMethod, id string,
+	params stow.PresignRequestParams) (url string, err error) {
+	if len(params.HttpMethod) == 0 {
+		switch clientMethod {
+		case stow.ClientMethodGet:
+			params.HttpMethod = http.MethodGet
+		case stow.ClientMethodPut:
+			params.HttpMethod = http.MethodPut
+		}
+	}
+
+	return storage.SignedURL(c.name, id, &storage.SignedURLOptions{
+		Method:  params.HttpMethod,
+		Expires: time.Now().Add(params.ExpiresIn),
+	})
+}
+
 // Item returns a stow.Item instance of a container based on the
 // name of the container
 func (c *Container) Item(id string) (stow.Item, error) {

local/container.go

@@ -1,7 +1,9 @@
 package local
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/url"
 	"os"
@@ -31,6 +33,11 @@ func (c *container) URL() *url.URL {
 	}
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 func (c *container) CreateItem(name string) (stow.Item, io.WriteCloser, error) {
 	path := filepath.Join(c.path, filepath.FromSlash(name))
 	item := &item{

oracle/container.go

@@ -1,6 +1,8 @@
 package oracle
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 
@@ -16,6 +18,11 @@ type container struct {
 
 var _ stow.Container = (*container)(nil)
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 // ID returns a string value representing a unique container, in this case it's
 // the Container's name.
 func (c *container) ID() string {

s3/container.go

@@ -1,9 +1,13 @@
 package s3
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/request"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -23,6 +27,30 @@ type container struct {
 	customEndpoint string
 }
 
+func (c *container) PreSignRequest(ctx context.Context, clientMethod stow.ClientMethod, id string,
+	params stow.PresignRequestParams) (url string, err error) {
+
+	var req *request.Request
+	switch clientMethod {
+	case stow.ClientMethodGet:
+		req, _ = c.client.GetObjectRequest(&s3.GetObjectInput{
+			Bucket: aws.String(c.name),
+			Key:    aws.String(id),
+		})
+	case stow.ClientMethodPut:
+		req, _ = c.client.PutObjectRequest(&s3.PutObjectInput{
+			Bucket: aws.String(c.name),
+			Key:    aws.String(id),
+		})
+	default:
+		return "", fmt.Errorf("unsupported client method [%v]", clientMethod.String())
+	}
+
+	req.SetContext(ctx)
+
+	return req.Presign(params.ExpiresIn)
+}
+
 // ID returns a string value which represents the name of the container.
 func (c *container) ID() string {
 	return c.name

s3/stow_test.go

@@ -2,6 +2,7 @@
 package s3
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -11,6 +12,9 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -37,6 +41,36 @@ func TestStow(t *testing.T) {
 	test.All(t, "s3", config)
 }
 
+func TestPreSignedURL(t *testing.T) {
+	is := is.New(t)
+	accessKeyId := os.Getenv("S3ACCESSKEYID")
+	secretKey := os.Getenv("S3SECRETKEY")
+	region := os.Getenv("S3REGION")
+
+	if accessKeyId == "" || secretKey == "" || region == "" {
+		t.Skip("skipping test because missing one or more of S3ACCESSKEYID S3SECRETKEY S3REGION")
+	}
+
+	config := stow.ConfigMap{
+		"access_key_id": accessKeyId,
+		"secret_key":    secretKey,
+		"region":        region,
+	}
+
+	location, err := stow.Dial("s3", config)
+	is.NoErr(err)
+
+	container, err := location.Container("flyte-demo")
+	ctx := context.Background()
+	res, err := container.PreSignRequest(ctx, stow.ClientMethodPut, "blah/bloh/fileon", stow.PresignRequestParams{
+		ExpiresIn: time.Hour,
+	})
+
+	is.NoErr(err)
+	t.Log(res)
+	assert.NotEmpty(t, res)
+}
+
 func TestEtagCleanup(t *testing.T) {
 	etagValue := "9c51403a2255f766891a1382288dece4"
 	permutations := []string{

s3/v2signer.go

@@ -231,4 +231,4 @@ const logSignInfoMsg = `DEBUG: Request Signature:
 func (v2 *signer) logSigningInfo() {
 	msg := fmt.Sprintf(logSignInfoMsg, v2.stringToSign, v2.signature)
 	v2.Logger.Log(msg)
-}
+}

sftp/container.go

@@ -1,7 +1,9 @@
 package sftp
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -25,6 +27,11 @@ func (c *container) Name() string {
 	return c.name
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 // Item returns a stow.Item instance of a container based on the name of the
 // container and the file.
 func (c *container) Item(id string) (stow.Item, error) {

stow.go

@@ -1,6 +1,7 @@
 package stow
 
 import (
+	"context"
 	"errors"
 	"io"
 	"net/url"
@@ -45,6 +46,19 @@ var (
 	NoPrefix = ""
 )
 
+// HttpMethod defines an alias type for string to represent http Methods. These are defined in RFC 7231 section 4.3.
+type HttpMethod = string
+
+//go:generate enumer --type=ClientMethod --trimprefix=ClientMethod -json
+
+// ClientMethod defines common client methods across storage providers
+type ClientMethod int
+
+const (
+	ClientMethodGet ClientMethod = iota
+	ClientMethodPut
+)
+
 // IsCursorEnd checks whether the cursor indicates there are no
 // more items or not.
 func IsCursorEnd(cursor string) bool {
@@ -77,6 +91,12 @@ type Location interface {
 	ItemByURL(url *url.URL) (Item, error)
 }
 
+type PresignRequestParams struct {
+	ExpiresIn   time.Duration
+	ExtraParams map[string]interface{}
+	HttpMethod  HttpMethod
+}
+
 // Container represents a container.
 type Container interface {
 	// ID gets a unique string describing this Container.
@@ -100,6 +120,9 @@ type Container interface {
 	// Put creates a new Item with the specified name, and contents
 	// read from the reader.
 	Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (Item, error)
+
+	// PreSignRequest generates a pre-signed url for the given id (key after bucket/container) and a given clientMethod.
+	PreSignRequest(ctx context.Context, clientMethod ClientMethod, id string, params PresignRequestParams) (url string, err error)
 }
 
 // Item represents an item inside a Container.

swift/container.go

@@ -1,6 +1,8 @@
 package swift
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"strings"
 
@@ -25,6 +27,11 @@ func (c *container) Name() string {
 	return c.id
 }
 
+func (c *container) PreSignRequest(_ context.Context, _ stow.ClientMethod, _ string,
+	_ stow.PresignRequestParams) (url string, err error) {
+	return "", fmt.Errorf("unsupported")
+}
+
 func (c *container) Item(id string) (stow.Item, error) {
 	return c.getItem(id)
 }