fix(aws): Get region from IMDS when missing for awssmfs, awssmpfs, an…

…d blobfs (s3://) (#911)

Signed-off-by: Dave Henderson <[email protected]>

awssmfs/awssm.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	smtypes "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
 	"github.com/hairyhenderson/go-fsimpl"
+	"github.com/hairyhenderson/go-fsimpl/awsimdsfs"
 	"github.com/hairyhenderson/go-fsimpl/internal"
 )
 
@@ -47,6 +48,7 @@ type awssmFS struct {
 	base       *url.URL
 	httpclient *http.Client
 	smclient   SecretsManagerClient
+	imdsfs     fs.FS
 	root       string
 }
 
@@ -66,10 +68,18 @@ func New(u *url.URL) (fs.FS, error) {
 		root = u.Opaque
 	}
 
+	iu, _ := url.Parse("aws+imds:")
+
+	imdsfs, err := awsimdsfs.New(iu)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err)
+	}
+
 	return &awssmFS{
-		ctx:  context.Background(),
-		base: u,
-		root: root,
+		ctx:    context.Background(),
+		base:   u,
+		root:   root,
+		imdsfs: imdsfs,
 	}, nil
 }
 
@@ -86,6 +96,7 @@ var (
 	_ internal.WithContexter    = (*awssmFS)(nil)
 	_ internal.WithHTTPClienter = (*awssmFS)(nil)
 	_ withSMClienter            = (*awssmFS)(nil)
+	_ internal.WithIMDSFSer     = (*awssmFS)(nil)
 )
 
 func (f awssmFS) URL() string {
@@ -125,6 +136,17 @@ func (f *awssmFS) WithSMClient(smclient SecretsManagerClient) fs.FS {
 	return &fsys
 }
 
+func (f *awssmFS) WithIMDSFS(imdsfs fs.FS) fs.FS {
+	if imdsfs == nil {
+		return f
+	}
+
+	fsys := *f
+	fsys.imdsfs = imdsfs
+
+	return &fsys
+}
+
 func (f *awssmFS) getClient(ctx context.Context) (SecretsManagerClient, error) {
 	if f.smclient != nil {
 		return f.smclient, nil
@@ -140,6 +162,16 @@ func (f *awssmFS) getClient(ctx context.Context) (SecretsManagerClient, error) {
 		return nil, err
 	}
 
+	if cfg.Region == "" && f.imdsfs != nil {
+		// if we have an IMDS filesystem, use it to get the region
+		region, err := fs.ReadFile(f.imdsfs, "meta-data/placement/region")
+		if err != nil {
+			return nil, fmt.Errorf("couldn't get region from IMDS: %w", err)
+		}
+
+		cfg.Region = string(region)
+	}
+
 	optFns := []func(*secretsmanager.Options){}
 
 	// setting a host in the URL is only intended for test purposes

awssmpfs/awssmp.go

@@ -18,6 +18,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
 	"github.com/aws/aws-sdk-go-v2/service/ssm/types"
 	"github.com/hairyhenderson/go-fsimpl"
+	"github.com/hairyhenderson/go-fsimpl/awsimdsfs"
 	"github.com/hairyhenderson/go-fsimpl/internal"
 )
 
@@ -51,6 +52,7 @@ type awssmpFS struct {
 	base       *url.URL
 	httpclient *http.Client
 	ssmclient  SSMClient
+	imdsfs     fs.FS
 	root       string
 }
 
@@ -71,10 +73,18 @@ func New(u *url.URL) (fs.FS, error) {
 		u.Path = "/"
 	}
 
+	iu, _ := url.Parse("aws+imds:")
+
+	imdsfs, err := awsimdsfs.New(iu)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err)
+	}
+
 	return &awssmpFS{
-		ctx:  context.Background(),
-		base: u,
-		root: u.Path,
+		ctx:    context.Background(),
+		base:   u,
+		root:   u.Path,
+		imdsfs: imdsfs,
 	}, nil
 }
 
@@ -91,6 +101,7 @@ var (
 	_ internal.WithContexter    = (*awssmpFS)(nil)
 	_ internal.WithHTTPClienter = (*awssmpFS)(nil)
 	_ withClienter              = (*awssmpFS)(nil)
+	_ internal.WithIMDSFSer     = (*awssmpFS)(nil)
 )
 
 func (f awssmpFS) URL() string {
@@ -130,6 +141,17 @@ func (f *awssmpFS) WithClient(ssmclient SSMClient) fs.FS {
 	return &fsys
 }
 
+func (f *awssmpFS) WithIMDSFS(imdsfs fs.FS) fs.FS {
+	if imdsfs == nil {
+		return f
+	}
+
+	fsys := *f
+	fsys.imdsfs = imdsfs
+
+	return &fsys
+}
+
 func (f *awssmpFS) getClient(ctx context.Context) (SSMClient, error) {
 	if f.ssmclient != nil {
 		return f.ssmclient, nil
@@ -145,6 +167,16 @@ func (f *awssmpFS) getClient(ctx context.Context) (SSMClient, error) {
 		return nil, err
 	}
 
+	if cfg.Region == "" && f.imdsfs != nil {
+		// if we have an IMDS filesystem, use it to get the region
+		region, err := fs.ReadFile(f.imdsfs, "meta-data/placement/region")
+		if err != nil {
+			return nil, fmt.Errorf("couldn't get region from IMDS: %w", err)
+		}
+
+		cfg.Region = string(region)
+	}
+
 	optFns := []func(*ssm.Options){}
 
 	// setting a host in the URL is only intended for test purposes

blobfs/blob.go

@@ -16,6 +16,7 @@ import (
 	azblobblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/hairyhenderson/go-fsimpl"
+	"github.com/hairyhenderson/go-fsimpl/awsimdsfs"
 	"github.com/hairyhenderson/go-fsimpl/internal"
 	"github.com/hairyhenderson/go-fsimpl/internal/env"
 	"gocloud.dev/blob"
@@ -32,6 +33,7 @@ type blobFS struct {
 	hclient *http.Client
 	bucket  *blob.Bucket
 	envfs   fs.FS
+	imdsfs  fs.FS
 	root    string
 }
 
@@ -54,12 +56,20 @@ func New(u *url.URL) (fs.FS, error) {
 
 	root := strings.TrimPrefix(u.Path, "/")
 
+	iu, _ := url.Parse("aws+imds:")
+
+	imdsfs, err := awsimdsfs.New(iu)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err)
+	}
+
 	return &blobFS{
 		ctx:     context.Background(),
 		base:    u,
 		hclient: http.DefaultClient,
 		root:    root,
 		envfs:   os.DirFS("/"),
+		imdsfs:  imdsfs,
 	}, nil
 }
 
@@ -74,6 +84,7 @@ var (
 	_ fs.SubFS                  = (*blobFS)(nil)
 	_ internal.WithContexter    = (*blobFS)(nil)
 	_ internal.WithHTTPClienter = (*blobFS)(nil)
+	_ internal.WithIMDSFSer     = (*blobFS)(nil)
 )
 
 func (f blobFS) URL() string {
@@ -102,6 +113,17 @@ func (f *blobFS) WithHTTPClient(client *http.Client) fs.FS {
 	return &fsys
 }
 
+func (f *blobFS) WithIMDSFS(imdsfs fs.FS) fs.FS {
+	if imdsfs == nil {
+		return f
+	}
+
+	fsys := *f
+	fsys.imdsfs = imdsfs
+
+	return &fsys
+}
+
 func (f *blobFS) openBucket() (*blob.Bucket, error) {
 	o, err := f.newOpener(f.ctx, f.base.Scheme)
 	if err != nil {
@@ -196,7 +218,7 @@ func (f *blobFS) newOpener(ctx context.Context, scheme string) (opener blob.Buck
 	switch scheme {
 	case s3blob.Scheme:
 		// see https://gocloud.dev/concepts/urls/#muxes
-		return &s3v2URLOpener{}, nil
+		return &s3v2URLOpener{imdsfs: f.imdsfs}, nil
 	case gcsblob.Scheme:
 		if env.GetenvFS(f.envfs, "GOOGLE_ANON") == "true" {
 			return &gcsblob.URLOpener{
@@ -224,24 +246,6 @@ func (f *blobFS) newOpener(ctx context.Context, scheme string) (opener blob.Buck
 	}
 }
 
-// initS3Session -
-// Deprecated: this is for v1, but kept here for posterity
-// func (f *blobFS) initS3Sessionv1() *session.Session {
-// 	config := aws.NewConfig()
-// 	config = config.WithHTTPClient(f.hclient)
-
-// 	if env.GetenvFS(f.envfs, "AWS_ANON") == "true" {
-// 		config = config.WithCredentials(credentials.AnonymousCredentials)
-// 	}
-
-// 	config = config.WithCredentialsChainVerboseErrors(true)
-
-// 	return session.Must(session.NewSessionWithOptions(session.Options{
-// 		Config:            *config,
-// 		SharedConfigState: session.SharedConfigEnable,
-// 	}))
-// }
-
 // copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse
 func (f *blobFS) cleanCdkURL(u url.URL) url.URL {
 	switch u.Scheme {

blobfs/s3opener.go

@@ -3,6 +3,7 @@ package blobfs
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"net/url"
 	"strconv"
 	"strings"
@@ -24,6 +25,7 @@ import (
 var _ blob.BucketURLOpener = (*s3v2URLOpener)(nil)
 
 type s3v2URLOpener struct {
+	imdsfs fs.FS
 	// Options specifies the options to pass to OpenBucket.
 	Options s3blob.Options
 }
@@ -120,6 +122,16 @@ func (o *s3v2URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bu
 		return nil, fmt.Errorf("open bucket %v: %w", u, err)
 	}
 
+	if cfg.Region == "" && o.imdsfs != nil {
+		// if we have an IMDS filesystem, use it to get the region
+		region, err := fs.ReadFile(o.imdsfs, "meta-data/placement/region")
+		if err != nil {
+			return nil, fmt.Errorf("couldn't get region from IMDS: %w", err)
+		}
+
+		cfg.Region = string(region)
+	}
+
 	clientV2 := s3v2.NewFromConfig(cfg, opts...)
 
 	return s3blob.OpenBucketV2(ctx, clientV2, u.Host, &o.Options)

internal/types.go

@@ -95,3 +95,10 @@ type WithHTTPClienter interface {
 type WithHeaderer interface {
 	WithHeader(headers http.Header) fs.FS
 }
+
+// WithIMDSFSer overrides the IMDS filesystem used by fs, if the filesystem
+// supports it (i.e. has a WithIMDSFS method). This can be used for overriding
+// the IMDS filesystem used by the fs to discover the region.
+type WithIMDSFSer interface {
+	WithIMDSFS(imdsfs fs.FS) fs.FS
+}