From 59f04488f68c6d4a04ae02d5358db3f43893b78b Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Mon, 16 Dec 2024 19:25:02 -0500 Subject: [PATCH] fix(aws): Get region from IMDS when missing for awssmfs, awssmpfs, and blobfs (s3://) (#911) Signed-off-by: Dave Henderson --- awssmfs/awssm.go | 38 +++++++++++++++++++++++++++++++++++--- awssmpfs/awssmp.go | 38 +++++++++++++++++++++++++++++++++++--- blobfs/blob.go | 42 +++++++++++++++++++++++------------------- blobfs/s3opener.go | 12 ++++++++++++ internal/types.go | 7 +++++++ 5 files changed, 112 insertions(+), 25 deletions(-) diff --git a/awssmfs/awssm.go b/awssmfs/awssm.go index 1a0ef366..d57f99a7 100644 --- a/awssmfs/awssm.go +++ b/awssmfs/awssm.go @@ -19,6 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/secretsmanager" smtypes "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/awsimdsfs" "github.com/hairyhenderson/go-fsimpl/internal" ) @@ -47,6 +48,7 @@ type awssmFS struct { base *url.URL httpclient *http.Client smclient SecretsManagerClient + imdsfs fs.FS root string } @@ -66,10 +68,18 @@ func New(u *url.URL) (fs.FS, error) { root = u.Opaque } + iu, _ := url.Parse("aws+imds:") + + imdsfs, err := awsimdsfs.New(iu) + if err != nil { + return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err) + } + return &awssmFS{ - ctx: context.Background(), - base: u, - root: root, + ctx: context.Background(), + base: u, + root: root, + imdsfs: imdsfs, }, nil } @@ -86,6 +96,7 @@ var ( _ internal.WithContexter = (*awssmFS)(nil) _ internal.WithHTTPClienter = (*awssmFS)(nil) _ withSMClienter = (*awssmFS)(nil) + _ internal.WithIMDSFSer = (*awssmFS)(nil) ) func (f awssmFS) URL() string { @@ -125,6 +136,17 @@ func (f *awssmFS) WithSMClient(smclient SecretsManagerClient) fs.FS { return &fsys } +func (f *awssmFS) WithIMDSFS(imdsfs fs.FS) fs.FS { + if imdsfs == nil { + return f + } + + fsys := *f + fsys.imdsfs = imdsfs + + return &fsys +} + func (f *awssmFS) getClient(ctx context.Context) (SecretsManagerClient, error) { if f.smclient != nil { return f.smclient, nil @@ -140,6 +162,16 @@ func (f *awssmFS) getClient(ctx context.Context) (SecretsManagerClient, error) { return nil, err } + if cfg.Region == "" && f.imdsfs != nil { + // if we have an IMDS filesystem, use it to get the region + region, err := fs.ReadFile(f.imdsfs, "meta-data/placement/region") + if err != nil { + return nil, fmt.Errorf("couldn't get region from IMDS: %w", err) + } + + cfg.Region = string(region) + } + optFns := []func(*secretsmanager.Options){} // setting a host in the URL is only intended for test purposes diff --git a/awssmpfs/awssmp.go b/awssmpfs/awssmp.go index 4e8a78e0..6e0f4423 100644 --- a/awssmpfs/awssmp.go +++ b/awssmpfs/awssmp.go @@ -18,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/awsimdsfs" "github.com/hairyhenderson/go-fsimpl/internal" ) @@ -51,6 +52,7 @@ type awssmpFS struct { base *url.URL httpclient *http.Client ssmclient SSMClient + imdsfs fs.FS root string } @@ -71,10 +73,18 @@ func New(u *url.URL) (fs.FS, error) { u.Path = "/" } + iu, _ := url.Parse("aws+imds:") + + imdsfs, err := awsimdsfs.New(iu) + if err != nil { + return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err) + } + return &awssmpFS{ - ctx: context.Background(), - base: u, - root: u.Path, + ctx: context.Background(), + base: u, + root: u.Path, + imdsfs: imdsfs, }, nil } @@ -91,6 +101,7 @@ var ( _ internal.WithContexter = (*awssmpFS)(nil) _ internal.WithHTTPClienter = (*awssmpFS)(nil) _ withClienter = (*awssmpFS)(nil) + _ internal.WithIMDSFSer = (*awssmpFS)(nil) ) func (f awssmpFS) URL() string { @@ -130,6 +141,17 @@ func (f *awssmpFS) WithClient(ssmclient SSMClient) fs.FS { return &fsys } +func (f *awssmpFS) WithIMDSFS(imdsfs fs.FS) fs.FS { + if imdsfs == nil { + return f + } + + fsys := *f + fsys.imdsfs = imdsfs + + return &fsys +} + func (f *awssmpFS) getClient(ctx context.Context) (SSMClient, error) { if f.ssmclient != nil { return f.ssmclient, nil @@ -145,6 +167,16 @@ func (f *awssmpFS) getClient(ctx context.Context) (SSMClient, error) { return nil, err } + if cfg.Region == "" && f.imdsfs != nil { + // if we have an IMDS filesystem, use it to get the region + region, err := fs.ReadFile(f.imdsfs, "meta-data/placement/region") + if err != nil { + return nil, fmt.Errorf("couldn't get region from IMDS: %w", err) + } + + cfg.Region = string(region) + } + optFns := []func(*ssm.Options){} // setting a host in the URL is only intended for test purposes diff --git a/blobfs/blob.go b/blobfs/blob.go index 6ffbf07b..42a8c7f6 100644 --- a/blobfs/blob.go +++ b/blobfs/blob.go @@ -16,6 +16,7 @@ import ( azblobblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/awsimdsfs" "github.com/hairyhenderson/go-fsimpl/internal" "github.com/hairyhenderson/go-fsimpl/internal/env" "gocloud.dev/blob" @@ -32,6 +33,7 @@ type blobFS struct { hclient *http.Client bucket *blob.Bucket envfs fs.FS + imdsfs fs.FS root string } @@ -54,12 +56,20 @@ func New(u *url.URL) (fs.FS, error) { root := strings.TrimPrefix(u.Path, "/") + iu, _ := url.Parse("aws+imds:") + + imdsfs, err := awsimdsfs.New(iu) + if err != nil { + return nil, fmt.Errorf("couldn't create IMDS filesystem: %w", err) + } + return &blobFS{ ctx: context.Background(), base: u, hclient: http.DefaultClient, root: root, envfs: os.DirFS("/"), + imdsfs: imdsfs, }, nil } @@ -74,6 +84,7 @@ var ( _ fs.SubFS = (*blobFS)(nil) _ internal.WithContexter = (*blobFS)(nil) _ internal.WithHTTPClienter = (*blobFS)(nil) + _ internal.WithIMDSFSer = (*blobFS)(nil) ) func (f blobFS) URL() string { @@ -102,6 +113,17 @@ func (f *blobFS) WithHTTPClient(client *http.Client) fs.FS { return &fsys } +func (f *blobFS) WithIMDSFS(imdsfs fs.FS) fs.FS { + if imdsfs == nil { + return f + } + + fsys := *f + fsys.imdsfs = imdsfs + + return &fsys +} + func (f *blobFS) openBucket() (*blob.Bucket, error) { o, err := f.newOpener(f.ctx, f.base.Scheme) if err != nil { @@ -196,7 +218,7 @@ func (f *blobFS) newOpener(ctx context.Context, scheme string) (opener blob.Buck switch scheme { case s3blob.Scheme: // see https://gocloud.dev/concepts/urls/#muxes - return &s3v2URLOpener{}, nil + return &s3v2URLOpener{imdsfs: f.imdsfs}, nil case gcsblob.Scheme: if env.GetenvFS(f.envfs, "GOOGLE_ANON") == "true" { return &gcsblob.URLOpener{ @@ -224,24 +246,6 @@ func (f *blobFS) newOpener(ctx context.Context, scheme string) (opener blob.Buck } } -// initS3Session - -// Deprecated: this is for v1, but kept here for posterity -// func (f *blobFS) initS3Sessionv1() *session.Session { -// config := aws.NewConfig() -// config = config.WithHTTPClient(f.hclient) - -// if env.GetenvFS(f.envfs, "AWS_ANON") == "true" { -// config = config.WithCredentials(credentials.AnonymousCredentials) -// } - -// config = config.WithCredentialsChainVerboseErrors(true) - -// return session.Must(session.NewSessionWithOptions(session.Options{ -// Config: *config, -// SharedConfigState: session.SharedConfigEnable, -// })) -// } - // copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse func (f *blobFS) cleanCdkURL(u url.URL) url.URL { switch u.Scheme { diff --git a/blobfs/s3opener.go b/blobfs/s3opener.go index 36a06451..89be320f 100644 --- a/blobfs/s3opener.go +++ b/blobfs/s3opener.go @@ -3,6 +3,7 @@ package blobfs import ( "context" "fmt" + "io/fs" "net/url" "strconv" "strings" @@ -24,6 +25,7 @@ import ( var _ blob.BucketURLOpener = (*s3v2URLOpener)(nil) type s3v2URLOpener struct { + imdsfs fs.FS // Options specifies the options to pass to OpenBucket. Options s3blob.Options } @@ -120,6 +122,16 @@ func (o *s3v2URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bu return nil, fmt.Errorf("open bucket %v: %w", u, err) } + if cfg.Region == "" && o.imdsfs != nil { + // if we have an IMDS filesystem, use it to get the region + region, err := fs.ReadFile(o.imdsfs, "meta-data/placement/region") + if err != nil { + return nil, fmt.Errorf("couldn't get region from IMDS: %w", err) + } + + cfg.Region = string(region) + } + clientV2 := s3v2.NewFromConfig(cfg, opts...) return s3blob.OpenBucketV2(ctx, clientV2, u.Host, &o.Options) diff --git a/internal/types.go b/internal/types.go index 7039dd6e..65a513db 100644 --- a/internal/types.go +++ b/internal/types.go @@ -95,3 +95,10 @@ type WithHTTPClienter interface { type WithHeaderer interface { WithHeader(headers http.Header) fs.FS } + +// WithIMDSFSer overrides the IMDS filesystem used by fs, if the filesystem +// supports it (i.e. has a WithIMDSFS method). This can be used for overriding +// the IMDS filesystem used by the fs to discover the region. +type WithIMDSFSer interface { + WithIMDSFS(imdsfs fs.FS) fs.FS +}