diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go index 6bde06ae1..189a13829 100644 --- a/data/datasource_awssmp.go +++ b/data/datasource_awssmp.go @@ -1,9 +1,13 @@ package data import ( + "context" + "net/url" "path" + "strings" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/ssm" "github.com/pkg/errors" @@ -13,32 +17,59 @@ import ( // awssmpGetter - A subset of SSM API for use in unit testing type awssmpGetter interface { GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) + GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error) + GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) } -func parseAWSSMPArgs(origPath string, args ...string) (paramPath string, err error) { - paramPath = origPath - if len(args) >= 1 { - paramPath = path.Join(paramPath, args[0]) - } - +func parseAWSSMPArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { if len(args) >= 2 { err = errors.New("Maximum two arguments to aws+smp datasource: alias, extraPath") + return nil, "", err + } + + p = sourceURL.Path + params = make(map[string]interface{}) + for key, val := range sourceURL.Query() { + params[key] = strings.Join(val, " ") + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, "", err + } + + if parsed.Path != "" { + p = path.Join(p, parsed.Path) + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } } - return + return params, p, err } -func readAWSSMP(source *Source, args ...string) (output []byte, err error) { +func readAWSSMP(source *Source, args ...string) (data []byte, err error) { + ctx := context.TODO() if source.asmpg == nil { source.asmpg = ssm.New(gaws.SDKSession()) } - paramPath, err := parseAWSSMPArgs(source.URL.Path, args...) + _, paramPath, err := parseAWSSMPArgs(source.URL, args...) if err != nil { return nil, err } source.mediaType = jsonMimetype - return readAWSSMPParam(source, paramPath) + switch { + case strings.HasSuffix(paramPath, "/"): + source.mediaType = jsonArrayMimetype + data, err = listAWSSMPParams(ctx, source, paramPath) + default: + data, err = readAWSSMPParam(source, paramPath) + } + return data, err } func readAWSSMPParam(source *Source, paramPath string) ([]byte, error) { @@ -58,3 +89,23 @@ func readAWSSMPParam(source *Source, paramPath string) ([]byte, error) { output, err := ToJSON(result) return []byte(output), err } + +// listAWSSMPParams - supports directory semantics, returns array +func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([]byte, error) { + input := &ssm.GetParametersByPathInput{ + Path: aws.String(paramPath), + } + + response, err := source.asmpg.GetParametersByPathWithContext(ctx, input) + if err != nil { + return nil, errors.Wrapf(err, "Error reading aws+smp from AWS using GetParameter with input %v", input) + } + + listing := make([]string, len(response.Parameters)) + for i, p := range response.Parameters { + listing[i] = (*p.Name)[len(paramPath):] + } + + output, err := ToJSON(listing) + return []byte(output), err +} diff --git a/data/datasource_awssmp_test.go b/data/datasource_awssmp_test.go index 8ae34172a..72acc11f5 100644 --- a/data/datasource_awssmp_test.go +++ b/data/datasource_awssmp_test.go @@ -1,12 +1,14 @@ package data import ( + "context" "encoding/json" "net/url" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/ssm" "github.com/stretchr/testify/assert" ) @@ -15,11 +17,16 @@ import ( type DummyParamGetter struct { t *testing.T param *ssm.Parameter + params []*ssm.Parameter err awserr.Error mockGetParameter func(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) } func (d DummyParamGetter) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { + return d.GetParameterWithContext(context.Background(), input) +} + +func (d DummyParamGetter) GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error) { if d.mockGetParameter != nil { output, err := d.mockGetParameter(input) return output, err @@ -33,6 +40,16 @@ func (d DummyParamGetter) GetParameter(input *ssm.GetParameterInput) (*ssm.GetPa }, nil } +func (d DummyParamGetter) GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) { + if d.err != nil { + return nil, d.err + } + assert.NotNil(d.t, d.params, "Must provide a param if no error!") + return &ssm.GetParametersByPathOutput{ + Parameters: d.params, + }, nil +} + func simpleAWSSourceHelper(dummy awssmpGetter) *Source { return &Source{ Alias: "foo", @@ -45,25 +62,29 @@ func simpleAWSSourceHelper(dummy awssmpGetter) *Source { } func TestAWSSMP_ParseArgsSimple(t *testing.T) { - paramPath, err := parseAWSSMPArgs("noddy") - assert.Equal(t, "noddy", paramPath) + u, _ := url.Parse("noddy") + _, p, err := parseAWSSMPArgs(u) + assert.Equal(t, "noddy", p) assert.Nil(t, err) } func TestAWSSMP_ParseArgsAppend(t *testing.T) { - paramPath, err := parseAWSSMPArgs("base", "extra") - assert.Equal(t, "base/extra", paramPath) + u, _ := url.Parse("base") + _, p, err := parseAWSSMPArgs(u, "extra") + assert.Equal(t, "base/extra", p) assert.Nil(t, err) } func TestAWSSMP_ParseArgsAppend2(t *testing.T) { - paramPath, err := parseAWSSMPArgs("/foo/", "/extra") - assert.Equal(t, "/foo/extra", paramPath) + u, _ := url.Parse("/foo/") + _, p, err := parseAWSSMPArgs(u, "/extra") + assert.Equal(t, "/foo/extra", p) assert.Nil(t, err) } func TestAWSSMP_ParseArgsTooMany(t *testing.T) { - _, err := parseAWSSMPArgs("base", "extra", "too many!") + u, _ := url.Parse("base") + _, _, err := parseAWSSMPArgs(u, "extra", "too many!") assert.Error(t, err) } @@ -117,3 +138,37 @@ func TestAWSSMP_GetParameterMissing(t *testing.T) { _, err := readAWSSMP(s, "") assert.Error(t, err, "Test of error message") } + +func TestAWSSMP_listAWSSMPParams(t *testing.T) { + ctx := context.Background() + s := simpleAWSSourceHelper(DummyParamGetter{ + t: t, + err: awserr.New("ParameterNotFound", "foo", nil), + }) + _, err := listAWSSMPParams(ctx, s, "") + assert.Error(t, err) + + s = simpleAWSSourceHelper(DummyParamGetter{ + t: t, + params: []*ssm.Parameter{ + {Name: aws.String("/a")}, + {Name: aws.String("/b")}, + {Name: aws.String("/c")}, + }, + }) + data, err := listAWSSMPParams(ctx, s, "/") + assert.NoError(t, err) + assert.Equal(t, []byte(`["a","b","c"]`), data) + + s = simpleAWSSourceHelper(DummyParamGetter{ + t: t, + params: []*ssm.Parameter{ + {Name: aws.String("/a/a")}, + {Name: aws.String("/a/b")}, + {Name: aws.String("/a/c")}, + }, + }) + data, err = listAWSSMPParams(ctx, s, "/a/") + assert.NoError(t, err) + assert.Equal(t, []byte(`["a","b","c"]`), data) +} diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 6bf721723..1d1e9778f 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -81,7 +81,8 @@ Currently the following datasources support directory semantics: When accessing a directory datasource, an array of key names is returned, and can be iterated through to access each individual value contained within. - [AWS S3](#using-s3-datasources) - [Google Cloud Storage](#using-google-cloud-storage-gs-datasources) -- [Git](#using-git-datasources) +- [Git](#using-git-datasources) +- [AWS Systems Manager Parameter Store](#using-aws-smp-datasources) For example, a group of configuration key/value pairs (named `one`, `two`, and `three`, with values `v1`, `v2`, and `v3` respectively) could be rendered like this: @@ -153,14 +154,16 @@ The [`github.com/joho/godotenv`](https://github.com/joho/godotenv) package is us The `aws+smp://` scheme can be used to retrieve data from the [AWS Systems Manager](https://aws.amazon.com/systems-manager/) (née AWS EC2 Simple Systems Manager) [Parameter Store](https://aws.amazon.com/systems-manager/features/#Parameter_Store). This hierarchically organized key/value store allows you to store text, lists or encrypted secrets for easy retrieval by AWS resources. See [the AWS Systems Manager documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-su-create.html#sysman-paramstore-su-create-about) for details on creating these parameters. - You must grant `gomplate` permission via IAM credentials for the [`ssm:GetParameter` action](https://docs.aws.amazon.com/systems-manager/latest/userguide/auth-and-access-control-permissions-reference.html). See details on how to configure gomplate's AWS support in [_Configuring AWS_](../functions/aws/#configuring-aws). ### URL Considerations -For `aws+smp`, only the _scheme_ and _path_ components are necessary to be defined. Other URL components are ignored. +The _scheme_ and _path_ URL components are used by this datasource. + +- the _scheme_ must be `aws+smp` +- the _path_ component is used to specify the path to the parameter. [Directory](#directory-datasources) semantics are available when the path ends with a `/` character. ### Output @@ -196,6 +199,12 @@ Bill,Ben $ echo '{{ (ds "foo" "/second/p1").Value }}' | gomplate -d foo=aws+smp:///foo/ aaa + +$ gomplate -d foo=aws+smp:///foo/first/ -i '{{ range (ds "foo") }} +{{ . }}: {{ (ds "foo" .).Value }} +{{- end }}' +others: Bill,Ben +password: super-secret ``` ## Using `aws+sm` datasource