Add directory datasource support to aws+smp

Signed-off-by: Dave Henderson <[email protected]>

data/datasource_awssmp.go

@@ -1,9 +1,13 @@
 package data
 
 import (
+	"context"
+	"net/url"
 	"path"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/pkg/errors"
 
@@ -13,32 +17,59 @@ import (
 // awssmpGetter - A subset of SSM API for use in unit testing
 type awssmpGetter interface {
 	GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error)
+	GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error)
+	GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error)
 }
 
-func parseAWSSMPArgs(origPath string, args ...string) (paramPath string, err error) {
-	paramPath = origPath
-	if len(args) >= 1 {
-		paramPath = path.Join(paramPath, args[0])
-	}
-
+func parseAWSSMPArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) {
 	if len(args) >= 2 {
 		err = errors.New("Maximum two arguments to aws+smp datasource: alias, extraPath")
+		return nil, "", err
+	}
+
+	p = sourceURL.Path
+	params = make(map[string]interface{})
+	for key, val := range sourceURL.Query() {
+		params[key] = strings.Join(val, " ")
+	}
+
+	if len(args) == 1 {
+		parsed, err := url.Parse(args[0])
+		if err != nil {
+			return nil, "", err
+		}
+
+		if parsed.Path != "" {
+			p = path.Join(p, parsed.Path)
+		}
+
+		for key, val := range parsed.Query() {
+			params[key] = strings.Join(val, " ")
+		}
 	}
-	return
+	return params, p, err
 }
 
-func readAWSSMP(source *Source, args ...string) (output []byte, err error) {
+func readAWSSMP(source *Source, args ...string) (data []byte, err error) {
+	ctx := context.TODO()
 	if source.asmpg == nil {
 		source.asmpg = ssm.New(gaws.SDKSession())
 	}
 
-	paramPath, err := parseAWSSMPArgs(source.URL.Path, args...)
+	_, paramPath, err := parseAWSSMPArgs(source.URL, args...)
 	if err != nil {
 		return nil, err
 	}
 
 	source.mediaType = jsonMimetype
-	return readAWSSMPParam(source, paramPath)
+	switch {
+	case strings.HasSuffix(paramPath, "/"):
+		source.mediaType = jsonArrayMimetype
+		data, err = listAWSSMPParams(ctx, source, paramPath)
+	default:
+		data, err = readAWSSMPParam(source, paramPath)
+	}
+	return data, err
 }
 
 func readAWSSMPParam(source *Source, paramPath string) ([]byte, error) {
@@ -58,3 +89,23 @@ func readAWSSMPParam(source *Source, paramPath string) ([]byte, error) {
 	output, err := ToJSON(result)
 	return []byte(output), err
 }
+
+// listAWSSMPParams - supports directory semantics, returns array
+func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([]byte, error) {
+	input := &ssm.GetParametersByPathInput{
+		Path: aws.String(paramPath),
+	}
+
+	response, err := source.asmpg.GetParametersByPathWithContext(ctx, input)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error reading aws+smp from AWS using GetParameter with input %v", input)
+	}
+
+	listing := make([]string, len(response.Parameters))
+	for i, p := range response.Parameters {
+		listing[i] = (*p.Name)[len(paramPath):]
+	}
+
+	output, err := ToJSON(listing)
+	return []byte(output), err
+}

data/datasource_awssmp_test.go

@@ -1,12 +1,14 @@
 package data
 
 import (
+	"context"
 	"encoding/json"
 	"net/url"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/stretchr/testify/assert"
 )
@@ -15,11 +17,16 @@ import (
 type DummyParamGetter struct {
 	t                *testing.T
 	param            *ssm.Parameter
+	params           []*ssm.Parameter
 	err              awserr.Error
 	mockGetParameter func(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error)
 }
 
 func (d DummyParamGetter) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) {
+	return d.GetParameterWithContext(context.Background(), input)
+}
+
+func (d DummyParamGetter) GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error) {
 	if d.mockGetParameter != nil {
 		output, err := d.mockGetParameter(input)
 		return output, err
@@ -33,6 +40,16 @@ func (d DummyParamGetter) GetParameter(input *ssm.GetParameterInput) (*ssm.GetPa
 	}, nil
 }
 
+func (d DummyParamGetter) GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) {
+	if d.err != nil {
+		return nil, d.err
+	}
+	assert.NotNil(d.t, d.params, "Must provide a param if no error!")
+	return &ssm.GetParametersByPathOutput{
+		Parameters: d.params,
+	}, nil
+}
+
 func simpleAWSSourceHelper(dummy awssmpGetter) *Source {
 	return &Source{
 		Alias: "foo",
@@ -45,25 +62,29 @@ func simpleAWSSourceHelper(dummy awssmpGetter) *Source {
 }
 
 func TestAWSSMP_ParseArgsSimple(t *testing.T) {
-	paramPath, err := parseAWSSMPArgs("noddy")
-	assert.Equal(t, "noddy", paramPath)
+	u, _ := url.Parse("noddy")
+	_, p, err := parseAWSSMPArgs(u)
+	assert.Equal(t, "noddy", p)
 	assert.Nil(t, err)
 }
 
 func TestAWSSMP_ParseArgsAppend(t *testing.T) {
-	paramPath, err := parseAWSSMPArgs("base", "extra")
-	assert.Equal(t, "base/extra", paramPath)
+	u, _ := url.Parse("base")
+	_, p, err := parseAWSSMPArgs(u, "extra")
+	assert.Equal(t, "base/extra", p)
 	assert.Nil(t, err)
 }
 
 func TestAWSSMP_ParseArgsAppend2(t *testing.T) {
-	paramPath, err := parseAWSSMPArgs("/foo/", "/extra")
-	assert.Equal(t, "/foo/extra", paramPath)
+	u, _ := url.Parse("/foo/")
+	_, p, err := parseAWSSMPArgs(u, "/extra")
+	assert.Equal(t, "/foo/extra", p)
 	assert.Nil(t, err)
 }
 
 func TestAWSSMP_ParseArgsTooMany(t *testing.T) {
-	_, err := parseAWSSMPArgs("base", "extra", "too many!")
+	u, _ := url.Parse("base")
+	_, _, err := parseAWSSMPArgs(u, "extra", "too many!")
 	assert.Error(t, err)
 }
 
@@ -117,3 +138,37 @@ func TestAWSSMP_GetParameterMissing(t *testing.T) {
 	_, err := readAWSSMP(s, "")
 	assert.Error(t, err, "Test of error message")
 }
+
+func TestAWSSMP_listAWSSMPParams(t *testing.T) {
+	ctx := context.Background()
+	s := simpleAWSSourceHelper(DummyParamGetter{
+		t:   t,
+		err: awserr.New("ParameterNotFound", "foo", nil),
+	})
+	_, err := listAWSSMPParams(ctx, s, "")
+	assert.Error(t, err)
+
+	s = simpleAWSSourceHelper(DummyParamGetter{
+		t: t,
+		params: []*ssm.Parameter{
+			{Name: aws.String("/a")},
+			{Name: aws.String("/b")},
+			{Name: aws.String("/c")},
+		},
+	})
+	data, err := listAWSSMPParams(ctx, s, "/")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(`["a","b","c"]`), data)
+
+	s = simpleAWSSourceHelper(DummyParamGetter{
+		t: t,
+		params: []*ssm.Parameter{
+			{Name: aws.String("/a/a")},
+			{Name: aws.String("/a/b")},
+			{Name: aws.String("/a/c")},
+		},
+	})
+	data, err = listAWSSMPParams(ctx, s, "/a/")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(`["a","b","c"]`), data)
+}

docs/content/datasources.md

@@ -81,7 +81,8 @@ Currently the following datasources support directory semantics:
 When accessing a directory datasource, an array of key names is returned, and can be iterated through to access each individual value contained within.
 - [AWS S3](#using-s3-datasources)
 - [Google Cloud Storage](#using-google-cloud-storage-gs-datasources)
-- [Git](#using-git-datasources)
+- [Git](#using-git-datasources) 
+- [AWS Systems Manager Parameter Store](#using-aws-smp-datasources)
 
 For example, a group of configuration key/value pairs (named `one`, `two`, and `three`, with values `v1`, `v2`, and `v3` respectively) could be rendered like this: 
 
@@ -153,14 +154,16 @@ The [`github.com/joho/godotenv`](https://github.com/joho/godotenv) package is us
 
 The `aws+smp://` scheme can be used to retrieve data from the [AWS Systems Manager](https://aws.amazon.com/systems-manager/) (née AWS EC2 Simple Systems Manager) [Parameter Store](https://aws.amazon.com/systems-manager/features/#Parameter_Store). This hierarchically organized key/value store allows you to store text, lists or encrypted secrets for easy retrieval by AWS resources. See [the AWS Systems Manager documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-su-create.html#sysman-paramstore-su-create-about) for details on creating these parameters.
 
-
 You must grant `gomplate` permission via IAM credentials for the [`ssm:GetParameter` action](https://docs.aws.amazon.com/systems-manager/latest/userguide/auth-and-access-control-permissions-reference.html). <!-- List support further requires `ssm.GetParameters` permissions. -->
 
 See details on how to configure gomplate's AWS support in [_Configuring AWS_](../functions/aws/#configuring-aws).
 
 ### URL Considerations
 
-For `aws+smp`, only the _scheme_ and _path_ components are necessary to be defined. Other URL components are ignored.
+The _scheme_ and _path_ URL components are used by this datasource.
+
+- the _scheme_ must be `aws+smp`
+- the _path_ component is used to specify the path to the parameter. [Directory](#directory-datasources) semantics are available when the path ends with a `/` character.
 
 ### Output
 
@@ -196,6 +199,12 @@ Bill,Ben
 
 $ echo '{{ (ds "foo" "/second/p1").Value }}' | gomplate -d foo=aws+smp:///foo/
 aaa
+
+$ gomplate -d foo=aws+smp:///foo/first/ -i '{{ range (ds "foo") }}
+{{ . }}: {{ (ds "foo" .).Value }}
+{{- end }}'
+others: Bill,Ben
+password: super-secret
 ```
 
 ## Using `aws+sm` datasource