diff --git a/data/datasource.go b/data/datasource.go index 17db245e3..3f5462a28 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -16,8 +16,6 @@ import ( "github.com/pkg/errors" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/blang/vfs" "github.com/hairyhenderson/gomplate/libkv" "github.com/hairyhenderson/gomplate/vault" @@ -83,18 +81,18 @@ func (d *Data) Cleanup() { } // NewData - constructor for Data -func NewData(datasourceArgs []string, headerArgs []string) (*Data, error) { +func NewData(datasourceArgs, headerArgs []string) (*Data, error) { sources := make(map[string]*Source) headers, err := parseHeaderArgs(headerArgs) if err != nil { return nil, err } for _, v := range datasourceArgs { - s, err := ParseSource(v) + s, err := parseSource(v) if err != nil { return nil, errors.Wrapf(err, "error parsing datasource") } - s.Header = headers[s.Alias] + s.header = headers[s.Alias] // pop the header out of the map, so we end up with only the unreferenced ones delete(headers, s.Alias) @@ -106,45 +104,28 @@ func NewData(datasourceArgs []string, headerArgs []string) (*Data, error) { }, nil } -// AWSSMPGetter - A subset of SSM API for use in unit testing -type AWSSMPGetter interface { - GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) -} - // Source - a data source type Source struct { - Alias string - URL *url.URL - Ext string - Type string - Params map[string]string - FS vfs.Filesystem // used for file: URLs, nil otherwise - HC *http.Client // used for http[s]: URLs, nil otherwise - VC *vault.Vault // used for vault: URLs, nil otherwise - KV *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise - ASMPG AWSSMPGetter // used for aws+smp:, nil otherwise - Header http.Header // used for http[s]: URLs, nil otherwise + Alias string + URL *url.URL + mediaType string + fs vfs.Filesystem // used for file: URLs, nil otherwise + hc *http.Client // used for http[s]: URLs, nil otherwise + vc *vault.Vault // used for vault: URLs, nil otherwise + kv *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise + asmpg awssmpGetter // used for aws+smp:, nil otherwise + header http.Header // used for http[s]: URLs, nil otherwise } func (s *Source) cleanup() { - if s.VC != nil { - s.VC.Logout() + if s.vc != nil { + s.vc.Logout() } - if s.KV != nil { - s.KV.Logout() + if s.kv != nil { + s.kv.Logout() } } -// NewSource - builds a &Source -func NewSource(alias string, URL *url.URL) (*Source, error) { - s := &Source{ - Alias: alias, - URL: URL, - } - - return s, nil -} - // mimeType returns the MIME type to use as a hint for parsing the datasource. // It's expected that the datasource will have already been read before // this function is called, and so the Source's Type property may be already set. @@ -155,26 +136,21 @@ func NewSource(alias string, URL *url.URL) (*Source, error) { // 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered // 4. otherwise, the default type of 'text/plain' is used func (s *Source) mimeType() (mimeType string, err error) { - ext := filepath.Ext(s.URL.Path) - // TODO: stop modifying s, also Ext is unused - s.Ext = ext - mediatype := s.URL.Query().Get("type") if mediatype == "" { - mediatype = s.Type + mediatype = s.mediaType } if mediatype == "" { + ext := filepath.Ext(s.URL.Path) mediatype = mime.TypeByExtension(ext) } if mediatype != "" { - t, params, err := mime.ParseMediaType(mediatype) + t, _, err := mime.ParseMediaType(mediatype) if err != nil { return "", err } mediatype = t - // TODO: stop modifying s, also Params is unused - s.Params = params return mediatype, nil } @@ -184,37 +160,34 @@ func (s *Source) mimeType() (mimeType string, err error) { // String is the method to format the flag's value, part of the flag.Value interface. // The String method's output will be used in diagnostics. func (s *Source) String() string { - return fmt.Sprintf("%s=%s (%s)", s.Alias, s.URL.String(), s.Type) + return fmt.Sprintf("%s=%s (%s)", s.Alias, s.URL.String(), s.mediaType) } -// ParseSource - -func ParseSource(value string) (*Source, error) { - var ( - alias string - srcURL *url.URL - err error - ) +// parseSource creates a *Source by parsing the value provided to the +// --datasource/-d commandline flag +func parseSource(value string) (source *Source, err error) { + source = &Source{} parts := strings.SplitN(value, "=", 2) if len(parts) == 1 { f := parts[0] - alias = strings.SplitN(value, ".", 2)[0] + source.Alias = strings.SplitN(value, ".", 2)[0] if path.Base(f) != f { err = errors.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory", value) return nil, err } - srcURL, err = absURL(f) + source.URL, err = absURL(f) if err != nil { return nil, err } } else if len(parts) == 2 { - alias = parts[0] - srcURL, err = parseSourceURL(parts[1]) + source.Alias = parts[0] + source.URL, err = parseSourceURL(parts[1]) if err != nil { return nil, err } } - return &Source{Alias: alias, URL: srcURL}, nil + return source, nil } func parseSourceURL(value string) (*url.URL, error) { @@ -266,7 +239,7 @@ func (d *Data) DefineDatasource(alias, value string) (*Source, error) { s := &Source{ Alias: alias, URL: srcURL, - Header: d.extraHeaders[alias], + header: d.extraHeaders[alias], } if d.Sources == nil { d.Sources = make(map[string]*Source) @@ -291,7 +264,7 @@ func (d *Data) lookupSource(alias string) (*Source, error) { source = &Source{ Alias: alias, URL: srcURL, - Header: d.extraHeaders[alias], + header: d.extraHeaders[alias], } d.Sources[alias] = source } @@ -304,7 +277,7 @@ func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { if err != nil { return nil, err } - b, err := d.ReadSource(source, args...) + b, err := d.readSource(source, args...) if err != nil { return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", alias) } @@ -344,7 +317,7 @@ func (d *Data) DatasourceReachable(alias string, args ...string) bool { if !ok { return false } - _, err := d.ReadSource(source, args...) + _, err := d.readSource(source, args...) return err == nil } @@ -354,15 +327,16 @@ func (d *Data) Include(alias string, args ...string) (string, error) { if !ok { return "", errors.Errorf("Undefined datasource '%s'", alias) } - b, err := d.ReadSource(source, args...) + b, err := d.readSource(source, args...) if err != nil { return "", errors.Wrapf(err, "Couldn't read datasource '%s'", alias) } return string(b), nil } -// ReadSource - -func (d *Data) ReadSource(source *Source, args ...string) ([]byte, error) { +// readSource returns the (possibly cached) data from the given source, +// as referenced by the given args +func (d *Data) readSource(source *Source, args ...string) ([]byte, error) { if d.cache == nil { d.cache = make(map[string][]byte) } @@ -398,15 +372,15 @@ func readStdin(source *Source, args ...string) ([]byte, error) { } func readHTTP(source *Source, args ...string) ([]byte, error) { - if source.HC == nil { - source.HC = &http.Client{Timeout: time.Second * 5} + if source.hc == nil { + source.hc = &http.Client{Timeout: time.Second * 5} } req, err := http.NewRequest("GET", source.URL.String(), nil) if err != nil { return nil, err } - req.Header = source.Header - res, err := source.HC.Do(req) + req.Header = source.header + res, err := source.hc.Do(req) if err != nil { return nil, err } @@ -424,20 +398,19 @@ func readHTTP(source *Source, args ...string) ([]byte, error) { } ctypeHdr := res.Header.Get("Content-Type") if ctypeHdr != "" { - mediatype, params, e := mime.ParseMediaType(ctypeHdr) + mediatype, _, e := mime.ParseMediaType(ctypeHdr) if e != nil { return nil, e } - source.Type = mediatype - source.Params = params + source.mediaType = mediatype } return body, nil } func readConsul(source *Source, args ...string) ([]byte, error) { - if source.KV == nil { - source.KV = libkv.NewConsul(source.URL) - err := source.KV.Login() + if source.kv == nil { + source.kv = libkv.NewConsul(source.URL) + err := source.kv.Login() if err != nil { return nil, err } @@ -448,7 +421,7 @@ func readConsul(source *Source, args ...string) ([]byte, error) { p = p + "/" + args[0] } - data, err := source.KV.Read(p) + data, err := source.kv.Read(p) if err != nil { return nil, err } @@ -457,8 +430,8 @@ func readConsul(source *Source, args ...string) ([]byte, error) { } func readBoltDB(source *Source, args ...string) ([]byte, error) { - if source.KV == nil { - source.KV = libkv.NewBoltDB(source.URL) + if source.kv == nil { + source.kv = libkv.NewBoltDB(source.URL) } if len(args) != 1 { @@ -466,7 +439,7 @@ func readBoltDB(source *Source, args ...string) ([]byte, error) { } p := args[0] - data, err := source.KV.Read(p) + data, err := source.kv.Read(p) if err != nil { return nil, err } diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go index 7002180ea..4fff2b2e3 100644 --- a/data/datasource_awssmp.go +++ b/data/datasource_awssmp.go @@ -10,6 +10,11 @@ import ( gaws "github.com/hairyhenderson/gomplate/aws" ) +// awssmpGetter - A subset of SSM API for use in unit testing +type awssmpGetter interface { + GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) +} + func parseAWSSMPArgs(origPath string, args ...string) (paramPath string, err error) { paramPath = origPath if len(args) >= 1 { @@ -23,8 +28,8 @@ func parseAWSSMPArgs(origPath string, args ...string) (paramPath string, err err } func readAWSSMP(source *Source, args ...string) (output []byte, err error) { - if source.ASMPG == nil { - source.ASMPG = ssm.New(gaws.SDKSession()) + if source.asmpg == nil { + source.asmpg = ssm.New(gaws.SDKSession()) } paramPath, err := parseAWSSMPArgs(source.URL.Path, args...) @@ -32,7 +37,7 @@ func readAWSSMP(source *Source, args ...string) (output []byte, err error) { return nil, err } - source.Type = jsonMimetype + source.mediaType = jsonMimetype return readAWSSMPParam(source, paramPath) } @@ -42,7 +47,7 @@ func readAWSSMPParam(source *Source, paramPath string) ([]byte, error) { WithDecryption: aws.Bool(true), } - response, err := source.ASMPG.GetParameter(input) + response, err := source.asmpg.GetParameter(input) if err != nil { return nil, errors.Wrapf(err, "Error reading aws+smp from AWS using GetParameter with input %v", input) diff --git a/data/datasource_awssmp_test.go b/data/datasource_awssmp_test.go index 7b095e9d4..c0fbb0725 100644 --- a/data/datasource_awssmp_test.go +++ b/data/datasource_awssmp_test.go @@ -35,14 +35,14 @@ func (d DummyParamGetter) GetParameter(input *ssm.GetParameterInput) (*ssm.GetPa }, nil } -func simpleAWSSourceHelper(dummy AWSSMPGetter) *Source { +func simpleAWSSourceHelper(dummy awssmpGetter) *Source { return &Source{ Alias: "foo", URL: &url.URL{ Scheme: "aws+smp", Path: "/foo", }, - ASMPG: dummy, + asmpg: dummy, } } @@ -106,7 +106,7 @@ func TestAWSSMP_GetParameterValidOutput(t *testing.T) { err = json.Unmarshal(output, &actual) assert.Nil(t, err) assert.Equal(t, expected, actual) - assert.Equal(t, jsonMimetype, s.Type) + assert.Equal(t, jsonMimetype, s.mediaType) } func TestAWSSMP_GetParameterMissing(t *testing.T) { diff --git a/data/datasource_file.go b/data/datasource_file.go index e66615c07..02c9cc816 100644 --- a/data/datasource_file.go +++ b/data/datasource_file.go @@ -15,8 +15,8 @@ import ( ) func readFile(source *Source, args ...string) ([]byte, error) { - if source.FS == nil { - source.FS = vfs.OS() + if source.fs == nil { + source.fs = vfs.OS() } p := filepath.FromSlash(source.URL.Path) @@ -33,20 +33,20 @@ func readFile(source *Source, args ...string) ([]byte, error) { } // make sure we can access the file - i, err := source.FS.Stat(p) + i, err := source.fs.Stat(p) if err != nil { return nil, errors.Wrapf(err, "Can't stat %s", p) } if strings.HasSuffix(p, "/") { - source.Type = jsonArrayMimetype + source.mediaType = jsonArrayMimetype if i.IsDir() { return readFileDir(source, p) } return nil, errors.Errorf("%s is not a directory", p) } - f, err := source.FS.OpenFile(p, os.O_RDONLY, 0) + f, err := source.fs.OpenFile(p, os.O_RDONLY, 0) if err != nil { return nil, errors.Wrapf(err, "Can't open %s", p) } @@ -59,7 +59,7 @@ func readFile(source *Source, args ...string) ([]byte, error) { } func readFileDir(source *Source, p string) ([]byte, error) { - names, err := source.FS.ReadDir(p) + names, err := source.fs.ReadDir(p) if err != nil { return nil, err } diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go index bfbaaedb8..5bc773969 100644 --- a/data/datasource_file_test.go +++ b/data/datasource_file_test.go @@ -25,25 +25,25 @@ func TestReadFile(t *testing.T) { _, _ = vfs.Create(fs, "/tmp/partial/baz.txt") source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")} - source.FS = fs + source.fs = fs actual, err := readFile(source) assert.NoError(t, err) assert.Equal(t, content, actual) source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")} - source.FS = fs + source.fs = fs _, err = readFile(source) assert.Error(t, err) source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")} - source.FS = fs + source.fs = fs actual, err = readFile(source, "foo.txt") assert.NoError(t, err) assert.Equal(t, content, actual) source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")} - source.FS = fs + source.fs = fs actual, err = readFile(source) assert.NoError(t, err) assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) diff --git a/data/datasource_test.go b/data/datasource_test.go index 7b24a36ac..d975d942a 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -28,46 +28,46 @@ func TestNewData(t *testing.T) { d, err = NewData([]string{"foo=http:///foo.json"}, []string{}) assert.NoError(t, err) assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Empty(t, d.Sources["foo"].Header) + assert.Empty(t, d.Sources["foo"].header) d, err = NewData([]string{"foo=http:///foo.json"}, []string{"bar=Accept: blah"}) assert.NoError(t, err) assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Empty(t, d.Sources["foo"].Header) + assert.Empty(t, d.Sources["foo"].header) d, err = NewData([]string{"foo=http:///foo.json"}, []string{"foo=Accept: blah"}) assert.NoError(t, err) assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Equal(t, "blah", d.Sources["foo"].Header["Accept"][0]) + assert.Equal(t, "blah", d.Sources["foo"].header["Accept"][0]) } func TestParseSourceNoAlias(t *testing.T) { - s, err := ParseSource("foo.json") + s, err := parseSource("foo.json") assert.NoError(t, err) assert.Equal(t, "foo", s.Alias) - _, err = ParseSource("../foo.json") + _, err = parseSource("../foo.json") assert.Error(t, err) - _, err = ParseSource("ftp://example.com/foo.yml") + _, err = parseSource("ftp://example.com/foo.yml") assert.Error(t, err) } func TestParseSourceWithAlias(t *testing.T) { - s, err := ParseSource("data=foo.json") + s, err := parseSource("data=foo.json") assert.NoError(t, err) assert.Equal(t, "data", s.Alias) assert.Equal(t, "file", s.URL.Scheme) assert.True(t, s.URL.IsAbs()) - s, err = ParseSource("data=/otherdir/foo.json") + s, err = parseSource("data=/otherdir/foo.json") assert.NoError(t, err) assert.Equal(t, "data", s.Alias) assert.Equal(t, "file", s.URL.Scheme) assert.True(t, s.URL.IsAbs()) assert.Equal(t, "/otherdir/foo.json", s.URL.Path) - s, err = ParseSource("data=sftp://example.com/blahblah/foo.json") + s, err = parseSource("data=sftp://example.com/blahblah/foo.json") assert.NoError(t, err) assert.Equal(t, "data", s.Alias) assert.Equal(t, "sftp", s.URL.Scheme) @@ -85,11 +85,10 @@ func TestDatasource(t *testing.T) { sources := map[string]*Source{ "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, - Ext: ext, - Type: mime, - FS: fs, + Alias: "foo", + URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, + mediaType: mime, + fs: fs, }, } return &Data{Sources: sources} @@ -123,16 +122,15 @@ func TestDatasourceReachable(t *testing.T) { sources := map[string]*Source{ "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, - Ext: "json", - Type: jsonMimetype, - FS: fs, + Alias: "foo", + URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, + mediaType: jsonMimetype, + fs: fs, }, "bar": { Alias: "bar", URL: &url.URL{Scheme: "file", Path: "/bogus"}, - FS: fs, + fs: fs, }, } data := &Data{Sources: sources} @@ -186,7 +184,7 @@ func TestHTTPFile(t *testing.T) { Host: "example.com", Path: "/foo", }, - HC: client, + hc: client, } data := &Data{ Sources: sources, @@ -217,8 +215,8 @@ func TestHTTPFileWithHeaders(t *testing.T) { Host: "example.com", Path: "/foo", }, - HC: client, - Header: http.Header{ + hc: client, + header: http.Header{ "Foo": {"bar"}, "foo": {"baz"}, "User-Agent": {}, @@ -305,11 +303,10 @@ func TestInclude(t *testing.T) { sources := map[string]*Source{ "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, - Ext: ext, - Type: textMimetype, - FS: fs, + Alias: "foo", + URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, + mediaType: textMimetype, + fs: fs, }, } data := &Data{ @@ -393,17 +390,17 @@ func TestMimeType(t *testing.T) { assert.NoError(t, err) assert.Equal(t, jsonMimetype, mt) - s = &Source{URL: mustParseURL("http://example.com/foo.json"), Type: "text/foo"} + s = &Source{URL: mustParseURL("http://example.com/foo.json"), mediaType: "text/foo"} mt, err = s.mimeType() assert.NoError(t, err) assert.Equal(t, "text/foo", mt) - s = &Source{URL: mustParseURL("http://example.com/foo.json"), Type: "text/foo"} + s = &Source{URL: mustParseURL("http://example.com/foo.json"), mediaType: "text/foo"} mt, err = s.mimeType() assert.NoError(t, err) assert.Equal(t, "text/foo", mt) - s = &Source{URL: mustParseURL("http://example.com/foo.json?type=application/yaml"), Type: "text/foo"} + s = &Source{URL: mustParseURL("http://example.com/foo.json?type=application/yaml"), mediaType: "text/foo"} mt, err = s.mimeType() assert.NoError(t, err) assert.Equal(t, "application/yaml", mt) diff --git a/data/datasource_vault.go b/data/datasource_vault.go index 46cd43c08..de497b363 100644 --- a/data/datasource_vault.go +++ b/data/datasource_vault.go @@ -34,9 +34,9 @@ func parseVaultParams(sourceURL *url.URL, args []string) (params map[string]inte } func readVault(source *Source, args ...string) ([]byte, error) { - if source.VC == nil { - source.VC = vault.New(source.URL) - source.VC.Login() + if source.vc == nil { + source.vc = vault.New(source.URL) + source.vc.Login() } params, p, err := parseVaultParams(source.URL, args) @@ -46,14 +46,14 @@ func readVault(source *Source, args ...string) ([]byte, error) { var data []byte - source.Type = jsonMimetype + source.mediaType = jsonMimetype if len(params) > 0 { - data, err = source.VC.Write(p, params) + data, err = source.vc.Write(p, params) } else if strings.HasSuffix(p, "/") { - source.Type = jsonArrayMimetype - data, err = source.VC.List(p) + source.mediaType = jsonArrayMimetype + data, err = source.vc.List(p) } else { - data, err = source.VC.Read(p) + data, err = source.vc.Read(p) } if err != nil { return nil, err diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go index 368d0ef3e..6bb5ce83e 100644 --- a/data/datasource_vault_test.go +++ b/data/datasource_vault_test.go @@ -14,11 +14,10 @@ func TestReadVault(t *testing.T) { defer server.Close() source := &Source{ - Alias: "foo", - URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, - Ext: "", - Type: textMimetype, - VC: v, + Alias: "foo", + URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, + mediaType: textMimetype, + vc: v, } r, err := readVault(source) @@ -39,7 +38,7 @@ func TestReadVault(t *testing.T) { assert.Equal(t, []byte(expected), r) expected = "[\"one\",\"two\"]\n" - server, source.VC = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) + server, source.vc = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) defer server.Close() source.URL, _ = url.Parse("vault:///secret/foo/") r, err = readVault(source)