From 8bcc8d006bb91fd690d270675034ea3d04f71fcf Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Tue, 4 Jan 2022 16:34:50 -0500 Subject: [PATCH] Use go-fsimpl to read from datasources Signed-off-by: Dave Henderson --- context_test.go | 6 + data/datasource.go | 310 ++++++----- data/datasource_aws_sm.go | 87 ---- data/datasource_aws_sm_test.go | 176 ------- data/datasource_awssmp.go | 40 ++ data/datasource_blob.go | 170 ------ data/datasource_blob_test.go | 135 ----- data/datasource_env.go | 19 - data/datasource_env_test.go | 55 -- data/datasource_file.go | 85 --- data/datasource_file_test.go | 70 --- data/datasource_git.go | 328 ------------ data/datasource_git_test.go | 486 ------------------ data/datasource_http.go | 63 --- data/datasource_http_test.go | 149 ------ data/datasource_merge.go | 30 +- data/datasource_merge_test.go | 45 +- data/datasource_test.go | 290 +++++------ data/datasource_vault.go | 48 -- data/datasource_vault_test.go | 50 -- data/mimetypes.go | 122 +++++ data/mimetypes_test.go | 108 +++- go.mod | 28 +- go.sum | 146 +++++- internal/datafs/envfs.go | 210 ++++++++ internal/datafs/envfs_test.go | 105 ++++ .../integration/datasources_vault_test.go | 5 +- 27 files changed, 1117 insertions(+), 2249 deletions(-) delete mode 100644 data/datasource_aws_sm.go delete mode 100644 data/datasource_aws_sm_test.go delete mode 100644 data/datasource_blob.go delete mode 100644 data/datasource_blob_test.go delete mode 100644 data/datasource_env.go delete mode 100644 data/datasource_env_test.go delete mode 100644 data/datasource_file.go delete mode 100644 data/datasource_file_test.go delete mode 100644 data/datasource_git.go delete mode 100644 data/datasource_git_test.go delete mode 100644 data/datasource_http.go delete mode 100644 data/datasource_http_test.go delete mode 100644 data/datasource_vault.go delete mode 100644 data/datasource_vault_test.go create mode 100644 internal/datafs/envfs.go create mode 100644 internal/datafs/envfs_test.go diff --git a/context_test.go b/context_test.go index 27cf7d94c..172591e1c 100644 --- a/context_test.go +++ b/context_test.go @@ -6,8 +6,10 @@ import ( "os" "testing" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/data" "github.com/hairyhenderson/gomplate/v3/internal/config" + "github.com/hairyhenderson/gomplate/v3/internal/datafs" "github.com/stretchr/testify/assert" ) @@ -31,6 +33,9 @@ func TestCreateContext(t *testing.T) { assert.NoError(t, err) assert.Empty(t, c) + fsmux := fsimpl.NewMux() + fsmux.Add(datafs.EnvFS) + fooURL := "env:///foo?type=application/yaml" barURL := "env:///bar?type=application/yaml" uf, _ := url.Parse(fooURL) @@ -40,6 +45,7 @@ func TestCreateContext(t *testing.T) { "foo": {URL: uf}, ".": {URL: ub}, }, + FSMux: fsmux, } os.Setenv("foo", "foo: bar") defer os.Unsetenv("foo") diff --git a/data/datasource.go b/data/datasource.go index 79c29e5fd..9f6e166ed 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -2,21 +2,28 @@ package data import ( "context" + "encoding/json" "fmt" + "io/fs" + "io/ioutil" "mime" "net/http" "net/url" - "path/filepath" "sort" "strings" - "github.com/spf13/afero" - "github.com/pkg/errors" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/awssmfs" + "github.com/hairyhenderson/go-fsimpl/blobfs" + "github.com/hairyhenderson/go-fsimpl/filefs" + "github.com/hairyhenderson/go-fsimpl/gitfs" + "github.com/hairyhenderson/go-fsimpl/httpfs" + "github.com/hairyhenderson/go-fsimpl/vaultfs" "github.com/hairyhenderson/gomplate/v3/internal/config" + "github.com/hairyhenderson/gomplate/v3/internal/datafs" "github.com/hairyhenderson/gomplate/v3/libkv" - "github.com/hairyhenderson/gomplate/v3/vault" ) func regExtension(ext, typ string) { @@ -41,27 +48,12 @@ func (d *Data) registerReaders() { d.sourceReaders = make(map[string]func(context.Context, *Source, ...string) ([]byte, error)) d.sourceReaders["aws+smp"] = readAWSSMP - d.sourceReaders["aws+sm"] = readAWSSecretsManager d.sourceReaders["boltdb"] = readBoltDB d.sourceReaders["consul"] = readConsul d.sourceReaders["consul+http"] = readConsul d.sourceReaders["consul+https"] = readConsul - d.sourceReaders["env"] = readEnv - d.sourceReaders["file"] = readFile - d.sourceReaders["http"] = readHTTP - d.sourceReaders["https"] = readHTTP d.sourceReaders["merge"] = d.readMerge d.sourceReaders["stdin"] = readStdin - d.sourceReaders["vault"] = readVault - d.sourceReaders["vault+http"] = readVault - d.sourceReaders["vault+https"] = readVault - d.sourceReaders["s3"] = readBlob - d.sourceReaders["gs"] = readBlob - d.sourceReaders["git"] = readGit - d.sourceReaders["git+file"] = readGit - d.sourceReaders["git+http"] = readGit - d.sourceReaders["git+https"] = readGit - d.sourceReaders["git+ssh"] = readGit } // lookupReader - return the reader function for the given scheme @@ -83,10 +75,17 @@ type Data struct { Sources map[string]*Source sourceReaders map[string]func(context.Context, *Source, ...string) ([]byte, error) - cache map[string][]byte + cache map[string]*fileContent // headers from the --datasource-header/-H option that don't reference datasources from the commandline extraHeaders map[string]http.Header + + FSMux fsimpl.FSMux +} + +type fileContent struct { + b []byte + contentType string } // Cleanup - clean up datasources before shutting the process down - things @@ -139,91 +138,25 @@ func FromConfig(ctx context.Context, cfg *config.Config) *Data { // Source - a data source type Source struct { - URL *url.URL - fs afero.Fs // used for file: URLs, nil otherwise - hc *http.Client // used for http[s]: URLs, nil otherwise - vc *vault.Vault // used for vault: URLs, nil otherwise - kv *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise - asmpg awssmpGetter // used for aws+smp:, nil otherwise - awsSecretsManager awsSecretsManagerGetter // used for aws+sm, nil otherwise - header http.Header // used for http[s]: URLs, nil otherwise - Alias string - mediaType string + URL *url.URL + kv *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise + asmpg awssmpGetter // used for aws+smp:, nil otherwise + header http.Header // used for http[s]: URLs, nil otherwise + Alias string + mediaType string } func (s *Source) inherit(parent *Source) { - s.fs = parent.fs - s.hc = parent.hc - s.vc = parent.vc s.kv = parent.kv s.asmpg = parent.asmpg } func (s *Source) cleanup() { - if s.vc != nil { - s.vc.Logout() - } if s.kv != nil { s.kv.Logout() } } -// mimeType returns the MIME type to use as a hint for parsing the datasource. -// It's expected that the datasource will have already been read before -// this function is called, and so the Source's Type property may be already set. -// -// The MIME type is determined by these rules: -// 1. the 'type' URL query parameter is used if present -// 2. otherwise, the Type property on the Source is used, if present -// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered -// 4. otherwise, the default type of 'text/plain' is used -func (s *Source) mimeType(arg string) (mimeType string, err error) { - if len(arg) > 0 { - if strings.HasPrefix(arg, "//") { - arg = arg[1:] - } - if !strings.HasPrefix(arg, "/") { - arg = "/" + arg - } - } - argURL, err := url.Parse(arg) - if err != nil { - return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) - } - mediatype := argURL.Query().Get("type") - if mediatype == "" { - mediatype = s.URL.Query().Get("type") - } - - if mediatype == "" { - mediatype = s.mediaType - } - - // make it so + doesn't need to be escaped - mediatype = strings.ReplaceAll(mediatype, " ", "+") - - if mediatype == "" { - ext := filepath.Ext(argURL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype == "" { - ext := filepath.Ext(s.URL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype != "" { - t, _, err := mime.ParseMediaType(mediatype) - if err != nil { - return "", errors.Wrapf(err, "MIME type was %q", mediatype) - } - mediatype = t - return mediatype, nil - } - - return textMimetype, nil -} - // String is the method to format the flag's value, part of the flag.Value interface. // The String method's output will be used in diagnostics. func (s *Source) String() string { @@ -280,41 +213,33 @@ func (d *Data) lookupSource(alias string) (*Source, error) { return source, nil } -func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (data, mimeType string, err error) { +func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (*fileContent, error) { source, err := d.lookupSource(alias) if err != nil { - return "", "", err + return nil, err } - b, err := d.readSource(ctx, source, args...) + fc, err := d.readSource(ctx, source, args...) if err != nil { - return "", "", errors.Wrapf(err, "Couldn't read datasource '%s'", alias) + return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", alias) } - subpath := "" - if len(args) > 0 { - subpath = args[0] - } - mimeType, err = source.mimeType(subpath) - if err != nil { - return "", "", err - } - return string(b), mimeType, nil + return fc, nil } // Include - func (d *Data) Include(alias string, args ...string) (string, error) { - data, _, err := d.readDataSource(d.ctx, alias, args...) - return data, err + fc, err := d.readDataSource(d.ctx, alias, args...) + return string(fc.b), err } // Datasource - func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { - data, mimeType, err := d.readDataSource(d.ctx, alias, args...) + fc, err := d.readDataSource(d.ctx, alias, args...) if err != nil { return nil, err } - return parseData(mimeType, data) + return parseData(fc.contentType, string(fc.b)) } func parseData(mimeType, s string) (out interface{}, err error) { @@ -360,9 +285,9 @@ func (d *Data) DatasourceReachable(alias string, args ...string) bool { // readSource returns the (possibly cached) data from the given source, // as referenced by the given args -func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ([]byte, error) { +func (d *Data) readSource(ctx context.Context, source *Source, args ...string) (*fileContent, error) { if d.cache == nil { - d.cache = make(map[string][]byte) + d.cache = make(map[string]*fileContent) } cacheKey := source.Alias for _, v := range args { @@ -372,16 +297,107 @@ func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ( if ok { return cached, nil } - r, err := d.lookupReader(source.URL.Scheme) + + var data []byte + + // TODO: initialize this elsewhere? + if d.FSMux == nil { + d.FSMux = fsimpl.NewMux() + d.FSMux.Add(filefs.FS) + d.FSMux.Add(httpfs.FS) + d.FSMux.Add(blobfs.FS) + d.FSMux.Add(gitfs.FS) + d.FSMux.Add(vaultfs.FS) + d.FSMux.Add(awssmfs.FS) + d.FSMux.Add(datafs.EnvFS) + } + + arg := "" + if len(args) > 0 { + arg = args[0] + } + + u, err := resolveURL(source.URL, arg) + if err != nil { + return nil, err + } + + // possible type hint + mimeType := u.Query().Get("type") + + u, fname := splitFSMuxURL(u) + + fsys, err := d.FSMux.Lookup(u.String()) + if err == nil { + f, err := fsys.Open(fname) + if err != nil { + return nil, fmt.Errorf("open (url: %q, name: %q): %w", u, fname, err) + } + + fi, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err) + } + + if mimeType == "" { + mimeType = fsimpl.ContentType(fi) + } + + if fi.IsDir() { + des, err := fs.ReadDir(fsys, fname) + if err != nil { + return nil, fmt.Errorf("readDir (url: %q, name: %s): %w", u, fname, err) + } + + entries := make([]string, len(des)) + for i, e := range des { + entries[i] = e.Name() + } + data, err = json.Marshal(entries) + if err != nil { + return nil, fmt.Errorf("json.Marshal: %w", err) + } + + mimeType = jsonArrayMimetype + } else { + data, err = ioutil.ReadAll(f) + + if err != nil { + return nil, fmt.Errorf("read (url: %q, name: %s): %w", u, fname, err) + } + } + + fc := &fileContent{data, mimeType} + d.cache[cacheKey] = fc + + return fc, nil + } + + // TODO: get rid of this, I guess? + r, err := d.lookupReader(u.Scheme) if err != nil { - return nil, errors.Wrap(err, "Datasource not yet supported") + return nil, fmt.Errorf("lookupReader (url: %q): %w", u, err) } - data, err := r(ctx, source, args...) + data, err = r(ctx, source, args...) if err != nil { return nil, err } - d.cache[cacheKey] = data - return data, nil + + if mimeType == "" { + subpath := "" + if len(args) > 0 { + subpath = args[0] + } + + mimeType, err = source.mimeType(subpath) + if err != nil { + return nil, err + } + } + + fc := &fileContent{data, mimeType} + d.cache[cacheKey] = fc + return fc, nil } // Show all datasources - @@ -393,3 +409,65 @@ func (d *Data) ListDatasources() []string { sort.Strings(datasources) return datasources } + +// resolveURL parses the relative URL rel against base, and returns the +// resolved URL. Differs from url.ResolveReference in that query parameters are +// added. In case of duplicates, params from rel are used. +func resolveURL(base *url.URL, rel string) (*url.URL, error) { + relURL, err := url.Parse(rel) + if err != nil { + return nil, err + } + + out := base.ResolveReference(relURL) + if base.RawQuery != "" { + bq := base.Query() + rq := relURL.Query() + for k := range rq { + bq.Set(k, rq.Get(k)) + } + out.RawQuery = bq.Encode() + } + + return out, nil +} + +// splitFSMuxURL splits a URL into a filesystem URL and a relative file path +func splitFSMuxURL(in *url.URL) (*url.URL, string) { + u := *in + + // base := path.Base(u.Path) + // if path.Dir(u.Path) == path.Clean(u.Path) { + // base = "." + // } + + base := strings.TrimPrefix(u.Path, "/") + + if base == "" && u.Opaque != "" { + base = u.Opaque + u.Opaque = "" + } + + if base == "" { + base = "." + } + + u.Path = "/" + + // handle some env-specific idiosyncrasies + // if u.Scheme == "env" { + // base = in.Path + // base = strings.TrimPrefix(base, "/") + // if base == "" { + // base = in.Opaque + // } + // } + // if u.Scheme == "vault" && !strings.HasSuffix(u.Path, "/") && u.Path != "" { + // u.Path += "/" + // } + // if u.Scheme == "s3" && !strings.HasSuffix(u.Path, "/") && u.Path != "" { + // u.Path += "/" + // } + + return &u, base +} diff --git a/data/datasource_aws_sm.go b/data/datasource_aws_sm.go deleted file mode 100644 index 8fce296ae..000000000 --- a/data/datasource_aws_sm.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "context" - "fmt" - "net/url" - "path" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - - gaws "github.com/hairyhenderson/gomplate/v3/aws" -) - -// awsSecretsManagerGetter - A subset of Secrets Manager API for use in unit testing -type awsSecretsManagerGetter interface { - GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) -} - -func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { - if len(args) >= 2 { - err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", - sourceURL.Scheme, len(args)) - return nil, "", err - } - - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if p == "" && sourceURL.Opaque != "" { - p = sourceURL.Opaque - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = path.Join(p, parsed.Path) - if strings.HasSuffix(parsed.Path, "/") { - p += "/" - } - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, nil -} - -func readAWSSecretsManager(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if source.awsSecretsManager == nil { - source.awsSecretsManager = secretsmanager.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - return readAWSSecretsManagerParam(ctx, source, paramPath) -} - -func readAWSSecretsManagerParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(paramPath), - } - - response, err := source.awsSecretsManager.GetSecretValueWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("reading aws+sm source %q: %w", source.Alias, err) - } - - if response.SecretString != nil { - return []byte(*response.SecretString), nil - } - - return response.SecretBinary, nil -} diff --git a/data/datasource_aws_sm_test.go b/data/datasource_aws_sm_test.go deleted file mode 100644 index c274dcf8c..000000000 --- a/data/datasource_aws_sm_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - "github.com/stretchr/testify/assert" -) - -// DummyAWSSecretsManagerSecretGetter - test double -type DummyAWSSecretsManagerSecretGetter struct { - t *testing.T - secretValut *secretsmanager.GetSecretValueOutput - err awserr.Error - mockGetSecretValue func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) -} - -func (d DummyAWSSecretsManagerSecretGetter) GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) { - if d.mockGetSecretValue != nil { - output, err := d.mockGetSecretValue(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.secretValut, "Must provide a param if no error!") - return d.secretValut, nil -} - -func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+sm", - Path: "/foo", - }, - awsSecretsManager: dummyGetter, - } -} - -func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) { - _, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!") - assert.Error(t, err) - - tplain := map[string]interface{}{"type": "text/plain"} - - data := []struct { - eParams map[string]interface{} - u string - ePath string - args string - }{ - {u: "noddy", ePath: "noddy"}, - {u: "base", ePath: "base/extra", args: "extra"}, - {u: "/foo/", ePath: "/foo/extra", args: "/extra"}, - {u: "aws+sm:///foo", ePath: "/foo/bar", args: "bar"}, - {u: "aws+sm:foo", ePath: "foo"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar"}, - {u: "aws+sm:foo", ePath: "foo/baz", args: "baz"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar/baz", args: "baz"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar/baz", args: "baz"}, - {u: "aws+sm:///foo", ePath: "/foo/dir/", args: "dir/"}, - {u: "aws+sm:///foo/", ePath: "/foo/"}, - {u: "aws+sm:///foo/", ePath: "/foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo?type=text/plain", ePath: "foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo/bar?type=text/plain", ePath: "foo/bar/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:/foo/bar?type=text/plain", ePath: "/foo/bar/baz", args: "baz"}, - { - eParams: map[string]interface{}{ - "type": "application/json", - "param": "quux", - }, - u: "aws+sm:/foo/bar?type=text/plain", - ePath: "/foo/bar/baz/qux", - args: "baz/qux?type=application/json¶m=quux", - }, - } - - for _, d := range data { - args := []string{d.args} - if d.args == "" { - args = nil - } - params, p, err := parseDatasourceURLArgs(mustParseURL(d.u), args...) - assert.NoError(t, err) - if d.eParams == nil { - assert.Empty(t, params) - } else { - assert.EqualValues(t, d.eParams, params) - } - assert.Equal(t, d.ePath, p) - } -} - -func TestAWSSecretsManager_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSecretsManager_GetParameterSetupWrongArgs(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar", "/foo", "/bla") - assert.False(t, calledOk) - assert.Error(t, err) -} - -func TestAWSSecretsManager_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSecretsManager_ReadSecret(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - output, err := readAWSSecretsManagerParam(context.Background(), s, "/foo/bar") - assert.True(t, calledOk) - assert.NoError(t, err) - assert.Equal(t, []byte("blub"), output) -} - -func TestAWSSecretsManager_ReadSecretBinary(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretBinary: []byte("supersecret")}, nil - }, - }) - - output, err := readAWSSecretsManagerParam(context.Background(), s, "/foo/bar") - assert.True(t, calledOk) - assert.NoError(t, err) - assert.Equal(t, []byte("supersecret"), output) -} diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go index d61affe82..3f9a14e5f 100644 --- a/data/datasource_awssmp.go +++ b/data/datasource_awssmp.go @@ -2,6 +2,9 @@ package data import ( "context" + "fmt" + "net/url" + "path" "strings" "github.com/aws/aws-sdk-go/aws" @@ -76,3 +79,40 @@ func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([] output, err := ToJSON(listing) return []byte(output), err } + +func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { + if len(args) >= 2 { + err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", + sourceURL.Scheme, len(args)) + return nil, "", err + } + + p = sourceURL.Path + params = make(map[string]interface{}) + for key, val := range sourceURL.Query() { + params[key] = strings.Join(val, " ") + } + + if p == "" && sourceURL.Opaque != "" { + p = sourceURL.Opaque + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, "", err + } + + if parsed.Path != "" { + p = path.Join(p, parsed.Path) + if strings.HasSuffix(parsed.Path, "/") { + p += "/" + } + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } + } + return params, p, nil +} diff --git a/data/datasource_blob.go b/data/datasource_blob.go deleted file mode 100644 index bb316e7f3..000000000 --- a/data/datasource_blob.go +++ /dev/null @@ -1,170 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "io" - "mime" - "net/url" - "path" - "strings" - - gaws "github.com/hairyhenderson/gomplate/v3/aws" - "github.com/hairyhenderson/gomplate/v3/env" - "github.com/pkg/errors" - - "gocloud.dev/blob" - "gocloud.dev/blob/gcsblob" - "gocloud.dev/blob/s3blob" - "gocloud.dev/gcp" -) - -func readBlob(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if len(args) >= 2 { - return nil, errors.New("maximum two arguments to blob datasource: alias, extraPath") - } - - key := source.URL.Path - if len(args) == 1 { - key = path.Join(key, args[0]) - } - - opener, err := newOpener(ctx, source.URL) - if err != nil { - return nil, err - } - - mux := blob.URLMux{} - mux.RegisterBucket(source.URL.Scheme, opener) - - u := blobURL(source.URL) - bucket, err := mux.OpenBucket(ctx, u) - if err != nil { - return nil, err - } - defer bucket.Close() - - var r func(context.Context, *blob.Bucket, string) (string, []byte, error) - if strings.HasSuffix(key, "/") { - r = listBucket - } else { - r = getBlob - } - - mediaType, data, err := r(ctx, bucket, key) - if mediaType != "" { - source.mediaType = mediaType - } - return data, err -} - -// create the correct kind of blob.BucketURLOpener for the given URL -func newOpener(ctx context.Context, u *url.URL) (opener blob.BucketURLOpener, err error) { - switch u.Scheme { - case "s3": - // set up a "regular" gomplate AWS SDK session - sess := gaws.SDKSession() - // see https://gocloud.dev/concepts/urls/#muxes - opener = &s3blob.URLOpener{ConfigProvider: sess} - case "gs": - creds, err := gcp.DefaultCredentials(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve GCP credentials") - } - - client, err := gcp.NewHTTPClient( - gcp.DefaultTransport(), - gcp.CredentialsTokenSource(creds)) - if err != nil { - return nil, errors.Wrap(err, "failed to create GCP HTTP client") - } - opener = &gcsblob.URLOpener{ - Client: client, - } - } - return opener, nil -} - -func getBlob(ctx context.Context, bucket *blob.Bucket, key string) (mediaType string, data []byte, err error) { - key = strings.TrimPrefix(key, "/") - attr, err := bucket.Attributes(ctx, key) - if err != nil { - return "", nil, errors.Wrapf(err, "failed to retrieve attributes for %s", key) - } - if attr.ContentType != "" { - mt, _, e := mime.ParseMediaType(attr.ContentType) - if e != nil { - return "", nil, e - } - mediaType = mt - } - data, err = bucket.ReadAll(ctx, key) - return mediaType, data, errors.Wrapf(err, "failed to read %s", key) -} - -// calls the bucket listing API, returning a JSON Array -func listBucket(ctx context.Context, bucket *blob.Bucket, path string) (mediaType string, data []byte, err error) { - path = strings.TrimPrefix(path, "/") - opts := &blob.ListOptions{ - Prefix: path, - Delimiter: "/", - } - li := bucket.List(opts) - keys := []string{} - for { - obj, err := li.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return "", nil, err - } - keys = append(keys, strings.TrimPrefix(obj.Key, path)) - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(keys); err != nil { - return "", nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - data = b[:len(b)-1] - return jsonArrayMimetype, data, nil -} - -// copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse -func blobURL(u *url.URL) string { - out := cloneURL(u) - q := out.Query() - - for param := range q { - switch u.Scheme { - case "s3": - switch param { - case "region", "endpoint", "disableSSL", "s3ForcePathStyle": - default: - q.Del(param) - } - case "gs": - switch param { - case "access_id", "private_key_path": - default: - q.Del(param) - } - } - } - - if u.Scheme == "s3" { - // handle AWS_S3_ENDPOINT env var - endpoint := env.Getenv("AWS_S3_ENDPOINT") - if endpoint != "" { - q.Set("endpoint", endpoint) - } - } - - out.RawQuery = q.Encode() - - return out.String() -} diff --git a/data/datasource_blob_test.go b/data/datasource_blob_test.go deleted file mode 100644 index 7763d8e91..000000000 --- a/data/datasource_blob_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package data - -import ( - "bytes" - "context" - "net/http/httptest" - "net/url" - "os" - "testing" - - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - - "github.com/stretchr/testify/assert" -) - -func setupTestBucket(t *testing.T) (*httptest.Server, *url.URL) { - backend := s3mem.New() - faker := gofakes3.New(backend) - ts := httptest.NewServer(faker.Server()) - - err := backend.CreateBucket("mybucket") - assert.NoError(t, err) - c := "hello" - err = putFile(backend, "mybucket", "file1", "text/plain", c) - assert.NoError(t, err) - - c = `{"value": "goodbye world"}` - err = putFile(backend, "mybucket", "file2", "application/json", c) - assert.NoError(t, err) - - c = `value: what a world` - err = putFile(backend, "mybucket", "file3", "application/yaml", c) - assert.NoError(t, err) - - c = `value: out of this world` - err = putFile(backend, "mybucket", "dir1/file1", "application/yaml", c) - assert.NoError(t, err) - - c = `value: foo` - err = putFile(backend, "mybucket", "dir1/file2", "application/yaml", c) - assert.NoError(t, err) - - u, _ := url.Parse(ts.URL) - return ts, u -} - -func putFile(backend gofakes3.Backend, bucket, file, mime, content string) error { - _, err := backend.PutObject( - bucket, - file, - map[string]string{"Content-Type": mime}, - bytes.NewBufferString(content), - int64(len(content)), - ) - return err -} - -func TestReadBlob(t *testing.T) { - _, err := readBlob(context.Background(), nil, "foo", "bar") - assert.Error(t, err) - - ts, u := setupTestBucket(t) - defer ts.Close() - - os.Setenv("AWS_ANON", "true") - defer os.Unsetenv("AWS_ANON") - - d, err := NewData([]string{"-d", "data=s3://mybucket/file1?region=us-east-1&disableSSL=true&s3ForcePathStyle=true&type=text/plain&endpoint=" + u.Host}, nil) - assert.NoError(t, err) - - var expected interface{} - expected = "hello" - out, err := d.Datasource("data") - assert.NoError(t, err) - assert.Equal(t, expected, out) - - os.Unsetenv("AWS_ANON") - - os.Setenv("AWS_ACCESS_KEY_ID", "fake") - os.Setenv("AWS_SECRET_ACCESS_KEY", "fake") - defer os.Unsetenv("AWS_ACCESS_KEY_ID") - defer os.Unsetenv("AWS_SECRET_ACCESS_KEY") - os.Setenv("AWS_S3_ENDPOINT", u.Host) - defer os.Unsetenv("AWS_S3_ENDPOINT") - - d, err = NewData([]string{"-d", "data=s3://mybucket/file2?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = map[string]interface{}{"value": "goodbye world"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.Equal(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = []interface{}{"dir1/", "file1", "file2", "file3"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.EqualValues(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/dir1/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = []interface{}{"file1", "file2"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.EqualValues(t, expected, out) -} - -func TestBlobURL(t *testing.T) { - data := []struct { - in string - expected string - }{ - {"s3://foo/bar/baz", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?type=hello/world", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?region=us-east-1", "s3://foo/bar/baz?region=us-east-1"}, - {"s3://foo/bar/baz?disableSSL=true&type=text/csv", "s3://foo/bar/baz?disableSSL=true"}, - {"s3://foo/bar/baz?type=text/csv&s3ForcePathStyle=true&endpoint=1.2.3.4", "s3://foo/bar/baz?endpoint=1.2.3.4&s3ForcePathStyle=true"}, - {"gs://foo/bar/baz", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?type=foo/bar", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?access_id=123", "gs://foo/bar/baz?access_id=123"}, - {"gs://foo/bar/baz?private_key_path=/foo/bar", "gs://foo/bar/baz?private_key_path=%2Ffoo%2Fbar"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar", "gs://foo/bar/baz?private_key_path=key.json"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar&access_id=abcd", "gs://foo/bar/baz?access_id=abcd&private_key_path=key.json"}, - } - - for _, d := range data { - u, _ := url.Parse(d.in) - out := blobURL(u) - assert.Equal(t, d.expected, out) - } -} diff --git a/data/datasource_env.go b/data/datasource_env.go deleted file mode 100644 index f1e2e5aad..000000000 --- a/data/datasource_env.go +++ /dev/null @@ -1,19 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v3/env" -) - -func readEnv(ctx context.Context, source *Source, args ...string) (b []byte, err error) { - n := source.URL.Path - n = strings.TrimPrefix(n, "/") - if n == "" { - n = source.URL.Opaque - } - - b = []byte(env.Getenv(n)) - return b, nil -} diff --git a/data/datasource_env_test.go b/data/datasource_env_test.go deleted file mode 100644 index c7723b3b2..000000000 --- a/data/datasource_env_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package data - -import ( - "context" - "net/url" - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func mustParseURL(in string) *url.URL { - u, _ := url.Parse(in) - return u -} - -func TestReadEnv(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - os.Setenv("HELLO_WORLD", "hello world") - defer os.Unsetenv("HELLO_WORLD") - os.Setenv("HELLO_UNIVERSE", "hello universe") - defer os.Unsetenv("HELLO_UNIVERSE") - - source := &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD")} - - actual, err := readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:/HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) -} diff --git a/data/datasource_file.go b/data/datasource_file.go deleted file mode 100644 index 7619f0399..000000000 --- a/data/datasource_file.go +++ /dev/null @@ -1,85 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" - - "github.com/pkg/errors" -) - -func readFile(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.fs == nil { - source.fs = afero.NewOsFs() - } - - p := filepath.FromSlash(source.URL.Path) - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = filepath.Join(p, parsed.Path) - } - - // reset the media type - it may have been set by a parent dir read - source.mediaType = "" - } - - // make sure we can access the file - i, err := source.fs.Stat(p) - if err != nil { - return nil, errors.Wrapf(err, "Can't stat %s", p) - } - - if strings.HasSuffix(p, string(filepath.Separator)) { - source.mediaType = jsonArrayMimetype - if i.IsDir() { - return readFileDir(source, p) - } - return nil, errors.Errorf("%s is not a directory", p) - } - - f, err := source.fs.OpenFile(p, os.O_RDONLY, 0) - if err != nil { - return nil, errors.Wrapf(err, "Can't open %s", p) - } - - defer f.Close() - - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, errors.Wrapf(err, "Can't read %s", p) - } - return b, nil -} - -func readFileDir(source *Source, p string) ([]byte, error) { - names, err := afero.ReadDir(source.fs, p) - if err != nil { - return nil, err - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go deleted file mode 100644 index 8f2a7f0a2..000000000 --- a/data/datasource_file_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package data - -import ( - "context" - "testing" - - "github.com/spf13/afero" - - "github.com/stretchr/testify/assert" -) - -func TestReadFile(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - fs := afero.NewMemMapFs() - - _ = fs.Mkdir("/tmp", 0777) - f, _ := fs.Create("/tmp/foo") - _, _ = f.Write(content) - - _ = fs.Mkdir("/tmp/partial", 0777) - f, _ = fs.Create("/tmp/partial/foo.txt") - _, _ = f.Write(content) - _, _ = fs.Create("/tmp/partial/bar.txt") - _, _ = fs.Create("/tmp/partial/baz.txt") - _ = f.Close() - - source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")} - source.fs = fs - - actual, err := readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")} - source.fs = fs - _, err = readFile(ctx, source) - assert.Error(t, err) - - source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")} - source.fs = fs - actual, err = readFile(ctx, source, "foo.txt") - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")} - source.fs = fs - actual, err = readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fs - actual, err = readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - mime, err := source.mimeType("") - assert.NoError(t, err) - assert.Equal(t, "application/json", mime) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fs - actual, err = readFile(ctx, source, "foo.txt") - assert.NoError(t, err) - assert.Equal(t, content, actual) - mime, err = source.mimeType("") - assert.NoError(t, err) - assert.Equal(t, "application/json", mime) -} diff --git a/data/datasource_git.go b/data/datasource_git.go deleted file mode 100644 index c2673dd3c..000000000 --- a/data/datasource_git.go +++ /dev/null @@ -1,328 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/url" - "os" - "path" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v3/base64" - "github.com/hairyhenderson/gomplate/v3/env" - "github.com/rs/zerolog" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/memory" -) - -func readGit(ctx context.Context, source *Source, args ...string) ([]byte, error) { - g := gitsource{} - - u := source.URL - repoURL, path, err := g.parseGitPath(u, args...) - if err != nil { - return nil, err - } - - depth := 1 - if u.Scheme == "git+file" { - // we can't do shallow clones for filesystem repos apparently - depth = 0 - } - - fs, _, err := g.clone(ctx, repoURL, depth) - if err != nil { - return nil, err - } - - mimeType, out, err := g.read(fs, path) - if mimeType != "" { - source.mediaType = mimeType - } - return out, err -} - -type gitsource struct { -} - -func (g gitsource) parseArgURL(arg string) (u *url.URL, err error) { - if strings.HasPrefix(arg, "//") { - u, err = url.Parse(arg[1:]) - u.Path = "/" + u.Path - } else { - u, err = url.Parse(arg) - } - - if err != nil { - return nil, fmt.Errorf("failed to parse arg %s: %w", arg, err) - } - return u, err -} - -func (g gitsource) parseQuery(orig, arg *url.URL) string { - q := orig.Query() - pq := arg.Query() - for k, vs := range pq { - for _, v := range vs { - q.Add(k, v) - } - } - return q.Encode() -} - -func (g gitsource) parseArgPath(u *url.URL, arg string) (repo, p string) { - // if the source URL already specified a repo and subpath, the whole - // arg is interpreted as subpath - if strings.Contains(u.Path, "//") || strings.HasPrefix(arg, "//") { - return "", arg - } - - parts := strings.SplitN(arg, "//", 2) - repo = parts[0] - if len(parts) == 2 { - p = "/" + parts[1] - } - return repo, p -} - -// Massage the URL and args together to produce the repo to clone, -// and the path to read. -// The path is delimited from the repo by '//' -func (g gitsource) parseGitPath(u *url.URL, args ...string) (out *url.URL, p string, err error) { - if u == nil { - return nil, "", fmt.Errorf("parseGitPath: no url provided (%v)", u) - } - // copy the input url so we can modify it - out = cloneURL(u) - - parts := strings.SplitN(out.Path, "//", 2) - switch len(parts) { - case 1: - p = "/" - case 2: - p = "/" + parts[1] - - i := strings.LastIndex(out.Path, p) - out.Path = out.Path[:i-1] - } - - if len(args) > 0 { - argURL, uerr := g.parseArgURL(args[0]) - if uerr != nil { - return nil, "", uerr - } - repo, argpath := g.parseArgPath(u, argURL.Path) - out.Path = path.Join(out.Path, repo) - p = path.Join(p, argpath) - - out.RawQuery = g.parseQuery(u, argURL) - - if argURL.Fragment != "" { - out.Fragment = argURL.Fragment - } - } - return out, p, err -} - -//nolint: interfacer -func cloneURL(u *url.URL) *url.URL { - out, _ := url.Parse(u.String()) - return out -} - -// refFromURL - extract the ref from the URL fragment if present -func (g gitsource) refFromURL(u *url.URL) plumbing.ReferenceName { - switch { - case strings.HasPrefix(u.Fragment, "refs/"): - return plumbing.ReferenceName(u.Fragment) - case u.Fragment != "": - return plumbing.NewBranchReferenceName(u.Fragment) - default: - return plumbing.ReferenceName("") - } -} - -// refFromRemoteHead - extract the ref from the remote HEAD, to work around -// hard-coded 'master' default branch in go-git. -// Should be unnecessary once https://github.com/go-git/go-git/issues/249 is -// fixed. -func (g gitsource) refFromRemoteHead(ctx context.Context, u *url.URL, auth transport.AuthMethod) (plumbing.ReferenceName, error) { - e, err := transport.NewEndpoint(u.String()) - if err != nil { - return "", err - } - - cli, err := client.NewClient(e) - if err != nil { - return "", err - } - - s, err := cli.NewUploadPackSession(e, auth) - if err != nil { - return "", err - } - - info, err := s.AdvertisedReferencesContext(ctx) - if err != nil { - return "", err - } - - refs, err := info.AllReferences() - if err != nil { - return "", err - } - - headRef, ok := refs["HEAD"] - if !ok { - return "", fmt.Errorf("no HEAD ref found") - } - - return headRef.Target(), nil -} - -// clone a repo for later reading through http(s), git, or ssh. u must be the URL to the repo -// itself, and must have any file path stripped -func (g gitsource) clone(ctx context.Context, repoURL *url.URL, depth int) (billy.Filesystem, *git.Repository, error) { - fs := memfs.New() - storer := memory.NewStorage() - - // preserve repoURL by cloning it - u := cloneURL(repoURL) - - auth, err := g.auth(u) - if err != nil { - return nil, nil, err - } - - if strings.HasPrefix(u.Scheme, "git+") { - scheme := u.Scheme[len("git+"):] - u.Scheme = scheme - } - - ref := g.refFromURL(u) - u.Fragment = "" - u.RawQuery = "" - - // attempt to get the ref from the remote so we don't default to master - if ref == "" { - ref, err = g.refFromRemoteHead(ctx, u, auth) - if err != nil { - zerolog.Ctx(ctx).Warn(). - Stringer("repoURL", u). - Err(err). - Msg("failed to get ref from remote, using default") - } - } - - opts := &git.CloneOptions{ - URL: u.String(), - Auth: auth, - Depth: depth, - ReferenceName: ref, - SingleBranch: true, - Tags: git.NoTags, - } - repo, err := git.CloneContext(ctx, storer, fs, opts) - if u.Scheme == "file" && err == transport.ErrRepositoryNotFound && !strings.HasSuffix(u.Path, ".git") { - // maybe this has a `.git` subdirectory... - u = cloneURL(repoURL) - u.Path = path.Join(u.Path, ".git") - return g.clone(ctx, u, depth) - } - if err != nil { - return nil, nil, fmt.Errorf("git clone for %v failed: %w", repoURL, err) - } - return fs, repo, nil -} - -// read - reads the provided path out of a git repo -func (g gitsource) read(fs billy.Filesystem, path string) (string, []byte, error) { - fi, err := fs.Stat(path) - if err != nil { - return "", nil, fmt.Errorf("can't stat %s: %w", path, err) - } - if fi.IsDir() || strings.HasSuffix(path, string(filepath.Separator)) { - out, rerr := g.readDir(fs, path) - return jsonArrayMimetype, out, rerr - } - - f, err := fs.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return "", nil, fmt.Errorf("can't open %s: %w", path, err) - } - - b, err := ioutil.ReadAll(f) - if err != nil { - return "", nil, fmt.Errorf("can't read %s: %w", path, err) - } - - return "", b, nil -} - -func (g gitsource) readDir(fs billy.Filesystem, path string) ([]byte, error) { - names, err := fs.ReadDir(path) - if err != nil { - return nil, fmt.Errorf("couldn't read dir %s: %w", path, err) - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} - -/* -auth methods: -- ssh named key (no password support) - - GIT_SSH_KEY (base64-encoded) or GIT_SSH_KEY_FILE (base64-encoded, or not) -- ssh agent auth (preferred) -- http basic auth (for github, gitlab, bitbucket tokens) -- http token auth (bearer token, somewhat unusual) -*/ -func (g gitsource) auth(u *url.URL) (auth transport.AuthMethod, err error) { - user := u.User.Username() - switch u.Scheme { - case "git+http", "git+https": - if pass, ok := u.User.Password(); ok { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if pass := env.Getenv("GIT_HTTP_PASSWORD"); pass != "" { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if tok := env.Getenv("GIT_HTTP_TOKEN"); tok != "" { - // note docs on TokenAuth - this is rarely to be used - auth = &http.TokenAuth{Token: tok} - } - case "git+ssh": - k := env.Getenv("GIT_SSH_KEY") - if k != "" { - var key []byte - key, err = base64.Decode(k) - if err != nil { - key = []byte(k) - } - auth, err = ssh.NewPublicKeys(user, key, "") - } else { - auth, err = ssh.NewSSHAgentAuth(user) - } - } - return auth, err -} diff --git a/data/datasource_git_test.go b/data/datasource_git_test.go deleted file mode 100644 index 0c20c5583..000000000 --- a/data/datasource_git_test.go +++ /dev/null @@ -1,486 +0,0 @@ -package data - -import ( - "context" - "encoding/base64" - "fmt" - "io/ioutil" - "net/url" - "os" - "strings" - "testing" - "time" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-billy/v5/osfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/server" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/filesystem" - - "golang.org/x/crypto/ssh/testdata" - - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -func TestParseArgPath(t *testing.T) { - t.Parallel() - g := gitsource{} - - data := []struct { - url string - arg string - repo, path string - }{ - {"git+file:///foo//foo", - "/bar", - "", "/bar"}, - {"git+file:///foo//bar", - "/baz//qux", - "", "/baz//qux"}, - {"git+https://example.com/foo", - "/bar", - "/bar", ""}, - {"git+https://example.com/foo", - "//bar", - "", "//bar"}, - {"git+https://example.com/foo//bar", - "//baz", - "", "//baz"}, - {"git+https://example.com/foo", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo?type=t", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo#master", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo", - "//bar", - "", "//bar"}, - {"git+https://example.com/foo?type=t", - "//baz", - "", "//baz"}, - {"git+https://example.com/foo?type=t#v1", - "//bar", - "", "//bar"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.arg, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - repo, path := g.parseArgPath(u, d.arg) - assert.Equal(t, d.repo, repo) - assert.Equal(t, d.path, path) - }) - } -} - -func TestParseGitPath(t *testing.T) { - t.Parallel() - g := gitsource{} - _, _, err := g.parseGitPath(nil) - assert.ErrorContains(t, err, "") - - u := mustParseURL("http://example.com//foo") - assert.Equal(t, "//foo", u.Path) - parts := strings.SplitN(u.Path, "//", 2) - assert.Equal(t, 2, len(parts)) - assert.DeepEqual(t, []string{"", "foo"}, parts) - - data := []struct { - url string - args string - repo, path string - }{ - {"git+https://github.com/hairyhenderson/gomplate//docs-src/content/functions/aws.yml", - "", - "git+https://github.com/hairyhenderson/gomplate", - "/docs-src/content/functions/aws.yml"}, - {"git+ssh://github.com/hairyhenderson/gomplate.git", - "", - "git+ssh://github.com/hairyhenderson/gomplate.git", - "/"}, - {"https://github.com", - "", - "https://github.com", - "/"}, - {"git://example.com/foo//file.txt#someref", - "", - "git://example.com/foo#someref", "/file.txt"}, - {"git+file:///home/foo/repo//file.txt#someref", - "", - "git+file:///home/foo/repo#someref", "/file.txt"}, - {"git+file:///repo", - "", - "git+file:///repo", "/"}, - {"git+file:///foo//foo", - "", - "git+file:///foo", "/foo"}, - {"git+file:///foo//foo", - "/bar", - "git+file:///foo", "/foo/bar"}, - {"git+file:///foo//bar", - // in this case the // is meaningless - "/baz//qux", - "git+file:///foo", "/bar/baz/qux"}, - {"git+https://example.com/foo", - "/bar", - "git+https://example.com/foo/bar", "/"}, - {"git+https://example.com/foo", - "//bar", - "git+https://example.com/foo", "/bar"}, - {"git+https://example.com//foo", - "/bar", - "git+https://example.com", "/foo/bar"}, - {"git+https://example.com/foo//bar", - "//baz", - "git+https://example.com/foo", "/bar/baz"}, - {"git+https://example.com/foo", - "/bar//baz", - "git+https://example.com/foo/bar", "/baz"}, - {"git+https://example.com/foo?type=t", - "/bar//baz", - "git+https://example.com/foo/bar?type=t", "/baz"}, - {"git+https://example.com/foo#master", - "/bar//baz", - "git+https://example.com/foo/bar#master", "/baz"}, - {"git+https://example.com/foo", - "/bar//baz?type=t", - "git+https://example.com/foo/bar?type=t", "/baz"}, - {"git+https://example.com/foo", - "/bar//baz#master", - "git+https://example.com/foo/bar#master", "/baz"}, - {"git+https://example.com/foo", - "//bar?type=t", - "git+https://example.com/foo?type=t", "/bar"}, - {"git+https://example.com/foo", - "//bar#master", - "git+https://example.com/foo#master", "/bar"}, - {"git+https://example.com/foo?type=t", - "//bar#master", - "git+https://example.com/foo?type=t#master", "/bar"}, - {"git+https://example.com/foo?type=t#v1", - "//bar?type=j#v2", - "git+https://example.com/foo?type=t&type=j#v2", "/bar"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.args, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - args := []string{d.args} - if d.args == "" { - args = nil - } - repo, path, err := g.parseGitPath(u, args...) - assert.NilError(t, err) - assert.Equal(t, d.repo, repo.String()) - assert.Equal(t, d.path, path) - }) - } -} - -func TestReadGitRepo(t *testing.T) { - g := gitsource{} - fs := setupGitRepo(t) - fs, err := fs.Chroot("/repo") - assert.NilError(t, err) - - _, _, err = g.read(fs, "/bogus") - assert.ErrorContains(t, err, "can't stat /bogus") - - mtype, out, err := g.read(fs, "/foo") - assert.NilError(t, err) - assert.Equal(t, `["bar"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar") - assert.NilError(t, err) - assert.Equal(t, `["hi.txt"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar/hi.txt") - assert.NilError(t, err) - assert.Equal(t, `hello world`, string(out)) - assert.Equal(t, "", mtype) -} - -var testHashes = map[string]string{} - -func setupGitRepo(t *testing.T) billy.Filesystem { - fs := memfs.New() - fs.MkdirAll("/repo/.git", os.ModeDir) - repo, _ := fs.Chroot("/repo") - dot, _ := repo.Chroot("/.git") - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - - r, err := git.Init(s, repo) - assert.NilError(t, err) - - // default to main - h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) - err = s.SetReference(h) - assert.NilError(t, err) - - // config needs to be created after setting up a "normal" fs repo - // this is possibly a bug in git-go? - c, err := r.Config() - assert.NilError(t, err) - - c.Init.DefaultBranch = "main" - - s.SetConfig(c) - assert.NilError(t, err) - - w, err := r.Worktree() - assert.NilError(t, err) - - repo.MkdirAll("/foo/bar", os.ModeDir) - f, err := repo.Create("/foo/bar/hi.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("hello world")) - assert.NilError(t, err) - _, err = w.Add(f.Name()) - assert.NilError(t, err) - hash, err := w.Commit("initial commit", &git.CommitOptions{Author: &object.Signature{}}) - assert.NilError(t, err) - - ref, err := r.CreateTag("v1", hash, nil) - assert.NilError(t, err) - testHashes["v1"] = hash.String() - - branchName := plumbing.NewBranchReferenceName("mybranch") - err = w.Checkout(&git.CheckoutOptions{ - Branch: branchName, - Hash: ref.Hash(), - Create: true, - }) - assert.NilError(t, err) - - f, err = repo.Create("/secondfile.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("another file\n")) - assert.NilError(t, err) - n := f.Name() - _, err = w.Add(n) - assert.NilError(t, err) - hash, err = w.Commit("second commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - }, - }) - ref = plumbing.NewHashReference(branchName, hash) - assert.NilError(t, err) - testHashes["mybranch"] = ref.Hash().String() - - // make the repo dirty - _, err = f.Write([]byte("dirty file")) - assert.NilError(t, err) - - // set up a bare repo - fs.MkdirAll("/bare.git", os.ModeDir) - fs.MkdirAll("/barewt", os.ModeDir) - repo, _ = fs.Chroot("/barewt") - dot, _ = fs.Chroot("/bare.git") - s = filesystem.NewStorage(dot, nil) - - r, err = git.Init(s, repo) - assert.NilError(t, err) - - w, err = r.Worktree() - assert.NilError(t, err) - - f, err = repo.Create("/hello.txt") - assert.NilError(t, err) - f.Write([]byte("hello world")) - w.Add(f.Name()) - _, err = w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@doe.org", - When: time.Now(), - }, - }) - assert.NilError(t, err) - - return fs -} - -func overrideFSLoader(fs billy.Filesystem) { - l := server.NewFilesystemLoader(fs) - client.InstallProtocol("file", server.NewClient(l)) -} - -func TestOpenFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fs, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0) - assert.NilError(t, err) - - f, err := fs.Open("/foo/bar/hi.txt") - assert.NilError(t, err) - b, _ := ioutil.ReadAll(f) - assert.Equal(t, "hello world", string(b)) - - _, repo, err := g.clone(ctx, mustParseURL("git+file:///repo#main"), 0) - assert.NilError(t, err) - - ref, err := repo.Reference(plumbing.NewBranchReferenceName("main"), true) - assert.NilError(t, err) - assert.Equal(t, "refs/heads/main", ref.Name().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo#refs/tags/v1"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, testHashes["v1"], ref.Hash().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo/#mybranch"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, "refs/heads/mybranch", ref.Name().String()) - assert.Equal(t, testHashes["mybranch"], ref.Hash().String()) -} - -func TestOpenBareFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fs, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0) - assert.NilError(t, err) - - f, err := fs.Open("/hello.txt") - assert.NilError(t, err) - b, _ := ioutil.ReadAll(f) - assert.Equal(t, "hello world", string(b)) -} - -func TestReadGit(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - s := &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git//hello.txt"), - } - b, err := readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "hello world", string(b)) - - s = &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git"), - } - b, err = readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "application/array+json", s.mediaType) - assert.Equal(t, `["hello.txt"]`, string(b)) -} - -func TestGitAuth(t *testing.T) { - g := gitsource{} - a, err := g.auth(mustParseURL("git+file:///bare.git")) - assert.NilError(t, err) - assert.Equal(t, nil, a) - - a, err = g.auth(mustParseURL("git+https://example.com/foo")) - assert.NilError(t, err) - assert.Assert(t, is.Nil(a)) - - a, err = g.auth(mustParseURL("git+https://user:swordfish@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - - os.Setenv("GIT_HTTP_PASSWORD", "swordfish") - defer os.Unsetenv("GIT_HTTP_PASSWORD") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - os.Unsetenv("GIT_HTTP_PASSWORD") - - os.Setenv("GIT_HTTP_TOKEN", "mytoken") - defer os.Unsetenv("GIT_HTTP_TOKEN") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.TokenAuth{Token: "mytoken"}, a) - os.Unsetenv("GIT_HTTP_TOKEN") - - if os.Getenv("SSH_AUTH_SOCK") == "" { - t.Log("no SSH_AUTH_SOCK - skipping ssh agent test") - } else { - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - sa, ok := a.(*ssh.PublicKeysCallback) - assert.Equal(t, true, ok) - assert.Equal(t, "git", sa.User) - } - - key := string(testdata.PEMBytes["ed25519"]) - os.Setenv("GIT_SSH_KEY", key) - defer os.Unsetenv("GIT_SSH_KEY") - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") - - key = base64.StdEncoding.EncodeToString(testdata.PEMBytes["ed25519"]) - os.Setenv("GIT_SSH_KEY", key) - defer os.Unsetenv("GIT_SSH_KEY") - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok = a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") -} - -func TestRefFromURL(t *testing.T) { - t.Parallel() - g := gitsource{} - data := []struct { - url, expected string - }{ - {"git://localhost:1234/foo/bar.git//baz", ""}, - {"git+http://localhost:1234/foo/bar.git//baz", ""}, - {"git+ssh://localhost:1234/foo/bar.git//baz", ""}, - {"git+file:///foo/bar.git//baz", ""}, - {"git://localhost:1234/foo/bar.git//baz#master", "refs/heads/master"}, - {"git+http://localhost:1234/foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - {"git+ssh://localhost:1234/foo/bar.git//baz#refs/tags/foo", "refs/tags/foo"}, - {"git+file:///foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - } - - for _, d := range data { - out := g.refFromURL(mustParseURL(d.url)) - assert.Equal(t, plumbing.ReferenceName(d.expected), out) - } -} diff --git a/data/datasource_http.go b/data/datasource_http.go deleted file mode 100644 index ed1c97b70..000000000 --- a/data/datasource_http.go +++ /dev/null @@ -1,63 +0,0 @@ -package data - -import ( - "context" - "io/ioutil" - "mime" - "net/http" - "net/url" - "time" - - "github.com/pkg/errors" -) - -func buildURL(base *url.URL, args ...string) (*url.URL, error) { - if len(args) == 0 { - return base, nil - } - p, err := url.Parse(args[0]) - if err != nil { - return nil, errors.Wrapf(err, "bad sub-path %s", args[0]) - } - return base.ResolveReference(p), nil -} - -func readHTTP(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.hc == nil { - source.hc = &http.Client{Timeout: time.Second * 5} - } - u, err := buildURL(source.URL, args...) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header = source.header - res, err := source.hc.Do(req) - if err != nil { - return nil, err - } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - err = res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - err := errors.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) - return nil, err - } - ctypeHdr := res.Header.Get("Content-Type") - if ctypeHdr != "" { - mediatype, _, e := mime.ParseMediaType(ctypeHdr) - if e != nil { - return nil, e - } - source.mediaType = mediatype - } - return body, nil -} diff --git a/data/datasource_http_test.go b/data/datasource_http_test.go deleted file mode 100644 index e2c13a8f5..000000000 --- a/data/datasource_http_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func must(r interface{}, err error) interface{} { - if err != nil { - panic(err) - } - return r -} - -func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - w.Header().Set("Content-Type", mimetype) - w.WriteHeader(code) - if body == "" { - // mirror back the headers - fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal))) - } else { - fmt.Fprintln(w, body) - } - })) - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - }, - } - - return server, client -} - -func TestHTTPFile(t *testing.T) { - server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - } - data := &Data{ - ctx: context.Background(), - Sources: sources, - } - - expected := map[string]interface{}{ - "hello": "world", - } - - actual, err := data.Datasource("foo") - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - actual, err = data.Datasource(server.URL) - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestHTTPFileWithHeaders(t *testing.T) { - server, client := setupHTTP(200, jsonMimetype, "") - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - header: http.Header{ - "Foo": {"bar"}, - "foo": {"baz"}, - "User-Agent": {}, - "Accept-Encoding": {"test"}, - }, - } - data := &Data{ - ctx: context.Background(), - Sources: sources, - } - expected := http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - } - actual, err := data.Datasource("foo") - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - expected = http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - "User-Agent": {"Go-http-client/1.1"}, - } - data = &Data{ - ctx: context.Background(), - Sources: sources, - extraHeaders: map[string]http.Header{server.URL: expected}, - } - actual, err = data.Datasource(server.URL) - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestBuildURL(t *testing.T) { - expected := "https://example.com/index.html" - base := mustParseURL(expected) - u, err := buildURL(base) - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/index.html" - base = mustParseURL("https://example.com") - u, err = buildURL(base, "index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/a/b/c/index.html" - base = mustParseURL("https://example.com/a/") - u, err = buildURL(base, "b/c/index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/bar/baz/index.html" - base = mustParseURL("https://example.com/foo") - u, err = buildURL(base, "bar/baz/index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) -} diff --git a/data/datasource_merge.go b/data/datasource_merge.go index 136a3779b..b0beaca19 100644 --- a/data/datasource_merge.go +++ b/data/datasource_merge.go @@ -2,8 +2,12 @@ package data import ( "context" + "fmt" + "io/fs" + "path" "strings" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/coll" "github.com/hairyhenderson/gomplate/v3/internal/config" @@ -24,7 +28,7 @@ func (d *Data) readMerge(ctx context.Context, source *Source, args ...string) ([ opaque := source.URL.Opaque parts := strings.Split(opaque, "|") if len(parts) < 2 { - return nil, errors.New("need at least 2 datasources to merge") + return nil, fmt.Errorf("need at least 2 datasources to merge") } data := make([]map[string]interface{}, len(parts)) for i, part := range parts { @@ -43,16 +47,32 @@ func (d *Data) readMerge(ctx context.Context, source *Source, args ...string) ([ } subSource.inherit(source) - b, err := d.readSource(ctx, subSource) + u := *subSource.URL + + base := path.Base(u.Path) + if base == "/" { + base = "." + } + + u.Path = path.Dir(u.Path) + + fsys, err := d.FSMux.Lookup(u.String()) if err != nil { - return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", part) + return nil, fmt.Errorf("lookup %s: %w", u.String(), err) } - mimeType, err := subSource.mimeType("") + b, err := fs.ReadFile(fsys, base) if err != nil { - return nil, errors.Wrapf(err, "failed to read datasource %s", subSource.URL) + return nil, fmt.Errorf("readFile (fs: %q, name: %q): %w", &u, base, err) } + fi, err := fs.Stat(fsys, base) + if err != nil { + return nil, fmt.Errorf("stat (fs: %q, name: %q): %w", &u, base, err) + } + + mimeType := fsimpl.ContentType(fi) + data[i], err = parseMap(mimeType, string(b)) if err != nil { return nil, err diff --git a/data/datasource_merge_test.go b/data/datasource_merge_test.go index 2d81c98a1..b951253cb 100644 --- a/data/datasource_merge_test.go +++ b/data/datasource_merge_test.go @@ -2,13 +2,15 @@ package data import ( "context" + "io/fs" "net/url" "os" + "path" "path/filepath" "testing" + "testing/fstest" - "github.com/spf13/afero" - + "github.com/hairyhenderson/go-fsimpl" "github.com/stretchr/testify/assert" ) @@ -21,31 +23,27 @@ func TestReadMerge(t *testing.T) { mergedContent := "goodnight: moon\nhello: world\n" - fs := afero.NewMemMapFs() - - _ = fs.Mkdir("/tmp", 0777) - f, _ := fs.Create("/tmp/jsonfile.json") - _, _ = f.WriteString(jsonContent) - f, _ = fs.Create("/tmp/array.json") - _, _ = f.WriteString(arrayContent) - f, _ = fs.Create("/tmp/yamlfile.yaml") - _, _ = f.WriteString(yamlContent) - f, _ = fs.Create("/tmp/textfile.txt") - _, _ = f.WriteString(`plain text...`) + fsys := fstest.MapFS{} + fsys["tmp"] = &fstest.MapFile{Mode: fs.ModeDir | 0777} + fsys["tmp/jsonfile.json"] = &fstest.MapFile{Data: []byte(jsonContent)} + fsys["tmp/array.json"] = &fstest.MapFile{Data: []byte(arrayContent)} + fsys["tmp/yamlfile.yaml"] = &fstest.MapFile{Data: []byte(yamlContent)} + fsys["tmp/textfile.txt"] = &fstest.MapFile{Data: []byte(`plain text...`)} + // workding dir with volume name trimmed wd, _ := os.Getwd() - _ = fs.Mkdir(wd, 0777) - f, _ = fs.Create(filepath.Join(wd, "jsonfile.json")) - _, _ = f.WriteString(jsonContent) - f, _ = fs.Create(filepath.Join(wd, "array.json")) - _, _ = f.WriteString(arrayContent) - f, _ = fs.Create(filepath.Join(wd, "yamlfile.yaml")) - _, _ = f.WriteString(yamlContent) - f, _ = fs.Create(filepath.Join(wd, "textfile.txt")) - _, _ = f.WriteString(`plain text...`) + vol := filepath.VolumeName(wd) + wd = wd[len(vol)+1:] + + fsys[path.Join(wd, "jsonfile.json")] = &fstest.MapFile{Data: []byte(jsonContent)} + fsys[path.Join(wd, "array.json")] = &fstest.MapFile{Data: []byte(arrayContent)} + fsys[path.Join(wd, "yamlfile.yaml")] = &fstest.MapFile{Data: []byte(yamlContent)} + fsys[path.Join(wd, "textfile.txt")] = &fstest.MapFile{Data: []byte(`plain text...`)} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(&fsys, "file")) source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")} - source.fs = fs d := &Data{ Sources: map[string]*Source{ "foo": source, @@ -56,6 +54,7 @@ func TestReadMerge(t *testing.T) { "badtype": {Alias: "badtype", URL: mustParseURL("file:///tmp/textfile.txt?type=foo/bar")}, "array": {Alias: "array", URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(jsonArrayMimetype))}, }, + FSMux: fsmux, } actual, err := d.readMerge(ctx, source) diff --git a/data/datasource_test.go b/data/datasource_test.go index aadcef0ea..beec012b0 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -3,18 +3,22 @@ package data import ( "context" "fmt" + "io/fs" "net/http" "net/url" - "runtime" "testing" + "testing/fstest" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/internal/config" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" ) -const osWindows = "windows" +func mustParseURL(in string) *url.URL { + u, _ := url.Parse(in) + return u +} func TestNewData(t *testing.T) { d, err := NewData(nil, nil) @@ -42,55 +46,50 @@ func TestNewData(t *testing.T) { } func TestDatasource(t *testing.T) { - setup := func(ext, mime string, contents []byte) *Data { + setup := func(t *testing.T, ext string, contents []byte) *Data { + t.Helper() fname := "foo." + ext - fs := afero.NewMemMapFs() - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write(contents) + + fsys := fstest.MapFS{} + fsys["tmp"] = &fstest.MapFile{Mode: fs.ModeDir | 0777} + fsys["tmp/"+fname] = &fstest.MapFile{Data: contents} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, - mediaType: mime, - fs: fs, + Alias: "foo", + URL: mustParseURL("file:///tmp/" + fname), }, } - return &Data{Sources: sources} + return &Data{Sources: sources, FSMux: fsmux} } - test := func(ext, mime string, contents []byte, expected interface{}) { - data := setup(ext, mime, contents) - actual, err := data.Datasource("foo") + test := func(t *testing.T, ext, mime string, contents []byte, expected interface{}) { + t.Helper() + data := setup(t, ext, contents) + + actual, err := data.Datasource("foo", "?type="+mime) assert.NoError(t, err) assert.Equal(t, expected, actual) } - testObj := func(ext, mime string, contents []byte) { - test(ext, mime, contents, + testObj := func(t *testing.T, ext, mime string, contents []byte) { + test(t, ext, mime, contents, map[string]interface{}{ "hello": map[string]interface{}{"cruel": "world"}, }) } - testObj("json", jsonMimetype, []byte(`{"hello":{"cruel":"world"}}`)) - testObj("yml", yamlMimetype, []byte("hello:\n cruel: world\n")) - test("json", jsonMimetype, []byte(`[1, "two", true]`), + testObj(t, "json", jsonMimetype, []byte(`{"hello":{"cruel":"world"}}`)) + testObj(t, "yml", yamlMimetype, []byte("hello:\n cruel: world\n")) + test(t, "json", jsonMimetype, []byte(`[1, "two", true]`), []interface{}{1, "two", true}) - test("yaml", yamlMimetype, []byte("---\n- 1\n- two\n- true\n"), + test(t, "yaml", yamlMimetype, []byte("---\n- 1\n- two\n- true\n"), []interface{}{1, "two", true}) - d := setup("", textMimetype, nil) + d := setup(t, "", nil) actual, err := d.Datasource("foo") assert.NoError(t, err) assert.Equal(t, "", actual) @@ -100,35 +99,24 @@ func TestDatasource(t *testing.T) { } func TestDatasourceReachable(t *testing.T) { - fname := "foo.json" - fs := afero.NewMemMapFs() - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write([]byte("{}")) + fsys := fstest.MapFS{} + fsys["tmp/foo.json"] = &fstest.MapFile{Data: []byte("{}")} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, + URL: mustParseURL("file:///tmp/foo.json"), mediaType: jsonMimetype, - fs: fs, }, "bar": { Alias: "bar", URL: &url.URL{Scheme: "file", Path: "/bogus"}, - fs: fs, }, } - data := &Data{Sources: sources} + data := &Data{Sources: sources, FSMux: fsmux} assert.True(t, data.DatasourceReachable("foo")) assert.False(t, data.DatasourceReachable("bar")) @@ -144,35 +132,22 @@ func TestDatasourceExists(t *testing.T) { } func TestInclude(t *testing.T) { - ext := "txt" contents := "hello world" - fname := "foo." + ext - fs := afero.NewMemMapFs() - - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write([]byte(contents)) + + fsys := fstest.MapFS{} + fsys["tmp/foo.txt"] = &fstest.MapFile{Data: []byte(contents)} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, + URL: mustParseURL("file:///tmp/foo.txt"), mediaType: textMimetype, - fs: fs, }, } - data := &Data{ - Sources: sources, - } + data := &Data{Sources: sources, FSMux: fsmux} actual, err := data.Include("foo") assert.NoError(t, err) assert.Equal(t, contents, actual) @@ -245,103 +220,6 @@ func TestDefineDatasource(t *testing.T) { assert.Equal(t, "application/x-env", m) } -func TestMimeType(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} - _, err := s.mimeType("") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - expected string - }{ - {"http://example.com/foo.json", - "", - jsonMimetype}, - {"http://example.com/foo.json", - "text/foo", - "text/foo"}, - {"http://example.com/foo.json?type=application/yaml", - "text/foo", - "application/yaml"}, - {"http://example.com/list?type=application/array%2Bjson", - "text/foo", - "application/array+json"}, - {"http://example.com/list?type=application/array+json", - "", - "application/array+json"}, - {"http://example.com/unknown", - "", - "text/plain"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType("") - assert.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - -func TestMimeTypeWithArg(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com")} - _, err := s.mimeType("h\nttp://foo") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - arg string - expected string - }{ - {"http://example.com/unknown", - "", - "/foo.json", - "application/json"}, - {"http://example.com/unknown", - "", - "foo.json", - "application/json"}, - {"http://example.com/", - "text/foo", - "/foo.json", - "text/foo"}, - {"git+https://example.com/myrepo", - "", - "//foo.yaml", - "application/yaml"}, - {"http://example.com/foo.json", - "", - "/foo.yaml", - "application/yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml", - "application/array+yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml?type=application/yaml", - "application/yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "text/plain", - "/foo.yaml?type=application/yaml", - "application/yaml"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType(d.arg) - assert.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - func TestFromConfig(t *testing.T) { ctx := context.Background() cfg := &config.Config{} @@ -422,3 +300,79 @@ func TestListDatasources(t *testing.T) { assert.Equal(t, []string{"bar", "foo"}, data.ListDatasources()) } + +func TestSplitFSMuxURL(t *testing.T) { + t.Skip() + testdata := []struct { + in string + arg string + url string + file string + }{ + {"http://example.com/foo.json", "", "http://example.com/", "foo.json"}, + { + "http://example.com/foo.json?type=application/array+yaml", + "", + "http://example.com/?type=application/array+yaml", + "foo.json", + }, + { + "vault:///secret/a/b/c", "", + "vault:///", + "secret/a/b/c", + }, + { + "vault:///secret/a/b/", "", + "vault:///", + "secret/a/b", + }, + { + "s3://bucket/a/b/", "", + "s3://bucket/", + "a/b", + }, + { + "vault:///", "foo/bar", + "vault:///", + "foo/bar", + }, + { + "consul://myhost/foo/?q=1", "bar/baz", + "consul://myhost/?q=1", + "foo/bar/baz", + }, + { + "consul://myhost/foo/?q=1", "bar/baz", + "consul://myhost/?q=1", + "foo/bar/baz", + }, + { + "git+https://example.com/myrepo", "//foo.yaml", + "git+https://example.com/myrepo", "foo.yaml", + }, + { + "ssh://git@github.com/hairyhenderson/go-which.git//a/b/", + "c/d?q=1", + "ssh://git@github.com/hairyhenderson/go-which.git?q=1", + "a/b/c/d", + }, + } + + for _, d := range testdata { + u, err := url.Parse(d.in) + assert.NoError(t, err) + url, file := splitFSMuxURL(u) + assert.Equal(t, d.url, url.String()) + assert.Equal(t, d.file, file) + } +} + +func TestResolveURL(t *testing.T) { + out, err := resolveURL(mustParseURL("http://example.com/foo.json"), "bar.json") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/bar.json", out.String()) + + out, err = resolveURL(mustParseURL("http://example.com/a/b/?n=2"), "bar.json?q=1") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/a/b/bar.json?n=2&q=1", out.String()) +} diff --git a/data/datasource_vault.go b/data/datasource_vault.go deleted file mode 100644 index 09f6dcb17..000000000 --- a/data/datasource_vault.go +++ /dev/null @@ -1,48 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/pkg/errors" - - "github.com/hairyhenderson/gomplate/v3/vault" -) - -func readVault(ctx context.Context, source *Source, args ...string) (data []byte, err error) { - if source.vc == nil { - source.vc, err = vault.New(source.URL) - if err != nil { - return nil, err - } - err = source.vc.Login() - if err != nil { - return nil, err - } - } - - params, p, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case len(params) > 0: - data, err = source.vc.Write(p, params) - case strings.HasSuffix(p, "/"): - source.mediaType = jsonArrayMimetype - data, err = source.vc.List(p) - default: - data, err = source.vc.Read(p) - } - if err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, errors.Errorf("no value found for path %s", p) - } - - return data, nil -} diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go deleted file mode 100644 index 8b470abc6..000000000 --- a/data/datasource_vault_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/hairyhenderson/gomplate/v3/vault" - "github.com/stretchr/testify/assert" -) - -func TestReadVault(t *testing.T) { - ctx := context.Background() - - expected := "{\"value\":\"foo\"}\n" - server, v := vault.MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - - source := &Source{ - Alias: "foo", - URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, - mediaType: textMimetype, - vc: v, - } - - r, err := readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "bar") - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "?param=value") - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - source.URL, _ = url.Parse("vault:///secret/foo?param1=value1¶m2=value2") - r, err = readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - expected = "[\"one\",\"two\"]\n" - server, source.vc = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) - defer server.Close() - source.URL, _ = url.Parse("vault:///secret/foo/") - r, err = readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) -} diff --git a/data/mimetypes.go b/data/mimetypes.go index bdc12ad4c..3db302cc1 100644 --- a/data/mimetypes.go +++ b/data/mimetypes.go @@ -1,5 +1,15 @@ package data +import ( + "fmt" + "mime" + "net/url" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + const ( textMimetype = "text/plain" csvMimetype = "text/csv" @@ -23,3 +33,115 @@ func mimeAlias(m string) string { } return m } + +// mimeType returns the MIME type to use as a hint for parsing the datasource. +// It's expected that the datasource will have already been read before +// this function is called, and so the Source's Type property may be already set. +// +// The MIME type is determined by these rules: +// 1. the 'type' URL query parameter is used if present +// 2. otherwise, the Type property on the Source is used, if present +// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered +// 4. otherwise, the default type of 'text/plain' is used +func (s *Source) mimeType(arg string) (mimeType string, err error) { + if len(arg) > 0 { + if strings.HasPrefix(arg, "//") { + arg = arg[1:] + } + if !strings.HasPrefix(arg, "/") { + arg = "/" + arg + } + } + argURL, err := url.Parse(arg) + if err != nil { + return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) + } + mediatype := argURL.Query().Get("type") + if mediatype == "" { + mediatype = s.URL.Query().Get("type") + } + + if mediatype == "" { + mediatype = s.mediaType + } + + // make it so + doesn't need to be escaped + mediatype = strings.ReplaceAll(mediatype, " ", "+") + + if mediatype == "" { + ext := filepath.Ext(argURL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype == "" { + ext := filepath.Ext(s.URL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype != "" { + t, _, err := mime.ParseMediaType(mediatype) + if err != nil { + return "", errors.Wrapf(err, "MIME type was %q", mediatype) + } + mediatype = t + return mediatype, nil + } + + return textMimetype, nil +} + +// mimeType returns the MIME type to use as a hint for parsing the datasource. +// It's expected that the datasource will have already been read before +// this function is called, and so the Source's Type property may be already set. +// +// The MIME type is determined by these rules: +// 1. the 'type' URL query parameter is used if present +// 2. otherwise, the Type property on the Source is used, if present +// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered +// 4. otherwise, the default type of 'text/plain' is used +func guessMimeType(base *url.URL, name, mimeGuess string) (mimeType string, err error) { + if len(name) > 0 { + if strings.HasPrefix(name, "//") { + name = name[1:] + } + if !strings.HasPrefix(name, "/") { + name = "/" + name + } + } + nameURL, err := url.Parse(name) + if err != nil { + return "", fmt.Errorf("mimeType: couldn't parse name %q: %w", name, err) + } + mediatype := nameURL.Query().Get("type") + if mediatype == "" { + mediatype = base.Query().Get("type") + } + + if mediatype == "" { + mediatype = mimeGuess + } + + // make it so + doesn't need to be escaped + mediatype = strings.ReplaceAll(mediatype, " ", "+") + + if mediatype == "" { + ext := filepath.Ext(nameURL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype == "" { + ext := filepath.Ext(base.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype != "" { + t, _, err := mime.ParseMediaType(mediatype) + if err != nil { + return "", errors.Wrapf(err, "MIME type was %q", mediatype) + } + mediatype = t + return mediatype, nil + } + + return textMimetype, nil +} diff --git a/data/mimetypes_test.go b/data/mimetypes_test.go index 0dd1ab052..8f36dc06d 100644 --- a/data/mimetypes_test.go +++ b/data/mimetypes_test.go @@ -1,9 +1,10 @@ package data import ( + "fmt" "testing" - "gotest.tools/v3/assert" + "github.com/stretchr/testify/assert" ) func TestMimeAlias(t *testing.T) { @@ -20,3 +21,108 @@ func TestMimeAlias(t *testing.T) { assert.Equal(t, d.out, mimeAlias(d.in)) } } + +func TestMimeType(t *testing.T) { + s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} + _, err := s.mimeType("") + assert.Error(t, err) + + data := []struct { + url string + mediaType string + expected string + }{ + {"http://example.com/foo.json", + "", + jsonMimetype}, + {"http://example.com/foo.json", + "text/foo", + "text/foo"}, + {"http://example.com/foo.json?type=application/yaml", + "text/foo", + "application/yaml"}, + {"http://example.com/list?type=application/array%2Bjson", + "text/foo", + "application/array+json"}, + {"http://example.com/list?type=application/array+json", + "", + "application/array+json"}, + {"http://example.com/unknown", + "", + "text/plain"}, + } + + for i, d := range data { + d := d + t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { + s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} + mt, err := s.mimeType("") + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + + mt, err = guessMimeType(mustParseURL(d.url), "", d.mediaType) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + }) + } +} + +func TestMimeTypeWithArg(t *testing.T) { + s := &Source{URL: mustParseURL("http://example.com")} + _, err := s.mimeType("h\nttp://foo") + assert.Error(t, err) + + data := []struct { + url string + mediaType string + arg string + expected string + }{ + {"http://example.com/unknown", + "", + "/foo.json", + "application/json"}, + {"http://example.com/unknown", + "", + "foo.json", + "application/json"}, + {"http://example.com/", + "text/foo", + "/foo.json", + "text/foo"}, + {"git+https://example.com/myrepo", + "", + "//foo.yaml", + "application/yaml"}, + {"http://example.com/foo.json", + "", + "/foo.yaml", + "application/yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "", + "/foo.yaml", + "application/array+yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "", + "/foo.yaml?type=application/yaml", + "application/yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "text/plain", + "/foo.yaml?type=application/yaml", + "application/yaml"}, + } + + for i, d := range data { + d := d + t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { + s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} + mt, err := s.mimeType(d.arg) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + + mt, err = guessMimeType(mustParseURL(d.url), d.arg, d.mediaType) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + }) + } +} diff --git a/go.mod b/go.mod index 1471a1c1d..728e51e5e 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,9 @@ require ( github.com/aws/aws-sdk-go v1.43.12 github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa - github.com/go-git/go-billy/v5 v5.3.1 - github.com/go-git/go-git/v5 v5.4.2 github.com/google/uuid v1.3.0 github.com/gosimple/slug v1.12.0 + github.com/hairyhenderson/go-fsimpl v0.0.0-20220206190937-a8132d614180 github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf github.com/hashicorp/consul/api v1.12.0 github.com/hashicorp/go-sockaddr v1.0.2 @@ -26,7 +25,6 @@ require ( github.com/ugorji/go/codec v1.2.7 github.com/zealic/xignore v0.3.3 go.etcd.io/bbolt v1.3.6 - gocloud.dev v0.24.0 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 @@ -38,8 +36,16 @@ require ( ) require ( - cloud.google.com/go v0.95.0 // indirect - cloud.google.com/go/storage v1.16.1 // indirect + cloud.google.com/go v0.97.0 // indirect + cloud.google.com/go/storage v1.17.0 // indirect + github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-storage-blob-go v0.14.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.21 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.16 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210920160938-87db9fbc61c7 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect @@ -51,6 +57,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.2.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.4.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.7.1 // indirect github.com/aws/smithy-go v1.8.0 // indirect @@ -60,6 +67,9 @@ require ( github.com/emirpasic/gods v1.12.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-git/v5 v5.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -91,6 +101,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.1.0 // indirect github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -110,14 +121,15 @@ require ( go.uber.org/atomic v1.9.0 // indirect go4.org/intern v0.0.0-20210108033219-3eb7198706b2 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 // indirect + gocloud.dev v0.24.0 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect + golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.1.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/api v0.57.0 // indirect + google.golang.org/api v0.58.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0 // indirect + google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4 // indirect google.golang.org/grpc v1.41.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 9a4007f34..dece7dccc 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -33,8 +34,8 @@ cloud.google.com/go v0.92.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.0/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.95.0 h1:JVWssQIj9cLwHmLjqWLptFa83o7HgqUictM6eyvGWJE= -cloud.google.com/go v0.95.0/go.mod h1:MzZUAH870Y7E+c14j23Ir66FC1+PK8WLG7OG4SjP+0k= +cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -59,8 +60,10 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.16.1 h1:sMEIc4wxvoY3NXG7Rn9iP7jb/2buJgWR1vNXCR/UPfs= +cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE= cloud.google.com/go/storage v1.16.1/go.mod h1:LaNorbty3ehnU3rEjXSNV/NRgQA0O8Y+uh6bPe5UOk4= +cloud.google.com/go/storage v1.17.0 h1:CDpe3jS3EiD5nGlbtvyA4EUfkF6k9GMrxLR8+hLmoec= +cloud.google.com/go/storage v1.17.0/go.mod h1:0wRtHSM3Npk/QJYdwcpRNVRVJlH2OxyWF9Dws3J+MtE= cloud.google.com/go/trace v0.1.0/go.mod h1:wxEwsoeRVPbeSkt7ZC9nWCgmoKQRAoySN7XHW2AmI7g= contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ= @@ -78,27 +81,39 @@ github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEew github.com/Azure/go-amqp v0.13.0/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs= github.com/Azure/go-amqp v0.13.11/go.mod h1:D5ZrjQqB1dyp1A+G73xeL/kNn7D5qHJIIsNNps7YNmk= github.com/Azure/go-amqp v0.13.12/go.mod h1:D5ZrjQqB1dyp1A+G73xeL/kNn7D5qHJIIsNNps7YNmk= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= +github.com/Azure/go-autorest/autorest v0.11.21 h1:w77zY/9RnUAWcIQyDC0Fc89mCvwftR8F+zsR/OH6enk= +github.com/Azure/go-autorest/autorest v0.11.21/go.mod h1:Do/yuMSW/13ayUkcVREpsMHGG+MvV81uzSCFgYPj4tM= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/adal v0.9.16 h1:P8An8Z9rH1ldbOLdFpxYorgOt2sywL9V24dAwWHPuGc= +github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 h1:DOhB+nXkF7LN0JfBGB5YtCF6QLK8mLe4psaHF7ZQEKM= github.com/Azure/go-autorest/autorest/azure/cli v0.4.3/go.mod h1:yAQ2b6eP/CmLPnmLvxtT1ALIY3OR1oFcCqVBi8vHiTc= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -107,9 +122,11 @@ github.com/GoogleCloudPlatform/cloudsql-proxy v1.24.0/go.mod h1:3tx938GhY4FC+E1K github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= @@ -130,6 +147,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= +github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -140,8 +159,11 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.40.58/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.43.12 h1:wOdx6+reSDpUBFEuJDA6edCrojzy8rOtMzhS2rD9+7M= github.com/aws/aws-sdk-go v1.43.12/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= @@ -163,6 +185,7 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1K github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.1 h1:APEjhKZLFlNVLATnA/TJyA+w1r/xd5r5ACWBDZ9aIvc= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.1/go.mod h1:Ve+eJOx9UWaT/lMVebnFhDhO49fSLVedHoA82+Rqme0= github.com/aws/aws-sdk-go-v2/service/kms v1.5.0/go.mod h1:w7JuP9Oq1IKMFQPkNe3V6s9rOssXzOVEMNEqK1L1bao= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.6.0 h1:3vxYnnbPWwECs3xN+cu/bRefhynMOH6elQAxuHES01Q= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.6.0/go.mod h1:B+7C5UKdVq1ylkI/A6O8wcurFtaux0R1njePNPtKwoA= github.com/aws/aws-sdk-go-v2/service/ssm v1.10.0/go.mod h1:4dXS5YNqI3SNbetQ7X7vfsMlX6ZnboJA2dulBwJx7+g= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= @@ -197,7 +220,18 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200709052629-daa8e1ccc0bc/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -209,10 +243,16 @@ github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27N github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 h1:x0AMRhackzbivKKiEeSMzH6gZmbALPXCBG0ecBmRlco= github.com/docker/libkv v0.2.2-0.20180912205406-458977154600/go.mod h1:r5hEwHwW8dr0TFBYGCarMNbrQOiwL1xoqDYZ/JqoTK0= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ= github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= @@ -237,6 +277,9 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -247,6 +290,8 @@ github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsouza/fake-gcs-server v1.30.1 h1:OBuje/XJiwwzGOuwEhPzZ8s2gF1vDHTZq1X+AhYEqjc= +github.com/fsouza/fake-gcs-server v1.30.1/go.mod h1:8S1lJH/fxjz4AJMhQJn5AUU4m6jPoCTHQituQinWTAQ= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= @@ -271,6 +316,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-ldap/ldap/v3 v3.1.3/go.mod h1:3rbOH3jRS2u6jg2rJnKAMLE/xQyCKIveG2Sa/Cohzb8= github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -285,17 +331,24 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0= +github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -330,6 +383,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -399,6 +453,10 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -408,6 +466,8 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hairyhenderson/go-fsimpl v0.0.0-20220206190937-a8132d614180 h1:VPUjkRXIafQ2x0aaaBJ+PLRwhQPL4sn64bI9+LV5y5o= +github.com/hairyhenderson/go-fsimpl v0.0.0-20220206190937-a8132d614180/go.mod h1:v6ot3ZgWfrsMmcbmEi3+sb0TXEaUhaYff/EtA+pyS2k= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbIt9i+hB1zfKw2mE8C12TuGxPiW7YmtLbPa4= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf/go.mod h1:jDHmWDKZY6MIIYltYYfW4Rs7hQ50oS4qf/6spSiZAxY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -423,12 +483,15 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.16.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= @@ -438,13 +501,16 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.6.2/go.mod h1:gEx6HMUGxYYhJScX7W1Il64m6cc2C1mDaW3NQ9sY1FY= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/base62 v0.1.1/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw= @@ -464,11 +530,13 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -482,8 +550,12 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f/go.mod h1:euTFbi2YJgwcju3imEt919lhJKF68nN1cQPq3aA+kBE= +github.com/hashicorp/vault/api v1.1.1/go.mod h1:29UXcn/1cLOPHQNMWA7bCz2By4PSd0VKPAydKXS5yN0= github.com/hashicorp/vault/api v1.4.1 h1:mWLfPT0RhxBitjKr6swieCEP2v5pp/M//t70S3kMLRo= github.com/hashicorp/vault/api v1.4.1/go.mod h1:LkMdrZnWNrFaQyYYazWVn7KshilfDidgVBq6YiTq/bM= +github.com/hashicorp/vault/sdk v0.1.14-0.20200519221530-14615acda45f/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10= +github.com/hashicorp/vault/sdk v0.2.1/go.mod h1:WfUiO1vYzfBkz1TmoE4ZGU7HD0T0Cl/rZwaxjBkgN4U= github.com/hashicorp/vault/sdk v0.4.1 h1:3SaHOJY687jY1fnB61PtL0cOkKItphrbLmux7T92HBo= github.com/hashicorp/vault/sdk v0.4.1/go.mod h1:aZ3fNuL5VNydQk8GcLJ2TV8YCRVvyaakYkhZRoVuhj0= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= @@ -504,6 +576,7 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -525,6 +598,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v1.1.0 h1:pH/t1WS9NzT8go394IqZeJTMHVm6Cr6ZJ6AQ+mdNo/o= github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -581,6 +656,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= @@ -593,6 +669,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -602,20 +679,31 @@ github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -626,15 +714,19 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -658,10 +750,12 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+ github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -675,9 +769,11 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.8.1 h1:izYHOT71f9iZ7iq37Uqjael60/vYC6vMtzedudZ0zEk= github.com/spf13/afero v1.8.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -702,6 +798,7 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo= @@ -731,6 +828,7 @@ go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -747,16 +845,19 @@ go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 h1:1tk03FU go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= gocloud.dev v0.24.0 h1:cNtHD07zQQiv02OiwwDyVMuHmR7iQt2RLkzoAgz7wBs= gocloud.dev v0.24.0/go.mod h1:uA+als++iBX5ShuG4upQo/3Zoz49iIPlYUWHV5mM8w8= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -766,6 +867,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= @@ -824,8 +926,10 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -839,6 +943,7 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -858,6 +963,7 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -876,10 +982,12 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -897,13 +1005,16 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -934,6 +1045,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -970,6 +1082,8 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -993,7 +1107,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= @@ -1007,12 +1123,14 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1088,14 +1206,16 @@ google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1Avk google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42L18= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.52.0/go.mod h1:Him/adpjt0sxtkWViy0b6xyKW/SD71CwdJ7HqJo7SrU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0 h1:4t9zuDlHLcIx0ZEhmXEeFVCRsiOgpgn2QOH9N0MNjPI= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0 h1:MDkAbYIB1JpSgCTOCYYoIec/coMlKK4oVbpnBLLcyT0= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1155,6 +1275,8 @@ google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624174822-c5cf32407d0a/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= @@ -1169,14 +1291,19 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0 h1:5Tbluzus3QxoAJx4IefGt1W0HQZW4nuMrVk684jI74Q= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4 h1:YXPV/eKW0ZWRdB5tyI6aPoaa2Wxb4OSlFrTREMdwn64= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1215,6 +1342,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1224,9 +1352,11 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1246,6 +1376,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/datafs/envfs.go b/internal/datafs/envfs.go new file mode 100644 index 000000000..e0979baa6 --- /dev/null +++ b/internal/datafs/envfs.go @@ -0,0 +1,210 @@ +package datafs + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "strings" + "time" + + "github.com/hairyhenderson/go-fsimpl" +) + +// NewEnvFS returns a filesystem (an fs.FS) that can be used to read data from +// environment variables. +func NewEnvFS(u *url.URL) (fs.FS, error) { + return &envFS{locfs: os.DirFS("/")}, nil +} + +type envFS struct { + locfs fs.FS +} + +//nolint:gochecknoglobals +var EnvFS = fsimpl.FSProviderFunc(NewEnvFS, "env") + +var _ fs.FS = (*envFS)(nil) + +func (f *envFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + + return &envFile{locfs: f.locfs, name: name}, nil +} + +type envFile struct { + locfs fs.FS + body io.Reader + name string + + dirents []fs.DirEntry + diroff int +} + +var ( + _ fs.File = (*envFile)(nil) + _ fs.ReadDirFile = (*envFile)(nil) +) + +func (e *envFile) Close() error { + e.body = nil + return nil +} + +func (e *envFile) envReader() (int, io.Reader, error) { + v, found := os.LookupEnv(e.name) + if found { + return len(v), bytes.NewBufferString(v), nil + } + + fname, found := os.LookupEnv(e.name + "_FILE") + if found && fname != "" { + fname = strings.TrimPrefix(fname, "/") + + b, err := fs.ReadFile(e.locfs, fname) + if err != nil { + return 0, nil, err + } + + b = bytes.TrimSpace(b) + + return len(b), bytes.NewBuffer(b), nil + } + + return 0, nil, fs.ErrNotExist +} + +func (e *envFile) Stat() (fs.FileInfo, error) { + n, _, err := e.envReader() + if err != nil { + return nil, err + } + + return FileInfo(e.name, int64(n), 0444, time.Time{}, ""), nil +} + +func (e *envFile) Read(p []byte) (int, error) { + if e.body == nil { + _, r, err := e.envReader() + if err != nil { + return 0, err + } + e.body = r + } + + return e.body.Read(p) +} + +func (e *envFile) ReadDir(n int) ([]fs.DirEntry, error) { + // envFS has no concept of subdirectories, but we can support a root + // directory by listing all environment variables. + if e.name != "." { + return nil, fmt.Errorf("%s: is not a directory", e.name) + } + + if e.dirents == nil { + envs := os.Environ() + e.dirents = make([]fs.DirEntry, len(envs)) + for i, env := range envs { + parts := strings.SplitN(env, "=", 2) + name, value := parts[0], parts[1] + + e.dirents[i] = FileInfoDirEntry( + FileInfo(name, int64(len(value)), 0444, time.Time{}, ""), + ) + } + } + + if n > 0 && e.diroff >= len(e.dirents) { + return nil, io.EOF + } + + low := e.diroff + high := e.diroff + n + + // clamp high at the max, and ensure it's higher than low + if high >= len(e.dirents) || high <= low { + high = len(e.dirents) + } + + entries := make([]fs.DirEntry, high-low) + copy(entries, e.dirents[e.diroff:]) + + e.diroff = high + + return entries, nil +} + +// FileInfo/DirInfo/FileInfoDirEntry/etc are taken from go-fsimpl's internal +// package, and may be exported in the future... + +// FileInfo creates a static fs.FileInfo with the given properties. +// The result is also a fs.DirEntry and can be safely cast. +func FileInfo(name string, size int64, mode fs.FileMode, modTime time.Time, contentType string) fs.FileInfo { + return &staticFileInfo{ + name: name, + size: size, + mode: mode, + modTime: modTime, + contentType: contentType, + } +} + +// DirInfo creates a fs.FileInfo for a directory with the given name. Use +// FileInfo to set other values. +func DirInfo(name string, modTime time.Time) fs.FileInfo { + return FileInfo(name, 0, fs.ModeDir, modTime, "") +} + +type staticFileInfo struct { + modTime time.Time + name string + contentType string + size int64 + mode fs.FileMode +} + +var ( + _ fs.FileInfo = (*staticFileInfo)(nil) + _ fs.DirEntry = (*staticFileInfo)(nil) +) + +func (fi staticFileInfo) ContentType() string { return fi.contentType } +func (fi staticFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi staticFileInfo) Mode() fs.FileMode { return fi.mode } +func (fi *staticFileInfo) ModTime() time.Time { return fi.modTime } +func (fi staticFileInfo) Name() string { return fi.name } +func (fi staticFileInfo) Size() int64 { return fi.size } +func (fi staticFileInfo) Sys() interface{} { return nil } +func (fi *staticFileInfo) Info() (fs.FileInfo, error) { return fi, nil } +func (fi staticFileInfo) Type() fs.FileMode { return fi.Mode().Type() } + +// FileInfoDirEntry adapts a fs.FileInfo into a fs.DirEntry. If it doesn't +// already implement fs.DirEntry, it will be wrapped to always return the +// same fs.FileInfo. +func FileInfoDirEntry(fi fs.FileInfo) fs.DirEntry { + de, ok := fi.(fs.DirEntry) + if ok { + return de + } + + return &fileinfoDirEntry{fi} +} + +// a wrapper to make a fs.FileInfo into an fs.DirEntry +type fileinfoDirEntry struct { + fs.FileInfo +} + +var _ fs.DirEntry = (*fileinfoDirEntry)(nil) + +func (fi *fileinfoDirEntry) Info() (fs.FileInfo, error) { return fi, nil } +func (fi *fileinfoDirEntry) Type() fs.FileMode { return fi.Mode().Type() } diff --git a/internal/datafs/envfs_test.go b/internal/datafs/envfs_test.go new file mode 100644 index 000000000..3f41c129f --- /dev/null +++ b/internal/datafs/envfs_test.go @@ -0,0 +1,105 @@ +package datafs + +import ( + "io/fs" + "net/url" + "os" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" +) + +func TestEnvFS_Open(t *testing.T) { + fsys, err := NewEnvFS(nil) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + f, err := fsys.Open("foo") + assert.NoError(t, err) + assert.IsType(t, &envFile{}, f) +} + +func TestEnvFile_Read(t *testing.T) { + content := `hello world` + os.Setenv("HELLO_WORLD", "hello world") + defer os.Unsetenv("HELLO_WORLD") + + f := &envFile{name: "HELLO_WORLD"} + b := make([]byte, len(content)) + n, err := f.Read(b) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + f = &envFile{name: "FOO", locfs: fsys} + + b = make([]byte, len(content)) + t.Logf("b len is %d", len(b)) + n, err = f.Read(b) + t.Logf("b len is %d", len(b)) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) +} + +func TestEnvFile_Stat(t *testing.T) { + content := []byte(`hello world`) + os.Setenv("HELLO_WORLD", "hello world") + defer os.Unsetenv("HELLO_WORLD") + + f := &envFile{name: "HELLO_WORLD"} + + fi, err := f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + f = &envFile{name: "FOO", locfs: fsys} + + fi, err = f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) +} + +func TestEnvFS(t *testing.T) { + u, _ := url.Parse("env:") + + lfsys := fstest.MapFS{} + lfsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello file\n")} + + fsys, err := NewEnvFS(u) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + envfs, ok := fsys.(*envFS) + assert.True(t, ok) + envfs.locfs = lfsys + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + b, err := fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello file", string(b)) + + os.Setenv("FOO", "hello world") + defer os.Unsetenv("FOO") + + b, err = fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello world", string(b)) + + assert.NoError(t, fstest.TestFS(fsys, "FOO", "FOO_FILE", "HOME", "USER")) +} diff --git a/internal/tests/integration/datasources_vault_test.go b/internal/tests/integration/datasources_vault_test.go index 798f8733e..2a4cdc7cd 100644 --- a/internal/tests/integration/datasources_vault_test.go +++ b/internal/tests/integration/datasources_vault_test.go @@ -4,6 +4,7 @@ package integration import ( + "fmt" "os" "os/user" "path" @@ -63,7 +64,7 @@ func startVault(t *testing.T) (*fs.Dir, *vaultClient) { "-dev", "-dev-root-token-id="+vaultRootToken, "-dev-leased-kv", - "-log-level=err", + "-log-level=trace", "-dev-listen-address="+vaultAddr, "-config="+tmpDir.Join("config.json"), ) @@ -85,6 +86,8 @@ func startVault(t *testing.T) (*fs.Dir, *vaultClient) { result.Assert(t, icmd.Expected{ExitCode: 0}) + fmt.Println(result.Combined()) + // restore old token if it was backed up u, _ := user.Current() homeDir := u.HomeDir