From d261bb9b0f3f908b7c0a3b4097f5b485e941c995 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sun, 2 Jun 2024 21:05:04 -0400 Subject: [PATCH] chore(api)!: Overhauling config and rendering types Signed-off-by: Dave Henderson --- gomplate.go | 46 ++++++++-- gomplate_test.go | 24 ++--- internal/cmd/config.go | 24 +++++ internal/cmd/config_test.go | 33 +++++-- internal/cmd/main.go | 12 ++- internal/config/configfile.go | 137 ++++++++++++++++++++++++----- internal/config/configfile_test.go | 32 +++++-- internal/datafs/context.go | 5 ++ render.go | 122 ++++++++----------------- render_test.go | 22 ++--- template.go | 21 +++-- template_test.go | 2 +- 12 files changed, 312 insertions(+), 168 deletions(-) diff --git a/gomplate.go b/gomplate.go index 61bda1fb3..1a1990a36 100644 --- a/gomplate.go +++ b/gomplate.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "fmt" + "log/slog" "path/filepath" "strings" "text/template" @@ -38,6 +39,14 @@ func Run(ctx context.Context, cfg *config.Config) error { return fmt.Errorf("failed to validate config: %w\n%+v", err, cfg) } + if cfg.Experimental { + slog.SetDefault(slog.With("experimental", true)) + slog.InfoContext(ctx, "experimental functions and features enabled!") + + ctx = SetExperimental(ctx) + } + + // bind plugins from the configuration to the funcMap funcMap := template.FuncMap{} err = bindPlugins(ctx, cfg, funcMap) if err != nil { @@ -47,14 +56,26 @@ func Run(ctx context.Context, cfg *config.Config) error { // if a custom Stdin is set in the config, inject it into the context now ctx = datafs.ContextWithStdin(ctx, cfg.Stdin) + // if a custom FSProvider is set in the context, use it, otherwise inject + // the default now - one is needed for the calls below to gatherTemplates + // as well as the rendering itself + if datafs.FSProviderFromContext(ctx) == nil { + ctx = datafs.ContextWithFSProvider(ctx, DefaultFSProvider) + } + + // extract the rendering options from the config opts := optionsFromConfig(cfg) opts.Funcs = funcMap tr := newRenderer(opts) start := time.Now() + // figure out how to name output files (only relevant if we're dealing with an InputDir) namer := chooseNamer(cfg, tr) + + // prepare to render templates (read them in, open output writers, etc) tmpl, err := gatherTemplates(ctx, cfg, namer) + Metrics.GatherDuration = time.Since(start) if err != nil { Metrics.Errors++ @@ -70,22 +91,33 @@ func Run(ctx context.Context, cfg *config.Config) error { return nil } -func chooseNamer(cfg *config.Config, tr *renderer) func(context.Context, string) (string, error) { +type outputNamer interface { + // Name the output file for the given input path + Name(ctx context.Context, inPath string) (string, error) +} + +type outputNamerFunc func(context.Context, string) (string, error) + +func (f outputNamerFunc) Name(ctx context.Context, inPath string) (string, error) { + return f(ctx, inPath) +} + +func chooseNamer(cfg *config.Config, tr *renderer) outputNamer { if cfg.OutputMap == "" { return simpleNamer(cfg.OutputDir) } return mappingNamer(cfg.OutputMap, tr) } -func simpleNamer(outDir string) func(ctx context.Context, inPath string) (string, error) { - return func(_ context.Context, inPath string) (string, error) { +func simpleNamer(outDir string) outputNamer { + return outputNamerFunc(func(_ context.Context, inPath string) (string, error) { outPath := filepath.Join(outDir, inPath) return filepath.Clean(outPath), nil - } + }) } -func mappingNamer(outMap string, tr *renderer) func(context.Context, string) (string, error) { - return func(ctx context.Context, inPath string) (string, error) { +func mappingNamer(outMap string, tr *renderer) outputNamer { + return outputNamerFunc(func(ctx context.Context, inPath string) (string, error) { tcontext, err := createTmplContext(ctx, tr.tctxAliases, tr.sr) if err != nil { return "", err @@ -114,5 +146,5 @@ func mappingNamer(outMap string, tr *renderer) func(context.Context, string) (st } return filepath.Clean(strings.TrimSpace(out.String())), nil - } + }) } diff --git a/gomplate_test.go b/gomplate_test.go index 5bca78cc7..e27dd3df8 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -29,7 +29,7 @@ func testTemplate(t *testing.T, tr *renderer, tmpl string) string { } func TestGetenvTemplates(t *testing.T) { - tr := newRenderer(Options{ + tr := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "getenv": env.Getenv, "bool": conv.ToBool, @@ -41,7 +41,7 @@ func TestGetenvTemplates(t *testing.T) { } func TestBoolTemplates(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "bool": conv.ToBool, }, @@ -55,7 +55,7 @@ func TestBoolTemplates(t *testing.T) { func TestEc2MetaTemplates(t *testing.T) { createGomplate := func(data map[string]string, region string) *renderer { ec2meta := aws.MockEC2Meta(data, nil, region) - return newRenderer(Options{Funcs: template.FuncMap{"ec2meta": ec2meta.Meta}}) + return newRenderer(RenderOptions{Funcs: template.FuncMap{"ec2meta": ec2meta.Meta}}) } g := createGomplate(nil, "") @@ -70,7 +70,7 @@ func TestEc2MetaTemplates(t *testing.T) { func TestEc2MetaTemplates_WithJSON(t *testing.T) { ec2meta := aws.MockEC2Meta(map[string]string{"obj": `"foo": "bar"`}, map[string]string{"obj": `"foo": "baz"`}, "") - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "ec2meta": ec2meta.Meta, "ec2dynamic": ec2meta.Dynamic, @@ -83,7 +83,7 @@ func TestEc2MetaTemplates_WithJSON(t *testing.T) { } func TestJSONArrayTemplates(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "jsonArray": parsers.JSONArray, }, @@ -94,7 +94,7 @@ func TestJSONArrayTemplates(t *testing.T) { } func TestYAMLTemplates(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "yaml": parsers.YAML, "yamlArray": parsers.YAMLArray, @@ -107,7 +107,7 @@ func TestYAMLTemplates(t *testing.T) { } func TestHasTemplate(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ Funcs: template.FuncMap{ "yaml": parsers.YAML, "has": conv.Has, @@ -141,7 +141,7 @@ func TestMissingKey(t *testing.T) { } for name, tt := range tests { t.Run(name, func(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ MissingKey: tt.MissingKey, }) tmpl := `{{ .name }}` @@ -151,7 +151,7 @@ func TestMissingKey(t *testing.T) { } func TestCustomDelim(t *testing.T) { - g := newRenderer(Options{ + g := newRenderer(RenderOptions{ LDelim: "[", RDelim: "]", }) @@ -171,7 +171,7 @@ func TestRunTemplates(t *testing.T) { func TestSimpleNamer(t *testing.T) { n := simpleNamer("out/") - out, err := n(context.Background(), "file") + out, err := n.Name(context.Background(), "file") require.NoError(t, err) expected := filepath.FromSlash("out/file") assert.Equal(t, expected, out) @@ -187,13 +187,13 @@ func TestMappingNamer(t *testing.T) { }, } n := mappingNamer("out/{{ .in }}", tr) - out, err := n(ctx, "file") + out, err := n.Name(ctx, "file") require.NoError(t, err) expected := filepath.FromSlash("out/file") assert.Equal(t, expected, out) n = mappingNamer("out/{{ foo }}{{ .in }}", tr) - out, err = n(ctx, "file") + out, err = n.Name(ctx, "file") require.NoError(t, err) expected = filepath.FromSlash("out/foofile") assert.Equal(t, expected, out) diff --git a/internal/cmd/config.go b/internal/cmd/config.go index fc64f8e42..d50a11c83 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -1,9 +1,12 @@ package cmd import ( + "bytes" "context" "fmt" + "io" "log/slog" + "os" "time" "github.com/hairyhenderson/gomplate/v4/conv" @@ -263,3 +266,24 @@ func applyEnvVars(_ context.Context, cfg *config.Config) (*config.Config, error) return cfg, nil } + +// postExecInput - return the input to be used after the post-exec command. The +// input config may be modified if ExecPipe is set (OutputFiles is set to "-"), +// and Stdout is redirected to a pipe. +func postExecInput(cfg *config.Config) io.Reader { + if cfg.ExecPipe { + pipe := &bytes.Buffer{} + cfg.OutputFiles = []string{"-"} + + // --exec-pipe redirects standard out to the out pipe + cfg.Stdout = pipe + + return pipe + } + + if cfg.Stdin != nil { + return cfg.Stdin + } + + return os.Stdin +} diff --git a/internal/cmd/config_test.go b/internal/cmd/config_test.go index 1c9adfe52..5ad5afbb0 100644 --- a/internal/cmd/config_test.go +++ b/internal/cmd/config_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "net/url" + "os" "testing" "testing/fstest" "time" @@ -109,18 +110,36 @@ func TestLoadConfig(t *testing.T) { cmd.ParseFlags([]string{"--in", "foo", "--exec-pipe", "--", "tr", "[a-z]", "[A-Z]"}) out, err = loadConfig(ctx, cmd, cmd.Flags().Args()) expected = &config.Config{ - Input: "foo", - ExecPipe: true, - PostExec: []string{"tr", "[a-z]", "[A-Z]"}, - PostExecInput: out.PostExecInput, - Stdin: stdin, - Stdout: out.Stdout, - Stderr: stderr, + Input: "foo", + ExecPipe: true, + PostExec: []string{"tr", "[a-z]", "[A-Z]"}, + Stdin: stdin, + Stdout: out.Stdout, + Stderr: stderr, } require.NoError(t, err) assert.EqualValues(t, expected, out) } +func TestPostExecInput(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ExecPipe: false} + assert.Equal(t, os.Stdin, postExecInput(cfg)) + + cfg = &config.Config{ExecPipe: true} + + pipe := postExecInput(cfg) + assert.IsType(t, &bytes.Buffer{}, pipe) + assert.Equal(t, []string{"-"}, cfg.OutputFiles) + assert.Equal(t, pipe, cfg.Stdout) + + stdin := &bytes.Buffer{} + cfg = &config.Config{ExecPipe: false, Stdin: stdin} + pipe = postExecInput(cfg) + assert.Equal(t, stdin, pipe) +} + func TestCobraConfig(t *testing.T) { t.Parallel() cmd := &cobra.Command{} diff --git a/internal/cmd/main.go b/internal/cmd/main.go index 223885218..fa7c97c3a 100644 --- a/internal/cmd/main.go +++ b/internal/cmd/main.go @@ -88,12 +88,8 @@ func NewGomplateCmd(stderr io.Writer) *cobra.Command { return err } - if cfg.Experimental { - slog.SetDefault(slog.With("experimental", true)) - slog.InfoContext(ctx, "experimental functions and features enabled!") - - ctx = gomplate.SetExperimental(ctx) - } + // get the post-exec reader now as this may modify cfg + postExecReader := postExecInput(cfg) slog.DebugContext(ctx, fmt.Sprintf("starting %s", cmd.Name())) slog.DebugContext(ctx, fmt.Sprintf("config is:\n%v", cfg), @@ -101,6 +97,7 @@ func NewGomplateCmd(stderr io.Writer) *cobra.Command { slog.String("build", version.GitCommit), ) + // run the main command err = gomplate.Run(ctx, cfg) cmd.SilenceErrors = true cmd.SilenceUsage = true @@ -113,7 +110,8 @@ func NewGomplateCmd(stderr io.Writer) *cobra.Command { if err != nil { return err } - return postRunExec(ctx, cfg.PostExec, cfg.PostExecInput, cmd.OutOrStdout(), cmd.ErrOrStderr()) + + return postRunExec(ctx, cfg.PostExec, postExecReader, cmd.OutOrStdout(), cmd.ErrOrStderr()) }, Args: optionalExecArgs, } diff --git a/internal/config/configfile.go b/internal/config/configfile.go index d255a36cb..7f48e1e94 100644 --- a/internal/config/configfile.go +++ b/internal/config/configfile.go @@ -1,7 +1,6 @@ package config import ( - "bytes" "context" "fmt" "io" @@ -31,23 +30,64 @@ func Parse(in io.Reader) (*Config, error) { return out, nil } -// Config - configures the gomplate execution +// Config models gomplate's configuration file and command-line options. It +// also contains some fields that can't be set in the config file. type Config struct { - Stdin io.Reader `yaml:"-"` + // Stdin - override for stdin:// URLs or the '-' input file. Can't be set in + // the config file. + // Usually this should be left as default - this will be set at runtime. + Stdin io.Reader `yaml:"-"` + + // Stdout - override for the '-' output file. Can't be set in the config + // file. + // Usually this should be left as default - this will be set at runtime. Stdout io.Writer `yaml:"-"` + + // Stderr - override for plugins to write to stderr. Can't be set in the + // config file. + // Usually this should be left as default - this will be set at runtime. Stderr io.Writer `yaml:"-"` + // ExtraHeaders - Extra HTTP headers not attached to pre-defined datsources. + // Potentially used by datasources defined in the template at runtime. Can't + // currently be set in the config file. + ExtraHeaders map[string]http.Header `yaml:"-"` + DataSources map[string]DataSource `yaml:"datasources,omitempty"` Context map[string]DataSource `yaml:"context,omitempty"` + Templates map[string]DataSource `yaml:"templates,omitempty"` Plugins map[string]PluginConfig `yaml:"plugins,omitempty"` - Templates Templates `yaml:"templates,omitempty"` - // Extra HTTP headers not attached to pre-defined datsources. Potentially - // used by datasources defined in the template. - ExtraHeaders map[string]http.Header `yaml:"-"` + Input string `yaml:"in,omitempty"` + InputDir string `yaml:"inputDir,omitempty"` + InputFiles []string `yaml:"inputFiles,omitempty,flow"` + ExcludeGlob []string `yaml:"excludes,omitempty"` + ExcludeProcessingGlob []string `yaml:"excludeProcessing,omitempty"` + + OutputDir string `yaml:"outputDir,omitempty"` + OutputMap string `yaml:"outputMap,omitempty"` + OutputFiles []string `yaml:"outputFiles,omitempty,flow"` + OutMode string `yaml:"chmod,omitempty"` - // internal use only, can't be injected in YAML - PostExecInput io.Reader `yaml:"-"` + LDelim string `yaml:"leftDelim,omitempty"` + RDelim string `yaml:"rightDelim,omitempty"` + + MissingKey string `yaml:"missingKey,omitempty"` + + PostExec []string `yaml:"postExec,omitempty,flow"` + + PluginTimeout time.Duration `yaml:"pluginTimeout,omitempty"` + + ExecPipe bool `yaml:"execPipe,omitempty"` + Experimental bool `yaml:"experimental,omitempty"` +} + +// TODO: remove when we remove the deprecated array format for templates +type rawConfig struct { + DataSources map[string]DataSource `yaml:"datasources,omitempty"` + Context map[string]DataSource `yaml:"context,omitempty"` + Templates Templates `yaml:"templates,omitempty"` + Plugins map[string]PluginConfig `yaml:"plugins,omitempty"` Input string `yaml:"in,omitempty"` InputDir string `yaml:"inputDir,omitempty"` @@ -73,6 +113,72 @@ type Config struct { Experimental bool `yaml:"experimental,omitempty"` } +// TODO: remove when we remove the deprecated array format for templates +// +// Deprecated: custom unmarshaling will be removed in the next version +func (c *Config) UnmarshalYAML(value *yaml.Node) error { + r := rawConfig{} + err := value.Decode(&r) + if err != nil { + return err + } + + *c = Config{ + DataSources: r.DataSources, + Context: r.Context, + Templates: r.Templates, + Plugins: r.Plugins, + Input: r.Input, + InputDir: r.InputDir, + InputFiles: r.InputFiles, + ExcludeGlob: r.ExcludeGlob, + ExcludeProcessingGlob: r.ExcludeProcessingGlob, + OutputDir: r.OutputDir, + OutputMap: r.OutputMap, + OutputFiles: r.OutputFiles, + OutMode: r.OutMode, + LDelim: r.LDelim, + RDelim: r.RDelim, + MissingKey: r.MissingKey, + PostExec: r.PostExec, + PluginTimeout: r.PluginTimeout, + ExecPipe: r.ExecPipe, + Experimental: r.Experimental, + } + + return nil +} + +// TODO: remove when we remove the deprecated array format for templates +// +// Deprecated: custom unmarshaling will be removed in the next version +func (c Config) MarshalYAML() (interface{}, error) { + aux := rawConfig{ + DataSources: c.DataSources, + Context: c.Context, + Templates: c.Templates, + Plugins: c.Plugins, + Input: c.Input, + InputDir: c.InputDir, + InputFiles: c.InputFiles, + ExcludeGlob: c.ExcludeGlob, + ExcludeProcessingGlob: c.ExcludeProcessingGlob, + OutputDir: c.OutputDir, + OutputMap: c.OutputMap, + OutputFiles: c.OutputFiles, + OutMode: c.OutMode, + LDelim: c.LDelim, + RDelim: c.RDelim, + MissingKey: c.MissingKey, + PostExec: c.PostExec, + PluginTimeout: c.PluginTimeout, + ExecPipe: c.ExecPipe, + Experimental: c.Experimental, + } + + return aux, nil +} + type experimentalCtxKey struct{} func SetExperimental(ctx context.Context) context.Context { @@ -538,7 +644,7 @@ func (c *Config) ApplyDefaults() { if c.Input == "" && c.InputDir == "" && len(c.InputFiles) == 0 { c.InputFiles = []string{"-"} } - if c.OutputDir == "" && c.OutputMap == "" && len(c.OutputFiles) == 0 && !c.ExecPipe { + if c.OutputDir == "" && c.OutputMap == "" && len(c.OutputFiles) == 0 { c.OutputFiles = []string{"-"} } if c.LDelim == "" { @@ -551,17 +657,6 @@ func (c *Config) ApplyDefaults() { c.MissingKey = "error" } - if c.ExecPipe { - pipe := &bytes.Buffer{} - c.PostExecInput = pipe - c.OutputFiles = []string{"-"} - - // --exec-pipe redirects standard out to the out pipe - c.Stdout = pipe - } else { - c.PostExecInput = c.Stdin - } - if c.PluginTimeout == 0 { c.PluginTimeout = 5 * time.Second } diff --git a/internal/config/configfile_test.go b/internal/config/configfile_test.go index 3cd1b1095..dca94a236 100644 --- a/internal/config/configfile_test.go +++ b/internal/config/configfile_test.go @@ -24,6 +24,24 @@ func TestParseConfigFile(t *testing.T) { require.NoError(t, err) assert.Equal(t, expected, cf) + // legacy array form for templates (will be removed in v4.1.0 or so) + in = `in: hello world +templates: + - foo=bar + - baz=https://example.com/baz.yaml +` + + expected = &Config{ + Input: "hello world", + Templates: map[string]DataSource{ + "foo": {URL: mustURL("bar")}, + "baz": {URL: mustURL("https://example.com/baz.yaml")}, + }, + } + cf, err = Parse(strings.NewReader(in)) + require.NoError(t, err) + assert.Equal(t, expected, cf) + in = `in: hello world outputFiles: [out.txt] chmod: 644 @@ -74,7 +92,7 @@ pluginTimeout: 2s Plugins: map[string]PluginConfig{ "foo": {Cmd: "echo", Pipe: true}, }, - Templates: Templates{"foo": DataSource{URL: mustURL("file:///tmp/foo.t")}}, + Templates: map[string]DataSource{"foo": {URL: mustURL("file:///tmp/foo.t")}}, PluginTimeout: 2 * time.Second, } @@ -386,7 +404,7 @@ func TestMergeFrom(t *testing.T) { cfg = &Config{ InputDir: "indir/", ExcludeGlob: []string{"*.txt"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": { URL: mustURL("file:///foo.yaml"), }, @@ -402,7 +420,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "baz": {URL: mustURL("vault:///baz")}, }, @@ -414,7 +432,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "bar": { URL: mustURL("stdin:///"), @@ -503,7 +521,7 @@ func TestParseDataSourceFlags(t *testing.T) { ) require.NoError(t, err) assert.EqualValues(t, &Config{ - Templates: Templates{ + Templates: map[string]DataSource{ "foo": { URL: mustURL("http://example.com"), Header: http.Header{"Accept": {"application/json"}}, @@ -551,7 +569,7 @@ pluginTimeout: 5s RDelim: "R", Input: "foo", OutputFiles: []string{"-"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, @@ -576,7 +594,7 @@ templates: RDelim: "R", Input: "long input that should be truncated", OutputFiles: []string{"-"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, diff --git a/internal/datafs/context.go b/internal/datafs/context.go index 30722f2b2..310d4445d 100644 --- a/internal/datafs/context.go +++ b/internal/datafs/context.go @@ -30,10 +30,15 @@ func WithDataSourceRegistryFS(registry Registry, fsys fs.FS) fs.FS { type stdinCtxKey struct{} +// ContextWithStdin injects an [io.Reader] into the context, which can be used +// to override the default stdin. func ContextWithStdin(ctx context.Context, r io.Reader) context.Context { return context.WithValue(ctx, stdinCtxKey{}, r) } +// StdinFromContext returns the io.Reader that should be used for stdin as +// injected by [ContextWithStdin]. If no reader has been injected, [os.Stdin] is +// returned. func StdinFromContext(ctx context.Context) io.Reader { if r, ok := ctx.Value(stdinCtxKey{}).(io.Reader); ok { return r diff --git a/render.go b/render.go index cf55e06ea..67a361f57 100644 --- a/render.go +++ b/render.go @@ -6,7 +6,6 @@ import ( "io" "io/fs" "net/http" - "net/url" "path" "slices" "strings" @@ -21,22 +20,17 @@ import ( "github.com/hairyhenderson/gomplate/v4/internal/funcs" ) -// Options for template rendering. -// -// Experimental: subject to breaking changes before the next major release -type Options struct { - // FSProvider - allows lookups of data source filesystems. Defaults to - // [DefaultFSProvider]. - FSProvider fsimpl.FSProvider - +// RenderOptions - options for controlling how templates are rendered, and +// what data are available. +type RenderOptions struct { // Datasources - map of datasources to be read on demand when the // 'datasource'/'ds'/'include' functions are used. - Datasources map[string]Datasource + Datasources map[string]DataSource // Context - map of datasources to be read immediately and added to the // template's context - Context map[string]Datasource + Context map[string]DataSource // Templates - map of templates that can be referenced as nested templates - Templates map[string]Datasource + Templates map[string]DataSource // Extra HTTP headers not attached to pre-defined datsources. Potentially // used by datasources defined in the template. @@ -55,62 +49,32 @@ type Options struct { // MissingKey controls the behavior during execution if a map is indexed with a key that is not present in the map MissingKey string - - // Experimental - enable experimental features - Experimental bool } -// optionsFromConfig - create a set of options from the internal config struct. +// optionsFromConfig - translate the internal config struct to a RenderOptions. // Does not set the Funcs field. -func optionsFromConfig(cfg *config.Config) Options { - ds := make(map[string]Datasource, len(cfg.DataSources)) - for k, v := range cfg.DataSources { - ds[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - cs := make(map[string]Datasource, len(cfg.Context)) - for k, v := range cfg.Context { - cs[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - ts := make(map[string]Datasource, len(cfg.Templates)) - for k, v := range cfg.Templates { - ts[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - - opts := Options{ - Datasources: ds, - Context: cs, - Templates: ts, +func optionsFromConfig(cfg *config.Config) RenderOptions { + opts := RenderOptions{ + Datasources: cfg.DataSources, + Context: cfg.Context, + Templates: cfg.Templates, ExtraHeaders: cfg.ExtraHeaders, LDelim: cfg.LDelim, RDelim: cfg.RDelim, MissingKey: cfg.MissingKey, - Experimental: cfg.Experimental, } return opts } -// Datasource - a datasource URL with optional headers +// DataSource - a datasource URL with optional headers // // Experimental: subject to breaking changes before the next major release -type Datasource struct { - URL *url.URL - Header http.Header -} +type DataSource = config.DataSource type renderer struct { sr datafs.DataSourceReader - fsp fsimpl.FSProvider - nested config.Templates + nested map[string]config.DataSource funcs template.FuncMap lDelim string rDelim string @@ -120,8 +84,6 @@ type renderer struct { // Renderer provides gomplate's core template rendering functionality. // See [NewRenderer]. -// -// Experimental: subject to breaking changes before the next major release type Renderer interface { // RenderTemplates renders a list of templates, parsing each template's // Text and executing it, outputting to its Writer. If a template's Writer @@ -140,11 +102,11 @@ type Renderer interface { // use. // // Experimental: subject to breaking changes before the next major release -func NewRenderer(opts Options) Renderer { +func NewRenderer(opts RenderOptions) Renderer { return newRenderer(opts) } -func newRenderer(opts Options) *renderer { +func newRenderer(opts RenderOptions) *renderer { if Metrics == nil { Metrics = newMetrics() } @@ -170,7 +132,7 @@ func newRenderer(opts Options) *renderer { // convert the internal config.Templates to a map[string]Datasource // TODO: simplify when config.Templates is removed - nested := config.Templates{} + nested := map[string]config.DataSource{} for alias, ds := range opts.Templates { nested[alias] = config.DataSource{ URL: ds.URL, @@ -191,28 +153,20 @@ func newRenderer(opts Options) *renderer { missingKey = "error" } - if opts.FSProvider == nil { - opts.FSProvider = DefaultFSProvider - } - - // TODO: move this in? sr := datafs.NewSourceReader(reg) return &renderer{ - nested: nested, + nested: opts.Templates, sr: sr, funcs: opts.Funcs, tctxAliases: tctxAliases, lDelim: opts.LDelim, rDelim: opts.RDelim, missingKey: missingKey, - fsp: opts.FSProvider, } } // Template contains the basic data needed to render a template with a Renderer -// -// Experimental: subject to breaking changes before the next major release type Template struct { // Writer is the writer to output the rendered template to. If this writer // is a non-os.Stdout io.Closer, it will be closed after the template is @@ -224,37 +178,37 @@ type Template struct { Text string } -func (t *renderer) RenderTemplates(ctx context.Context, templates []Template) error { +func (r *renderer) RenderTemplates(ctx context.Context, templates []Template) error { if datafs.FSProviderFromContext(ctx) == nil { - ctx = datafs.ContextWithFSProvider(ctx, t.fsp) + ctx = datafs.ContextWithFSProvider(ctx, DefaultFSProvider) } // configure the template context with the refreshed Data value // only done here because the data context may have changed - tmplctx, err := createTmplContext(ctx, t.tctxAliases, t.sr) + tmplctx, err := createTmplContext(ctx, r.tctxAliases, r.sr) if err != nil { return err } - return t.renderTemplatesWithData(ctx, templates, tmplctx) + return r.renderTemplatesWithData(ctx, templates, tmplctx) } -func (t *renderer) renderTemplatesWithData(ctx context.Context, templates []Template, tmplctx interface{}) error { +func (r *renderer) renderTemplatesWithData(ctx context.Context, templates []Template, tmplctx interface{}) error { // update funcs with the current context // only done here to ensure the context is properly set in func namespaces f := CreateFuncs(ctx) // add datasource funcs here because they need to share the source reader - addToMap(f, funcs.CreateDataSourceFuncs(ctx, t.sr)) + addToMap(f, funcs.CreateDataSourceFuncs(ctx, r.sr)) // add user-defined funcs last so they override the built-in funcs - addToMap(f, t.funcs) + addToMap(f, r.funcs) // track some metrics for debug output start := time.Now() defer func() { Metrics.TotalRenderDuration = time.Since(start) }() for _, template := range templates { - err := t.renderTemplate(ctx, template, f, tmplctx) + err := r.renderTemplate(ctx, template, f, tmplctx) if err != nil { return fmt.Errorf("renderTemplate: %w", err) } @@ -262,7 +216,7 @@ func (t *renderer) renderTemplatesWithData(ctx context.Context, templates []Temp return nil } -func (t *renderer) renderTemplate(ctx context.Context, template Template, f template.FuncMap, tmplctx interface{}) error { +func (r *renderer) renderTemplate(ctx context.Context, template Template, f template.FuncMap, tmplctx interface{}) error { if template.Writer != nil { if wr, ok := template.Writer.(io.Closer); ok { defer wr.Close() @@ -270,7 +224,7 @@ func (t *renderer) renderTemplate(ctx context.Context, template Template, f temp } tstart := time.Now() - tmpl, err := t.parseTemplate(ctx, template.Name, template.Text, f, tmplctx) + tmpl, err := r.parseTemplate(ctx, template.Name, template.Text, f, tmplctx) if err != nil { return fmt.Errorf("parse template %s: %w", template.Name, err) } @@ -286,17 +240,17 @@ func (t *renderer) renderTemplate(ctx context.Context, template Template, f temp return nil } -func (t *renderer) Render(ctx context.Context, name, text string, wr io.Writer) error { - return t.RenderTemplates(ctx, []Template{ +func (r *renderer) Render(ctx context.Context, name, text string, wr io.Writer) error { + return r.RenderTemplates(ctx, []Template{ {Name: name, Text: text, Writer: wr}, }) } // parseTemplate - parses text as a Go template with the given name and options -func (t *renderer) parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}) (tmpl *template.Template, err error) { +func (r *renderer) parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}) (tmpl *template.Template, err error) { tmpl = template.New(name) - missingKey := t.missingKey + missingKey := r.missingKey if missingKey == "" { missingKey = "error" } @@ -313,13 +267,13 @@ func (t *renderer) parseTemplate(ctx context.Context, name, text string, funcs t // the "tmpl" funcs get added here because they need access to the root template and context addTmplFuncs(funcMap, tmpl, tmplctx, name) tmpl.Funcs(funcMap) - tmpl.Delims(t.lDelim, t.rDelim) + tmpl.Delims(r.lDelim, r.rDelim) _, err = tmpl.Parse(text) if err != nil { return nil, err } - err = t.parseNestedTemplates(ctx, tmpl) + err = r.parseNestedTemplates(ctx, tmpl) if err != nil { return nil, fmt.Errorf("parse nested templates: %w", err) } @@ -327,10 +281,10 @@ func (t *renderer) parseTemplate(ctx context.Context, name, text string, funcs t return tmpl, nil } -func (t *renderer) parseNestedTemplates(ctx context.Context, tmpl *template.Template) error { +func (r *renderer) parseNestedTemplates(ctx context.Context, tmpl *template.Template) error { fsp := datafs.FSProviderFromContext(ctx) - for alias, n := range t.nested { + for alias, n := range r.nested { u := *n.URL fname := path.Base(u.Path) @@ -361,7 +315,7 @@ func (t *renderer) parseNestedTemplates(ctx context.Context, tmpl *template.Temp // inject context & header in case they're useful... fsys = fsimpl.WithContextFS(ctx, fsys) fsys = fsimpl.WithHeaderFS(n.Header, fsys) - fsys = datafs.WithDataSourceRegistryFS(t.sr, fsys) + fsys = datafs.WithDataSourceRegistryFS(r.sr, fsys) // valid fs.FS paths have no trailing slash fname = strings.TrimRight(fname, "/") diff --git a/render_test.go b/render_test.go index 1a4d00168..936e0b014 100644 --- a/render_test.go +++ b/render_test.go @@ -31,7 +31,7 @@ func TestRenderTemplate(t *testing.T) { ctx := datafs.ContextWithFSProvider(context.Background(), fsp) // no options - built-in function - tr := NewRenderer(Options{}) + tr := NewRenderer(RenderOptions{}) out := &bytes.Buffer{} err := tr.Render(ctx, "test", "{{ `hello world` | toUpper }}", out) require.NoError(t, err) @@ -43,11 +43,11 @@ func TestRenderTemplate(t *testing.T) { t.Setenv("WORLD", "world") - tr = NewRenderer(Options{ - Context: map[string]Datasource{ + tr = NewRenderer(RenderOptions{ + Context: map[string]DataSource{ "hi": {URL: hu}, }, - Datasources: map[string]Datasource{ + Datasources: map[string]DataSource{ "world": {URL: wu}, }, }) @@ -62,8 +62,8 @@ func TestRenderTemplate(t *testing.T) { fsys["nested.tmpl"] = &fstest.MapFile{Data: []byte( `<< . | toUpper >>`)} - tr = NewRenderer(Options{ - Templates: map[string]Datasource{ + tr = NewRenderer(RenderOptions{ + Templates: map[string]DataSource{ "nested": {URL: nu}, }, LDelim: "<<", @@ -75,7 +75,7 @@ func TestRenderTemplate(t *testing.T) { assert.Equal(t, "HELLO", out.String()) // errors contain the template name - tr = NewRenderer(Options{}) + tr = NewRenderer(RenderOptions{}) err = tr.Render(ctx, "foo", `{{ bogus }}`, &bytes.Buffer{}) assert.ErrorContains(t, err, "template: foo:") } @@ -86,7 +86,7 @@ func ExampleRenderer() { ctx := context.Background() // create a new template renderer - tr := NewRenderer(Options{}) + tr := NewRenderer(RenderOptions{}) // render a template to stdout err := tr.Render(ctx, "mytemplate", @@ -104,7 +104,7 @@ func ExampleRenderer_manyTemplates() { ctx := context.Background() // create a new template renderer - tr := NewRenderer(Options{}) + tr := NewRenderer(RenderOptions{}) templates := []Template{ { @@ -145,8 +145,8 @@ func ExampleRenderer_datasources() { // a datasource that retrieves JSON from a public API u, _ := url.Parse("https://ipinfo.io/1.1.1.1") - tr := NewRenderer(Options{ - Context: map[string]Datasource{ + tr := NewRenderer(RenderOptions{ + Context: map[string]DataSource{ "info": {URL: u}, }, }) diff --git a/template.go b/template.go index bb8976cfd..9ee51056d 100644 --- a/template.go +++ b/template.go @@ -45,9 +45,7 @@ func copyFuncMap(funcMap template.FuncMap) template.FuncMap { } // gatherTemplates - gather and prepare templates for rendering -// -//nolint:gocyclo -func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func(context.Context, string) (string, error)) ([]Template, error) { +func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer outputNamer) ([]Template, error) { mode, modeOverride, err := cfg.GetMode() if err != nil { return nil, err @@ -56,7 +54,6 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( var templates []Template switch { - // the arg-provided input string gets a special name case cfg.Input != "": // open the output file - no need to close it, as it will be closed by the // caller later @@ -66,6 +63,7 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( } templates = []Template{{ + // the arg-provided input string gets a special name Name: "", Text: cfg.Input, Writer: target, @@ -76,7 +74,7 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( if err != nil { return nil, fmt.Errorf("walkDir: %w", err) } - case cfg.Input == "": + case len(cfg.InputFiles) > 0: templates = make([]Template, len(cfg.InputFiles)) for i, f := range cfg.InputFiles { templates[i], err = fileToTemplate(ctx, cfg, f, cfg.OutputFiles[i], mode, modeOverride) @@ -92,7 +90,7 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( // walkDir - given an input dir `dir` and an output dir `outDir`, and a list // of .gomplateignore and exclude globs (if any), walk the input directory and create a list of // tplate objects, and an error, if any. -func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer func(context.Context, string) (string, error), excludeGlob []string, excludeProcessingGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) { +func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer outputNamer, excludeGlob []string, excludeProcessingGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) { dir = filepath.ToSlash(filepath.Clean(dir)) // get a filesystem rooted in the same volume as dir (or / on non-Windows) @@ -157,7 +155,7 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f inPath = filepath.ToSlash(inPath) // but outFileNamer expects only the filename itself - outFile, err := outFileNamer(ctx, file) + outFile, err := outFileNamer.Name(ctx, file) if err != nil { return nil, fmt.Errorf("outFileNamer: %w", err) } @@ -192,13 +190,14 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f return templates, nil } -func readInFile(ctx context.Context, cfg *config.Config, inFile string, mode os.FileMode) (source string, newmode os.FileMode, err error) { +func readInFile(ctx context.Context, inFile string, mode os.FileMode) (source string, newmode os.FileMode, err error) { newmode = mode var b []byte //nolint:nestif if inFile == "-" { - b, err = io.ReadAll(cfg.Stdin) + stdin := datafs.StdinFromContext(ctx) + b, err = io.ReadAll(stdin) if err != nil { return source, newmode, fmt.Errorf("read from stdin: %w", err) } @@ -244,7 +243,7 @@ func getOutfileHandler(ctx context.Context, cfg *config.Config, outFile string, } func copyFileToOutDir(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) error { - sourceStr, newmode, err := readInFile(ctx, cfg, inFile, mode) + sourceStr, newmode, err := readInFile(ctx, inFile, mode) if err != nil { return err } @@ -264,7 +263,7 @@ func copyFileToOutDir(ctx context.Context, cfg *config.Config, inFile, outFile s } func fileToTemplate(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) (Template, error) { - source, newmode, err := readInFile(ctx, cfg, inFile, mode) + source, newmode, err := readInFile(ctx, inFile, mode) if err != nil { return Template{}, err } diff --git a/template_test.go b/template_test.go index c9d716ad0..391307546 100644 --- a/template_test.go +++ b/template_test.go @@ -195,7 +195,7 @@ func TestParseNestedTemplates(t *testing.T) { // simple test with single template u, _ := url.Parse("foo.t") - nested := config.Templates{"foo": {URL: u}} + nested := map[string]config.DataSource{"foo": {URL: u}} tmpl, _ := template.New("root").Parse(`{{ template "foo" }}`)