Add --exclude-processing option (#1963)

* add --exclude-processing option

* update documentation

* add tests and address some PR comments

* fix linting errors

* TODO comment

---------

Co-authored-by: Dave Henderson <[email protected]>

config.go

@@ -16,15 +16,16 @@ import (
 // [github.com/hairyhenderson/gomplate/v4/internal/config.Config] is used
 // everywhere else, and will be exposed as API in a future version
 type Config struct {
-	Input       string
-	InputFiles  []string
-	InputDir    string
-	ExcludeGlob []string
-	OutputFiles []string
-	OutputDir   string
-	OutputMap   string
-	OutMode     string
-	Out         io.Writer
+	Input                 string
+	InputFiles            []string
+	InputDir              string
+	ExcludeGlob           []string
+	ExcludeProcessingGlob []string
+	OutputFiles           []string
+	OutputDir             string
+	OutputMap             string
+	OutMode               string
+	Out                   io.Writer
 
 	DataSources       []string
 	DataSourceHeaders []string
@@ -76,6 +77,10 @@ func (o *Config) String() string {
 		c += "\nexclude: " + strings.Join(o.ExcludeGlob, ", ")
 	}
 
+	if len(o.ExcludeProcessingGlob) > 0 {
+		c += "\nexcludeProcessing: " + strings.Join(o.ExcludeProcessingGlob, ", ")
+	}
+
 	c += "\noutput: "
 	switch {
 	case o.InputDir != "" && o.OutputDir != ".":
@@ -119,19 +124,20 @@ func (o *Config) String() string {
 
 func (o *Config) toNewConfig() (*config.Config, error) {
 	cfg := &config.Config{
-		Input:       o.Input,
-		InputFiles:  o.InputFiles,
-		InputDir:    o.InputDir,
-		ExcludeGlob: o.ExcludeGlob,
-		OutputFiles: o.OutputFiles,
-		OutputDir:   o.OutputDir,
-		OutputMap:   o.OutputMap,
-		OutMode:     o.OutMode,
-		LDelim:      o.LDelim,
-		RDelim:      o.RDelim,
-		Stdin:       os.Stdin,
-		Stdout:      &iohelpers.NopCloser{Writer: o.Out},
-		Stderr:      os.Stderr,
+		Input:                 o.Input,
+		InputFiles:            o.InputFiles,
+		InputDir:              o.InputDir,
+		ExcludeGlob:           o.ExcludeGlob,
+		ExcludeProcessingGlob: o.ExcludeProcessingGlob,
+		OutputFiles:           o.OutputFiles,
+		OutputDir:             o.OutputDir,
+		OutputMap:             o.OutputMap,
+		OutMode:               o.OutMode,
+		LDelim:                o.LDelim,
+		RDelim:                o.RDelim,
+		Stdin:                 os.Stdin,
+		Stdout:                &iohelpers.NopCloser{Writer: o.Out},
+		Stderr:                os.Stderr,
 	}
 	err := cfg.ParsePluginFlags(o.Plugins)
 	if err != nil {

docs/content/config.md

@@ -139,6 +139,20 @@ excludes:
 This will skip all files with the extension `.txt`, except for files named
 `include-this.txt`, which will be processed.
 
+## `excludeProcessing`
+
+See [`--exclude-processing`](../usage/#exclude-processing).
+
+This is an array of exclude patterns, used in conjunction with [`inputDir`](#inputdir).
+The matching files will be copied to the output directory without template rendering. 
+
+```yaml
+excludeProcessing:
+  - '*.jpg'
+```
+
+This will copy all files with the extension `.jpg` to the output directory.
+
 ## `execPipe`
 
 See [`--exec-pipe`](../usage/#exec-pipe).

docs/content/usage.md

@@ -130,6 +130,20 @@ $ gomplate --include *.tmpl --exclude foo*.tmpl --input-dir in/ --output-dir out
 
 This will cause only files ending in `.tmpl` to be processed, except for files with names beginning with `foo`: `template.tmpl` will be included, but `foo-template.tmpl` will not.
 
+### `--exclude-processing`
+
+When using the [`--input-dir`](#input-dir-and-output-dir) argument, it can be useful to skip some files from processing and copy them directly to the output directory. Like the `--exclude` flag, it takes a [`.gitignore`][]-style pattern, and any files match the pattern will be copied.
+
+_Note:_ These patterns are _not_ treated as filesystem globs, and so a pattern like `/foo/bar.json` will match relative to the input directory, not the root of the filesystem as they may appear!
+
+Examples:
+
+```console
+$ gomplate --exclude-processing `*.png` --input-dir in/ --output-dir out/
+```
+
+This will skip all `*.png` files in the `in/` directory from being processed, and copy them to the `out/` directory.
+
 #### `.gomplateignore` files
 
 You can also use a file named `.gomplateignore` containing one exclude pattern on each line. This has the same syntax as a [`.gitignore`][] file.

internal/cmd/config.go

@@ -114,6 +114,11 @@ func cobraConfig(cmd *cobra.Command, args []string) (cfg *config.Config, err err
 	if err != nil {
 		return nil, err
 	}
+	cfg.ExcludeProcessingGlob, err = getStringSlice(cmd, "exclude-processing")
+	if err != nil {
+		return nil, err
+	}
+
 	includesFlag, err := getStringSlice(cmd, "include")
 	if err != nil {
 		return nil, err

internal/cmd/main.go

@@ -139,6 +139,7 @@ func InitFlags(command *cobra.Command) {
 	command.Flags().String("input-dir", "", "`directory` which is examined recursively for templates (alternative to --file and --in)")
 
 	command.Flags().StringSlice("exclude", []string{}, "glob of files to not parse")
+	command.Flags().StringSlice("exclude-processing", []string{}, "glob of files to be copied without parsing")
 	command.Flags().StringSlice("include", []string{}, "glob of files to parse")
 
 	command.Flags().StringSliceP("out", "o", []string{"-"}, "output `file` name. Omit to use standard output.")

internal/config/configfile.go

@@ -49,10 +49,11 @@ type Config struct {
 	// internal use only, can't be injected in YAML
 	PostExecInput io.Reader `yaml:"-"`
 
-	Input       string   `yaml:"in,omitempty"`
-	InputDir    string   `yaml:"inputDir,omitempty"`
-	InputFiles  []string `yaml:"inputFiles,omitempty,flow"`
-	ExcludeGlob []string `yaml:"excludes,omitempty"`
+	Input                 string   `yaml:"in,omitempty"`
+	InputDir              string   `yaml:"inputDir,omitempty"`
+	InputFiles            []string `yaml:"inputFiles,omitempty,flow"`
+	ExcludeGlob           []string `yaml:"excludes,omitempty"`
+	ExcludeProcessingGlob []string `yaml:"excludeProcessing,omitempty"`
 
 	OutputDir   string   `yaml:"outputDir,omitempty"`
 	OutputMap   string   `yaml:"outputMap,omitempty"`
@@ -246,6 +247,9 @@ func (c *Config) MergeFrom(o *Config) *Config {
 	if !isZero(o.ExcludeGlob) {
 		c.ExcludeGlob = o.ExcludeGlob
 	}
+	if !isZero(o.ExcludeProcessingGlob) {
+		c.ExcludeProcessingGlob = o.ExcludeProcessingGlob
+	}
 	if !isZero(o.OutMode) {
 		c.OutMode = o.OutMode
 	}

internal/tests/integration/gomplateignore_test.go

@@ -279,3 +279,34 @@ func TestGomplateignore_WithIncludes(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, fromSlashes("rules/index.csv"), files)
 }
+
+func TestGomplateignore_WithExcludeProcessing(t *testing.T) {
+	files, err := executeOpts(t, `.gomplateignore
+*.log
+`, []string{
+		"--exclude-processing", "crash.bin",
+		"--exclude-processing", "log/*.zip",
+		"--exclude", "rules/*.txt",
+		"--exclude", "sprites/*.ini",
+	},
+		tfs.WithDir("logs",
+			tfs.WithFile("archive.zip", ""),
+			tfs.WithFile("engine.log", ""),
+			tfs.WithFile("skills.log", "")),
+		tfs.WithDir("rules",
+			tfs.WithFile("index.csv", ""),
+			tfs.WithFile("fire.txt", ""),
+			tfs.WithFile("earth.txt", "")),
+		tfs.WithDir("sprites",
+			tfs.WithFile("human.csv", ""),
+			tfs.WithFile("demon.xml", ""),
+			tfs.WithFile("alien.ini", "")),
+		tfs.WithFile("manifest.json", ""),
+		tfs.WithFile("crash.bin", ""),
+	)
+
+	require.NoError(t, err)
+	assert.Equal(t, fromSlashes(
+		"crash.bin", "logs/archive.zip", "manifest.json", "rules/index.csv",
+		"sprites/demon.xml", "sprites/human.csv"), files)
+}

template.go

@@ -204,7 +204,7 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func(
 		}}
 	case cfg.InputDir != "":
 		// input dirs presume output dirs are set too
-		templates, err = walkDir(ctx, cfg, cfg.InputDir, outFileNamer, cfg.ExcludeGlob, mode, modeOverride)
+		templates, err = walkDir(ctx, cfg, cfg.InputDir, outFileNamer, cfg.ExcludeGlob, cfg.ExcludeProcessingGlob, mode, modeOverride)
 		if err != nil {
 			return nil, fmt.Errorf("walkDir: %w", err)
 		}
@@ -224,7 +224,7 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func(
 // walkDir - given an input dir `dir` and an output dir `outDir`, and a list
 // of .gomplateignore and exclude globs (if any), walk the input directory and create a list of
 // tplate objects, and an error, if any.
-func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer func(context.Context, string) (string, error), excludeGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) {
+func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer func(context.Context, string) (string, error), excludeGlob []string, excludeProcessingGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) {
 	dir = filepath.ToSlash(filepath.Clean(dir))
 
 	// get a filesystem rooted in the same volume as dir (or / on non-Windows)
@@ -256,7 +256,7 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f
 	templates := make([]Template, 0)
 	matcher := xignore.NewMatcher(subfsys)
 
-	matches, err := matcher.Matches(".", &xignore.MatchesOptions{
+	excludeMatches, err := matcher.Matches(".", &xignore.MatchesOptions{
 		Ignorefile:    gomplateignore,
 		Nested:        true, // allow nested ignorefile
 		AfterPatterns: excludeGlob,
@@ -265,8 +265,25 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f
 		return nil, fmt.Errorf("ignore matching failed for %s: %w", dir, err)
 	}
 
+	excludeProcessingMatches, err := matcher.Matches(".", &xignore.MatchesOptions{
+		// TODO: fix or replace xignore module so we can avoid attempting to read the .gomplateignore file for both exclude and excludeProcessing patterns
+		Ignorefile:    gomplateignore,
+		Nested:        true, // allow nested ignorefile
+		AfterPatterns: excludeProcessingGlob,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("passthough matching failed for %s: %w", dir, err)
+	}
+
+	passthroughFiles := make(map[string]bool)
+
+	for _, file := range excludeProcessingMatches.MatchedFiles {
+		// files that need to be directly copied
+		passthroughFiles[file] = true
+	}
+
 	// Unmatched ignorefile rules's files
-	for _, file := range matches.UnmatchedFiles {
+	for _, file := range excludeMatches.UnmatchedFiles {
 		// we want to pass an absolute (as much as possible) path to fileToTemplate
 		inPath := filepath.Join(dir, file)
 		inPath = filepath.ToSlash(inPath)
@@ -277,6 +294,16 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f
 			return nil, fmt.Errorf("outFileNamer: %w", err)
 		}
 
+		_, ok := passthroughFiles[file]
+		if ok {
+			err = copyFileToOutDir(ctx, cfg, inPath, outFile, mode, modeOverride)
+			if err != nil {
+				return nil, fmt.Errorf("copyFileToOutDir: %w", err)
+			}
+
+			continue
+		}
+
 		tpl, err := fileToTemplate(ctx, cfg, inPath, outFile, mode, modeOverride)
 		if err != nil {
 			return nil, fmt.Errorf("fileToTemplate: %w", err)
@@ -297,46 +324,86 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f
 	return templates, nil
 }
 
-func fileToTemplate(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) (Template, error) {
-	source := ""
+func readInFile(ctx context.Context, cfg *config.Config, inFile string, mode os.FileMode) (source string, newmode os.FileMode, err error) {
+	newmode = mode
+	var b []byte
 
 	//nolint:nestif
 	if inFile == "-" {
-		b, err := io.ReadAll(cfg.Stdin)
+		b, err = io.ReadAll(cfg.Stdin)
 		if err != nil {
-			return Template{}, fmt.Errorf("read from stdin: %w", err)
+			return source, newmode, fmt.Errorf("read from stdin: %w", err)
 		}
 
 		source = string(b)
 	} else {
-		fsys, err := datafs.FSysForPath(ctx, inFile)
+		var fsys fs.FS
+		var si fs.FileInfo
+		fsys, err = datafs.FSysForPath(ctx, inFile)
 		if err != nil {
-			return Template{}, fmt.Errorf("fsysForPath: %w", err)
+			return source, newmode, fmt.Errorf("fsysForPath: %w", err)
 		}
 
-		si, err := fs.Stat(fsys, inFile)
+		si, err = fs.Stat(fsys, inFile)
 		if err != nil {
-			return Template{}, fmt.Errorf("stat %q: %w", inFile, err)
+			return source, newmode, fmt.Errorf("stat %q: %w", inFile, err)
 		}
 		if mode == 0 {
-			mode = si.Mode()
+			newmode = si.Mode()
 		}
 
 		// we read the file and store in memory immediately, to prevent leaking
 		// file descriptors.
-		b, err := fs.ReadFile(fsys, inFile)
+		b, err = fs.ReadFile(fsys, inFile)
 		if err != nil {
-			return Template{}, fmt.Errorf("readAll %q: %w", inFile, err)
+			return source, newmode, fmt.Errorf("readAll %q: %w", inFile, err)
 		}
 
 		source = string(b)
 	}
+	return source, newmode, err
+}
 
+func getOutfileHandler(ctx context.Context, cfg *config.Config, outFile string, mode os.FileMode, modeOverride bool) (io.Writer, error) {
 	// open the output file - no need to close it, as it will be closed by the
 	// caller later
 	target, err := openOutFile(ctx, outFile, 0o755, mode, modeOverride, cfg.Stdout, cfg.SuppressEmpty)
 	if err != nil {
-		return Template{}, fmt.Errorf("openOutFile: %w", err)
+		return nil, fmt.Errorf("openOutFile: %w", err)
+	}
+
+	return target, nil
+}
+
+func copyFileToOutDir(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) error {
+	sourceStr, newmode, err := readInFile(ctx, cfg, inFile, mode)
+	if err != nil {
+		return err
+	}
+
+	outFH, err := getOutfileHandler(ctx, cfg, outFile, newmode, modeOverride)
+	if err != nil {
+		return err
+	}
+
+	wr, ok := outFH.(io.Closer)
+	if ok && wr != os.Stdout {
+		defer wr.Close()
+	}
+
+	_, err = outFH.Write([]byte(sourceStr))
+	return err
+}
+
+func fileToTemplate(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) (Template, error) {
+	source, newmode, err := readInFile(ctx, cfg, inFile, mode)
+	if err != nil {
+		return Template{}, err
+	}
+
+	target, err := getOutfileHandler(ctx, cfg, outFile, newmode, modeOverride)
+	if err != nil {
+		return Template{}, err
 	}
 
 	tmpl := Template{

template_unix_test.go

@@ -23,7 +23,7 @@ func TestWalkDir_UNIX(t *testing.T) {
 
 	cfg := &config.Config{}
 
-	_, err := walkDir(ctx, cfg, "/indir", simpleNamer("/outdir"), nil, 0, false)
+	_, err := walkDir(ctx, cfg, "/indir", simpleNamer("/outdir"), nil, nil, 0, false)
 	assert.Error(t, err)
 
 	err = hackpadfs.MkdirAll(fsys, "/indir/one", 0o777)
@@ -37,7 +37,7 @@ func TestWalkDir_UNIX(t *testing.T) {
 	err = hackpadfs.WriteFullFile(fsys, "/indir/two/baz", []byte("baz"), 0o644)
 	require.NoError(t, err)
 
-	templates, err := walkDir(ctx, cfg, "/indir", simpleNamer("/outdir"), []string{"*/two"}, 0, false)
+	templates, err := walkDir(ctx, cfg, "/indir", simpleNamer("/outdir"), []string{"*/two"}, []string{}, 0, false)
 	require.NoError(t, err)
 
 	expected := []Template{

template_windows_test.go

@@ -32,7 +32,7 @@ func TestWalkDir_Windows(t *testing.T) {
 
 	cfg := &config.Config{}
 
-	_, err := walkDir(ctx, cfg, `C:\indir`, simpleNamer(`C:/outdir`), nil, 0, false)
+	_, err := walkDir(ctx, cfg, `C:\indir`, simpleNamer(`C:/outdir`), nil, nil, 0, false)
 	assert.Error(t, err)
 
 	err = hackpadfs.MkdirAll(fsys, `C:\indir\one`, 0o777)
@@ -50,7 +50,7 @@ func TestWalkDir_Windows(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "baz", fi.Name())
 
-	templates, err := walkDir(ctx, cfg, `C:\indir`, simpleNamer(`C:/outdir`), []string{`*\two`}, 0, false)
+	templates, err := walkDir(ctx, cfg, `C:\indir`, simpleNamer(`C:/outdir`), []string{`*\two`}, []string{}, 0, false)
 	require.NoError(t, err)
 
 	expected := []Template{