diff --git a/CHANGELOG.md b/CHANGELOG.md index fe14c26419..06ee8afebb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - Fields `cache_control`, `content_disposition`, `content_language` and `website_redirect_location` added to the `aws_s3` output. - Field `cors.enabled` and `cors.allowed_origins` added to the server wide `http` config. +- Allow mapping imports in Bloblang environments to be disabled. ### Fixed diff --git a/internal/bloblang/environment.go b/internal/bloblang/environment.go index 1aa19e3a31..944af5b1ba 100644 --- a/internal/bloblang/environment.go +++ b/internal/bloblang/environment.go @@ -10,8 +10,7 @@ import ( // Environment provides an isolated Bloblang environment where the available // features, functions and methods can be modified. type Environment struct { - functions *query.FunctionSet - methods *query.MethodSet + pCtx parser.Context } // GlobalEnvironment returns the global default environment. Modifying this @@ -20,8 +19,7 @@ type Environment struct { // changes. func GlobalEnvironment() *Environment { return &Environment{ - functions: query.AllFunctions, - methods: query.AllMethods, + pCtx: parser.GlobalContext(), } } @@ -44,8 +42,7 @@ func NewEnvironment() *Environment { // empty, where no functions or methods are initially available. func NewEmptyEnvironment() *Environment { return &Environment{ - functions: query.NewFunctionSet(), - methods: query.NewMethodSet(), + pCtx: parser.EmptyContext(), } } @@ -55,12 +52,7 @@ func NewEmptyEnvironment() *Environment { // When a parsing error occurs the returned error will be a *parser.Error type, // which allows you to gain positional and structured error messages. func (e *Environment) NewField(expr string) (*field.Expression, error) { - pCtx := parser.GlobalContext() - if e != nil { - pCtx.Functions = e.functions - pCtx.Methods = e.methods - } - f, err := parser.ParseField(pCtx, expr) + f, err := parser.ParseField(e.pCtx, expr) if err != nil { return nil, err } @@ -73,13 +65,8 @@ func (e *Environment) NewField(expr string) (*field.Expression, error) { // When a parsing error occurs the error will be the type *parser.Error, which // gives access to the line and column where the error occurred, as well as a // method for creating a well formatted error message. -func (e *Environment) NewMapping(path, blobl string) (*mapping.Executor, error) { - pCtx := parser.GlobalContext() - if e != nil { - pCtx.Functions = e.functions - pCtx.Methods = e.methods - } - exec, err := parser.ParseMapping(pCtx, path, blobl) +func (e *Environment) NewMapping(blobl string) (*mapping.Executor, error) { + exec, err := parser.ParseMapping(e.pCtx, blobl) if err != nil { return nil, err } @@ -97,28 +84,57 @@ func (e *Environment) NewMapping(path, blobl string) (*mapping.Executor, error) // that is independent of the source. func (e *Environment) Deactivated() *Environment { return &Environment{ - functions: e.functions.Deactivated(), - methods: e.methods.Deactivated(), + pCtx: e.pCtx.Deactivated(), } } // RegisterMethod adds a new Bloblang method to the environment. func (e *Environment) RegisterMethod(spec query.MethodSpec, ctor query.MethodCtor) error { - return e.methods.Add(spec, ctor) + return e.pCtx.Methods.Add(spec, ctor) } // RegisterFunction adds a new Bloblang function to the environment. func (e *Environment) RegisterFunction(spec query.FunctionSpec, ctor query.FunctionCtor) error { - return e.functions.Add(spec, ctor) + return e.pCtx.Functions.Add(spec, ctor) +} + +// WithImporter returns a new environment where Bloblang imports are performed +// from a new importer. +func (e *Environment) WithImporter(importer parser.Importer) *Environment { + nextCtx := e.pCtx.WithImporter(importer) + return &Environment{ + pCtx: nextCtx, + } +} + +// WithImporterRelativeToFile returns a new environment where any relative +// imports will be made from the directory of the provided file path. The +// provided path can itself be relative (to the current importer directory) or +// absolute. +func (e *Environment) WithImporterRelativeToFile(filePath string) *Environment { + nextCtx := e.pCtx.WithImporterRelativeToFile(filePath) + return &Environment{ + pCtx: nextCtx, + } +} + +// WithDisabledImports returns a version of the environment where imports within +// mappings are disabled entirely. This prevents mappings from accessing files +// from the host disk. +func (e *Environment) WithDisabledImports() *Environment { + return &Environment{ + pCtx: e.pCtx.DisabledImports(), + } } // WithoutMethods returns a copy of the environment but with a variadic list of // method names removed. Instantiation of these removed methods within a mapping // will cause errors at parse time. func (e *Environment) WithoutMethods(names ...string) *Environment { + nextCtx := e.pCtx + nextCtx.Methods = e.pCtx.Methods.Without(names...) return &Environment{ - functions: e.functions, - methods: e.methods.Without(names...), + pCtx: nextCtx, } } @@ -126,8 +142,9 @@ func (e *Environment) WithoutMethods(names ...string) *Environment { // of function names removed. Instantiation of these removed functions within a // mapping will cause errors at parse time. func (e *Environment) WithoutFunctions(names ...string) *Environment { + nextCtx := e.pCtx + nextCtx.Functions = e.pCtx.Functions.Without(names...) return &Environment{ - functions: e.functions.Without(names...), - methods: e.methods, + pCtx: nextCtx, } } diff --git a/internal/bloblang/package.go b/internal/bloblang/package.go index b5e5da9d50..894ebdf03a 100644 --- a/internal/bloblang/package.go +++ b/internal/bloblang/package.go @@ -27,6 +27,6 @@ func NewField(expr string) (*field.Expression, error) { // // When a parsing error occurs the returned error may be a *parser.Error type, // which allows you to gain positional and structured error messages. -func NewMapping(path, expr string) (*mapping.Executor, error) { - return GlobalEnvironment().NewMapping(path, expr) +func NewMapping(expr string) (*mapping.Executor, error) { + return GlobalEnvironment().NewMapping(expr) } diff --git a/internal/bloblang/package_test.go b/internal/bloblang/package_test.go index d4d2a6e8c8..9cb0bb7674 100644 --- a/internal/bloblang/package_test.go +++ b/internal/bloblang/package_test.go @@ -85,7 +85,7 @@ func TestMappings(t *testing.T) { for name, test := range tests { test := test t.Run(name, func(t *testing.T) { - m, err := NewMapping("", test.mapping) + m, err := NewMapping(test.mapping) require.NoError(t, err) assert.Equal(t, test.assignmentTargets, m.AssignmentTargets()) @@ -129,7 +129,7 @@ func TestMappingParallelExecution(t *testing.T) { for name, test := range tests { test := test t.Run(name, func(t *testing.T) { - m, err := NewMapping("", test.mapping) + m, err := NewMapping(test.mapping) require.NoError(t, err) startChan := make(chan struct{}) diff --git a/internal/bloblang/parser/context.go b/internal/bloblang/parser/context.go new file mode 100644 index 0000000000..ea12648f41 --- /dev/null +++ b/internal/bloblang/parser/context.go @@ -0,0 +1,179 @@ +package parser + +import ( + "errors" + "io" + "os" + "path/filepath" + + "github.com/Jeffail/benthos/v3/internal/bloblang/query" +) + +// Context contains context used throughout a Bloblang parser for +// accessing function and method constructors. +type Context struct { + Functions *query.FunctionSet + Methods *query.MethodSet + namedContext *namedContext + importer Importer +} + +// EmptyContext returns a parser context with no functions, methods or import +// capabilities. +func EmptyContext() Context { + return Context{ + Functions: query.NewFunctionSet(), + Methods: query.NewMethodSet(), + importer: newOSImporter(), + } +} + +// GlobalContext returns a parser context with globally defined functions and +// methods. +func GlobalContext() Context { + return Context{ + Functions: query.AllFunctions, + Methods: query.AllMethods, + importer: newOSImporter(), + } +} + +type namedContext struct { + name string + next *namedContext +} + +// WithNamedContext returns a Context with a named execution context. +func (pCtx Context) WithNamedContext(name string) Context { + next := pCtx.namedContext + pCtx.namedContext = &namedContext{name, next} + return pCtx +} + +// HasNamedContext returns true if a given name exists as a named context. +func (pCtx Context) HasNamedContext(name string) bool { + tmp := pCtx.namedContext + for tmp != nil { + if tmp.name == name { + return true + } + tmp = tmp.next + } + return false +} + +// InitFunction attempts to initialise a function from the available +// constructors of the parser context. +func (pCtx Context) InitFunction(name string, args *query.ParsedParams) (query.Function, error) { + return pCtx.Functions.Init(name, args) +} + +// InitMethod attempts to initialise a method from the available constructors of +// the parser context. +func (pCtx Context) InitMethod(name string, target query.Function, args *query.ParsedParams) (query.Function, error) { + return pCtx.Methods.Init(name, target, args) +} + +// WithImporter returns a Context where imports are made from the provided +// Importer implementation. +func (pCtx Context) WithImporter(importer Importer) Context { + pCtx.importer = importer + return pCtx +} + +// WithImporterRelativeToFile returns a Context where any relative imports will +// be made from the directory of the provided file path. The provided path can +// itself be relative (to the current importer directory) or absolute. +func (pCtx Context) WithImporterRelativeToFile(pathStr string) Context { + pCtx.importer = pCtx.importer.RelativeToFile(pathStr) + return pCtx +} + +// Deactivated returns a version of the parser context where all functions and +// methods exist but can no longer be instantiated. This means it's possible to +// parse and validate mappings but not execute them. If the context also has an +// importer then it will also be replaced with an implementation that always +// returns empty files. +func (pCtx Context) Deactivated() Context { + nextCtx := pCtx + nextCtx.Functions = pCtx.Functions.Deactivated() + nextCtx.Methods = pCtx.Methods.Deactivated() + return nextCtx +} + +// DisabledImports returns a version of the parser context where file imports +// are entirely disabled. Any import statement within parsed mappings will +// return parse errors explaining that file imports are disabled. +func (pCtx Context) DisabledImports() Context { + nextCtx := pCtx + nextCtx.importer = disabledImporter{} + return nextCtx +} + +//------------------------------------------------------------------------------ + +// Importer represents a repository of bloblang files that can be imported by +// mappings. It's possible for mappings to import files using relative paths, if +// the import is from a mapping which was itself imported then the path should +// be interpretted as relative to that file. +type Importer interface { + // Import a file from a relative or absolute path. + Import(pathStr string) ([]byte, error) + + // Derive a new importer where relative import paths are resolved from the + // directory of the provided file path. The provided path could be absolute, + // or relative itself in which case it should be resolved from the + // pre-existing relative directory. + RelativeToFile(filePath string) Importer +} + +//------------------------------------------------------------------------------ + +type osImporter struct { + relativePath string +} + +func newOSImporter() Importer { + pwd, _ := os.Getwd() + return &osImporter{ + relativePath: pwd, + } +} + +func (i *osImporter) Import(pathStr string) ([]byte, error) { + if !filepath.IsAbs(pathStr) { + pathStr = filepath.Join(i.relativePath, pathStr) + } + + f, err := os.Open(pathStr) + if err != nil { + return nil, err + } + return io.ReadAll(f) +} + +func (i *osImporter) RelativeToFile(filePath string) Importer { + dir := filepath.Dir(filePath) + if dir == "" || dir == "." { + return i + } + + pathStr := filepath.Dir(filePath) + if !filepath.IsAbs(pathStr) && i.relativePath != "" { + pathStr = filepath.Join(i.relativePath, pathStr) + } + + newI := *i + newI.relativePath = pathStr + return &newI +} + +type disabledImporter struct{} + +func (d disabledImporter) Import(pathStr string) ([]byte, error) { + return nil, errors.New("imports are disabled in this context") +} + +func (d disabledImporter) RelativeToFile(filePath string) Importer { + return d +} diff --git a/internal/bloblang/parser/context_test.go b/internal/bloblang/parser/context_test.go new file mode 100644 index 0000000000..8ff6031454 --- /dev/null +++ b/internal/bloblang/parser/context_test.go @@ -0,0 +1,116 @@ +package parser + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestContextImportIsolation(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "context_import_isolation") + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(tmpDir) + }) + + content := `map foo { root = this }` + fileName := "foo.blobl" + fullPath := filepath.Join(tmpDir, fileName) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) + + for _, srcCtx := range []Context{GlobalContext(), EmptyContext()} { + relCtx := srcCtx.WithImporterRelativeToFile(fullPath) + isoCtx := srcCtx.DisabledImports() + + // Source context only works with full path + _, err = srcCtx.importer.Import(fileName) + assert.Error(t, err) + + out, err := srcCtx.importer.Import(fullPath) + assert.NoError(t, err) + assert.Equal(t, content, string(out)) + + // Relative context works with full or relative path + out, err = relCtx.importer.Import(fullPath) + assert.NoError(t, err) + assert.Equal(t, content, string(out)) + + out, err = relCtx.importer.Import(fileName) + assert.NoError(t, err) + assert.Equal(t, content, string(out)) + + // Isolated context doesn't work with any path + _, err = isoCtx.importer.Import(fileName) + assert.Error(t, err) + + _, err = isoCtx.importer.Import(fullPath) + assert.Error(t, err) + } +} + +func TestContextImportRelativity(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "context_import_relativity") + require.NoError(t, err) + t.Cleanup(func() { + os.RemoveAll(tmpDir) + }) + + for path, content := range map[string]string{ + "mappings/foo.blobl": `map foo { root.foo = this.foo }`, + "mappings/first/bar.blobl": `map bar { root.bar = this.bar }`, + "mappings/first/second/baz.blobl": `map baz { root.baz = this.baz }`, + } { + osPath := filepath.FromSlash(path) + dirPath := filepath.Dir(osPath) + + require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, dirPath), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, osPath), []byte(content), 0644)) + } + + for _, srcCtx := range []Context{GlobalContext(), EmptyContext()} { + relCtxOne := srcCtx.WithImporterRelativeToFile(filepath.Join(tmpDir, "mappings", "foo.blobl")) + relCtxTwo := relCtxOne.WithImporterRelativeToFile(filepath.Join("first", "bar.blobl")) + + // foo.blobl + content, err := srcCtx.importer.Import(filepath.Join(tmpDir, "mappings", "foo.blobl")) + require.NoError(t, err) + assert.Equal(t, `map foo { root.foo = this.foo }`, string(content)) + + content, err = relCtxOne.importer.Import("foo.blobl") + require.NoError(t, err) + assert.Equal(t, `map foo { root.foo = this.foo }`, string(content)) + + content, err = relCtxTwo.importer.Import("../foo.blobl") + require.NoError(t, err) + assert.Equal(t, `map foo { root.foo = this.foo }`, string(content)) + + // bar.blobl + content, err = srcCtx.importer.Import(filepath.Join(tmpDir, "mappings", "first", "bar.blobl")) + require.NoError(t, err) + assert.Equal(t, `map bar { root.bar = this.bar }`, string(content)) + + content, err = relCtxOne.importer.Import("./first/bar.blobl") + require.NoError(t, err) + assert.Equal(t, `map bar { root.bar = this.bar }`, string(content)) + + content, err = relCtxTwo.importer.Import("bar.blobl") + require.NoError(t, err) + assert.Equal(t, `map bar { root.bar = this.bar }`, string(content)) + + // baz.blobl + content, err = srcCtx.importer.Import(filepath.Join(tmpDir, "mappings", "first", "second", "baz.blobl")) + require.NoError(t, err) + assert.Equal(t, `map baz { root.baz = this.baz }`, string(content)) + + content, err = relCtxOne.importer.Import("./first/second/baz.blobl") + require.NoError(t, err) + assert.Equal(t, `map baz { root.baz = this.baz }`, string(content)) + + content, err = relCtxTwo.importer.Import("second/baz.blobl") + require.NoError(t, err) + assert.Equal(t, `map baz { root.baz = this.baz }`, string(content)) + } +} diff --git a/internal/bloblang/parser/mapping_parser.go b/internal/bloblang/parser/mapping_parser.go index e12e7b2b95..c41adb0a26 100644 --- a/internal/bloblang/parser/mapping_parser.go +++ b/internal/bloblang/parser/mapping_parser.go @@ -3,9 +3,6 @@ package parser import ( "errors" "fmt" - "io/ioutil" - "path" - "path/filepath" "strings" "github.com/Jeffail/benthos/v3/internal/bloblang/mapping" @@ -18,14 +15,10 @@ import ( // // The filepath is optional and used for relative file imports and error // messages. -func ParseMapping(pCtx Context, filepath, expr string) (*mapping.Executor, *Error) { +func ParseMapping(pCtx Context, expr string) (*mapping.Executor, *Error) { in := []rune(expr) - dir := "" - if len(filepath) > 0 { - dir = path.Dir(filepath) - } - resDirectImport := singleRootImport(dir, pCtx)(in) + resDirectImport := singleRootImport(pCtx)(in) if resDirectImport.Err != nil && resDirectImport.Err.IsFatal() { return nil, resDirectImport.Err } @@ -33,7 +26,7 @@ func ParseMapping(pCtx Context, filepath, expr string) (*mapping.Executor, *Erro return resDirectImport.Payload.(*mapping.Executor), nil } - resExe := parseExecutor(dir, pCtx)(in) + resExe := parseExecutor(pCtx)(in) if resExe.Err != nil && resExe.Err.IsFatal() { return nil, resExe.Err } @@ -48,7 +41,7 @@ func ParseMapping(pCtx Context, filepath, expr string) (*mapping.Executor, *Erro //------------------------------------------------------------------------------' -func parseExecutor(baseDir string, pCtx Context) Func { +func parseExecutor(pCtx Context) Func { newline := NewlineAllowComment() whitespace := SpacesAndTabs() allWhitespace := DiscardAll(OneOf(whitespace, newline)) @@ -58,7 +51,7 @@ func parseExecutor(baseDir string, pCtx Context) Func { statements := []mapping.Statement{} statement := OneOf( - importParser(baseDir, maps, pCtx), + importParser(maps, pCtx), mapParser(maps, pCtx), letStatementParser(pCtx), metaStatementParser(false, pCtx), @@ -102,7 +95,7 @@ func parseExecutor(baseDir string, pCtx Context) Func { } } -func singleRootImport(baseDir string, pCtx Context) Func { +func singleRootImport(pCtx Context) Func { whitespace := SpacesAndTabs() allWhitespace := DiscardAll(OneOf(whitespace, Newline())) @@ -121,17 +114,15 @@ func singleRootImport(baseDir string, pCtx Context) Func { } fpath := res.Payload.([]interface{})[3].(string) - if !filepath.IsAbs(fpath) { - fpath = path.Join(baseDir, fpath) - } - - contents, err := ioutil.ReadFile(fpath) + contents, err := pCtx.importer.Import(fpath) if err != nil { return Fail(NewFatalError(input, fmt.Errorf("failed to read import: %w", err)), input) } + nextCtx := pCtx.WithImporterRelativeToFile(fpath) + importContent := []rune(string(contents)) - execRes := parseExecutor(path.Dir(fpath), pCtx)(importContent) + execRes := parseExecutor(nextCtx)(importContent) if execRes.Err != nil { return Fail(NewFatalError(input, NewImportError(fpath, importContent, execRes.Err)), input) } @@ -189,7 +180,7 @@ func varNameParser() Func { ) } -func importParser(baseDir string, maps map[string]query.Function, pCtx Context) Func { +func importParser(maps map[string]query.Function, pCtx Context) Func { p := Sequence( Term("import"), SpacesAndTabs(), @@ -208,16 +199,15 @@ func importParser(baseDir string, maps map[string]query.Function, pCtx Context) } fpath := res.Payload.([]interface{})[2].(string) - if !filepath.IsAbs(fpath) { - fpath = path.Join(baseDir, fpath) - } - contents, err := ioutil.ReadFile(fpath) + contents, err := pCtx.importer.Import(fpath) if err != nil { return Fail(NewFatalError(input, fmt.Errorf("failed to read import: %w", err)), input) } + nextCtx := pCtx.WithImporterRelativeToFile(fpath) + importContent := []rune(string(contents)) - execRes := parseExecutor(path.Dir(fpath), pCtx)(importContent) + execRes := parseExecutor(nextCtx)(importContent) if execRes.Err != nil { return Fail(NewFatalError(input, NewImportError(fpath, importContent, execRes.Err)), input) } diff --git a/internal/bloblang/parser/mapping_parser_test.go b/internal/bloblang/parser/mapping_parser_test.go index d8fa7e469a..c8559f27b7 100644 --- a/internal/bloblang/parser/mapping_parser_test.go +++ b/internal/bloblang/parser/mapping_parser_test.go @@ -28,29 +28,29 @@ func TestMappingErrors(t *testing.T) { require.NoError(t, ioutil.WriteFile(goodMapFile, []byte(`map foo { foo = "this is valid" }`), 0777)) tests := map[string]struct { - mapping string - err string + mapping string + errContains string }{ "bad variable name": { - mapping: `let foo+bar = baz`, - err: "line 1 char 8: expected whitespace", + mapping: `let foo+bar = baz`, + errContains: "line 1 char 8: expected whitespace", }, "bad meta name": { - mapping: `meta foo+bar = baz`, - err: "line 1 char 9: expected =", + mapping: `meta foo+bar = baz`, + errContains: "line 1 char 9: expected =", }, "no mappings": { - mapping: ``, - err: `line 1 char 1: expected import, map, or assignment`, + mapping: ``, + errContains: `line 1 char 1: expected import, map, or assignment`, }, "no mappings 2": { mapping: ` `, - err: `line 2 char 4: expected import, map, or assignment`, + errContains: `line 2 char 4: expected import, map, or assignment`, }, "double mapping": { - mapping: `foo = bar bar = baz`, - err: `line 1 char 11: expected line break`, + mapping: `foo = bar bar = baz`, + errContains: `line 1 char 11: expected line break`, }, "double mapping line breaks": { mapping: ` @@ -58,50 +58,50 @@ func TestMappingErrors(t *testing.T) { foo = bar bar = baz `, - err: `line 3 char 11: expected line break`, + errContains: `line 3 char 11: expected line break`, }, "double mapping line 2": { mapping: `let a = "a" foo = bar bar = baz`, - err: `line 2 char 11: expected line break`, + errContains: `line 2 char 11: expected line break`, }, "double mapping line 3": { mapping: `let a = "a" foo = bar bar = baz let a = "a"`, - err: "line 2 char 11: expected line break", + errContains: "line 2 char 11: expected line break", }, "bad mapping": { - mapping: `foo wat bar`, - err: `line 1 char 5: expected =`, + mapping: `foo wat bar`, + errContains: `line 1 char 5: expected =`, }, "bad char": { - mapping: `!foo = bar`, - err: "line 1 char 6: expected the mapping to end here as the beginning is shorthand for `root = !foo`, but this shorthand form cannot be followed with more assignments", + mapping: `!foo = bar`, + errContains: "line 1 char 6: expected the mapping to end here as the beginning is shorthand for `root = !foo`, but this shorthand form cannot be followed with more assignments", }, "bad inline query": { mapping: `content().uppercase().lowercase() meta foo = "bar"`, - err: "line 2 char 1: expected the mapping to end here as the beginning is shorthand for `root = content().up...`, but this shorthand form cannot be followed with more assignments", + errContains: "line 2 char 1: expected the mapping to end here as the beginning is shorthand for `root = content().up...`, but this shorthand form cannot be followed with more assignments", }, "bad char 2": { mapping: `let foo = bar !foo = bar`, - err: `line 2 char 1: expected import, map, or assignment`, + errContains: `line 2 char 1: expected import, map, or assignment`, }, "bad char 3": { mapping: `let foo = bar !foo = bar this = that`, - err: `line 2 char 1: expected import, map, or assignment`, + errContains: `line 2 char 1: expected import, map, or assignment`, }, "bad query": { - mapping: `foo = blah.`, - err: `line 1 char 12: required: expected method or field path`, + mapping: `foo = blah.`, + errContains: `line 1 char 12: required: expected method or field path`, }, "bad variable assign": { - mapping: `let = blah`, - err: `line 1 char 5: required: expected variable name`, + mapping: `let = blah`, + errContains: `line 1 char 5: required: expected variable name`, }, "double map definition": { mapping: `map foo { @@ -111,39 +111,39 @@ map foo { foo = bar } foo = bar.apply("foo")`, - err: `line 4 char 1: map name collision: foo`, + errContains: `line 4 char 1: map name collision: foo`, }, "map contains meta assignment": { mapping: `map foo { meta foo = "bar" } foo = bar.apply("foo")`, - err: `line 2 char 3: setting meta fields from within a map is not allowed`, + errContains: `line 2 char 3: setting meta fields from within a map is not allowed`, }, "no name map definition": { mapping: `map { foo = bar } foo = bar.apply("foo")`, - err: `line 1 char 5: required: expected map name`, + errContains: `line 1 char 5: required: expected map name`, }, "no file import": { mapping: `import "this file doesnt exist (i hope)" foo = bar.apply("from_import")`, - err: `line 1 char 1: failed to read import: open this file doesnt exist (i hope): no such file or directory`, + errContains: `this file doesnt exist (i hope): no such file or directory`, }, "bad file import": { mapping: fmt.Sprintf(`import "%v" foo = bar.apply("from_import")`, badMapFile), - err: fmt.Sprintf(`line 1 char 1: failed to parse import '%v': line 1 char 5: expected =`, badMapFile), + errContains: fmt.Sprintf(`line 1 char 1: failed to parse import '%v': line 1 char 5: expected =`, badMapFile), }, "no maps file import": { mapping: fmt.Sprintf(`import "%v" foo = bar.apply("from_import")`, noMapsFile), - err: fmt.Sprintf(`line 1 char 1: no maps to import from '%v'`, noMapsFile), + errContains: fmt.Sprintf(`line 1 char 1: no maps to import from '%v'`, noMapsFile), }, "colliding maps file import": { mapping: fmt.Sprintf(`map "foo" { this = that } @@ -151,21 +151,21 @@ foo = bar.apply("from_import")`, noMapsFile), import "%v" foo = bar.apply("foo")`, goodMapFile), - err: fmt.Sprintf(`line 3 char 1: map name collisions from import '%v': [foo]`, goodMapFile), + errContains: fmt.Sprintf(`line 3 char 1: map name collisions from import '%v': [foo]`, goodMapFile), }, "quotes at root": { mapping: ` "root.something" = 5 + 2`, - err: "line 2 char 1: expected import, map, or assignment", + errContains: "line 2 char 1: expected import, map, or assignment", }, } for name, test := range tests { test := test t.Run(name, func(t *testing.T) { - exec, err := ParseMapping(GlobalContext(), "", test.mapping) + exec, err := ParseMapping(GlobalContext(), test.mapping) require.NotNil(t, err) - assert.Equal(t, test.err, err.ErrorAtPosition([]rune(test.mapping))) + assert.Contains(t, err.ErrorAtPosition([]rune(test.mapping)), test.errContains) assert.Nil(t, exec) }) } @@ -491,7 +491,7 @@ root = this.apply("foo")`, goodMapFile), test.output.Meta = map[string]string{} } - exec, perr := ParseMapping(GlobalContext(), "", test.mapping) + exec, perr := ParseMapping(GlobalContext(), test.mapping) require.Nil(t, perr) resPart, err := exec.MapPart(test.index, msg) diff --git a/internal/bloblang/parser/query_parser.go b/internal/bloblang/parser/query_parser.go index 8e5a4fd2a7..b0eb2a625b 100644 --- a/internal/bloblang/parser/query_parser.go +++ b/internal/bloblang/parser/query_parser.go @@ -4,71 +4,6 @@ import ( "github.com/Jeffail/benthos/v3/internal/bloblang/query" ) -// FunctionSet provides constructors to the functions available in this query. -type FunctionSet interface { - Params(name string) (query.Params, error) - Init(name string, args *query.ParsedParams) (query.Function, error) -} - -// MethodSet provides constructors to the methods available in this query. -type MethodSet interface { - Params(name string) (query.Params, error) - Init(name string, target query.Function, args *query.ParsedParams) (query.Function, error) -} - -// Context contains context used throughout a Bloblang parser for -// accessing function and method constructors. -type Context struct { - Functions FunctionSet - Methods MethodSet - namedContext *namedContext -} - -// GlobalContext returns a parser context with globally defined functions and -// methods. -func GlobalContext() Context { - return Context{ - Functions: query.AllFunctions, - Methods: query.AllMethods, - } -} - -type namedContext struct { - name string - next *namedContext -} - -// WithNamedContext returns a Context with a named execution context. -func (pCtx Context) WithNamedContext(name string) Context { - next := pCtx.namedContext - pCtx.namedContext = &namedContext{name, next} - return pCtx -} - -// HasNamedContext returns true if a given name exists as a named context. -func (pCtx Context) HasNamedContext(name string) bool { - tmp := pCtx.namedContext - for tmp != nil { - if tmp.name == name { - return true - } - tmp = tmp.next - } - return false -} - -// InitFunction attempts to initialise a function from the available -// constructors of the parser context. -func (pCtx Context) InitFunction(name string, args *query.ParsedParams) (query.Function, error) { - return pCtx.Functions.Init(name, args) -} - -// InitMethod attempts to initialise a method from the available constructors of -// the parser context. -func (pCtx Context) InitMethod(name string, target query.Function, args *query.ParsedParams) (query.Function, error) { - return pCtx.Methods.Init(name, target, args) -} - func queryParser(pCtx Context) func(input []rune) Result { rootParser := parseWithTails(Expect( OneOf( @@ -137,5 +72,3 @@ func tryParseQuery(expr string, deprecated bool) (query.Function, *Error) { } return res.Payload.(query.Function), nil } - -//------------------------------------------------------------------------------ diff --git a/internal/bloblang/plugins/bloblang.go b/internal/bloblang/plugins/bloblang.go index 929b9c6d82..5c3a1b4d07 100644 --- a/internal/bloblang/plugins/bloblang.go +++ b/internal/bloblang/plugins/bloblang.go @@ -11,12 +11,12 @@ import ( func Register() error { dynamicBloblangParserContext := parser.Context{ Functions: query.AllFunctions.OnlyPure().NoMessage(), - Methods: query.AllMethods, - } + Methods: query.AllMethods.OnlyPure(), + }.DisabledImports() return query.AllMethods.Add( query.NewMethodSpec( - "bloblang", "Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings.", + "bloblang", "Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Imports and functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings.", ).InCategory( query.MethodCategoryParsing, "", query.NewExampleSpec( @@ -33,7 +33,7 @@ func Register() error { if err != nil { return nil, err } - exec, parserErr := parser.ParseMapping(dynamicBloblangParserContext, "", mappingStr) + exec, parserErr := parser.ParseMapping(dynamicBloblangParserContext, mappingStr) if parserErr != nil { return nil, parserErr } diff --git a/internal/bloblang/query/docs.go b/internal/bloblang/query/docs.go index 474f65ba80..755d8655db 100644 --- a/internal/bloblang/query/docs.go +++ b/internal/bloblang/query/docs.go @@ -173,6 +173,10 @@ type MethodSpec struct { // Categories that this method fits within. Categories []MethodCatSpec `json:"categories"` + + // Impure indicates that a method accesses or interacts with the outter + // environment, and is therefore unsafe to execute in shared environments. + Impure bool `json:"impure"` } // NewMethodSpec creates a new method spec. @@ -213,6 +217,13 @@ func (m MethodSpec) Beta() MethodSpec { return m } +// MarkImpure flags the method as being impure, meaning it access or interacts +// with the environment. +func (m MethodSpec) MarkImpure() MethodSpec { + m.Impure = true + return m +} + // Param adds a parameter to the function. func (m MethodSpec) Param(def ParamDefinition) MethodSpec { m.Params = m.Params.Add(def) diff --git a/internal/bloblang/query/method_set.go b/internal/bloblang/query/method_set.go index 127fe26930..e4082cd365 100644 --- a/internal/bloblang/query/method_set.go +++ b/internal/bloblang/query/method_set.go @@ -98,6 +98,18 @@ func (m *MethodSet) Without(methods ...string) *MethodSet { return &MethodSet{m.disableCtors, constructors, specs} } +// OnlyPure creates a clone of the methods set that can be mutated in isolation, +// where all impure methods are removed. +func (m *MethodSet) OnlyPure() *MethodSet { + var excludes []string + for _, v := range m.specs { + if v.Impure { + excludes = append(excludes, v.Name) + } + } + return m.Without(excludes...) +} + // Deactivated returns a version of the method set where constructors are // disabled, allowing mappings to be parsed and validated but not executed. // diff --git a/internal/bloblang/query/parsed_test.go b/internal/bloblang/query/parsed_test.go index 9fbe66682f..392309a0d7 100644 --- a/internal/bloblang/query/parsed_test.go +++ b/internal/bloblang/query/parsed_test.go @@ -35,7 +35,7 @@ func TestFunctionExamples(t *testing.T) { t.Run(spec.Name, func(t *testing.T) { t.Parallel() for i, e := range spec.Examples { - m, err := bloblang.NewMapping("", e.Mapping) + m, err := bloblang.NewMapping(e.Mapping) require.NoError(t, err) for j, io := range e.Results { @@ -83,7 +83,7 @@ func TestMethodExamples(t *testing.T) { t.Run(spec.Name, func(t *testing.T) { t.Parallel() for i, e := range spec.Examples { - m, err := bloblang.NewMapping("", e.Mapping) + m, err := bloblang.NewMapping(e.Mapping) require.NoError(t, err) for j, io := range e.Results { @@ -104,7 +104,7 @@ func TestMethodExamples(t *testing.T) { } for _, target := range spec.Categories { for i, e := range target.Examples { - m, err := bloblang.NewMapping("", e.Mapping) + m, err := bloblang.NewMapping(e.Mapping) require.NoError(t, err) for j, io := range e.Results { @@ -159,7 +159,7 @@ func TestMappings(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - m, err := bloblang.NewMapping("", test.mapping) + m, err := bloblang.NewMapping(test.mapping) require.NoError(t, err) for i, io := range test.inputOutputs { diff --git a/internal/component/input/span_reader.go b/internal/component/input/span_reader.go index 58af3974d4..2f220a9c3f 100644 --- a/internal/component/input/span_reader.go +++ b/internal/component/input/span_reader.go @@ -36,7 +36,7 @@ type SpanReader struct { // NewSpanReader wraps an async reader with a mechanism for extracting tracing // spans from the consumed message using a Bloblang mapping. func NewSpanReader(inputName, mapping string, rdr reader.Async, mgr types.Manager, logger log.Modular) (reader.Async, error) { - exe, err := bloblang.NewMapping("", mapping) + exe, err := bloblang.NewMapping(mapping) if err != nil { return nil, err } diff --git a/internal/component/metrics/mapping.go b/internal/component/metrics/mapping.go index 2ca25667b4..e336d91165 100644 --- a/internal/component/metrics/mapping.go +++ b/internal/component/metrics/mapping.go @@ -35,7 +35,7 @@ func NewMapping(mapping string, logger log.Modular) (*Mapping, error) { if mapping == "" { return &Mapping{m: nil, logger: logger}, nil } - m, err := bloblang.NewMapping("", mapping) + m, err := bloblang.NewMapping(mapping) if err != nil { if perr, ok := err.(*parser.Error); ok { return nil, fmt.Errorf("%v", perr.ErrorAtPosition([]rune(mapping))) diff --git a/internal/docs/bloblang.go b/internal/docs/bloblang.go index babd9ab5b6..968d053628 100644 --- a/internal/docs/bloblang.go +++ b/internal/docs/bloblang.go @@ -20,7 +20,7 @@ func LintBloblangMapping(ctx LintContext, line, col int, v interface{}) []Lint { if str == "" { return nil } - _, err := ctx.BloblangEnv.NewMapping("", str) + _, err := ctx.BloblangEnv.NewMapping(str) if err == nil { return nil } diff --git a/internal/impl/mongodb/output.go b/internal/impl/mongodb/output.go index ea94239101..b15c974702 100644 --- a/internal/impl/mongodb/output.go +++ b/internal/impl/mongodb/output.go @@ -138,7 +138,7 @@ func NewWriter( if conf.FilterMap == "" { return nil, errors.New("mongodb filter_map must be specified") } - if db.filterMap, err = bloblang.NewMapping("", conf.FilterMap); err != nil { + if db.filterMap, err = bloblang.NewMapping(conf.FilterMap); err != nil { return nil, fmt.Errorf("failed to parse filter_map: %v", err) } } else if conf.FilterMap != "" { @@ -149,7 +149,7 @@ func NewWriter( if conf.DocumentMap == "" { return nil, errors.New("mongodb document_map must be specified") } - if db.documentMap, err = bloblang.NewMapping("", conf.DocumentMap); err != nil { + if db.documentMap, err = bloblang.NewMapping(conf.DocumentMap); err != nil { return nil, fmt.Errorf("failed to parse document_map: %v", err) } } else if conf.DocumentMap != "" { @@ -157,7 +157,7 @@ func NewWriter( } if hintAllowed && conf.HintMap != "" { - if db.hintMap, err = bloblang.NewMapping("", conf.HintMap); err != nil { + if db.hintMap, err = bloblang.NewMapping(conf.HintMap); err != nil { return nil, fmt.Errorf("failed to parse hint_map: %v", err) } } else if conf.HintMap != "" { diff --git a/internal/impl/mongodb/processor.go b/internal/impl/mongodb/processor.go index 8e2db0945c..2072846bc3 100644 --- a/internal/impl/mongodb/processor.go +++ b/internal/impl/mongodb/processor.go @@ -180,7 +180,7 @@ func NewProcessor( if conf.MongoDB.FilterMap == "" { return nil, errors.New("mongodb filter_map must be specified") } - if m.filterMap, err = bloblang.NewMapping("", conf.MongoDB.FilterMap); err != nil { + if m.filterMap, err = bloblang.NewMapping(conf.MongoDB.FilterMap); err != nil { return nil, fmt.Errorf("failed to parse filter_map: %v", err) } } else if conf.MongoDB.FilterMap != "" { @@ -191,7 +191,7 @@ func NewProcessor( if conf.MongoDB.DocumentMap == "" { return nil, errors.New("mongodb document_map must be specified") } - if m.documentMap, err = bloblang.NewMapping("", conf.MongoDB.DocumentMap); err != nil { + if m.documentMap, err = bloblang.NewMapping(conf.MongoDB.DocumentMap); err != nil { return nil, fmt.Errorf("failed to parse document_map: %v", err) } } else if conf.MongoDB.DocumentMap != "" { @@ -199,7 +199,7 @@ func NewProcessor( } if hintAllowed && conf.MongoDB.HintMap != "" { - if m.hintMap, err = bloblang.NewMapping("", conf.MongoDB.HintMap); err != nil { + if m.hintMap, err = bloblang.NewMapping(conf.MongoDB.HintMap); err != nil { return nil, fmt.Errorf("failed to parse hint_map: %v", err) } } else if conf.MongoDB.HintMap != "" { diff --git a/internal/template/config.go b/internal/template/config.go index 9000a2fcc0..9ddf18b002 100644 --- a/internal/template/config.go +++ b/internal/template/config.go @@ -105,7 +105,7 @@ func (c Config) compile() (*compiled, error) { if err != nil { return nil, err } - mapping, err := bloblang.NewMapping("", c.Mapping) + mapping, err := bloblang.NewMapping(c.Mapping) if err != nil { var perr *parser.Error if errors.As(err, &perr) { diff --git a/lib/bloblang/package.go b/lib/bloblang/package.go index 615c5736fa..1f67934d4f 100644 --- a/lib/bloblang/package.go +++ b/lib/bloblang/package.go @@ -97,7 +97,7 @@ func (w *mappingWrap) MapPart(index int, msg Message) (types.Part, error) { // When a parsing error occurs the returned error may be a *parser.Error type, // which allows you to gain positional and structured error messages. func NewMapping(expr string) (Mapping, error) { - e, err := parser.ParseMapping(parser.GlobalContext(), "", expr) + e, err := parser.ParseMapping(parser.GlobalContext(), expr) if err != nil { return nil, err } diff --git a/lib/condition/bloblang.go b/lib/condition/bloblang.go index 92c487b4cf..cd67a5cefd 100644 --- a/lib/condition/bloblang.go +++ b/lib/condition/bloblang.go @@ -64,7 +64,7 @@ type Bloblang struct { func NewBloblang( conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, ) (Type, error) { - fn, err := bloblang.NewMapping("", string(conf.Bloblang)) + fn, err := bloblang.NewMapping(string(conf.Bloblang)) if err != nil { if perr, ok := err.(*parser.Error); ok { return nil, fmt.Errorf("%v", perr.ErrorAtPosition([]rune(conf.Bloblang))) diff --git a/lib/input/generate.go b/lib/input/generate.go index 56d33c7738..e4ea768009 100644 --- a/lib/input/generate.go +++ b/lib/input/generate.go @@ -188,7 +188,7 @@ func newBloblang(conf BloblangConfig) (*Bloblang, error) { } timer = time.NewTicker(duration) } - exec, err := bloblang.NewMapping("", conf.Mapping) + exec, err := bloblang.NewMapping(conf.Mapping) if err != nil { if perr, ok := err.(*parser.Error); ok { return nil, fmt.Errorf("failed to parse mapping: %v", perr.ErrorAtPosition([]rune(conf.Mapping))) diff --git a/lib/input/read_until.go b/lib/input/read_until.go index 79445a0821..2b9343205d 100644 --- a/lib/input/read_until.go +++ b/lib/input/read_until.go @@ -193,7 +193,7 @@ func NewReadUntil( var check *mapping.Executor if len(conf.ReadUntil.Check) > 0 { - if check, err = bloblang.NewMapping("", conf.ReadUntil.Check); err != nil { + if check, err = bloblang.NewMapping(conf.ReadUntil.Check); err != nil { return nil, fmt.Errorf("failed to parse check query: %w", err) } } diff --git a/lib/message/batch/policy.go b/lib/message/batch/policy.go index edc59c7b30..38648923fd 100644 --- a/lib/message/batch/policy.go +++ b/lib/message/batch/policy.go @@ -178,7 +178,7 @@ func NewPolicy( } var check *mapping.Executor if len(conf.Check) > 0 { - if check, err = bloblang.NewMapping("", conf.Check); err != nil { + if check, err = bloblang.NewMapping(conf.Check); err != nil { return nil, fmt.Errorf("failed to parse check: %v", err) } } diff --git a/lib/metrics/path_mapping.go b/lib/metrics/path_mapping.go index 86e0d961a7..8a0e10e6ef 100644 --- a/lib/metrics/path_mapping.go +++ b/lib/metrics/path_mapping.go @@ -51,7 +51,7 @@ func newPathMapping(mapping string, logger log.Modular) (*pathMapping, error) { if mapping == "" { return &pathMapping{m: nil, logger: logger}, nil } - m, err := bloblang.NewMapping("", mapping) + m, err := bloblang.NewMapping(mapping) if err != nil { if perr, ok := err.(*parser.Error); ok { return nil, fmt.Errorf("%v", perr.ErrorAtPosition([]rune(mapping))) diff --git a/lib/output/async_writer.go b/lib/output/async_writer.go index ffb94bd8eb..1a020bcec8 100644 --- a/lib/output/async_writer.go +++ b/lib/output/async_writer.go @@ -80,7 +80,7 @@ func NewAsyncWriter( // into messages. func (w *AsyncWriter) SetInjectTracingMap(mapping string) error { var err error - w.injectTracingMap, err = bloblang.NewMapping("", mapping) + w.injectTracingMap, err = bloblang.NewMapping(mapping) return err } diff --git a/lib/output/cassandra.go b/lib/output/cassandra.go index b268844407..f01007dc14 100644 --- a/lib/output/cassandra.go +++ b/lib/output/cassandra.go @@ -249,7 +249,7 @@ func (c *cassandraWriter) parseArgs() error { if c.conf.ArgsMapping != "" { var err error - if c.argsMapping, err = bloblang.NewMapping("", c.conf.ArgsMapping); err != nil { + if c.argsMapping, err = bloblang.NewMapping(c.conf.ArgsMapping); err != nil { return fmt.Errorf("parsing args_mapping: %w", err) } } diff --git a/lib/output/sql.go b/lib/output/sql.go index 79b38ca57f..0bf352d83a 100644 --- a/lib/output/sql.go +++ b/lib/output/sql.go @@ -202,7 +202,7 @@ func newSQLWriter(conf SQLConfig, log log.Modular) (*sqlWriter, error) { var argsMapping *mapping.Executor if conf.ArgsMapping != "" { var err error - if argsMapping, err = bloblang.NewMapping("", conf.ArgsMapping); err != nil { + if argsMapping, err = bloblang.NewMapping(conf.ArgsMapping); err != nil { return nil, fmt.Errorf("failed to parse `args_mapping`: %w", err) } } diff --git a/lib/output/switch.go b/lib/output/switch.go index 4f5d60a695..5283727ef3 100644 --- a/lib/output/switch.go +++ b/lib/output/switch.go @@ -328,7 +328,7 @@ func NewSwitch( return nil, fmt.Errorf("failed to create case '%v' output type '%v': %v", i, cConf.Output.Type, err) } if len(cConf.Check) > 0 { - if o.checks[i], err = bloblang.NewMapping("", cConf.Check); err != nil { + if o.checks[i], err = bloblang.NewMapping(cConf.Check); err != nil { return nil, fmt.Errorf("failed to parse case '%v' check mapping: %v", i, err) } } diff --git a/lib/processor/bloblang.go b/lib/processor/bloblang.go index 589524ca4c..ba3a327a3b 100644 --- a/lib/processor/bloblang.go +++ b/lib/processor/bloblang.go @@ -152,7 +152,7 @@ type Bloblang struct { func NewBloblang( conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, ) (Type, error) { - exec, err := bloblang.NewMapping("", string(conf.Bloblang)) + exec, err := bloblang.NewMapping(string(conf.Bloblang)) if err != nil { if perr, ok := err.(*parser.Error); ok { return nil, fmt.Errorf("%v", perr.ErrorAtPosition([]rune(conf.Bloblang))) diff --git a/lib/processor/branch.go b/lib/processor/branch.go index 6e88768240..3673e5681b 100644 --- a/lib/processor/branch.go +++ b/lib/processor/branch.go @@ -284,12 +284,12 @@ func newBranch( var err error if len(conf.RequestMap) > 0 { - if b.requestMap, err = bloblang.NewMapping("", conf.RequestMap); err != nil { + if b.requestMap, err = bloblang.NewMapping(conf.RequestMap); err != nil { return nil, fmt.Errorf("failed to parse request mapping: %w", err) } } if len(conf.ResultMap) > 0 { - if b.resultMap, err = bloblang.NewMapping("", conf.ResultMap); err != nil { + if b.resultMap, err = bloblang.NewMapping(conf.ResultMap); err != nil { return nil, fmt.Errorf("failed to parse result mapping: %w", err) } } diff --git a/lib/processor/group_by.go b/lib/processor/group_by.go index accce9fa3d..abd0e1e3d5 100644 --- a/lib/processor/group_by.go +++ b/lib/processor/group_by.go @@ -163,7 +163,7 @@ func NewGroupBy( } if len(gConf.Check) > 0 { - if groups[i].Check, err = bloblang.NewMapping("", gConf.Check); err != nil { + if groups[i].Check, err = bloblang.NewMapping(gConf.Check); err != nil { return nil, fmt.Errorf("failed to parse check for group '%v': %v", i, err) } } diff --git a/lib/processor/log.go b/lib/processor/log.go index c6c20f1644..fd4ddee13f 100644 --- a/lib/processor/log.go +++ b/lib/processor/log.go @@ -162,7 +162,7 @@ func NewLog( if l.loggerWith, ok = logger.(logWith); !ok { return nil, errors.New("the provided logger does not support structured fields required for `fields_mapping`") } - if l.fieldsMapping, err = bloblang.NewMapping("", conf.Log.FieldsMapping); err != nil { + if l.fieldsMapping, err = bloblang.NewMapping(conf.Log.FieldsMapping); err != nil { return nil, fmt.Errorf("failed to parse fields mapping: %w", err) } } diff --git a/lib/processor/sql.go b/lib/processor/sql.go index 76a8c090ef..6df54088dd 100644 --- a/lib/processor/sql.go +++ b/lib/processor/sql.go @@ -231,7 +231,7 @@ func NewSQL( return nil, errors.New("the field `args_mapping` cannot be used when running the `sql` processor in deprecated mode (using the `dsn` field), use the `data_source_name` field instead") } var err error - if argsMapping, err = bloblang.NewMapping("", conf.SQL.ArgsMapping); err != nil { + if argsMapping, err = bloblang.NewMapping(conf.SQL.ArgsMapping); err != nil { return nil, fmt.Errorf("failed to parse `args_mapping`: %w", err) } } diff --git a/lib/processor/switch.go b/lib/processor/switch.go index 1dae2af5cd..7e081ef583 100644 --- a/lib/processor/switch.go +++ b/lib/processor/switch.go @@ -205,7 +205,7 @@ func NewSwitch( var procs []types.Processor if len(caseConf.Check) > 0 { - if check, err = bloblang.NewMapping("", caseConf.Check); err != nil { + if check, err = bloblang.NewMapping(caseConf.Check); err != nil { return nil, fmt.Errorf("failed to parse case %v check: %w", i, err) } } diff --git a/lib/processor/while.go b/lib/processor/while.go index 9eac386e81..8d90f86ff6 100644 --- a/lib/processor/while.go +++ b/lib/processor/while.go @@ -121,7 +121,7 @@ func NewWhile( } } if len(conf.While.Check) > 0 { - if check, err = bloblang.NewMapping("", conf.While.Check); err != nil { + if check, err = bloblang.NewMapping(conf.While.Check); err != nil { return nil, fmt.Errorf("failed to parse check query: %w", err) } } diff --git a/lib/service/blobl/cli.go b/lib/service/blobl/cli.go index 0bd8c1b57f..ecd107e777 100644 --- a/lib/service/blobl/cli.go +++ b/lib/service/blobl/cli.go @@ -228,7 +228,8 @@ func run(c *cli.Context) error { m = string(mappingBytes) } - exec, err := bloblang.NewMapping(file, m) + bEnv := bloblang.NewEnvironment().WithImporterRelativeToFile(file) + exec, err := bEnv.NewMapping(m) if err != nil { if perr, ok := err.(*parser.Error); ok { fmt.Fprintf(os.Stderr, "%v %v\n", red("failed to parse mapping:"), perr.ErrorAtPositionStructured("", []rune(m))) diff --git a/lib/service/blobl/server.go b/lib/service/blobl/server.go index 7887a26bf1..8f42289027 100644 --- a/lib/service/blobl/server.go +++ b/lib/service/blobl/server.go @@ -361,7 +361,7 @@ func runServer(c *cli.Context) error { w.Write(resBytes) }() - exec, err := bloblang.NewMapping("", req.Mapping) + exec, err := bloblang.NewMapping(req.Mapping) if err != nil { if perr, ok := err.(*parser.Error); ok { res.ParseError = fmt.Sprintf("failed to parse mapping: %v\n", perr.ErrorAtPositionStructured("", []rune(req.Mapping))) diff --git a/lib/service/test/condition.go b/lib/service/test/condition.go index 664c3d387c..8f02774258 100644 --- a/lib/service/test/condition.go +++ b/lib/service/test/condition.go @@ -131,7 +131,7 @@ func parseBloblangCondition(n yaml.Node) (*bloblangCondition, error) { return nil, err } - m, err := bloblang.NewMapping("", expr) + m, err := bloblang.NewMapping(expr) if err != nil { return nil, err } diff --git a/lib/service/test/processors_provider.go b/lib/service/test/processors_provider.go index ceb8a95a39..019f1b8a1f 100644 --- a/lib/service/test/processors_provider.go +++ b/lib/service/test/processors_provider.go @@ -89,17 +89,18 @@ func (p *ProcessorsProvider) ProvideMocked(jsonPtr string, environment map[strin // ProvideBloblang attempts to parse a Bloblang mapping and returns a processor // slice that executes it. -func (p *ProcessorsProvider) ProvideBloblang(path string) ([]types.Processor, error) { - if !filepath.IsAbs(path) { - path = filepath.Join(filepath.Dir(p.targetPath), path) +func (p *ProcessorsProvider) ProvideBloblang(pathStr string) ([]types.Processor, error) { + if !filepath.IsAbs(pathStr) { + pathStr = filepath.Join(filepath.Dir(p.targetPath), pathStr) } - mappingBytes, err := ioutil.ReadFile(path) + mappingBytes, err := ioutil.ReadFile(pathStr) if err != nil { return nil, err } - exec, mapErr := parser.ParseMapping(parser.GlobalContext(), path, string(mappingBytes)) + pCtx := parser.GlobalContext().WithImporterRelativeToFile(pathStr) + exec, mapErr := parser.ParseMapping(pCtx, string(mappingBytes)) if mapErr != nil { return nil, mapErr } diff --git a/public/bloblang/environment.go b/public/bloblang/environment.go index 96c2d810d6..568939c3be 100644 --- a/public/bloblang/environment.go +++ b/public/bloblang/environment.go @@ -52,7 +52,7 @@ func NewEmptyEnvironment() *Environment { // gives access to the line and column where the error occurred, as well as a // method for creating a well formatted error message. func (e *Environment) Parse(blobl string) (*Executor, error) { - exec, err := e.env.NewMapping("", blobl) + exec, err := e.env.NewMapping(blobl) if err != nil { if pErr, ok := err.(*parser.Error); ok { return nil, internalToPublicParserError([]rune(blobl), pErr) @@ -160,6 +160,14 @@ func (e *Environment) WithoutFunctions(names ...string) *Environment { } } +// WithDisabledImports returns a copy of the environment where imports within +// mappings are disabled. +func (e *Environment) WithDisabledImports() *Environment { + return &Environment{ + env: e.env.Deactivated(), + } +} + //------------------------------------------------------------------------------ // Parse a Bloblang mapping allowing the use of the globally accessible range of @@ -169,7 +177,7 @@ func (e *Environment) WithoutFunctions(names ...string) *Environment { // gives access to the line and column where the error occurred, as well as a // method for creating a well formatted error message. func Parse(blobl string) (*Executor, error) { - exec, err := parser.ParseMapping(parser.GlobalContext(), "", blobl) + exec, err := parser.ParseMapping(parser.GlobalContext(), blobl) if err != nil { return nil, internalToPublicParserError([]rune(blobl), err) } diff --git a/public/service/message_test.go b/public/service/message_test.go index f98ed78d02..3d3d62400b 100644 --- a/public/service/message_test.go +++ b/public/service/message_test.go @@ -296,7 +296,7 @@ func BenchmarkMessageMappingOld(b *testing.B) { msg := message.New(nil) msg.Append(part) - blobl, err := ibloblang.NewMapping("", "root.new_content = this.content.uppercase()") + blobl, err := ibloblang.NewMapping("root.new_content = this.content.uppercase()") require.NoError(b, err) b.ResetTimer() diff --git a/website/docs/guides/bloblang/methods.md b/website/docs/guides/bloblang/methods.md index 277cb3b0a9..454a6db0b2 100644 --- a/website/docs/guides/bloblang/methods.md +++ b/website/docs/guides/bloblang/methods.md @@ -1785,7 +1785,7 @@ root = this.without("inner.a","inner.c","d") BETA: This method is mostly stable but breaking changes could still be made outside of major version releases if a fundamental problem with it is found. -Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings. +Executes an argument Bloblang mapping on the target. This method can be used in order to execute dynamic mappings. Imports and functions that interact with the environment, such as `file` and `env`, or that access message information directly, such as `content` or `json`, are not enabled for dynamic Bloblang mappings. #### Parameters