diff --git a/coll/jq.go b/coll/jq.go new file mode 100644 index 000000000..16dc8f0ed --- /dev/null +++ b/coll/jq.go @@ -0,0 +1,96 @@ +package coll + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + "github.com/itchyny/gojq" +) + +// JQ - +func JQ(ctx context.Context, jqExpr string, in interface{}) (interface{}, error) { + query, err := gojq.Parse(jqExpr) + if err != nil { + return nil, fmt.Errorf("jq parsing expression %q: %w", jqExpr, err) + } + + // convert input to a supported type, if necessary + in, err = jqConvertType(in) + if err != nil { + return nil, fmt.Errorf("jq type conversion: %w", err) + } + + iter := query.RunWithContext(ctx, in) + var out interface{} + a := []interface{}{} + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return nil, fmt.Errorf("jq execution: %w", err) + } + a = append(a, v) + } + if len(a) == 1 { + out = a[0] + } else { + out = a + } + + return out, nil +} + +// jqConvertType converts the input to a map[string]interface{}, []interface{}, +// or other supported primitive JSON types. +func jqConvertType(in interface{}) (interface{}, error) { + // if it's already a supported type, pass it through + switch in.(type) { + case map[string]interface{}, []interface{}, + string, []byte, + nil, bool, + int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, + float32, float64: + return in, nil + } + + inType := reflect.TypeOf(in) + value := reflect.ValueOf(in) + + // pointers need to be dereferenced first + if inType.Kind() == reflect.Ptr { + inType = inType.Elem() + value = value.Elem() + } + + mapType := reflect.TypeOf(map[string]interface{}{}) + sliceType := reflect.TypeOf([]interface{}{}) + // if it can be converted to a map or slice, do that + if inType.ConvertibleTo(mapType) { + return value.Convert(mapType).Interface(), nil + } else if inType.ConvertibleTo(sliceType) { + return value.Convert(sliceType).Interface(), nil + } + + // if it's a struct, the simplest (though not necessarily most efficient) + // is to JSON marshal/unmarshal it + if inType.Kind() == reflect.Struct { + b, err := json.Marshal(in) + if err != nil { + return nil, fmt.Errorf("json marshal struct: %w", err) + } + var m map[string]interface{} + err = json.Unmarshal(b, &m) + if err != nil { + return nil, fmt.Errorf("json unmarshal struct: %w", err) + } + return m, nil + } + + // we maybe don't need to convert the value, so return it as-is + return in, nil +} diff --git a/coll/jq_test.go b/coll/jq_test.go new file mode 100644 index 000000000..7f71891c9 --- /dev/null +++ b/coll/jq_test.go @@ -0,0 +1,236 @@ +package coll + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJQ(t *testing.T) { + ctx := context.Background() + in := map[string]interface{}{ + "store": map[string]interface{}{ + "book": []interface{}{ + map[string]interface{}{ + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95, + }, + map[string]interface{}{ + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99, + }, + map[string]interface{}{ + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99, + }, + map[string]interface{}{ + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99, + }, + }, + "bicycle": map[string]interface{}{ + "color": "red", + "price": 19.95, + }, + }, + } + out, err := JQ(ctx, ".store.bicycle.color", in) + assert.NoError(t, err) + assert.Equal(t, "red", out) + + out, err = JQ(ctx, ".store.bicycle.price", in) + assert.NoError(t, err) + assert.Equal(t, 19.95, out) + + out, err = JQ(ctx, ".store.bogus", in) + assert.NoError(t, err) + assert.Nil(t, out) + + _, err = JQ(ctx, "{.store.unclosed", in) + assert.Error(t, err) + + out, err = JQ(ctx, ".store", in) + assert.NoError(t, err) + assert.EqualValues(t, in["store"], out) + + out, err = JQ(ctx, ".store.book[].author", in) + assert.NoError(t, err) + assert.Len(t, out, 4) + assert.Contains(t, out, "Nigel Rees") + assert.Contains(t, out, "Evelyn Waugh") + assert.Contains(t, out, "Herman Melville") + assert.Contains(t, out, "J. R. R. Tolkien") + + out, err = JQ(ctx, ".store.book[]|select(.price < 10.0 )", in) + assert.NoError(t, err) + expected := []interface{}{ + map[string]interface{}{ + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95, + }, + map[string]interface{}{ + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99, + }, + } + assert.EqualValues(t, expected, out) + + in = map[string]interface{}{ + "a": map[string]interface{}{ + "aa": map[string]interface{}{ + "foo": map[string]interface{}{ + "aaa": map[string]interface{}{ + "aaaa": map[string]interface{}{ + "bar": 1234, + }, + }, + }, + }, + "ab": map[string]interface{}{ + "aba": map[string]interface{}{ + "foo": map[string]interface{}{ + "abaa": true, + "abab": "baz", + }, + }, + }, + }, + } + out, err = JQ(ctx, `tostream|select((.[0]|index("foo")) and (.[0][-1]!="foo") and (.[1])) as $s|($s[0]|index("foo")+1) as $ind|($ind|truncate_stream($s)) as $newstream|$newstream|reduce . as [$p,$v] ({};setpath($p;$v))|add`, in) + assert.NoError(t, err) + assert.Len(t, out, 3) + assert.Contains(t, out, map[string]interface{}{"aaaa": map[string]interface{}{"bar": 1234}}) + assert.Contains(t, out, true) + assert.Contains(t, out, "baz") +} + +func TestJQ_typeConversions(t *testing.T) { + ctx := context.Background() + + type bicycleType struct { + Color string + } + type storeType struct { + Bicycle *bicycleType + safe interface{} + } + + structIn := &storeType{ + Bicycle: &bicycleType{ + Color: "red", + }, + safe: "hidden", + } + + out, err := JQ(ctx, ".Bicycle.Color", structIn) + assert.NoError(t, err) + assert.Equal(t, "red", out) + + out, err = JQ(ctx, ".safe", structIn) + assert.NoError(t, err) + assert.Nil(t, out) + + _, err = JQ(ctx, ".*", structIn) + assert.Error(t, err) + + // a type with an underlying type of map[string]interface{}, just like + // gomplate.tmplctx + type mapType map[string]interface{} + + out, err = JQ(ctx, ".foo", mapType{"foo": "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", out) + + // sometimes it'll be a pointer... + out, err = JQ(ctx, ".foo", &mapType{"foo": "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", out) + + // underlying slice type + type sliceType []interface{} + + out, err = JQ(ctx, ".[1]", sliceType{"foo", "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", out) + + out, err = JQ(ctx, ".[2]", &sliceType{"foo", "bar", "baz"}) + assert.NoError(t, err) + assert.Equal(t, "baz", out) + + // other basic types + out, err = JQ(ctx, ".", []byte("hello")) + assert.NoError(t, err) + assert.EqualValues(t, "hello", out) + + out, err = JQ(ctx, ".", "hello") + assert.NoError(t, err) + assert.EqualValues(t, "hello", out) + + out, err = JQ(ctx, ".", 1234) + assert.NoError(t, err) + assert.EqualValues(t, 1234, out) + + out, err = JQ(ctx, ".", true) + assert.NoError(t, err) + assert.EqualValues(t, true, out) + + out, err = JQ(ctx, ".", nil) + assert.NoError(t, err) + assert.Nil(t, out) + + // underlying basic types + type intType int + out, err = JQ(ctx, ".", intType(1234)) + assert.NoError(t, err) + assert.EqualValues(t, 1234, out) + + type byteArrayType []byte + out, err = JQ(ctx, ".", byteArrayType("hello")) + assert.NoError(t, err) + assert.EqualValues(t, "hello", out) +} + +func TestJQConvertType_passthroughTypes(t *testing.T) { + // non-marshalable values, like recursive structs, can't be used + type recursive struct{ Self *recursive } + v := &recursive{} + v.Self = v + _, err := jqConvertType(v) + assert.Error(t, err) + + testdata := []interface{}{ + map[string]interface{}{"foo": 1234}, + []interface{}{"foo", "bar", "baz", 1, 2, 3}, + "foo", + []byte("foo"), + json.RawMessage(`{"foo": "bar"}`), + true, + nil, + int(1234), int8(123), int16(123), int32(123), int64(123), + uint(123), uint8(123), uint16(123), uint32(123), uint64(123), + float32(123.45), float64(123.45), + } + + for _, d := range testdata { + out, err := jqConvertType(d) + assert.NoError(t, err) + assert.Equal(t, d, out) + } +} diff --git a/docs-src/content/functions/coll.yml b/docs-src/content/functions/coll.yml index eedfd57aa..046f7d74b 100644 --- a/docs-src/content/functions/coll.yml +++ b/docs-src/content/functions/coll.yml @@ -136,6 +136,35 @@ funcs: - | $ gomplate -i '{{ .books | jsonpath `$..works[?( @.edition_count > 400 )].title` }}' -c books=https://openlibrary.org/subjects/fantasy.json [Alice's Adventures in Wonderland Gulliver's Travels] + - name: coll.JQ + alias: jq + description: | + Filters an input object or list using the [jq](https://stedolan.github.io/jq/) language, as implemented by [gojq](https://github.com/itchyny/gojq). + + Any JSON datatype may be used as input (NOTE: strings are not JSON-parsed but passed in as is). + If the expression results in multiple items (no matter if streamed or as an array) they are wrapped in an array. + Otherwise a single item is returned (even if resulting in an array with a single contained element). + + JQ filter expressions can be tested at https://jqplay.org/ + + See also: + + - [jq manual](https://stedolan.github.io/jq/manual/) + - [gojq differences to jq](https://github.com/itchyny/gojq#difference-to-jq) + pipeline: true + arguments: + - name: expression + required: true + description: The JQ expression + - name: in + required: true + description: The object or list to query + examples: + - | + $ gomplate \ + -i '{{ .books | jq `[.works[]|{"title":.title,"authors":[.authors[].name],"published":.first_publish_year}][0]` }}' \ + -c books=https://openlibrary.org/subjects/fantasy.json + map[authors:[Lewis Carroll] published:1865 title:Alice's Adventures in Wonderland] - name: coll.Keys alias: keys description: | diff --git a/docs/content/functions/coll.md b/docs/content/functions/coll.md index e441da9b1..55f2b9cff 100644 --- a/docs/content/functions/coll.md +++ b/docs/content/functions/coll.md @@ -200,6 +200,48 @@ $ gomplate -i '{{ .books | jsonpath `$..works[?( @.edition_count > 400 )].title` [Alice's Adventures in Wonderland Gulliver's Travels] ``` +## `coll.JQ` + +**Alias:** `jq` + +Filters an input object or list using the [jq](https://stedolan.github.io/jq/) language, as implemented by [gojq](https://github.com/itchyny/gojq). + +Any JSON datatype may be used as input (NOTE: strings are not JSON-parsed but passed in as is). +If the expression results in multiple items (no matter if streamed or as an array) they are wrapped in an array. +Otherwise a single item is returned (even if resulting in an array with a single contained element). + +JQ filter expressions can be tested at https://jqplay.org/ + +See also: + +- [jq manual](https://stedolan.github.io/jq/manual/) +- [gojq differences to jq](https://github.com/itchyny/gojq#difference-to-jq) + +### Usage + +```go +coll.JQ expression in +``` +```go +in | coll.JQ expression +``` + +### Arguments + +| name | description | +|------|-------------| +| `expression` | _(required)_ The JQ expression | +| `in` | _(required)_ The object or list to query | + +### Examples + +```console +$ gomplate \ + -i '{{ .books | jq `[.works[]|{"title":.title,"authors":[.authors[].name],"published":.first_publish_year}][0]` }}' \ + -c books=https://openlibrary.org/subjects/fantasy.json +map[authors:[Lewis Carroll] published:1865 title:Alice's Adventures in Wonderland] +``` + ## `coll.Keys` **Alias:** `keys` diff --git a/funcs/coll.go b/funcs/coll.go index 4a3ca5ab0..97bf911af 100644 --- a/funcs/coll.go +++ b/funcs/coll.go @@ -47,6 +47,7 @@ func CreateCollFuncs(ctx context.Context) map[string]interface{} { f["merge"] = ns.Merge f["sort"] = ns.Sort f["jsonpath"] = ns.JSONPath + f["jq"] = ns.JQ f["flatten"] = ns.Flatten return f } @@ -142,6 +143,11 @@ func (CollFuncs) JSONPath(p string, in interface{}) (interface{}, error) { return coll.JSONPath(p, in) } +// JQ - +func (f *CollFuncs) JQ(jqExpr string, in interface{}) (interface{}, error) { + return coll.JQ(f.ctx, jqExpr, in) +} + // Flatten - func (CollFuncs) Flatten(args ...interface{}) ([]interface{}, error) { if len(args) == 0 || len(args) > 2 { diff --git a/go.mod b/go.mod index 61c87d62b..3c5c25c2d 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/hashicorp/consul/api v1.18.0 github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/vault/api v1.8.2 + github.com/itchyny/gojq v0.12.10 github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 github.com/joho/godotenv v1.4.0 github.com/pkg/errors v0.9.1 @@ -105,11 +106,12 @@ require ( github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect diff --git a/go.sum b/go.sum index f8a2a61cf..9dd159808 100644 --- a/go.sum +++ b/go.sum @@ -976,6 +976,10 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/ionos-cloud/sdk-go/v6 v6.1.0/go.mod h1:Ox3W0iiEz0GHnfY9e5LmAxwklsxguuNFEUSu0gVRTME= +github.com/itchyny/gojq v0.12.10 h1:6TcS0VYWS6wgntpF/4tnrzwdCMjiTxRAxIqZWfDsDQU= +github.com/itchyny/gojq v0.12.10/go.mod h1:o3FT8Gkbg/geT4pLI0tF3hvip5F3Y/uskjRz9OYa38g= +github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -1124,8 +1128,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -1977,6 +1982,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/tests/integration/collection_test.go b/internal/tests/integration/collection_test.go index cc83e14d6..5175ef64f 100644 --- a/internal/tests/integration/collection_test.go +++ b/internal/tests/integration/collection_test.go @@ -98,3 +98,8 @@ func TestColl_Pick(t *testing.T) { func TestColl_Omit(t *testing.T) { inOutTest(t, `{{ $data := dict "foo" 1 "bar" 2 "baz" 3 }}{{ coll.Omit "foo" "baz" $data }}`, "map[bar:2]") } + +func TestColl_JQ(t *testing.T) { + inOutTest(t, `{{ coll.JQ ".foo" (dict "foo" 1 "bar" 2 "baz" 3) }}`, "1") + inOutTest(t, `{{ coll.Slice "one" 2 "three" 4.0 | jq ".[2]" }}`, `three`) +}