Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add coll.JQ using gojq library #1585

Merged
merged 16 commits into from
Dec 29, 2022
Merged

Add coll.JQ using gojq library #1585

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
76ae022
feat: add coll.JQ using gojq library
ahochsteger Dec 21, 2022
d4931ba
fix: jq function naming (gojq -> jq)
ahochsteger Dec 21, 2022
4828db1
test: add tests (take from jsonpath_test.go)
ahochsteger Dec 22, 2022
d627ef5
chore: add TODO for nil values (are they allowed?)
ahochsteger Dec 22, 2022
c3ff3d4
refactor: use fmt.Errorf instead of errors.Wrapf
ahochsteger Dec 29, 2022
e5f3b16
fix: wrong alias for coll.JQ
ahochsteger Dec 29, 2022
6e7a093
docs: add links to JQ
ahochsteger Dec 29, 2022
6c0701b
test: add assertions after json marshal/unmarshal
ahochsteger Dec 29, 2022
db8f0c2
refactor: use fmt.Errorf instead of errors.Wrapf
ahochsteger Dec 29, 2022
ce7c606
Merge branch 'main' into main
ahochsteger Dec 29, 2022
5b50110
fix: test syntax and null handling
ahochsteger Dec 29, 2022
a3a57e7
docs: improve documentation
ahochsteger Dec 29, 2022
97946ab
docs: add blank line
ahochsteger Dec 29, 2022
1f4818c
Support cancellation
hairyhenderson Dec 29, 2022
e7f569b
Support (almost) all types, not just map[string]interface{} and []int…
hairyhenderson Dec 29, 2022
600790b
add an integration test for coll.JQ
hairyhenderson Dec 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions coll/jq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package coll

import (
"github.com/itchyny/gojq"
"github.com/pkg/errors"
)

// JQ -
func JQ(jqExpr string, in interface{}) (interface{}, error) {
query, err := gojq.Parse(jqExpr)
if err != nil {
return nil, fmt.Errorf("couldn't parse JQ %s: %w", jqExpr, err)
}
iter := query.Run(in)
hairyhenderson marked this conversation as resolved.
Show resolved Hide resolved
var out interface{}
a := []interface{}{}
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
return nil, errors.Wrap(err, "executing JQ failed")
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved
}
if v != nil { // TODO: Check, if nil may be a valid result
a = append(a, v)
}
}
if len(a) == 1 {
out = a[0]
} else {
out = a
}
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved

return out, nil
}
150 changes: 150 additions & 0 deletions coll/jq_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package coll

import (
"encoding/json"
"testing"

"github.com/stretchr/testify/assert"
)

func TestJQ(t *testing.T) {
in := map[string]interface{}{
"store": map[string]interface{}{
"book": []interface{}{
map[string]interface{}{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95,
},
map[string]interface{}{
"category": "fiction",
"author": "Evelyn Waugh",
"title": "Sword of Honour",
"price": 12.99,
},
map[string]interface{}{
"category": "fiction",
"author": "Herman Melville",
"title": "Moby Dick",
"isbn": "0-553-21311-3",
"price": 8.99,
},
map[string]interface{}{
"category": "fiction",
"author": "J. R. R. Tolkien",
"title": "The Lord of the Rings",
"isbn": "0-395-19395-8",
"price": 22.99,
},
},
"bicycle": map[string]interface{}{
"color": "red",
"price": 19.95,
},
},
}
out, err := JQ(".store.bicycle.color", in)
assert.NoError(t, err)
assert.Equal(t, "red", out)

out, err = JQ(".store.bicycle.price", in)
assert.NoError(t, err)
assert.Equal(t, 19.95, out)

out, err = JQ(".store.bogus", in)
assert.NoError(t, err)
assert.Nil(t, out)

_, err = JQ("{.store.unclosed", in)
assert.Error(t, err)

out, err = JQ(".store", in)
assert.NoError(t, err)
assert.EqualValues(t, in["store"], out)

out, err = JQ(".store.book[].author", in)
assert.NoError(t, err)
assert.Len(t, out, 4)
assert.Contains(t, out, "Nigel Rees")
assert.Contains(t, out, "Evelyn Waugh")
assert.Contains(t, out, "Herman Melville")
assert.Contains(t, out, "J. R. R. Tolkien")

out, err = JQ(".store.book[]|select(.price < 10.0 )", in)
assert.NoError(t, err)
expected := []interface{}{
map[string]interface{}{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95,
},
map[string]interface{}{
"category": "fiction",
"author": "Herman Melville",
"title": "Moby Dick",
"isbn": "0-553-21311-3",
"price": 8.99,
},
}
assert.EqualValues(t, expected, out)

in = map[string]interface{}{
"a": map[string]interface{}{
"aa": map[string]interface{}{
"foo": map[string]interface{}{
"aaa": map[string]interface{}{
"aaaa": map[string]interface{}{
"bar": 1234,
},
},
},
},
"ab": map[string]interface{}{
"aba": map[string]interface{}{
"foo": map[string]interface{}{
"abaa": true,
"abab": "baz",
},
},
},
},
}
out, err = JQ(`tostream|select((.[0]|index("foo")) and (.[0][-1]!="foo") and (.[1])) as $s|($s[0]|index("foo")+1) as $ind|($ind|truncate_stream($s)) as $newstream|$newstream|reduce . as [$p,$v] ({};setpath($p;$v))|add`, in)
assert.NoError(t, err)
assert.Len(t, out, 3)
assert.Contains(t, out, map[string]interface{}{"aaaa": map[string]interface{}{"bar": 1234}})
assert.Contains(t, out, true)
assert.Contains(t, out, "baz")

type bicycleType struct {
Color string
}
type storeType struct {
Bicycle *bicycleType
safe interface{}
}

structIn := &storeType{
Bicycle: &bicycleType{
Color: "red",
},
safe: "hidden",
}

// TODO: Check if this is a valid test case (taken from jsonpath_test.go) since the struct
// had to be converted to JSON and parsed from it again to be able to process using gojq.
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔

I suppose this is because JQ can't be used with arbitrary structs or other types, which is perhaps something we should allow...

In the same vein, it might be useful to be able to pass the template context (gomplate.tmplctx) as the input for something like:

$ gomplate -c books=https://openlibrary.org/subjects/fantasy.json -i '{{ jq `.works[].title` . }}'

Right now this fails with:

executing JQ failed: %!v(PANIC=Error method: invalid type: *gomplate.tmplctx (&map[books:map[key:/subjects/fantasy...

What's interesting in that particular case is that gomplate.tmplctx is a map[string]interface{}, which is in theory OK.

I'm going to hack on this a bit and make a commit if I can figure It out...

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I've gotten this to work:

$ bin/gomplate -c books=https://openlibrary.org/subjects/fantasy.json \
-i '{{ range jq `.books.works[].title` . -}}
title: {{.}}
{{end}}'
title: Alice's Adventures in Wonderland
title: Gulliver's Travels
title: Treasure Island
title: Through the Looking-Glass
title: A Midsummer Night's Dream
title: Il principe
title: The Wonderful Wizard of Oz
title: Avventure di Pinocchio
title: Alice's Adventures in Wonderland / Through the Looking Glass
title: Five Children and It
title: The Hobbit
title: Harry Potter and the Philosopher's Stone

Note that I've referenced the books datasource in the . context passed through as an argument.

v := map[string]interface{}{}
b, err := json.Marshal(structIn)
err = json.Unmarshal(b, &v)
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved
out, err = JQ(".Bicycle.Color", v)
assert.NoError(t, err)
assert.Equal(t, "red", out)

_, err = JQ(".safe", structIn)
assert.Error(t, err)

_, err = JQ(".*", structIn)
assert.Error(t, err)
}
31 changes: 31 additions & 0 deletions docs-src/content/functions/coll.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,37 @@ funcs:
- |
$ gomplate -i '{{ .books | jsonpath `$..works[?( @.edition_count > 400 )].title` }}' -c books=https://openlibrary.org/subjects/fantasy.json
[Alice's Adventures in Wonderland Gulliver's Travels]
- name: coll.JQ
alias: jq
description: |
Filters an input object or list using the [jq](https://stedolan.github.io/jq/) language, as implemented by [gojq](https://github.com/itchyny/gojq).

Any object or list may be used as input. The output depends somewhat on the expression; if multiple items are matched, an array is returned.
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved

JQ filter expressions can be tested at https://jqplay.org/

[jq Manual]: https://stedolan.github.io/jq/manual/
[gojq differences to jq]: https://github.com/itchyny/gojq#difference-to-jq
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved
pipeline: true
arguments:
- name: expression
required: true
description: The JQ expression
- name: in
required: true
description: The object or list to query
examples:
- |
$ gomplate \
-i '{{ .books | jq `[.works[]|{"title":.title,"authors":[.authors[].name],"published":.first_publish_year}][0]` }}' \
-c books=https://openlibrary.org/subjects/fantasy.json
{
"authors": [
"Lewis Carroll"
],
"published": 1865,
"title": "Alice's Adventures in Wonderland"
}
- name: coll.Keys
alias: keys
description: |
Expand Down
45 changes: 45 additions & 0 deletions docs/content/functions/coll.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,51 @@ $ gomplate -i '{{ .books | jsonpath `$..works[?( @.edition_count > 400 )].title`
[Alice's Adventures in Wonderland Gulliver's Travels]
```

## `coll.JQ`

**Alias:** `jq`

Filters an input object or list using the JQ language implemented by gojq.

Any object or list may be used as input. The output depends somewhat on the expression; if multiple items are matched, an array is returned.

JQ filter expressions can be tested at https://jqplay.org/

[jq Manual]: https://stedolan.github.io/jq/manual/
[gojq library]: https://github.com/itchyny/gojq
[gojq differences to jq]: https://github.com/itchyny/gojq#difference-to-jq
ahochsteger marked this conversation as resolved.
Show resolved Hide resolved

### Usage

```go
coll.JQ expression in
```
```go
in | coll.JQ expression
```

### Arguments

| name | description |
|------|-------------|
| `expression` | _(required)_ The JQ expression |
| `in` | _(required)_ The object or list to query |

### Examples

```console
$ gomplate \
-i '{{ .books | coll.JQ `[
.works[]
| {
"title": .title,
"authors": [.authors[].name],
"published": .first_publish_year
}][0]` }}' \
-c books=https://openlibrary.org/subjects/fantasy.json
map[authors:[Lewis Carroll] published:1865 title:Alice's Adventures in Wonderland]
```

## `coll.Keys`

**Alias:** `keys`
Expand Down
6 changes: 6 additions & 0 deletions funcs/coll.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func CreateCollFuncs(ctx context.Context) map[string]interface{} {
f["merge"] = ns.Merge
f["sort"] = ns.Sort
f["jsonpath"] = ns.JSONPath
f["jq"] = ns.JQ
f["flatten"] = ns.Flatten
return f
}
Expand Down Expand Up @@ -142,6 +143,11 @@ func (CollFuncs) JSONPath(p string, in interface{}) (interface{}, error) {
return coll.JSONPath(p, in)
}

// JQ -
func (CollFuncs) JQ(jqExpr string, in interface{}) (interface{}, error) {
return coll.JQ(jqExpr, in)
}

// Flatten -
func (CollFuncs) Flatten(args ...interface{}) ([]interface{}, error) {
if len(args) == 0 || len(args) > 2 {
Expand Down
4 changes: 3 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/hashicorp/consul/api v1.18.0
github.com/hashicorp/go-sockaddr v1.0.2
github.com/hashicorp/vault/api v1.8.2
github.com/itchyny/gojq v0.12.10
github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2
github.com/joho/godotenv v1.4.0
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -105,11 +106,12 @@ require (
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
Expand Down
8 changes: 7 additions & 1 deletion go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -976,6 +976,10 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
github.com/ionos-cloud/sdk-go/v6 v6.1.0/go.mod h1:Ox3W0iiEz0GHnfY9e5LmAxwklsxguuNFEUSu0gVRTME=
github.com/itchyny/gojq v0.12.10 h1:6TcS0VYWS6wgntpF/4tnrzwdCMjiTxRAxIqZWfDsDQU=
github.com/itchyny/gojq v0.12.10/go.mod h1:o3FT8Gkbg/geT4pLI0tF3hvip5F3Y/uskjRz9OYa38g=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
Expand Down Expand Up @@ -1124,8 +1128,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
Expand Down Expand Up @@ -1977,6 +1982,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down