diff --git a/README.md b/README.md index 3a4667d5c..142fa4dee 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ _Read the docs at [docs.gomplate.ca][docs-url], chat with developers and communi [![Discussions][discussions-image]][discussions-url] `gomplate` is a template renderer which supports a growing list of datasources, -such as: JSON (_including EJSON - encrypted JSON_), YAML, AWS EC2 metadata, [BoltDB](https://pkg.go.dev/go.etcd.io/bbolt), +such as: JSON (_including EJSON - encrypted JSON_), YAML, AWS EC2 metadata, [Hashicorp Consul](https://www.consul.io/) and [Hashicorp Vault](https://www.vaultproject.io/) secrets. Come chat with developers and community in the [#gomplate channel][] on [Gophers Slack][] and on [GitHub Discussions][discussions-url]! diff --git a/data/datasource.go b/data/datasource.go index a28a66f7d..a8ce5e742 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -42,8 +42,6 @@ func (d *Data) registerReaders() { d.sourceReaders["aws+smp"] = readAWSSMP d.sourceReaders["aws+sm"] = readAWSSecretsManager - // Deprecated: don't use - d.sourceReaders["boltdb"] = readBoltDB d.sourceReaders["consul"] = readConsul d.sourceReaders["consul+http"] = readConsul d.sourceReaders["consul+https"] = readConsul @@ -148,7 +146,7 @@ type Source struct { fs afero.Fs // used for file: URLs, nil otherwise hc *http.Client // used for http[s]: URLs, nil otherwise vc *vault.Vault // used for vault: URLs, nil otherwise - kv *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise + kv *libkv.LibKV // used for consul:, etcd:, zookeeper: URLs, nil otherwise asmpg awssmpGetter // used for aws+smp:, nil otherwise awsSecretsManager awsSecretsManagerGetter // used for aws+sm, nil otherwise mediaType string diff --git a/data/datasource_boltdb.go b/data/datasource_boltdb.go deleted file mode 100644 index 63be1e646..000000000 --- a/data/datasource_boltdb.go +++ /dev/null @@ -1,32 +0,0 @@ -package data - -import ( - "context" - - "github.com/hairyhenderson/gomplate/v3/internal/deprecated" - "github.com/hairyhenderson/gomplate/v3/libkv" - "github.com/pkg/errors" -) - -// Deprecated: don't use -func readBoltDB(ctx context.Context, source *Source, args ...string) (data []byte, err error) { - deprecated.WarnDeprecated(ctx, "boltdb support is deprecated and will be removed in a future major version of gomplate") - if source.kv == nil { - source.kv, err = libkv.NewBoltDB(source.URL) - if err != nil { - return nil, err - } - } - - if len(args) != 1 { - return nil, errors.New("missing key") - } - p := args[0] - - data, err = source.kv.Read(p) - if err != nil { - return nil, err - } - - return data, nil -} diff --git a/docs/content/_index.md b/docs/content/_index.md index cae47fa29..d1bac2805 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -8,7 +8,7 @@ menu: --- `gomplate` is a template renderer which supports a growing list of datasources, -such as: JSON (_including EJSON - encrypted JSON_), YAML, AWS EC2 metadata, [BoltDB](https://pkg.go.dev/go.etcd.io/bbolt), +such as: JSON (_including EJSON - encrypted JSON_), YAML, AWS EC2 metadata, [Hashicorp Consul](https://www.consul.io/) and [Hashicorp Vault](https://www.vaultproject.io/) secrets. Come chat with developers and community in the [#gomplate channel][] on [Gophers Slack][] and on [GitHub Discussions][discussions-url]! diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 6f9f02f80..aac4b0797 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -31,7 +31,7 @@ For our purposes, the _scheme_ and the _path_ components are especially importan | _authority_ | Used only by remote datasources, and can be omitted in some of those cases. Consists of _userinfo_ (`user:pass`), _host_, and _port_. | | _path_ | Can be omitted, but usually used as the basis of the locator for the datasource. If the path ends with a `/` character, [directory](#directory-datasources) semantics are used. | | _query_ | Used rarely for datasources where information must be provided in order to get a reasonable reply (such as generating dynamic secrets with Vault), or for [overriding MIME types](#overriding-mime-types) | -| _fragment_ | Used rarely for accessing a subset of the given path (such as a bucket name in a BoltDB database) | +| _fragment_ | Used rarely for accessing a subset of the given path | ### Opaque URIs @@ -58,7 +58,6 @@ Gomplate supports a number of datasources, each specified with a particular URL | [AWS Systems Manager Parameter Store](#using-aws-smp-datasources) | `aws+smp` | [AWS Systems Manager Parameter Store][AWS SMP] is a hierarchically-organized key/value store which allows storage of text, lists, or encrypted secrets for retrieval by AWS resources | | [AWS Secrets Manager](#using-aws-sm-datasource) | `aws+sm` | [AWS Secrets Manager][] helps you protect secrets needed to access your applications, services, and IT resources. | | [Amazon S3](#using-s3-datasources) | `s3` | [Amazon S3][] is a popular object storage service. | -| [BoltDB](#using-boltdb-datasources) | `boltdb` | [BoltDB][] is a simple local key/value store used by many Go tools | | [Consul](#using-consul-datasources) | `consul`, `consul+http`, `consul+https` | [HashiCorp Consul][] provides (among many other features) a key/value store | | [Environment](#using-env-datasources) | `env` | Environment variables can be used as datasources - useful for testing | | [File](#using-file-datasources) | `file` | Files can be read in any of the [supported formats](#mime-types), including by piping through standard input (`Stdin`). [Directories](#directory-datasources) are also supported. | @@ -309,32 +308,6 @@ $ gomplate -d bucket=s3://my-bucket/?region=eu-west-1&endpoint=my-test-site& -i Hello world ``` -## Using `boltdb` datasources - -[BoltDB][] is a simple local key/value store used by many Go tools. The `boltdb://` scheme can be used to access values stored in a BoltDB database file. The full path is provided in the URL, and the bucket name can be specified using a URL fragment (e.g. `boltdb:///tmp/database.db#bucket`). - -**Note:** Access is implemented through [`libkv`](https://github.com/docker/libkv), and as such, the first 8 bytes of all values are used as an incrementing last modified index value. All values must therefore be at least 9 bytes long, with the first 8 being ignored. - -The following environment variables can be set: - -| name | usage | -|------|-------| -| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. | -| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | - -### URL Considerations - -For `boltdb`, the _scheme_, _path_, and _fragment_ are used. - -The _path_ must point to a BoltDB database on the local file system, while the _fragment_ must provide the name of the bucket to use. - -### Example - -```console -$ gomplate -d config=boltdb:///tmp/config.db#Bucket1 -i '{{(datasource "config" "foo")}}' -bar -``` - ## Using `consul` datasources Gomplate supports retrieving data from [HashiCorp Consul][]'s [KV Store](https://www.consul.io/api/kv.html). @@ -778,7 +751,6 @@ The file `/tmp/vault-aws-nonce` will be created if it didn't already exist, and [AWS SMP]: https://aws.amazon.com/systems-manager/features#Parameter_Store [AWS Secrets Manager]: https://aws.amazon.com/secrets-manager -[BoltDB]: https://pkg.go.dev/go.etcd.io/bbolt [HashiCorp Consul]: https://consul.io [HashiCorp Vault]: https://vaultproject.io [JSON]: https://json.org diff --git a/go.mod b/go.mod index c051259e9..83234dd5f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,6 @@ require ( github.com/stretchr/testify v1.7.2 github.com/ugorji/go/codec v1.2.7 github.com/zealic/xignore v0.3.3 - go.etcd.io/bbolt v1.3.6 gocloud.dev v0.25.1-0.20220408200107-09b10f7359f7 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a diff --git a/go.sum b/go.sum index 4d268f583..aeee53066 100644 --- a/go.sum +++ b/go.sum @@ -802,8 +802,6 @@ github.com/zealic/xignore v0.3.3 h1:EpLXUgZY/JEzFkTc+Y/VYypzXtNz+MSOMVCGW5Q4CKQ= github.com/zealic/xignore v0.3.3/go.mod h1:lhS8V7fuSOtJOKsvKI7WfsZE276/7AYEqokv3UiqEAU= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -1042,7 +1040,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/tests/integration/datasources_boltdb_test.go b/internal/tests/integration/datasources_boltdb_test.go deleted file mode 100644 index 848a3119d..000000000 --- a/internal/tests/integration/datasources_boltdb_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package integration - -import ( - "testing" - - "github.com/stretchr/testify/require" - "go.etcd.io/bbolt" - "gotest.tools/v3/fs" -) - -func setupDatasourcesBoltDBTest(t *testing.T) *fs.Dir { - tmpDir := fs.NewDir(t, "gomplate-inttests") - t.Cleanup(tmpDir.Remove) - - db, err := bbolt.Open(tmpDir.Join("config.db"), 0600, nil) - require.NoError(t, err) - defer db.Close() - - err = db.Update(func(tx *bbolt.Tx) error { - var b *bbolt.Bucket - b, err = tx.CreateBucket([]byte("Bucket1")) - if err != nil { - return err - } - // the first 8 bytes are ignored when read by libkv, so we prefix with gibberish - err = b.Put([]byte("foo"), []byte("00000000bar")) - if err != nil { - return err - } - - b, err = tx.CreateBucket([]byte("Bucket2")) - if err != nil { - return err - } - err = b.Put([]byte("foobar"), []byte("00000000baz")) - return err - }) - require.NoError(t, err) - - return tmpDir -} - -func TestDatasources_BoltDB_Datasource(t *testing.T) { - tmpDir := setupDatasourcesBoltDBTest(t) - - // ignore the stderr output, it'll contain boltdb deprecation warning - o, _, err := cmd(t, "-d", "config=boltdb://"+tmpDir.Join("config.db#Bucket1"), - "-i", `{{(ds "config" "foo")}}`).run() - assertSuccess(t, o, "", err, "bar") - - o, _, err = cmd(t, "-d", "config=boltdb://"+tmpDir.Join("config.db#Bucket1"), - "-d", "config2=boltdb://"+tmpDir.Join("config.db#Bucket2"), - "-i", `{{(ds "config" "foo")}}-{{(ds "config2" "foobar")}}`).run() - assertSuccess(t, o, "", err, "bar-baz") -} diff --git a/libkv/boltdb.go b/libkv/boltdb.go deleted file mode 100644 index c8d111ba2..000000000 --- a/libkv/boltdb.go +++ /dev/null @@ -1,42 +0,0 @@ -package libkv - -import ( - "net/url" - "time" - - "github.com/docker/libkv" - "github.com/docker/libkv/store" - "github.com/docker/libkv/store/boltdb" - "github.com/hairyhenderson/gomplate/v3/conv" - "github.com/hairyhenderson/gomplate/v3/env" - "github.com/pkg/errors" -) - -// NewBoltDB - initialize a new BoltDB datasource handler -// Deprecated: don't use -func NewBoltDB(u *url.URL) (*LibKV, error) { - boltdb.Register() - - config, err := setupBoltDB(u.Fragment) - if err != nil { - return nil, err - } - kv, err := libkv.NewStore(store.BOLTDB, []string{u.Path}, config) - if err != nil { - return nil, errors.Wrapf(err, "BoltDB store creation failed") - } - return &LibKV{kv}, nil -} - -func setupBoltDB(bucket string) (*store.Config, error) { - if bucket == "" { - return nil, errors.New("missing bucket - must specify BoltDB bucket in URL fragment") - } - - t := conv.MustParseInt(env.Getenv("BOLTDB_TIMEOUT"), 10, 16) - return &store.Config{ - Bucket: bucket, - ConnectionTimeout: time.Duration(t) * time.Second, - PersistConnection: conv.Bool(env.Getenv("BOLTDB_PERSIST")), - }, nil -} diff --git a/libkv/boltdb_test.go b/libkv/boltdb_test.go deleted file mode 100644 index 7965ae43b..000000000 --- a/libkv/boltdb_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package libkv - -import ( - "net/url" - "os" - "testing" - "time" - - "github.com/docker/libkv/store" - "github.com/stretchr/testify/assert" -) - -func TestSetupBoltDB(t *testing.T) { - _, err := setupBoltDB("") - assert.Error(t, err) - - expectedConfig := &store.Config{Bucket: "foo"} - actualConfig, err := setupBoltDB("foo") - assert.NoError(t, err) - assert.Equal(t, expectedConfig, actualConfig) - - expectedConfig = &store.Config{ - Bucket: "bar", - ConnectionTimeout: 42 * time.Second, - } - os.Setenv("BOLTDB_TIMEOUT", "42") - defer os.Unsetenv("BOLTDB_TIMEOUT") - actualConfig, err = setupBoltDB("bar") - assert.NoError(t, err) - assert.Equal(t, expectedConfig, actualConfig) - - expectedConfig = &store.Config{ - Bucket: "bar", - ConnectionTimeout: 42 * time.Second, - PersistConnection: true, - } - os.Setenv("BOLTDB_PERSIST", "true") - defer os.Unsetenv("BOLTDB_PERSIST") - actualConfig, err = setupBoltDB("bar") - assert.NoError(t, err) - assert.Equal(t, expectedConfig, actualConfig) -} - -func TestNewBoltDB(t *testing.T) { - u, _ := url.Parse("boltdb:///bolt.db") - _, err := NewBoltDB(u) - assert.Error(t, err) -}