Skip to content

Commit

Permalink
Merge pull request #74 from hairyhenderson/vault-datasource-54
Browse files Browse the repository at this point in the history
Support for Vault datasources (app-id & token auth)
  • Loading branch information
hairyhenderson authored Nov 15, 2016
2 parents 02e71e4 + 765dea9 commit 0612cf1
Show file tree
Hide file tree
Showing 10 changed files with 696 additions and 10 deletions.
38 changes: 37 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ Hello world

Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument).

Currently, `file://`, `http://` and `https://` URLs are supported.
Currently, `file://`, `http://`, `https://`, and `vault://` URLs are supported.

Currently-supported formats are JSON and YAML.

Expand Down Expand Up @@ -249,6 +249,42 @@ $ echo 'Hello there, {{(datasource "foo").headers.Host}}...' | gomplate -d foo=h
Hello there, httpbin.org...
```

###### Usage with Vault data

The special `vault://` URL scheme can be used to retrieve data from [Hashicorp
Vault](https://vaultproject.io). To use this, you must put the Vault server's
URL in the `$VAULT_ADDR` environment variable.

Currently, the [`app-id`](https://www.vaultproject.io/docs/auth/app-id.html)
auth backend is supported, as well as Vault tokens obtained through external
means.

To use a Vault datasource with a single secret, just use a URL of
`vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left
empty.

```console
$ echo 'My voice is my passport. {{(datasource "vault").value}}' \
| gomplate -d vault=vault:///secret/sneakers
My voice is my passport. Verify me.
```

You can also specify the secret path in the template by using a URL of `vault://`
(or `vault:///`, or `vault:`):
```console
$ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \
| gomplate -d vault=vault://
My voice is my passport. Verify me.
```

And the two can be mixed to scope secrets to a specific namespace:

```console
$ echo 'db_password={{(datasource "vault" "db/pass").value}}' \
| gomplate -d vault=vault:///secret/production
db_password=prodsecret
```

#### `ec2meta`

Queries AWS [EC2 Instance Metadata](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for information. This only retrieves data in the `meta-data` path -- for data in the `dynamic` path use `ec2dynamic`.
Expand Down
13 changes: 13 additions & 0 deletions cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package main

var cleanupHooks = make([]func(), 0)

func addCleanupHook(hook func()) {
cleanupHooks = append(cleanupHooks, hook)
}

func runCleanupHooks() {
for _, hook := range cleanupHooks {
hook()
}
}
42 changes: 33 additions & 9 deletions data.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"time"

"github.com/blang/vfs"
"github.com/hairyhenderson/gomplate/vault"
)

func init() {
Expand All @@ -22,18 +23,19 @@ func init() {
mime.AddExtensionType(".yml", "application/yaml")
mime.AddExtensionType(".yaml", "application/yaml")

sourceReaders = make(map[string]func(*Source) ([]byte, error))
sourceReaders = make(map[string]func(*Source, ...string) ([]byte, error))

// Register our source-reader functions
addSourceReader("http", readHTTP)
addSourceReader("https", readHTTP)
addSourceReader("file", readFile)
addSourceReader("vault", readVault)
}

var sourceReaders map[string]func(*Source) ([]byte, error)
var sourceReaders map[string]func(*Source, ...string) ([]byte, error)

// addSourceReader -
func addSourceReader(scheme string, readFunc func(*Source) ([]byte, error)) {
func addSourceReader(scheme string, readFunc func(*Source, ...string) ([]byte, error)) {
sourceReaders[scheme] = readFunc
}

Expand Down Expand Up @@ -67,6 +69,7 @@ type Source struct {
Type string
FS vfs.Filesystem // used for file: URLs, nil otherwise
HC *http.Client // used for http[s]: URLs, nil otherwise
VC *vault.Client //used for vault: URLs, nil otherwise
}

// NewSource - builds a &Source
Expand Down Expand Up @@ -141,9 +144,9 @@ func absURL(value string) *url.URL {
}

// Datasource -
func (d *Data) Datasource(alias string) map[string]interface{} {
func (d *Data) Datasource(alias string, args ...string) map[string]interface{} {
source := d.Sources[alias]
b, err := d.ReadSource(source.FS, source)
b, err := d.ReadSource(source.FS, source, args...)
if err != nil {
log.Fatalf("Couldn't read datasource '%s': %s", alias, err)
}
Expand All @@ -160,7 +163,7 @@ func (d *Data) Datasource(alias string) map[string]interface{} {
}

// ReadSource -
func (d *Data) ReadSource(fs vfs.Filesystem, source *Source) ([]byte, error) {
func (d *Data) ReadSource(fs vfs.Filesystem, source *Source, args ...string) ([]byte, error) {
if d.cache == nil {
d.cache = make(map[string][]byte)
}
Expand All @@ -169,7 +172,7 @@ func (d *Data) ReadSource(fs vfs.Filesystem, source *Source) ([]byte, error) {
return cached, nil
}
if r, ok := sourceReaders[source.URL.Scheme]; ok {
data, err := r(source)
data, err := r(source, args...)
if err != nil {
return nil, err
}
Expand All @@ -181,7 +184,7 @@ func (d *Data) ReadSource(fs vfs.Filesystem, source *Source) ([]byte, error) {
return nil, nil
}

func readFile(source *Source) ([]byte, error) {
func readFile(source *Source, args ...string) ([]byte, error) {
if source.FS == nil {
source.FS = vfs.OS()
}
Expand All @@ -207,7 +210,7 @@ func readFile(source *Source) ([]byte, error) {
return b, nil
}

func readHTTP(source *Source) ([]byte, error) {
func readHTTP(source *Source, args ...string) ([]byte, error) {
if source.HC == nil {
source.HC = &http.Client{Timeout: time.Second * 5}
}
Expand All @@ -234,3 +237,24 @@ func readHTTP(source *Source) ([]byte, error) {
}
return body, nil
}

func readVault(source *Source, args ...string) ([]byte, error) {
if source.VC == nil {
source.VC = vault.NewClient()
source.VC.Login()
addCleanupHook(source.VC.RevokeToken)
}

p := source.URL.Path
if len(args) == 1 {
p = p + "/" + args[0]
}

data, err := source.VC.Read(p)
if err != nil {
return nil, err
}
source.Type = "application/json"

return data, nil
}
1 change: 1 addition & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func NewGomplate(data *Data) *Gomplate {
}

func runTemplate(c *cli.Context) error {
defer runCleanupHooks()
data := NewData(c.StringSlice("datasource"))

g := NewGomplate(data)
Expand Down
88 changes: 88 additions & 0 deletions vault/app-id_strategy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package vault

import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"time"
)

// AppIDAuthStrategy - an AuthStrategy that uses Vault's app-id authentication backend.
type AppIDAuthStrategy struct {
AppID string `json:"app_id"`
UserID string `json:"user_id"`
hc *http.Client
}

// NewAppIDAuthStrategy - create an AuthStrategy that uses Vault's app-id auth
// backend.
func NewAppIDAuthStrategy() *AppIDAuthStrategy {
appID := os.Getenv("VAULT_APP_ID")
userID := os.Getenv("VAULT_USER_ID")
if appID != "" && userID != "" {
return &AppIDAuthStrategy{appID, userID, nil}
}
return nil
}

// GetToken - log in to the app-id auth backend and return the client token
func (a *AppIDAuthStrategy) GetToken(addr *url.URL) (string, error) {
if a.hc == nil {
a.hc = &http.Client{Timeout: time.Second * 5}
}
client := a.hc

buf := new(bytes.Buffer)
json.NewEncoder(buf).Encode(&a)

u := &url.URL{}
*u = *addr
u.Path = "/v1/auth/app-id/login"
res, err := client.Post(u.String(), "application/json; charset=utf-8", buf)
if err != nil {
return "", err
}
response := &AuthResponse{}
err = json.NewDecoder(res.Body).Decode(response)
res.Body.Close()
if err != nil {
return "", err
}
if res.StatusCode != 200 {
err := fmt.Errorf("Unexpected HTTP status %d on AppId login to %s: %s", res.StatusCode, u, response)
return "", err
}
return response.Auth.ClientToken, nil
}

// Revokable -
func (a *AppIDAuthStrategy) Revokable() bool {
return true
}

func (a *AppIDAuthStrategy) String() string {
return fmt.Sprintf("app-id: %s, user-id: %s", a.AppID, a.UserID)
}

// AuthResponse - the Auth response from /v1/auth/app-id/login
type AuthResponse struct {
Auth struct {
ClientToken string `json:"client_token"`
LeaseDuration int64 `json:"lease_duration"`
Metadata struct {
AppID string `json:"app-id"`
UserID string `json:"user-id"`
} `json:"metadata"`
Policies []string `json:"policies"`
Renewable bool `json:"renewable"`
} `json:"auth"`
}

func (a *AuthResponse) String() string {
buf := new(bytes.Buffer)
json.NewEncoder(buf).Encode(&a)
return string(buf.Bytes())
}
111 changes: 111 additions & 0 deletions vault/app-id_strategy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package vault

import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"

"github.com/stretchr/testify/assert"
)

func TestNewAppIDAuthStrategy(t *testing.T) {
os.Unsetenv("VAULT_APP_ID")
os.Unsetenv("VAULT_USER_ID")
assert.Nil(t, NewAppIDAuthStrategy())

os.Setenv("VAULT_APP_ID", "foo")
assert.Nil(t, NewAppIDAuthStrategy())

os.Unsetenv("VAULT_APP_ID")
os.Setenv("VAULT_USER_ID", "bar")
assert.Nil(t, NewAppIDAuthStrategy())

os.Setenv("VAULT_APP_ID", "foo")
os.Setenv("VAULT_USER_ID", "bar")
auth := NewAppIDAuthStrategy()
assert.Equal(t, "foo", auth.AppID)
assert.Equal(t, "bar", auth.UserID)
}

func TestGetToken_AppIDErrorsGivenNetworkError(t *testing.T) {
server, client := setupErrorHTTP()
defer server.Close()

vaultURL, _ := url.Parse("http://vault:8200")

auth := &AppIDAuthStrategy{"foo", "bar", client}
_, err := auth.GetToken(vaultURL)
assert.Error(t, err)
}

func TestGetToken_AppIDErrorsGivenHTTPErrorStatus(t *testing.T) {
server, client := setupHTTP(500, "application/json; charset=utf-8", `{}`)
defer server.Close()

vaultURL, _ := url.Parse("http://vault:8200")

auth := &AppIDAuthStrategy{"foo", "bar", client}
_, err := auth.GetToken(vaultURL)
assert.Error(t, err)
}

func TestGetToken_AppIDErrorsGivenBadJSON(t *testing.T) {
server, client := setupHTTP(200, "application/json; charset=utf-8", `{`)
defer server.Close()

vaultURL, _ := url.Parse("http://vault:8200")

auth := &AppIDAuthStrategy{"foo", "bar", client}
_, err := auth.GetToken(vaultURL)
assert.Error(t, err)
}

func TestGetToken_AppID(t *testing.T) {
server, client := setupHTTP(200, "application/json; charset=utf-8", `{"auth": {"client_token": "baz"}}`)
defer server.Close()

vaultURL, _ := url.Parse("http://vault:8200")

auth := &AppIDAuthStrategy{"foo", "bar", client}
token, err := auth.GetToken(vaultURL)
assert.NoError(t, err)

assert.Equal(t, "baz", token)
}

func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", mimetype)
w.WriteHeader(code)
fmt.Fprintln(w, body)
}))

client := &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL)
},
},
}

return server, client
}

func setupErrorHTTP() (*httptest.Server, *http.Client) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
panic("boo")
}))

client := &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL)
},
},
}

return server, client
}
Loading

0 comments on commit 0612cf1

Please sign in to comment.