Skip to content

Commit

Permalink
Vault AWS EC2 auth (#190)
Browse files Browse the repository at this point in the history
  • Loading branch information
stuart-c authored and hairyhenderson committed Aug 8, 2017
1 parent cbc2af1 commit d2cf55b
Show file tree
Hide file tree
Showing 19 changed files with 2,489 additions and 6 deletions.
8 changes: 7 additions & 1 deletion Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 24 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ clean:
rm -Rf $(PREFIX)/bin/*
rm -f $(PREFIX)/test/integration/gomplate
rm -f $(PREFIX)/test/integration/mirror
rm -f $(PREFIX)/test/integration/meta
rm -f $(PREFIX)/test/integration/aws

build-x: $(patsubst %,$(PREFIX)/bin/$(PKG_NAME)_%,$(platforms))

Expand All @@ -58,6 +60,12 @@ $(PREFIX)/bin/$(PKG_NAME)_%: $(shell find $(PREFIX) -type f -name '*.go' -not -p
$(PREFIX)/bin/mirror_%: $(shell find $(PREFIX)/test/integration/mirrorsvc -type f -name '*.go')
$(call gocross-tool,$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\1/'),$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\2/'),mirror)

$(PREFIX)/bin/meta_%: $(shell find $(PREFIX)/test/integration/metasvc -type f -name '*.go')
$(call gocross-tool,$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\1/'),$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\2/'),meta)

$(PREFIX)/bin/aws_%: $(shell find $(PREFIX)/test/integration/awssvc -type f -name '*.go')
$(call gocross-tool,$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\1/'),$(shell echo $* | sed 's/\([^-]*\)-\([^.]*\).*/\2/'),aws)

$(PREFIX)/bin/$(PKG_NAME)$(call extension,$(GOOS)): $(shell find $(PREFIX) -type f -name '*.go' -not -path "$(PREFIX)/test/*")
CGO_ENABLED=0 \
$(GO) build -ldflags "-w -s $(COMMIT_FLAG) $(VERSION_FLAG)" -o $@
Expand All @@ -66,22 +74,36 @@ $(PREFIX)/bin/mirror$(call extension,$(GOOS)): $(shell find $(PREFIX)/test/integ
CGO_ENABLED=0 \
$(GO) build -ldflags "-w -s $(COMMIT_FLAG) $(VERSION_FLAG)" -o $@ $(PREFIX)/test/integration/mirrorsvc

$(PREFIX)/bin/meta$(call extension,$(GOOS)): $(shell find $(PREFIX)/test/integration/metasvc -type f -name '*.go')
CGO_ENABLED=0 \
$(GO) build -ldflags "-w -s $(COMMIT_FLAG) $(VERSION_FLAG)" -o $@ $(PREFIX)/test/integration/metasvc

$(PREFIX)/bin/aws$(call extension,$(GOOS)): $(shell find $(PREFIX)/test/integration/awssvc -type f -name '*.go')
CGO_ENABLED=0 \
$(GO) build -ldflags "-w -s $(COMMIT_FLAG) $(VERSION_FLAG)" -o $@ $(PREFIX)/test/integration/awssvc

build: $(PREFIX)/bin/$(PKG_NAME)$(call extension,$(GOOS))

build-mirror: $(PREFIX)/bin/mirror$(call extension,$(GOOS))

build-meta: $(PREFIX)/bin/meta$(call extension,$(GOOS))

build-aws: $(PREFIX)/bin/aws$(call extension,$(GOOS))

test:
$(GO) test -v -race `glide novendor`

build-integration-image: $(PREFIX)/bin/$(PKG_NAME)_linux-amd64$(call extension,$(GOOS)) $(PREFIX)/bin/mirror_linux-amd64$(call extension,$(GOOS))
build-integration-image: $(PREFIX)/bin/$(PKG_NAME)_linux-amd64$(call extension,$(GOOS)) $(PREFIX)/bin/mirror_linux-amd64$(call extension,$(GOOS)) $(PREFIX)/bin/meta_linux-amd64$(call extension,$(GOOS)) $(PREFIX)/bin/aws_linux-amd64$(call extension,$(GOOS))
cp $(PREFIX)/bin/$(PKG_NAME)_linux-amd64 test/integration/gomplate
cp $(PREFIX)/bin/mirror_linux-amd64 test/integration/mirror
cp $(PREFIX)/bin/meta_linux-amd64 test/integration/meta
cp $(PREFIX)/bin/aws_linux-amd64 test/integration/aws
docker build -f test/integration/Dockerfile -t gomplate-test test/integration/

test-integration-docker: build-integration-image
docker run -it --rm gomplate-test

test-integration: build build-mirror
test-integration: build build-mirror build-meta build-aws
@test/integration/test.sh

gen-changelog:
Expand Down
8 changes: 7 additions & 1 deletion aws/ec2meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import (
"net/http"
"strings"
"time"

"github.com/hairyhenderson/gomplate/env"
)

// DefaultEndpoint -
const DefaultEndpoint = "http://169.254.169.254"
var DefaultEndpoint = "http://169.254.169.254"

// Ec2Meta -
type Ec2Meta struct {
Expand All @@ -23,6 +25,10 @@ type Ec2Meta struct {

// NewEc2Meta -
func NewEc2Meta(options ClientOptions) *Ec2Meta {
if endpoint := env.Getenv("AWS_META_ENDPOINT"); endpoint != "" {
DefaultEndpoint = endpoint
}

return &Ec2Meta{cache: make(map[string]string), options: options}
}

Expand Down
11 changes: 11 additions & 0 deletions docs/content/functions/general.md
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,7 @@ This table describes the currently-supported authentication mechanisms and how t
| [`github`](https://www.vaultproject.io/docs/auth/github.html) | Environment variable `$VAULT_AUTH_GITHUB_TOKEN` must be set to an appropriate value.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_GITHUB_MOUNT`. |
| [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) | Environment variables `$VAULT_AUTH_USERNAME` and `$VAULT_AUTH_PASSWORD` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_USERPASS_MOUNT`. |
| [`token`](https://www.vaultproject.io/docs/auth/token.html) | Determined from either the `$VAULT_TOKEN` environment variable, or read from the file `~/.vault-token` |
| [`aws`](https://www.vaultproject.io/docs/auth/aws.html) | As a final option authentication will be attempted using the AWS auth backend. See below for more details. |

_**Note:**_ The secret values listed in the above table can either be set in environment
variables or provided in files. This can increase security when using
Expand Down Expand Up @@ -637,6 +638,16 @@ $ echo 'otp={{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").ke
otp=604a4bd5-7afd-30a2-d2d8-80c4aebc6183
```

#### Authentication using AWS details

If running on an EC2 instance authentication will be attempted using the AWS auth backend. The
optional `VAULT_AUTH_AWS_MOUNT` environment variable can be used to set the mount point to use if
it differs from the default of `aws`. Additionally `AWS_TIMEOUT` can be set (in seconds) to a value
to wait for AWS to respond before skipping the attempt.

If set, the `VAULT_AUTH_AWS_ROLE` environment variable will be used to specify the role to authenticate
using. If not set the AMI ID of the EC2 instance will be used by Vault.

## `datasourceExists`

Tests whether or not a given datasource was defined on the commandline (with the
Expand Down
2 changes: 2 additions & 0 deletions test/integration/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
gomplate
mirror
meta
aws
4 changes: 3 additions & 1 deletion test/integration/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM alpine:edge

ENV VAULT_VER 0.7.0
ENV VAULT_VER 0.7.3
ENV CONSUL_VER 0.9.0
RUN apk add --no-cache \
curl \
Expand All @@ -21,6 +21,8 @@ RUN mkdir /lib64 \

COPY gomplate /bin/gomplate
COPY mirror /bin/mirror
COPY meta /bin/meta
COPY aws /bin/aws
COPY *.sh /tests/
COPY *.bash /tests/
COPY *.bats /tests/
Expand Down
190 changes: 190 additions & 0 deletions test/integration/awssvc/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
package main

import (
"encoding/json"
"flag"
"log"
"net"
"net/http"
)

// Req -
type Req struct {
Headers http.Header `json:"headers"`
}

var port string

func main() {
flag.StringVar(&port, "p", "8082", "Port to listen to")
flag.Parse()

l, err := net.Listen("tcp", ":"+port)
if err != nil {
log.Fatal(err)
}
// defer l.Close()
http.HandleFunc("/", rootHandler)

http.HandleFunc("/sts/", stsHandler)
http.HandleFunc("/ec2/", ec2Handler)
http.HandleFunc("/quit", quitHandler(l))

http.Serve(l, nil)
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
req := Req{r.Header}
b, err := json.Marshal(req)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusBadRequest)
}
w.Header().Set("Content-Type", "application/json")
w.Write(b)
}

func stsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml")
w.Write([]byte(`<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<GetCallerIdentityResult>
<Arn>arn:aws:iam::1:user/Test</Arn>
<UserId>AKIAI44QH8DHBEXAMPLE</UserId>
<Account>1</Account>
</GetCallerIdentityResult>
<ResponseMetadata>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ResponseMetadata>
</GetCallerIdentityResponse>`))
}

func ec2Handler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/xml")
w.Write([]byte(`<DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
<requestId>8f7724cf-496f-496e-8fe3-example</requestId>
<reservationSet>
<item>
<reservationId>r-1234567890abcdef0</reservationId>
<ownerId>123456789012</ownerId>
<groupSet/>
<instancesSet>
<item>
<instanceId>i-00000000000000000</instanceId>
<imageId>ami-00000000</imageId>
<instanceState>
<code>16</code>
<name>running</name>
</instanceState>
<privateDnsName>ip-192-168-1-88.eu-west-1.compute.internal</privateDnsName>
<dnsName>ec2-54-194-252-215.eu-west-1.compute.amazonaws.com</dnsName>
<reason/>
<keyName>my_keypair</keyName>
<amiLaunchIndex>0</amiLaunchIndex>
<productCodes/>
<instanceType>t2.micro</instanceType>
<launchTime>2015-12-22T10:44:05.000Z</launchTime>
<placement>
<availabilityZone>eu-west-1c</availabilityZone>
<groupName/>
<tenancy>default</tenancy>
</placement>
<monitoring>
<state>disabled</state>
</monitoring>
<subnetId>subnet-56f5f633</subnetId>
<vpcId>vpc-11112222</vpcId>
<privateIpAddress>192.168.1.88</privateIpAddress>
<ipAddress>54.194.252.215</ipAddress>
<sourceDestCheck>true</sourceDestCheck>
<groupSet>
<item>
<groupId>sg-e4076980</groupId>
<groupName>SecurityGroup1</groupName>
</item>
</groupSet>
<architecture>x86_64</architecture>
<rootDeviceType>ebs</rootDeviceType>
<rootDeviceName>/dev/xvda</rootDeviceName>
<blockDeviceMapping>
<item>
<deviceName>/dev/xvda</deviceName>
<ebs>
<volumeId>vol-1234567890abcdef0</volumeId>
<status>attached</status>
<attachTime>2015-12-22T10:44:09.000Z</attachTime>
<deleteOnTermination>true</deleteOnTermination>
</ebs>
</item>
</blockDeviceMapping>
<virtualizationType>hvm</virtualizationType>
<clientToken>xMcwG14507example</clientToken>
<tagSet>
<item>
<key>Name</key>
<value>Server_1</value>
</item>
</tagSet>
<hypervisor>xen</hypervisor>
<networkInterfaceSet>
<item>
<networkInterfaceId>eni-551ba033</networkInterfaceId>
<subnetId>subnet-56f5f633</subnetId>
<vpcId>vpc-11112222</vpcId>
<description>Primary network interface</description>
<ownerId>123456789012</ownerId>
<status>in-use</status>
<macAddress>02:dd:2c:5e:01:69</macAddress>
<privateIpAddress>192.168.1.88</privateIpAddress>
<privateDnsName>ip-192-168-1-88.eu-west-1.compute.internal</privateDnsName>
<sourceDestCheck>true</sourceDestCheck>
<groupSet>
<item>
<groupId>sg-e4076980</groupId>
<groupName>SecurityGroup1</groupName>
</item>
</groupSet>
<attachment>
<attachmentId>eni-attach-39697adc</attachmentId>
<deviceIndex>0</deviceIndex>
<status>attached</status>
<attachTime>2015-12-22T10:44:05.000Z</attachTime>
<deleteOnTermination>true</deleteOnTermination>
</attachment>
<association>
<publicIp>54.194.252.215</publicIp>
<publicDnsName>ec2-54-194-252-215.eu-west-1.compute.amazonaws.com</publicDnsName>
<ipOwnerId>amazon</ipOwnerId>
</association>
<privateIpAddressesSet>
<item>
<privateIpAddress>192.168.1.88</privateIpAddress>
<privateDnsName>ip-192-168-1-88.eu-west-1.compute.internal</privateDnsName>
<primary>true</primary>
<association>
<publicIp>54.194.252.215</publicIp>
<publicDnsName>ec2-54-194-252-215.eu-west-1.compute.amazonaws.com</publicDnsName>
<ipOwnerId>amazon</ipOwnerId>
</association>
</item>
</privateIpAddressesSet>
<ipv6AddressesSet>
<item>
<ipv6Address>2001:db8:1234:1a2b::123</ipv6Address>
</item>
</ipv6AddressesSet>
</item>
</networkInterfaceSet>
<ebsOptimized>false</ebsOptimized>
</item>
</instancesSet>
</item>
</reservationSet>
</DescribeInstancesResponse>`))
}

func quitHandler(l net.Listener) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
l.Close()
w.WriteHeader(http.StatusNoContent)
}
}
21 changes: 21 additions & 0 deletions test/integration/datasources_vault.bats
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,15 @@ path "*" {
}
EOF
tmpdir=$(mktemp -d)
cp ~/.vault-token ~/.vault-token.bak
start_meta_svc
start_aws_svc
}

function teardown () {
mv ~/.vault-token.bak ~/.vault-token
stop_meta_svc
stop_aws_svc
rm -rf $tmpdir
unset VAULT_TOKEN
vault delete secret/foo
Expand All @@ -27,6 +33,7 @@ function teardown () {
vault auth-disable approle2
vault auth-disable app-id
vault auth-disable app-id2
vault auth-disable aws
vault policy-delete writepol
vault policy-delete readpol
vault unmount ssh
Expand Down Expand Up @@ -122,6 +129,20 @@ function teardown () {
[[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
}

@test "Testing ec2 vault auth" {
vault write secret/foo value="$BATS_TEST_DESCRIPTION"
vault auth-enable aws
vault write auth/aws/config/client secret_key=secret access_key=access endpoint=http://127.0.0.1:8082/ec2 iam_endpoint=http://127.0.0.1:8082/iam sts_endpoint=http://127.0.0.1:8082/sts
curl -o $tmpdir/certificate -s -f http://127.0.0.1:8081/certificate
vault write auth/aws/config/certificate/testcert type=pkcs7 aws_public_cert=@$tmpdir/certificate
vault write auth/aws/role/ami-00000000 auth_type=ec2 bound_ami_id=ami-00000000 policies=readpol
unset VAULT_TOKEN
rm ~/.vault-token
AWS_META_ENDPOINT=http://127.0.0.1:8081 gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
[ "$status" -eq 0 ]
[[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
}

@test "Testing vault auth with dynamic secret" {
vault mount ssh
vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8
Expand Down
Loading

0 comments on commit d2cf55b

Please sign in to comment.