Adding ability to expand environment variables used in config file

CHANGELOG.asciidoc

@@ -53,6 +53,7 @@ https://github.com/elastic/beats/compare/1.0.0...master[Check the HEAD diff]
 *Affecting all Beats*
 - Update builds to Golang version 1.5.2
 - Make logstash output compression level configurable. {pull}630[630]
+- Add ability to override configuration settings using environment variables {issue}114[114]
 
 *Packetbeat*
 - Add support for capturing DNS over TCP network traffic. {pull}486[486] {pull}554[554]

libbeat/cfgfile/cfgfile.go

@@ -6,7 +6,9 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/elastic/beats/libbeat/logp"
 	"gopkg.in/yaml.v2"
 )
 
@@ -48,10 +50,12 @@ func Read(out interface{}, path string) error {
 	}
 
 	filecontent, err := ioutil.ReadFile(path)
-
 	if err != nil {
 		return fmt.Errorf("Failed to read %s: %v. Exiting.", path, err)
 	}
+
+	filecontent = expandEnv(filecontent)
+
 	if err = yaml.Unmarshal(filecontent, out); err != nil {
 		return fmt.Errorf("YAML config parsing failed on %s: %v. Exiting.", path, err)
 	}
@@ -62,3 +66,27 @@ func Read(out interface{}, path string) error {
 func IsTestConfig() bool {
 	return *testConfig
 }
+
+// expandEnv replaces ${var} or $var in config according to the values of the
+// current environment variables. The replacement is case-sensitive. References
+// to undefined variables are replaced by the empty string. A default value
+// can be given by using the form ${var:default value}.
+func expandEnv(config []byte) []byte {
+	return []byte(os.Expand(string(config), func(key string) string {
+		keyAndDefault := strings.SplitN(key, ":", 2)
+		key = keyAndDefault[0]
+
+		v := os.Getenv(key)
+		if v == "" && len(keyAndDefault) == 2 {
+			// Set value to the default.
+			v = keyAndDefault[1]
+			logp.Info("Replacing config environment variable '${%s}' with "+
+				"default '%s'", key, keyAndDefault[1])
+		} else {
+			logp.Info("Replacing config environment variable '${%s}' with '%s'",
+				key, v)
+		}
+
+		return v
+	}))
+}

libbeat/cfgfile/cfgfile_test.go

@@ -1,6 +1,7 @@
 package cfgfile
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -37,3 +38,32 @@ func TestRead(t *testing.T) {
 	// Chat that it is integer
 	assert.Equal(t, 9200, config.Output.Elasticsearch.Port)
 }
+
+func TestExpandEnv(t *testing.T) {
+	var tests = []struct {
+		in  string
+		out string
+	}{
+		// Environment variables can be specified as ${env} or $env.
+		{"x$y", "xy"},
+		{"x${y}", "xy"},
+
+		// Environment variables are case-sensitive. Neither are replaced.
+		{"x$Y", "x"},
+		{"x${Y}", "x"},
+
+		// Defaults can only be specified when using braces.
+		{"x${Z:D}", "xD"},
+		{"x${Z:A B C D}", "xA B C D"}, // Spaces are allowed in the default.
+		{"x${Z:}", "x"},
+
+		// Defaults don't work unless braces are used.
+		{"x$y:D", "xy:D"},
+	}
+
+	for _, test := range tests {
+		os.Setenv("y", "y")
+		output := expandEnv([]byte(test.in))
+		assert.Equal(t, test.out, string(output), "Input: %s", test.in)
+	}
+}