Update configuration to load from yaml or env vars.

.gitignore

@@ -21,3 +21,4 @@ release.properties
 
 # Documentation build
 /docs/_site/
+.pr-train.yml

agent/pom.xml

@@ -92,6 +92,11 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.stefanbirkner</groupId>
+      <artifactId>system-rules</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <!-- semantic metrics -->
     <dependency>
@@ -119,6 +124,18 @@
             <goals>
               <goal>shade</goal>
             </goals>
+            <configuration>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+            </configuration>
           </execution>
         </executions>
       </plugin>

agent/src/main/java/com/spotify/ffwd/FastForwardAgent.java

@@ -29,7 +29,6 @@
 import com.spotify.metrics.jvm.MemoryUsageGaugeSet;
 import com.spotify.metrics.jvm.ThreadStatesMetricSet;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -52,13 +51,11 @@ public static void main(String[] argv) {
             path = Optional.of(Paths.get(argv[0]));
         }
 
-        final FastForwardAgent agent = setup(path, Optional.empty());
+        final FastForwardAgent agent = setup(path);
         run(agent);
     }
 
-    static FastForwardAgent setup(
-        final Optional<Path> configPath, final Optional<InputStream> configStream
-    ) {
+    static FastForwardAgent setup(final Optional<Path> configPath) {
         // needed for HTTP content decompression in:
         // com.spotify.ffwd.http.HttpModule
         System.setProperty("io.netty.noJdkZlibDecoder", "false");
@@ -102,12 +99,9 @@ static FastForwardAgent setup(
         final AgentCore.Builder builder = AgentCore.builder()
             .modules(modules)
             .statistics(statistics.statistics);
-
-        configStream.map(builder::configStream);
         configPath.map(builder::configPath);
 
         final AgentCore core = builder.build();
-
         return new FastForwardAgent(statistics, core);
     }
 

agent/src/test/java/com/spotify/ffwd/FfwdConfigurationTest.java

@@ -23,28 +23,28 @@
 import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
-import com.google.inject.Injector;
 import com.spotify.ffwd.output.BatchingPluginSink;
 import com.spotify.ffwd.output.CoreOutputManager;
 import com.spotify.ffwd.output.FilteringPluginSink;
 import com.spotify.ffwd.output.OutputManager;
 import com.spotify.ffwd.output.PluginSink;
-import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
-import java.util.function.Supplier;
-import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.runner.RunWith;
 import org.mockito.runners.MockitoJUnitRunner;
 
 @RunWith(MockitoJUnitRunner.class)
 public class FfwdConfigurationTest {
 
-    @Before
-    public void setup() {
-    }
+    @Rule
+    public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
 
     @Test
     public void testConfAllPluginsEnabled() {
@@ -90,17 +90,43 @@ public void testConfMixedPluginsEnabled() {
             "ffwd-mixed-plugins.yaml", expectedSinks);
     }
 
+    @Test
+    public void testConfigFromEnvVars() {
+        environmentVariables.set("FFWD_TTL", "100");
+        CoreOutputManager outputManager = getOutputManager(null);
+        assertEquals(100, outputManager.getTtl());
+    }
+
+    @Test
+    public void testIgnoreUnknownFields() {
+        Path configPath = resource("invalid.yaml");
+
+        String host = getOutputManager(configPath).getHost();
+        assertEquals("jimjam", host);
+    }
+
+    @Test
+    public void testMergeOrder() {
+        environmentVariables.set("FFWD_TTL", "100");
+        Path configPath = resource("basic-settings.yaml");
+        CoreOutputManager outputManager = getOutputManager(configPath);
+
+        assertEquals(100, outputManager.getTtl());
+        assertEquals("jimjam", outputManager.getHost());
+    }
+
+    private CoreOutputManager getOutputManager(final Path configPath) {
+        final FastForwardAgent agent = FastForwardAgent.setup(Optional.ofNullable(configPath));
+        return (CoreOutputManager) agent.getCore().getPrimaryInjector().getInstance(OutputManager.class);
+    }
+
     private void verifyLoadedSinksForConfig(
-        final String expectationString, final String configName,
+        final String expectationString,
+        final String configName,
         final List<List<String>> expectedSinks
     ) {
-        final InputStream configStream = stream(configName).get();
-
-        final FastForwardAgent agent =
-            FastForwardAgent.setup(Optional.empty(), Optional.of(configStream));
-        final Injector primaryInjector = agent.getCore().getPrimaryInjector();
-        final CoreOutputManager outputManager =
-            (CoreOutputManager) primaryInjector.getInstance(OutputManager.class);
+        final Path configPath = resource(configName);
+        final CoreOutputManager outputManager = getOutputManager(configPath);
         final List<PluginSink> sinks = outputManager.getSinks();
 
         final List<List<String>> sinkChains = new ArrayList<>();
@@ -130,7 +156,8 @@ private List<String> extractSinkChain(final PluginSink sink) {
         return sinkChain;
     }
 
-    private Supplier<InputStream> stream(String name) {
-        return () -> getClass().getClassLoader().getResourceAsStream(name);
+    private Path resource(String name) {
+        final String path = Objects.requireNonNull(getClass().getClassLoader().getResource(name)).getPath();
+        return Paths.get(path);
     }
 }

agent/src/test/resources/basic-settings.yaml

@@ -0,0 +1,2 @@
+ttl: 50
+host: jimjam

agent/src/test/resources/invalid.yaml

@@ -0,0 +1,3 @@
+unknown_fake_property: true
+
+host: jimjam

core/pom.xml

@@ -30,6 +30,10 @@
       <groupId>org.jetbrains.kotlin</groupId>
       <artifactId>kotlin-stdlib-jdk8</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.uchuhimo</groupId>
+      <artifactId>konf</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>com.fasterxml.jackson.module</groupId>

core/src/main/java/com/spotify/ffwd/AgentConfig.kt

@@ -16,53 +16,83 @@
 
 package com.spotify.ffwd
 
-import com.fasterxml.jackson.annotation.JsonCreator
-import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.databind.module.SimpleModule
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
 import com.spotify.ffwd.domain.SearchDomainDiscovery
 import com.spotify.ffwd.input.InputManagerModule
 import com.spotify.ffwd.output.OutputManagerModule
+import com.uchuhimo.konf.Config
+import com.uchuhimo.konf.ConfigSpec
 import java.net.InetAddress
 import java.net.InetSocketAddress
 import java.net.UnknownHostException
 import java.nio.file.Path
-import java.nio.file.Paths
 
-data class AgentConfig(
-    @JsonProperty("debug") var debug: Debug?,
-    @JsonProperty("host") var host: String = buildDefaultHost(),
-    @JsonProperty("tags") var tags: Map<String, String> = emptyMap(),
-    @JsonProperty("tagsToResource") var tagsToResource: Map<String, String> = emptyMap(),
-    @JsonProperty("riemannTags") var riemannTags: Set<String> = emptySet(),
-    @JsonProperty("skipTagsForKeys") var skipTagsForKeys: Set<String> = emptySet(),
-    @JsonProperty("automaticHostTag") var automaticHostTag: Boolean = true,
-    var input: InputManagerModule =
-        InputManagerModule.supplyDefault().get(),
-    @JsonProperty("output") var output: OutputManagerModule =
-        OutputManagerModule.supplyDefault().get(),
-    @JsonProperty("searchDomain") var searchDomain: SearchDomainDiscovery =
-        SearchDomainDiscovery.supplyDefault(),
-    @JsonProperty("asyncThreads") var asyncThreads: Int = 4,
-    @JsonProperty("schedulerThreads") var schedulerThreads: Int = 4,
-    @JsonProperty("bossThreads") var bossThreads: Int = 2,
-    @JsonProperty("workerThreads") var workerThreads: Int = 4,
-    @JsonProperty("ttl") var ttl: Long = 0,
-    // NB(hexedpackets): qlog is unused and can be removed once the config parser ignores unknown
-    // properties.
-    @JsonProperty("qlog") var qlog: String?
-)
+// Helper class to make interop with java easier. The configuration loading is done through the
+// static object.
+class AgentConfig(val config: Config) {
+    fun hasDebug(): Boolean = config.contains(Debug.host) or config.contains(Debug.port)
+
+    val debugLocalAddress =  config[Debug.localAddress]
+    val host = config[AgentConfig.host]
+    val tags = config[AgentConfig.tags]
+    val tagsToResource = config[AgentConfig.tagsToResource]
+    val riemannTags = config[AgentConfig.riemannTags]
+    val skipTagsForKeys = config[AgentConfig.skipTagsForKeys]
+    val automaticHostTag = config[AgentConfig.automaticHostTag]
+    val input: InputManagerModule = config[AgentConfig.input]
+    val output: OutputManagerModule = config[AgentConfig.output]
+    val searchDomain = config[AgentConfig.searchDomain]
+    val asyncThreads = config[AgentConfig.asyncThreads]
+    val schedulerThreads = config[AgentConfig.schedulerThreads]
+    val bossThreads = config[AgentConfig.bossThreads]
+    val workerThreads = config[AgentConfig.workerThreads]
+    val ttl = config[AgentConfig.ttl]
+
+    companion object : ConfigSpec("") {
+        object Debug : ConfigSpec() {
+            val host by optional("localhost")
+            val port by optional(19001)
+            val localAddress by lazy { InetSocketAddress(it[host], it[port]) }
+        }
+
+        val host by lazy { buildDefaultHost() }
+        val tags by optional(emptyMap<String, String>())
+        val tagsToResource by optional(emptyMap<String, String>())
+        val riemannTags by optional(emptySet<String>())
+        val skipTagsForKeys by optional(emptySet<String>())
+        val automaticHostTag by optional(true)
+        val input by lazy { InputManagerModule.supplyDefault().get() }
+        val output by lazy { OutputManagerModule.supplyDefault().get() }
+
+        val searchDomain by lazy { SearchDomainDiscovery.supplyDefault() }
+        val asyncThreads by optional(4)
+        val schedulerThreads by optional(4)
+        val bossThreads by optional(2)
+        val workerThreads by optional(4)
+        val ttl by optional(0)
+
+        @JvmStatic
+        fun load(path: Path, extraModule: SimpleModule): Config {
+            val config = Config { addSpec(AgentConfig) }
+            config.mapper
+                .registerModule(Jdk8Module())
+                .registerModule(extraModule)
+
+            // Load yaml config files with no prefix, then set it to "ffwd" for other sources.
+            return config
+                .from.yaml.file(path.toFile())
+                .withPrefix("ffwd")
+                .from.env()
+                .from.systemProperties()
+        }
+    }
+}
 
 private fun buildDefaultHost(): String {
     try {
         return InetAddress.getLocalHost().hostName
     } catch (e: UnknownHostException) {
         throw RuntimeException("unable to get local host", e)
     }
-}
-
-data class Debug(
-    val localAddress: InetSocketAddress
-) {
-    @JsonCreator
-    constructor(@JsonProperty("host") host: String?, @JsonProperty port: Int?)
-        : this(InetSocketAddress(host ?: "localhost", port ?: 19001))
 }