From 81952c429385eb03cf1070c82608f679af7163a4 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Wed, 21 Jun 2017 22:32:43 +0200 Subject: [PATCH] Support prioritization of DockerClientProviderStrategies (#362) * Support prioritization of DockerClientProviderStrategies * Support storing the global configuration in user's home folder. Store selected DockerClientProviderStrategy globally. * Add changelog and priority JavaDoc --- CHANGELOG.md | 2 + .../DockerClientProviderStrategy.java | 137 ++++++++++++------ .../DockerMachineClientProviderStrategy.java | 16 +- ...dSystemPropertyClientProviderStrategy.java | 27 +++- ...oxiedUnixSocketClientProviderStrategy.java | 25 +++- .../UnixSocketClientProviderStrategy.java | 17 ++- .../WindowsClientProviderStrategy.java | 6 + .../utility/TestcontainersConfiguration.java | 115 +++++++++++---- 8 files changed, 248 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48d631b3b4b..ebe1a458678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. - Added pre-flight checks (can be disabled with `checks.disable` configuration property) (#363) - Removed unused Jersey dependencies (#361) - Fixed non-POSIX fallback for file attribute reading (#371) +- Improved startup time by adding dynamic priorities to DockerClientProviderStrategy (#362) +- Added global configuration file `~/.testcontainers.properties` (#362) ## [1.3.0] - 2017-06-05 ### Fixed diff --git a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java index 48b556d68a5..bc1cf2ea174 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java @@ -5,16 +5,22 @@ import com.github.dockerjava.core.DockerClientConfig; import com.github.dockerjava.netty.NettyDockerCmdExecFactory; import com.google.common.base.Throwables; +import org.apache.commons.io.IOUtils; import org.jetbrains.annotations.Nullable; +import org.rnorth.ducttape.TimeoutException; import org.rnorth.ducttape.ratelimits.RateLimiter; import org.rnorth.ducttape.ratelimits.RateLimiterBuilder; import org.rnorth.ducttape.unreliables.Unreliables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.utility.TestcontainersConfiguration; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; /** * Mechanism to find a viable Docker client configuration according to the host system environment. @@ -39,6 +45,17 @@ public abstract class DockerClientProviderStrategy { */ public abstract String getDescription(); + protected boolean isApplicable() { + return true; + } + + /** + * @return highest to lowest priority value + */ + protected int getPriority() { + return 0; + } + protected static final Logger LOGGER = LoggerFactory.getLogger(DockerClientProviderStrategy.class); /** @@ -49,45 +66,70 @@ public abstract class DockerClientProviderStrategy { public static DockerClientProviderStrategy getFirstValidStrategy(List strategies) { List configurationFailures = new ArrayList<>(); - for (DockerClientProviderStrategy strategy : strategies) { - try { - strategy.test(); - LOGGER.info("Looking for Docker environment. Tried {}", strategy.getDescription()); - return strategy; - } catch (Exception | ExceptionInInitializerError | NoClassDefFoundError e) { - @Nullable String throwableMessage = e.getMessage(); - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - Throwable rootCause = Throwables.getRootCause(e); - @Nullable String rootCauseMessage = rootCause.getMessage(); - - String failureDescription; - if (throwableMessage != null && throwableMessage.equals(rootCauseMessage)) { - failureDescription = String.format("%s: failed with exception %s (%s)", - strategy.getClass().getSimpleName(), - e.getClass().getSimpleName(), - throwableMessage); - } else { - failureDescription = String.format("%s: failed with exception %s (%s). Root cause %s (%s)", - strategy.getClass().getSimpleName(), - e.getClass().getSimpleName(), - throwableMessage, - rootCause.getClass().getSimpleName(), - rootCauseMessage - ); - } - configurationFailures.add(failureDescription); - - LOGGER.debug(failureDescription); - } - } - - LOGGER.error("Could not find a valid Docker environment. Please check configuration. Attempted configurations were:"); - for (String failureMessage : configurationFailures) { - LOGGER.error(" " + failureMessage); - } - LOGGER.error("As no valid configuration was found, execution cannot continue"); - - throw new IllegalStateException("Could not find a valid Docker environment. Please see logs and check configuration"); + return Stream + .concat( + Stream + .of(TestcontainersConfiguration.getInstance().getDockerClientStrategyClassName()) + .filter(Objects::nonNull) + .flatMap(it -> { + try { + Class strategyClass = (Class) Thread.currentThread().getContextClassLoader().loadClass(it); + return Stream.of(strategyClass.newInstance()); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + LOGGER.warn("Can't instantiate a strategy from " + it, e); + return Stream.empty(); + } + }), + strategies + .stream() + .filter(DockerClientProviderStrategy::isApplicable) + .sorted(Comparator.comparing(DockerClientProviderStrategy::getPriority).reversed()) + ) + .flatMap(strategy -> { + try { + strategy.test(); + LOGGER.info("Found Docker environment with {}", strategy.getDescription()); + + TestcontainersConfiguration.getInstance().updateGlobalConfig("docker.client.strategy", strategy.getClass().getName()); + + return Stream.of(strategy); + } catch (Exception | ExceptionInInitializerError | NoClassDefFoundError e) { + @Nullable String throwableMessage = e.getMessage(); + @SuppressWarnings("ThrowableResultOfMethodCallIgnored") + Throwable rootCause = Throwables.getRootCause(e); + @Nullable String rootCauseMessage = rootCause.getMessage(); + + String failureDescription; + if (throwableMessage != null && throwableMessage.equals(rootCauseMessage)) { + failureDescription = String.format("%s: failed with exception %s (%s)", + strategy.getClass().getSimpleName(), + e.getClass().getSimpleName(), + throwableMessage); + } else { + failureDescription = String.format("%s: failed with exception %s (%s). Root cause %s (%s)", + strategy.getClass().getSimpleName(), + e.getClass().getSimpleName(), + throwableMessage, + rootCause.getClass().getSimpleName(), + rootCauseMessage + ); + } + configurationFailures.add(failureDescription); + + LOGGER.debug(failureDescription); + return Stream.empty(); + } + }) + .findAny() + .orElseThrow(() -> { + LOGGER.error("Could not find a valid Docker environment. Please check configuration. Attempted configurations were:"); + for (String failureMessage : configurationFailures) { + LOGGER.error(" " + failureMessage); + } + LOGGER.error("As no valid configuration was found, execution cannot continue"); + + return new IllegalStateException("Could not find a valid Docker environment. Please see logs and check configuration"); + }); } /** @@ -105,13 +147,18 @@ protected DockerClient getClientForConfig(DockerClientConfig config) { } protected void ping(DockerClient client, int timeoutInSeconds) { - Unreliables.retryUntilSuccess(timeoutInSeconds, TimeUnit.SECONDS, () -> { - return PING_RATE_LIMITER.getWhenReady(() -> { - LOGGER.debug("Pinging docker daemon..."); - client.pingCmd().exec(); - return true; + try { + Unreliables.retryUntilSuccess(timeoutInSeconds, TimeUnit.SECONDS, () -> { + return PING_RATE_LIMITER.getWhenReady(() -> { + LOGGER.debug("Pinging docker daemon..."); + client.pingCmd().exec(); + return true; + }); }); - }); + } catch (TimeoutException e) { + IOUtils.closeQuietly(client); + throw e; + } } public String getDockerHostIpAddress() { diff --git a/core/src/main/java/org/testcontainers/dockerclient/DockerMachineClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/DockerMachineClientProviderStrategy.java index 59211e317a2..b06dc0fa697 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/DockerMachineClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/DockerMachineClientProviderStrategy.java @@ -1,6 +1,7 @@ package org.testcontainers.dockerclient; import com.github.dockerjava.core.DefaultDockerClientConfig; +import lombok.extern.slf4j.Slf4j; import org.testcontainers.utility.CommandLine; import org.testcontainers.utility.DockerMachineClient; @@ -13,10 +14,21 @@ /** * Use Docker machine (if available on the PATH) to locate a Docker environment. */ +@Slf4j public class DockerMachineClientProviderStrategy extends DockerClientProviderStrategy { private static final String PING_TIMEOUT_DEFAULT = "30"; private static final String PING_TIMEOUT_PROPERTY_NAME = "testcontainers.dockermachineprovider.timeout"; + @Override + protected boolean isApplicable() { + return DockerMachineClient.instance().isInstalled(); + } + + @Override + protected int getPriority() { + return ProxiedUnixSocketClientProviderStrategy.PRIORITY - 10; + } + @Override public void test() throws InvalidConfigurationException { @@ -28,13 +40,13 @@ public void test() throws InvalidConfigurationException { checkArgument(machineNameOptional.isPresent(), "docker-machine is installed but no default machine could be found"); String machineName = machineNameOptional.get(); - LOGGER.info("Found docker-machine, and will use machine named {}", machineName); + log.info("Found docker-machine, and will use machine named {}", machineName); DockerMachineClient.instance().ensureMachineRunning(machineName); String dockerDaemonIpAddress = DockerMachineClient.instance().getDockerDaemonIpAddress(machineName); - LOGGER.info("Docker daemon IP address for docker machine {} is {}", machineName, dockerDaemonIpAddress); + log.info("Docker daemon IP address for docker machine {} is {}", machineName, dockerDaemonIpAddress); config = DefaultDockerClientConfig.createDefaultConfigBuilder() .withDockerHost("tcp://" + dockerDaemonIpAddress + ":2376") diff --git a/core/src/main/java/org/testcontainers/dockerclient/EnvironmentAndSystemPropertyClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/EnvironmentAndSystemPropertyClientProviderStrategy.java index ed482abc36b..195a07a4024 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/EnvironmentAndSystemPropertyClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/EnvironmentAndSystemPropertyClientProviderStrategy.java @@ -2,31 +2,50 @@ import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.SystemUtils; /** * Use environment variables and system properties (as supported by the underlying DockerClient DefaultConfigBuilder) * to try and locate a docker environment. */ +@Slf4j public class EnvironmentAndSystemPropertyClientProviderStrategy extends DockerClientProviderStrategy { + + public static final int PRIORITY = 100; + private static final String PING_TIMEOUT_DEFAULT = "10"; private static final String PING_TIMEOUT_PROPERTY_NAME = "testcontainers.environmentprovider.timeout"; + public EnvironmentAndSystemPropertyClientProviderStrategy() { + // Try using environment variables + config = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); + } + + @Override + protected boolean isApplicable() { + return "tcp".equalsIgnoreCase(config.getDockerHost().getScheme()) || SystemUtils.IS_OS_LINUX; + } + + @Override + protected int getPriority() { + return PRIORITY; + } + @Override public void test() throws InvalidConfigurationException { try { - // Try using environment variables - config = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); client = getClientForConfig(config); final int timeout = Integer.parseInt(System.getProperty(PING_TIMEOUT_PROPERTY_NAME, PING_TIMEOUT_DEFAULT)); ping(client, timeout); } catch (Exception | UnsatisfiedLinkError e) { - LOGGER.error("ping failed with configuration {} due to {}", getDescription(), e.toString(), e); + log.error("ping failed with configuration {} due to {}", getDescription(), e.toString(), e); throw new InvalidConfigurationException("ping failed"); } - LOGGER.info("Found docker client settings from environment"); + log.info("Found docker client settings from environment"); } @Override diff --git a/core/src/main/java/org/testcontainers/dockerclient/ProxiedUnixSocketClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/ProxiedUnixSocketClientProviderStrategy.java index 7036e048b34..ffd689fcb9e 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/ProxiedUnixSocketClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/ProxiedUnixSocketClientProviderStrategy.java @@ -1,27 +1,38 @@ package org.testcontainers.dockerclient; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.SystemUtils; import org.rnorth.tcpunixsocketproxy.TcpToUnixSocketProxy; import java.io.File; +@Slf4j public class ProxiedUnixSocketClientProviderStrategy extends UnixSocketClientProviderStrategy { + public static final int PRIORITY = EnvironmentAndSystemPropertyClientProviderStrategy.PRIORITY - 10; + + private final File socketFile = new File(DOCKER_SOCK_PATH); + @Override - public void test() throws InvalidConfigurationException { + protected boolean isApplicable() { + return !SystemUtils.IS_OS_LINUX && socketFile.exists(); + } - String osName = System.getProperty("os.name").toLowerCase(); - if (!osName.contains("mac") && !osName.contains("linux")) { - throw new InvalidConfigurationException("this strategy is only applicable to OS X and Linux"); - } + @Override + protected int getPriority() { + return PRIORITY; + } - TcpToUnixSocketProxy proxy = new TcpToUnixSocketProxy(new File(DOCKER_SOCK_PATH)); + @Override + public void test() throws InvalidConfigurationException { + TcpToUnixSocketProxy proxy = new TcpToUnixSocketProxy(socketFile); try { int proxyPort = proxy.start().getPort(); config = tryConfiguration("tcp://localhost:" + proxyPort); - LOGGER.info("Accessing unix domain socket via TCP proxy (" + DOCKER_SOCK_PATH + " via localhost:" + proxyPort + ")"); + log.debug("Accessing unix domain socket via TCP proxy (" + DOCKER_SOCK_PATH + " via localhost:" + proxyPort + ")"); } catch (Exception e) { proxy.stop(); diff --git a/core/src/main/java/org/testcontainers/dockerclient/UnixSocketClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/UnixSocketClientProviderStrategy.java index e4aac02e6ed..774f3184ef3 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/UnixSocketClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/UnixSocketClientProviderStrategy.java @@ -2,6 +2,8 @@ import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.SystemUtils; import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -9,6 +11,7 @@ import java.nio.file.Path; import java.nio.file.Paths; +@Slf4j public class UnixSocketClientProviderStrategy extends DockerClientProviderStrategy { protected static final String DOCKER_SOCK_PATH = "/var/run/docker.sock"; private static final String SOCKET_LOCATION = "unix://" + DOCKER_SOCK_PATH; @@ -16,18 +19,16 @@ public class UnixSocketClientProviderStrategy extends DockerClientProviderStrate private static final String PING_TIMEOUT_DEFAULT = "10"; private static final String PING_TIMEOUT_PROPERTY_NAME = "testcontainers.unixsocketprovider.timeout"; - @Override - public void test() - throws InvalidConfigurationException { - - if (!System.getProperty("os.name").toLowerCase().contains("linux")) { - throw new InvalidConfigurationException("this strategy is only applicable to Linux"); - } + protected boolean isApplicable() { + return SystemUtils.IS_OS_LINUX; + } + @Override + public void test() throws InvalidConfigurationException { try { config = tryConfiguration(SOCKET_LOCATION); - LOGGER.info("Accessing docker with local Unix socket"); + log.info("Accessing docker with local Unix socket"); } catch (Exception | UnsatisfiedLinkError e) { throw new InvalidConfigurationException("ping failed", e); } diff --git a/core/src/main/java/org/testcontainers/dockerclient/WindowsClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/WindowsClientProviderStrategy.java index 15cd35c48bf..f53ac10d044 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/WindowsClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/WindowsClientProviderStrategy.java @@ -2,6 +2,7 @@ import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientConfig; +import org.apache.commons.lang.SystemUtils; import org.jetbrains.annotations.NotNull; public class WindowsClientProviderStrategy extends DockerClientProviderStrategy { @@ -9,6 +10,11 @@ public class WindowsClientProviderStrategy extends DockerClientProviderStrategy private static final int PING_TIMEOUT_DEFAULT = 5; private static final String PING_TIMEOUT_PROPERTY_NAME = "testcontainers.windowsprovider.timeout"; + @Override + protected boolean isApplicable() { + return SystemUtils.IS_OS_WINDOWS; + } + @Override public void test() throws InvalidConfigurationException { config = tryConfiguration("tcp://localhost:2375"); diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 1c90a1fabae..a47b2b90637 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -1,57 +1,110 @@ package org.testcontainers.utility; -import com.google.common.base.MoreObjects; -import lombok.AccessLevel; -import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; +import lombok.*; import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; +import java.io.*; +import java.net.MalformedURLException; +import java.util.Objects; import java.util.Properties; +import java.util.stream.Stream; /** * Provides a mechanism for fetching configuration/defaults from the classpath. */ @Data -@Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class TestcontainersConfiguration { + private static String PROPERTIES_FILE_NAME = "testcontainers.properties"; + + private static File GLOBAL_CONFIG_FILE = new File(System.getProperty("user.home"), "." + PROPERTIES_FILE_NAME); + @Getter(lazy = true) private static final TestcontainersConfiguration instance = loadConfiguration(); - private String ambassadorContainerImage = "richnorth/ambassador:latest"; - private String vncRecordedContainerImage = "richnorth/vnc-recorder:latest"; - private String tinyImage = "alpine:3.5"; - private boolean disableChecks = false; + private final Properties properties; - private static TestcontainersConfiguration loadConfiguration() { - final TestcontainersConfiguration config = new TestcontainersConfiguration(); + public String getAmbassadorContainerImage() { + return (String) properties.getOrDefault("ambassador.container.image", "richnorth/ambassador:latest"); + } - ClassLoader loader = MoreObjects.firstNonNull( - Thread.currentThread().getContextClassLoader(), - TestcontainersConfiguration.class.getClassLoader()); - final URL configOverrides = loader.getResource("testcontainers.properties"); - if (configOverrides != null) { + public String getVncRecordedContainerImage() { + return (String) properties.getOrDefault("vncrecorder.container.image", "richnorth/vnc-recorder:latest"); + } - log.debug("Testcontainers configuration overrides will be loaded from {}", configOverrides); + public String getTinyImage() { + return (String) properties.getOrDefault("tinyimage.container.image", "alpine:3.5"); + } - final Properties properties = new Properties(); - try (final InputStream inputStream = configOverrides.openStream()) { - properties.load(inputStream); + public boolean isDisableChecks() { + return Boolean.parseBoolean((String) properties.getOrDefault("checks.disable", "false")); + } - config.ambassadorContainerImage = properties.getProperty("ambassador.container.image", config.ambassadorContainerImage); - config.vncRecordedContainerImage = properties.getProperty("vncrecorder.container.image", config.vncRecordedContainerImage); - config.tinyImage = properties.getProperty("tinyimage.container.image", config.tinyImage); - config.disableChecks = Boolean.parseBoolean(properties.getProperty("checks.disable", config.disableChecks + "")); + public String getDockerClientStrategyClassName() { + return (String) properties.get("docker.client.strategy"); + } - log.debug("Testcontainers configuration overrides loaded from {}: {}", configOverrides, config); + @Synchronized + public boolean updateGlobalConfig(@NonNull String prop, @NonNull String value) { + try { + Properties globalProperties = new Properties(); + GLOBAL_CONFIG_FILE.createNewFile(); + try (InputStream inputStream = new FileInputStream(GLOBAL_CONFIG_FILE)) { + globalProperties.load(inputStream); + } - } catch (IOException e) { - log.error("Testcontainers config override was found on classpath but could not be loaded", e); + if (value.equals(globalProperties.get(prop))) { + return false; } + + globalProperties.setProperty(prop, value); + + try (OutputStream outputStream = new FileOutputStream(GLOBAL_CONFIG_FILE)) { + globalProperties.store(outputStream, "Modified by Testcontainers"); + } + + // Update internal state only if global config was successfully updated + properties.setProperty(prop, value); + return true; + } catch (Exception e) { + log.debug("Can't store global property {} in {}", prop, GLOBAL_CONFIG_FILE); + return false; + } + } + + @SneakyThrows(MalformedURLException.class) + private static TestcontainersConfiguration loadConfiguration() { + final TestcontainersConfiguration config = new TestcontainersConfiguration( + Stream + .of( + TestcontainersConfiguration.class.getClassLoader().getResource(PROPERTIES_FILE_NAME), + Thread.currentThread().getContextClassLoader().getResource(PROPERTIES_FILE_NAME), + GLOBAL_CONFIG_FILE.toURI().toURL() + ) + .filter(Objects::nonNull) + .map(it -> { + log.debug("Testcontainers configuration overrides will be loaded from {}", it); + + final Properties subProperties = new Properties(); + try (final InputStream inputStream = it.openStream()) { + subProperties.load(inputStream); + } catch (FileNotFoundException e) { + log.trace("Testcontainers config override was found on " + it + " but the file was not found", e); + } catch (IOException e) { + log.warn("Testcontainers config override was found on " + it + " but could not be loaded", e); + } + return subProperties; + }) + .reduce(new Properties(), (a, b) -> { + a.putAll(b); + return a; + }) + ); + + if (!config.getProperties().isEmpty()) { + log.debug("Testcontainers configuration overrides loaded from {}", config); } return config;