Skip to content

Commit

Permalink
pre-create Aurora clusters to speed up integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sergiyv-improving
sergiyv-improving committed Jun 20, 2023
1 parent d006fdd commit dbf12ba
Show file tree
Hide file tree
Showing 6 changed files with 194 additions and 17 deletions.
13 changes: 13 additions & 0 deletions wrapper/src/test/java/integration/refactored/TestEnvironmentRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
@JsonIgnoreProperties(ignoreUnknown = true)
public class TestEnvironmentRequest {

@JsonIgnore
private int envPreCreateIndex;

@JsonProperty("engine")
private DatabaseEngine engine;

Expand Down Expand Up @@ -105,4 +108,14 @@ public String getDisplayName() {
"Test environment [%s, %s, %s, %s, %d, %s]",
deployment, engine, targetJvm, instances, numOfInstances, features);
}

@JsonIgnore
public int getEnvPreCreateIndex() {
return this.envPreCreateIndex;
}

@JsonIgnore
public void setEnvPreCreateIndex(int index) {
this.envPreCreateIndex = index;
}
}
1 change: 1 addition & 0 deletions wrapper/src/test/java/integration/refactored/container/TestDriverProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ public void beforeEach(ExtensionContext context) throws Exception {
instanceIDs = new ArrayList<>();
}
}
assertTrue(instanceIDs.size() > 0);
assertTrue(
auroraUtil.isDBInstanceWriter(
testInfo.getAuroraClusterName(),
Expand Down
2 changes: 1 addition & 1 deletion wrapper/src/test/java/integration/refactored/container/tests/AuroraFailoverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ public void testServerFailoverWithIdleConnections() throws SQLException, Interru
conn));

// Sleep for a second to allow daemon threads to finish running.
Thread.sleep(1000);
Thread.sleep(10000);

// Ensure that all idle connections are closed.
for (Connection idleConnection : idleConnections) {
Expand Down
129 changes: 122 additions & 7 deletions wrapper/src/test/java/integration/refactored/host/TestEnvironment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import integration.refactored.TestEnvironmentRequest;
import integration.refactored.TestInstanceInfo;
import integration.refactored.TestProxyDatabaseInfo;
import integration.refactored.host.TestEnvironmentProvider.EnvPreCreateInfo;
import integration.util.AuroraTestUtility;
import integration.util.ContainerHelper;
import java.io.IOException;
Expand All @@ -37,6 +38,10 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
Expand All @@ -49,6 +54,9 @@
public class TestEnvironment implements AutoCloseable {

private static final Logger LOGGER = Logger.getLogger(TestEnvironment.class.getName());
private static final int NUM_OR_ENV_PRE_CREATE = 1; // create this number of environments in advance

private static final ExecutorService envPreCreateExecutor = Executors.newCachedThreadPool();

private static final String DATABASE_CONTAINER_NAME_PREFIX = "database-container-";
private static final String TEST_CONTAINER_NAME = "test-container";
Expand Down Expand Up @@ -86,10 +94,14 @@ private TestEnvironment(TestEnvironmentRequest request) {
}

public static TestEnvironment build(TestEnvironmentRequest request) {
TestEnvironment env = new TestEnvironment(request);
LOGGER.finest("Building test env: " + request.getEnvPreCreateIndex());
preCreateEnvironment(request.getEnvPreCreateIndex());

TestEnvironment env;

switch (request.getDatabaseEngineDeployment()) {
case DOCKER:
env = new TestEnvironment(request);
initDatabaseParams(env);
createDatabaseContainers(env);

Expand All @@ -104,14 +116,11 @@ public static TestEnvironment build(TestEnvironmentRequest request) {

break;
case AURORA:
initDatabaseParams(env);
createAuroraDbCluster(env);

if (request.getFeatures().contains(TestEnvironmentFeatures.IAM)) {
configureIamAccess(env);
}
env = createAuroraEnvironment(request);
authorizeIP(env);

break;

default:
throw new NotImplementedException(request.getDatabaseEngineDeployment().toString());
}
Expand All @@ -125,6 +134,61 @@ public static TestEnvironment build(TestEnvironmentRequest request) {
return env;
}

private static TestEnvironment createAuroraEnvironment(TestEnvironmentRequest request) {

EnvPreCreateInfo preCreateInfo =
TestEnvironmentProvider.preCreateInfos.get(request.getEnvPreCreateIndex());

if (preCreateInfo.envPreCreateFuture != null) {
/*
This environment has being created in advance.
We need to wait for results and apply details of newly created environment to the current
environment.
*/
Object result;
try {
// Effectively waits till the future completes and returns results.
final long startTime = System.nanoTime();
result = preCreateInfo.envPreCreateFuture.get();
final long duration = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime);
LOGGER.finest(() ->
String.format("Additional wait time for test environment to be ready (pre-create): %d sec", duration));
} catch (ExecutionException | InterruptedException ex) {
throw new RuntimeException("Test environment create error.", ex);
}

preCreateInfo.envPreCreateFuture = null;

if (result == null) {
throw new RuntimeException("Test environment create error. Results are empty.");
}
if (result instanceof Exception) {
throw new RuntimeException((Exception) result);
}
if (result instanceof TestEnvironment) {
TestEnvironment resultTestEnvironment = (TestEnvironment) result;
LOGGER.finer(() -> String.format("Use pre-created DB cluster: %s.cluster-%s",
resultTestEnvironment.auroraClusterName, resultTestEnvironment.auroraClusterDomain));

return resultTestEnvironment;
}
throw new RuntimeException(
"Test environment create error. Unrecognized result type: " + result.getClass().getName());

} else {
TestEnvironment env = new TestEnvironment(request);
initDatabaseParams(env);
createAuroraDbCluster(env);

if (request.getFeatures().contains(TestEnvironmentFeatures.IAM)) {
configureIamAccess(env);
}

return env;
}

}

private static void createDatabaseContainers(TestEnvironment env) {
ContainerHelper containerHelper = new ContainerHelper();

Expand Down Expand Up @@ -352,6 +416,10 @@ private static void createAuroraDbCluster(TestEnvironment env, int numOfInstance
env.info.getDatabaseInfo().getInstances().clear();
env.info.getDatabaseInfo().getInstances().addAll(instances);

authorizeIP(env);
}

private static void authorizeIP(TestEnvironment env) {
try {
env.runnerIP = env.auroraUtil.getPublicIPAddress();
} catch (UnknownHostException e) {
Expand Down Expand Up @@ -762,4 +830,51 @@ private void deleteAuroraDbCluster() {
LOGGER.finest("Deleted cluster " + this.auroraClusterName + ".cluster-" + this.auroraClusterDomain);
}
}

private static void preCreateEnvironment(int currentEnvIndex) {
int index = currentEnvIndex + 1; // inclusive
int endIndex = index + NUM_OR_ENV_PRE_CREATE; // exclusive
if (endIndex > TestEnvironmentProvider.preCreateInfos.size()) {
endIndex = TestEnvironmentProvider.preCreateInfos.size();
}

while (index < endIndex) {
EnvPreCreateInfo preCreateInfo = TestEnvironmentProvider.preCreateInfos.get(index);

if (preCreateInfo.envPreCreateFuture == null
&& (preCreateInfo.request.getDatabaseEngineDeployment() == DatabaseEngineDeployment.AURORA
|| preCreateInfo.request.getDatabaseEngineDeployment() == DatabaseEngineDeployment.RDS)) {

// run environment creation in advance
int finalIndex = index;
LOGGER.finest(() -> String.format("Pre-create environment for [%d] - %s",
finalIndex, preCreateInfo.request.getDisplayName()));

final TestEnvironment env = new TestEnvironment(preCreateInfo.request);

preCreateInfo.envPreCreateFuture = envPreCreateExecutor.submit(() -> {
final long startTime = System.nanoTime();
try {
initDatabaseParams(env);
createAuroraDbCluster(env);
if (env.info.getRequest().getFeatures().contains(TestEnvironmentFeatures.IAM)) {
configureIamAccess(env);
}
return env;

} catch (Exception ex) {
return ex;
} finally {
final long duration = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime);
LOGGER.finest(() -> String.format(
"Pre-create environment task [%d] run in background for %d sec.",
finalIndex,
duration));
}
});

}
index++;
}
}
}
28 changes: 27 additions & 1 deletion wrapper/src/test/java/integration/refactored/host/TestEnvironmentProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Future;
import java.util.logging.Logger;
import java.util.stream.Stream;
import org.junit.jupiter.api.extension.Extension;
Expand All @@ -36,6 +37,7 @@

public class TestEnvironmentProvider implements TestTemplateInvocationContextProvider {

static final ArrayList<EnvPreCreateInfo> preCreateInfos = new ArrayList<>();
private static final Logger LOGGER = Logger.getLogger(TestEnvironmentProvider.class.getName());

@Override
Expand All @@ -46,6 +48,8 @@ public boolean supportsTestTemplate(ExtensionContext context) {
@Override
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(
ExtensionContext context) {

preCreateInfos.clear();
ArrayList<TestTemplateInvocationContext> resultContextList = new ArrayList<>();

final boolean noDocker = Boolean.parseBoolean(System.getProperty("test-no-docker", "false"));
Expand Down Expand Up @@ -362,7 +366,8 @@ public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContex
}

private TestTemplateInvocationContext getEnvironment(TestEnvironmentRequest info) {
return new TestTemplateInvocationContext() {
return new AwsWrapperTestTemplateInvocationContext(info) {

@Override
public String getDisplayName(int invocationIndex) {
return String.format("[%d] - %s", invocationIndex, info.getDisplayName());
Expand All @@ -374,4 +379,25 @@ public List<Extension> getAdditionalExtensions() {
}
};
}

public abstract static class AwsWrapperTestTemplateInvocationContext
implements TestTemplateInvocationContext {
AwsWrapperTestTemplateInvocationContext(final TestEnvironmentRequest info) {
int index = preCreateInfos.size();
info.setEnvPreCreateIndex(index);

EnvPreCreateInfo envPreCreateInfo = new EnvPreCreateInfo();
envPreCreateInfo.request = info;
preCreateInfos.add(envPreCreateInfo);
}

public abstract String getDisplayName(int invocationIndex);

public abstract List<Extension> getAdditionalExtensions();
}

public static class EnvPreCreateInfo {
public TestEnvironmentRequest request;
public Future envPreCreateFuture;
}
}
38 changes: 30 additions & 8 deletions wrapper/src/test/java/integration/util/AuroraTestUtility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
Expand All @@ -66,6 +67,7 @@
import software.amazon.awssdk.services.rds.model.DBClusterMember;
import software.amazon.awssdk.services.rds.model.DBInstance;
import software.amazon.awssdk.services.rds.model.DbClusterNotFoundException;
import software.amazon.awssdk.services.rds.model.DeleteDbClusterResponse;
import software.amazon.awssdk.services.rds.model.DeleteDbInstanceRequest;
import software.amazon.awssdk.services.rds.model.DescribeDbClustersRequest;
import software.amazon.awssdk.services.rds.model.DescribeDbClustersResponse;
Expand Down Expand Up @@ -115,7 +117,7 @@ public AuroraTestUtility() {
* Initializes an AmazonRDS & AmazonEC2 client.
*
* @param region define AWS Regions, refer to
* https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html
* <a href="https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html">Regions, Availability Zones, and Local Zones</a>
*/
public AuroraTestUtility(Region region) {
this(region, DefaultCredentialsProvider.create());
Expand All @@ -125,7 +127,7 @@ public AuroraTestUtility(Region region) {
* Initializes an AmazonRDS & AmazonEC2 client.
*
* @param region define AWS Regions, refer to
* https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html
* <a href="https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html">Regions, Availability Zones, and Local Zones</a>
*/
public AuroraTestUtility(String region) {
this(getRegionInternal(region), DefaultCredentialsProvider.create());
Expand All @@ -146,7 +148,7 @@ public AuroraTestUtility(
* Initializes an AmazonRDS & AmazonEC2 client.
*
* @param region define AWS Regions, refer to
* https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html
* <a href="https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html">Regions, Availability Zones, and Local Zones</a>
* @param credentialsProvider Specific AWS credential provider
*/
public AuroraTestUtility(Region region, AwsCredentialsProvider credentialsProvider) {
Expand Down Expand Up @@ -480,8 +482,22 @@ public void deleteCluster() {
}

// Tear down cluster
rdsClient.deleteDBCluster(
(builder -> builder.skipFinalSnapshot(true).dbClusterIdentifier(dbIdentifier)));
int remainingAttempts = 5;
while (--remainingAttempts > 0) {
try {
DeleteDbClusterResponse response = rdsClient.deleteDBCluster(
(builder -> builder.skipFinalSnapshot(true).dbClusterIdentifier(dbIdentifier)));
if (response.sdkHttpResponse().isSuccessful()) {
break;
}
TimeUnit.SECONDS.sleep(30);

} catch (DbClusterNotFoundException ex) {
// ignore
} catch (Exception ex) {
LOGGER.warning("Error deleting db cluster " + dbIdentifier + ": " + ex);
}
}
}

public boolean doesClusterExist(final String clusterId) {
Expand Down Expand Up @@ -652,17 +668,23 @@ protected void makeSureInstancesUp(List<String> instances, boolean finalCheck)
break;
} catch (final SQLException ex) {
// Continue waiting until instance is up.
LOGGER.log(Level.FINEST, "Exception while trying to connect to instance " + id, ex);
} catch (final Exception ex) {
System.out.println("Exception: " + ex);
LOGGER.log(Level.SEVERE, "Exception:", ex);
break;
}
TimeUnit.MILLISECONDS.sleep(1000);
TimeUnit.MILLISECONDS.sleep(5000);
}
return null;
});
}
executorService.shutdown();
executorService.awaitTermination(5, TimeUnit.MINUTES);
boolean isDone = executorService.awaitTermination(7, TimeUnit.MINUTES);

if (!isDone) {
LOGGER.finest("Some task are not completed. Shutting down them now.");
executorService.shutdownNow();
}

if (finalCheck) {
assertTrue(
Expand Down

0 comments on commit dbf12ba

Please sign in to comment.