Skip to content

Commit

Permalink
Validating fetch config before trying to fetch artifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
manojlds committed Mar 3, 2015
1 parent f2d1f59 commit 684071d
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 57 deletions.
64 changes: 64 additions & 0 deletions fetch/src/main/java/com/indix/gocd/s3fetch/FetchConfig.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package com.indix.gocd.s3fetch;

import com.amazonaws.util.StringUtils;
import com.indix.gocd.utils.GoEnvironment;
import com.thoughtworks.go.plugin.api.response.validation.ValidationError;
import com.thoughtworks.go.plugin.api.response.validation.ValidationResult;
import com.thoughtworks.go.plugin.api.task.TaskConfig;
import com.thoughtworks.go.plugin.api.task.TaskExecutionContext;

import static com.indix.gocd.utils.Constants.*;

public class FetchConfig {
private final String materialLabel;
private final String pipeline;
private final String stage;
private final String job;
private GoEnvironment env;

public FetchConfig(TaskConfig config, TaskExecutionContext context) {
this.env = new GoEnvironment();
env.putAll(context.environment().asMap());

String repoName = config.getValue(FetchTask.REPO).toUpperCase();
String packageName = config.getValue(FetchTask.PACKAGE).toUpperCase();
this.materialLabel = env.get(String.format("GO_PACKAGE_%s_%s_LABEL", repoName, packageName));
this.pipeline = env.get(String.format("GO_PACKAGE_%s_%s_PIPELINE_NAME", repoName, packageName));
this.stage = env.get(String.format("GO_PACKAGE_%s_%s_STAGE_NAME", repoName, packageName));
this.job = env.get(String.format("GO_PACKAGE_%s_%s_JOB_NAME", repoName, packageName));
}

public ValidationResult validate() {
ValidationResult validationResult = new ValidationResult();
if (env.isAbsent(AWS_ACCESS_KEY_ID)) validationResult.addError(envNotFound(AWS_ACCESS_KEY_ID));
if (env.isAbsent(AWS_SECRET_ACCESS_KEY)) validationResult.addError(envNotFound(AWS_SECRET_ACCESS_KEY));
if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) validationResult.addError(envNotFound(GO_ARTIFACTS_S3_BUCKET));
if (StringUtils.isNullOrEmpty(materialLabel))
validationResult.addError(new ValidationError("Please check Repository name or Package name configuration. Also ensure that the appropriate S3 material is configured for the pipeline."));

return validationResult;
}

public String getArtifactsLocationTemplate() {
String[] counters = materialLabel.split("\\.");
String pipelineCounter = counters[0];
String stageCounter = counters[1];
return env.artifactsLocationTemplate(pipeline, stage, job, pipelineCounter, stageCounter);
}

public String getAWSAccessKeyId() {
return env.get(AWS_ACCESS_KEY_ID);
}

public String getAWSSecretAccessKey() {
return env.get(AWS_SECRET_ACCESS_KEY);
}

public String getS3Bucket() {
return env.get(GO_ARTIFACTS_S3_BUCKET);
}

private ValidationError envNotFound(String environmentVariable) {
return new ValidationError(environmentVariable, String.format("%s environment variable not present", environmentVariable));
}
}
46 changes: 12 additions & 34 deletions fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.indix.gocd.utils.GoEnvironment;
import com.indix.gocd.utils.store.S3ArtifactStore;
import com.thoughtworks.go.plugin.api.logging.Logger;
import com.thoughtworks.go.plugin.api.response.execution.ExecutionResult;
import com.thoughtworks.go.plugin.api.response.validation.ValidationResult;
import com.thoughtworks.go.plugin.api.task.TaskConfig;
import com.thoughtworks.go.plugin.api.task.TaskExecutionContext;
import com.thoughtworks.go.plugin.api.task.TaskExecutor;
Expand All @@ -14,24 +14,21 @@
import java.io.File;
import java.io.IOException;

import static com.indix.gocd.utils.Constants.*;

public class FetchExecutor implements TaskExecutor {
private static Logger logger = Logger.getLoggerFor(FetchTask.class);

@Override
public ExecutionResult execute(TaskConfig config, final TaskExecutionContext context) {
final GoEnvironment env = new GoEnvironment();
env.putAll(context.environment().asMap());
final FetchConfig fetchConfig = new FetchConfig(config, context);

if (env.isAbsent(AWS_ACCESS_KEY_ID)) return envNotFound(AWS_ACCESS_KEY_ID);
if (env.isAbsent(AWS_SECRET_ACCESS_KEY)) return envNotFound(AWS_SECRET_ACCESS_KEY);
if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
ValidationResult validationResult = fetchConfig.validate();
if(!validationResult.isSuccessful()) {
return ExecutionResult.failure(validationResult.getMessages().toString());
}

final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
final S3ArtifactStore store = s3ArtifactStore(env, bucket);
final S3ArtifactStore store = s3ArtifactStore(fetchConfig);

String artifactPathOnS3 = getS3ArtifactPath(config, env);
String artifactPathOnS3 = fetchConfig.getArtifactsLocationTemplate();
context.console().printLine(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3)));
String destination = String.format("%s/%s", context.workingDir(), config.getValue(FetchTask.DESTINATION));
setupDestinationDirectory(destination);
Expand All @@ -47,20 +44,6 @@ public ExecutionResult execute(TaskConfig config, final TaskExecutionContext con
return ExecutionResult.success("Fetched all artifacts");
}

private String getS3ArtifactPath(TaskConfig config, GoEnvironment env) {
String repoName = config.getValue(FetchTask.REPO).toUpperCase();
String packageName = config.getValue(FetchTask.PACKAGE).toUpperCase();

String materialLabel = env.get(String.format("GO_PACKAGE_%s_%s_LABEL", repoName, packageName));
String[] counters = materialLabel.split("\\.");
String pipelineCounter = counters[0];
String stageCounter = counters[1];
String pipeline = env.get(String.format("GO_PACKAGE_%s_%s_PIPELINE_NAME", repoName, packageName));
String stage = env.get(String.format("GO_PACKAGE_%s_%s_STAGE_NAME", repoName, packageName));
String job = env.get(String.format("GO_PACKAGE_%s_%s_JOB_NAME", repoName, packageName));
return env.artifactsLocationTemplate(pipeline, stage, job, pipelineCounter, stageCounter);
}

private void setupDestinationDirectory(String destination) {
File destinationDirectory = new File(destination);
try {
Expand All @@ -74,18 +57,13 @@ private void setupDestinationDirectory(String destination) {
}
}

public S3ArtifactStore s3ArtifactStore(GoEnvironment env, String bucket) {
return new S3ArtifactStore(s3Client(env), bucket);
public S3ArtifactStore s3ArtifactStore(FetchConfig config) {
return new S3ArtifactStore(s3Client(config), config.getS3Bucket());
}

public AmazonS3Client s3Client(GoEnvironment env) {
String accessKey = env.get(AWS_ACCESS_KEY_ID);
String secretKey = env.get(AWS_SECRET_ACCESS_KEY);
return new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
public AmazonS3Client s3Client(FetchConfig config) {
return new AmazonS3Client(new BasicAWSCredentials(config.getAWSAccessKeyId(), config.getAWSSecretAccessKey()));
}

private ExecutionResult envNotFound(String environmentVariable) {
return ExecutionResult.failure(String.format("%s environment variable not present", environmentVariable));
}
}

138 changes: 138 additions & 0 deletions fetch/src/test/java/com/indix/gocd/s3fetch/FetchConfigTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package com.indix.gocd.s3fetch;

import com.indix.gocd.utils.mocks.MockTaskExecutionContext;
import com.indix.gocd.utils.utils.Maps;
import com.thoughtworks.go.plugin.api.response.validation.ValidationResult;
import com.thoughtworks.go.plugin.api.task.TaskConfig;
import com.thoughtworks.go.plugin.api.task.TaskExecutionContext;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import static com.indix.gocd.utils.Constants.*;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class FetchConfigTest {
private final String bucket = "gocd";
Maps.MapBuilder<String, String> mockEnvironmentVariables;
private TaskConfig config;
private FetchConfig fetchConfig;
private final String secretKey = "secretKey";
private final String accessId = "accessId";

@Before
public void setUp() throws Exception {
config = mock(TaskConfig.class);
when(config.getValue(FetchTask.REPO)).thenReturn(bucket);
when(config.getValue(FetchTask.PACKAGE)).thenReturn("TestPublishS3Artifacts");
mockEnvironmentVariables = Maps.<String, String>builder()
.with(AWS_SECRET_ACCESS_KEY, secretKey)
.with(AWS_ACCESS_KEY_ID, accessId)
.with(GO_ARTIFACTS_S3_BUCKET, bucket)
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1")
.with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish")
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage")
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob");
}

@Test
public void shouldGetAWSSecretAccessKey() {
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
String awsSecretAccessKey = fetchConfig.getAWSSecretAccessKey();
assertThat(awsSecretAccessKey, is(secretKey));
}

@Test
public void shouldGetAWSAccessKeyId() {
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
String awsSecretAccessKey = fetchConfig.getAWSAccessKeyId();
assertThat(awsSecretAccessKey, is(accessId));
}

@Test
public void shouldS3Bucket() {
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
String awsSecretAccessKey = fetchConfig.getS3Bucket();
assertThat(awsSecretAccessKey, is(bucket));
}

@Test
public void shouldGetArtifactLocation() {
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
String location = fetchConfig.getArtifactsLocationTemplate();
assertThat(location, is("TestPublish/defaultStage/defaultJob/20.1"));
}

@Test
public void shouldBeValid() {
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
ValidationResult validationResult = fetchConfig.validate();
assertTrue(validationResult.isSuccessful());
}

@Test
public void shouldNotBeValidIfAWSSecretAccessKeyNotPresent() {
fetchConfig = new FetchConfig(config, mockContext( mockEnvironmentVariables.remove(AWS_SECRET_ACCESS_KEY).build()));
ValidationResult validationResult = fetchConfig.validate();
assertFalse(validationResult.isSuccessful());
ArrayList<String> messages = new ArrayList<String>();
messages.add("AWS_SECRET_ACCESS_KEY environment variable not present");
assertThat(validationResult.getMessages(), Matchers.<List<String>>is(messages));
}

@Test
public void shouldNotBeValidIfAWSAccessKeyIdNotPresent() {
fetchConfig = new FetchConfig(config, mockContext( mockEnvironmentVariables.remove(AWS_ACCESS_KEY_ID).build()));
ValidationResult validationResult = fetchConfig.validate();
assertFalse(validationResult.isSuccessful());
ArrayList<String> messages = new ArrayList<String>();
messages.add("AWS_ACCESS_KEY_ID environment variable not present");
assertThat(validationResult.getMessages(), Matchers.<List<String>>is(messages));
}

@Test
public void shouldNotBeValidIfS3BucketNotPresent() {
fetchConfig = new FetchConfig(config, mockContext( mockEnvironmentVariables.remove(GO_ARTIFACTS_S3_BUCKET).build()));
ValidationResult validationResult = fetchConfig.validate();
assertFalse(validationResult.isSuccessful());
ArrayList<String> messages = new ArrayList<String>();
messages.add("GO_ARTIFACTS_S3_BUCKET environment variable not present");
assertThat(validationResult.getMessages(), Matchers.<List<String>>is(messages));
}

@Test
public void shouldNotBeValidIfRepoConfigIsNotValid() {
when(config.getValue(FetchTask.REPO)).thenReturn("Wrong");
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
ValidationResult validationResult = fetchConfig.validate();
assertFalse(validationResult.isSuccessful());
ArrayList<String> messages = new ArrayList<String>();
messages.add("Please check Repository name or Package name configuration. Also ensure that the appropriate S3 material is configured for the pipeline.");
assertThat(validationResult.getMessages(), Matchers.<List<String>>is(messages));
}

@Test
public void shouldNotBeValidIfPackageConfigIsNotValid() {
when(config.getValue(FetchTask.PACKAGE)).thenReturn("Wrong");
fetchConfig = new FetchConfig(config, mockContext(mockEnvironmentVariables.build()));
ValidationResult validationResult = fetchConfig.validate();
assertFalse(validationResult.isSuccessful());
ArrayList<String> messages = new ArrayList<String>();
messages.add("Please check Repository name or Package name configuration. Also ensure that the appropriate S3 material is configured for the pipeline.");
assertThat(validationResult.getMessages(), Matchers.<List<String>>is(messages));
}

private TaskExecutionContext mockContext(final Map<String, String> environmentMap) {
return new MockTaskExecutionContext(environmentMap);
}
}
27 changes: 4 additions & 23 deletions fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.indix.gocd.utils.GoEnvironment;
import com.indix.gocd.utils.mocks.MockTaskExecutionContext;
import com.indix.gocd.utils.store.S3ArtifactStore;
import com.indix.gocd.utils.utils.Maps;
Expand Down Expand Up @@ -50,37 +49,19 @@ public void setUp() throws Exception {
}

@Test
public void shouldThrowIfAWS_ACCESS_KEY_IDNotPresent() {
public void shouldBeFailureIfFetchConfigNotValid() {
Map<String, String> mockVariables = mockEnvironmentVariables.remove(AWS_ACCESS_KEY_ID).build();

ExecutionResult executionResult = fetchExecutor.execute(config, mockContext(mockVariables));
assertFalse(executionResult.isSuccessful());
assertThat(executionResult.getMessagesForDisplay(), is("AWS_ACCESS_KEY_ID environment variable not present"));
}

@Test
public void shouldThrowIfAWS_SECRET_ACCESS_KEYNotPresent() {
Map<String, String> mockVariables = mockEnvironmentVariables.remove(AWS_SECRET_ACCESS_KEY).build();

ExecutionResult executionResult = fetchExecutor.execute(config, mockContext(mockVariables));
assertFalse(executionResult.isSuccessful());
assertThat(executionResult.getMessagesForDisplay(), is("AWS_SECRET_ACCESS_KEY environment variable not present"));
}

@Test
public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
Map<String, String> mockVariables = mockEnvironmentVariables.remove(GO_ARTIFACTS_S3_BUCKET).build();

ExecutionResult executionResult = fetchExecutor.execute(config, mockContext(mockVariables));
assertFalse(executionResult.isSuccessful());
assertThat(executionResult.getMessagesForDisplay(), is("GO_ARTIFACTS_S3_BUCKET environment variable not present"));
assertThat(executionResult.getMessagesForDisplay(), is("[AWS_ACCESS_KEY_ID environment variable not present]"));
}

@Test
public void shouldBeFailureIfUnableToFetchArtifacts() {
Map<String, String> mockVariables = mockEnvironmentVariables.build();
AmazonS3Client mockClient = mockClient();
doReturn(mockClient).when(fetchExecutor).s3Client(any(GoEnvironment.class));
doReturn(mockClient).when(fetchExecutor).s3Client(any(FetchConfig.class));
doThrow(new AmazonClientException("Exception message")).when(mockClient).listObjects(any(ListObjectsRequest.class));

ExecutionResult executionResult = fetchExecutor.execute(config, mockContext(mockVariables));
Expand All @@ -93,7 +74,7 @@ public void shouldBeFailureIfUnableToFetchArtifacts() {
public void shouldBeSuccessResultOnSuccessfulFetch() {
Map<String, String> mockVariables = mockEnvironmentVariables.build();
S3ArtifactStore mockStore = mockStore();
doReturn(mockStore).when(fetchExecutor).s3ArtifactStore(any(GoEnvironment.class), eq(bucket));
doReturn(mockStore).when(fetchExecutor).s3ArtifactStore(any(FetchConfig.class));

ExecutionResult executionResult = fetchExecutor.execute(config, mockContext(mockVariables));

Expand Down

0 comments on commit 684071d

Please sign in to comment.