Skip to content

Commit

Permalink
Merge pull request #56 from indix/multiple-bucket-support
Browse files Browse the repository at this point in the history 
Multiple bucket support
  • Loading branch information
@manojlds
manojlds authored Jul 13, 2017
2 parents 982b3b5 + 7dc98bd commit ed4b365
Show file tree
Hide file tree
Showing 9 changed files with 105 additions and 14 deletions.
20 changes: 17 additions & 3 deletions fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,11 @@ public class FetchExecutor {
private static Logger logger = Logger.getLoggerFor(FetchTask.class);

public TaskExecutionResult execute(Config config, final Context context) {
final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);

try {
final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
String artifactPathOnS3 = getArtifactsLocationTemplate(config, env);
final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
final String bucket = getBucket(config, env);
final S3ArtifactStore store = getS3ArtifactStore(env, bucket);

context.printMessage(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3)));
Expand Down Expand Up @@ -54,6 +53,21 @@ private void setupDestinationDirectory(String destination) {
}
}

private String getBucket(Config config, GoEnvironment env) {
String repoName = config.getRepo();
String packageName = config.getPkg();
String bucketFromMaterial = env.get(String.format("GO_REPO_%s_%s_S3_BUCKET", repoName, packageName));
if(bucketFromMaterial != null) {
return bucketFromMaterial;
}

if(env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) {
throw new RuntimeException("S3 bucket to fetch from should be from material plugin or GO_ARTIFACTS_S3_BUCKET env var.");
}

return env.get(GO_ARTIFACTS_S3_BUCKET);
}

private String getArtifactsLocationTemplate(Config config, GoEnvironment env) {
String repoName = config.getRepo();
String packageName = config.getPkg();
Expand Down
3 changes: 0 additions & 3 deletions fetch/src/main/resources/views/task.template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,3 @@
<input type="text" ng-model="Destination" ng-required="true">
<span class="form_error" ng-show="GOINPUTNAME[Destination].$error.server">{{ GOINPUTNAME[Destination].$error.server }}</span>
</div>
<div class="form_item_block">
<p class="required">Make sure GO_ARTIFACTS_S3_BUCKET environment variables is present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
</div>
22 changes: 18 additions & 4 deletions fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public void setUp() throws Exception {
.with(GO_ARTIFACTS_S3_BUCKET, bucket)
.with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153")
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1")
.with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)

.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish")
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage")
.with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob");
Expand Down Expand Up @@ -85,9 +85,6 @@ public void shouldBeFailureIfUnableToFetchArtifacts() {
@Test
public void shouldBeSuccessResultOnSuccessfulFetch() {
Map<String, String> mockVariables = mockEnvironmentVariables.build();
AmazonS3Client mockClient = mockClient();
S3ArtifactStore store = new S3ArtifactStore(mockClient, bucket);
doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
S3ArtifactStore mockStore = mockStore();

doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), any(String.class));
Expand All @@ -99,6 +96,23 @@ public void shouldBeSuccessResultOnSuccessfulFetch() {

}

@Test
public void shouldGetBucketInfoFromMaterialEnvVars() {
Map<String, String> mockVariables = mockEnvironmentVariables
.with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
.with(GO_ARTIFACTS_S3_BUCKET, "")
.build();
S3ArtifactStore mockStore = mockStore();

doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables));

assertTrue(result.isSuccessful());
assertThat(result.message(), is("Fetched all artifacts"));
verify(mockStore, times(1)).getPrefix("TestPublish/defaultStage/defaultJob/20.1", "here/artifacts");

}

@Test
public void shouldBeAbleToHandleTaskConfigEntriesWithDashesInTheName() {
Map<String, String> mockVariables = mockEnvironmentVariables
Expand Down
4 changes: 3 additions & 1 deletion publish/src/main/java/com/indix/gocd/s3publish/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,20 @@
import java.util.List;
import java.util.Map;

import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;

public class Config {

public String sourceDestinationsJson;
public String destinationPrefix;

public String artifactsBucket;

public Config(Map config) {
sourceDestinationsJson = getValue(config, SOURCEDESTINATIONS);
destinationPrefix = getValue(config, DESTINATION_PREFIX);
artifactsBucket = getValue(config, ARTIFACTS_BUCKET);
}

public List<SourceDestination> sourceDestinations() throws JsonSyntaxException {
Expand Down
14 changes: 12 additions & 2 deletions publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ public class PublishExecutor {
public TaskExecutionResult execute(Config config, final Context context) {
try {
final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
if (env.isAbsent(GO_SERVER_DASHBOARD_URL)) return envNotFound(GO_SERVER_DASHBOARD_URL);

final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
String bucket = getBucket(env, config);
if(bucket == null) return envNotFound(GO_ARTIFACTS_S3_BUCKET);

final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD));

Expand Down Expand Up @@ -69,6 +70,15 @@ public TaskExecutionResult execute(Config config, final Context context) {
}
}

private String getBucket(GoEnvironment env, Config config) {
if(StringUtils.isNotBlank(config.artifactsBucket)) {
return config.artifactsBucket;
} else if(env.has(GO_ARTIFACTS_S3_BUCKET)) {
return env.get(GO_ARTIFACTS_S3_BUCKET);
}
return null;
}

protected S3ArtifactStore getS3ArtifactStore(GoEnvironment env, String bucket) {
return new S3ArtifactStore(env, bucket);
}
Expand Down
7 changes: 7 additions & 0 deletions publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.List;
import java.util.Map;

import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
import static com.indix.gocd.utils.utils.Lists.foreach;
Expand Down Expand Up @@ -110,6 +111,7 @@ public void execute(SourceDestination input) {

private GoPluginApiResponse handleGetConfigRequest() {
HashMap config = new HashMap();

HashMap sourceDestinations = new HashMap();
sourceDestinations.put("default-value", "");
sourceDestinations.put("required", true);
Expand All @@ -120,6 +122,11 @@ private GoPluginApiResponse handleGetConfigRequest() {
destinationPrefix.put("required", false);
config.put(DESTINATION_PREFIX, destinationPrefix);

HashMap artifactsBucket = new HashMap();
artifactsBucket.put("default-value", "");
artifactsBucket.put("required", false);
config.put(ARTIFACTS_BUCKET, artifactsBucket);

return createResponse(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE, config);
}

Expand Down
13 changes: 12 additions & 1 deletion publish/src/main/resources/views/task.template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,18 @@
</p>
</div>
<div class="form_item_block">
<p class="required">Make sure GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
<label for="artifactsBucket">Artifacts Bucket</label>
<input id="artifactsBucket" type="text" ng-model="artifactsBucket" />
<span class="form_error" ng-show="GOINPUTNAME[artifactsBucket].$error.server">{{ GOINPUTNAME[artifactsBucket].$error.server }}</span>
</div>
<div class="form_item_block">
<p>
<span ng-show="artifactsBucket">The artifacts will be pushed to the bucket {{artifactsBucket}}</span>
<span ng-show="!artifactsBucket">GO_ARTIFACTS_S3_BUCKET env var must be set if the above is not configured.</span>
</p>
</div>
<div class="form_item_block">
<p class="required">Make sure GO_SERVER_DASHBOARD_URL environment variable is present with appropriate value on any of pipeline / Go Environments / on all agent machines. </p>
</div>
<script type="text/javascript">
jQuery(document).ready(function() {
Expand Down
35 changes: 35 additions & 0 deletions publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public void shouldFailIfNoFilesToUploadBasedOnSource() {
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand All @@ -83,6 +84,7 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand All @@ -100,18 +102,46 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
Map<String, String> mockVariables = mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "").build();

Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = publishExecutor.execute(config, mockContext(mockVariables));
assertFalse(result.isSuccessful());
assertThat(result.message(), is("GO_ARTIFACTS_S3_BUCKET environment variable is not set"));
}

@Test
public void shouldGetBucketFromConfig() {
AmazonS3Client mockClient = mockClient();

Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", testS3Bucket).build())
.build());

TaskExecutionResult result = executeMockPublish(
mockClient,
config,
new String[]{"README.md"},
mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "")
);

assertTrue(result.isSuccessful());
assertThat(result.message(), is("Published all artifacts to S3 successfully"));
}

@Test
public void shouldGetDisplayMessageAfterUpload() {
AmazonS3Client mockClient = mockClient();

Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand All @@ -131,6 +161,7 @@ public void shouldUploadALocalFileToS3WithDefaultPrefix() {
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand Down Expand Up @@ -173,6 +204,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefix() {
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "destinationPrefix").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand Down Expand Up @@ -204,6 +236,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefixUsingEnvVariable() {
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "test/${GO_PIPELINE_COUNTER}/").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand Down Expand Up @@ -235,6 +268,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
Config config = new Config(Maps.builder()
.with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
.with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "/").build())
.with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
.build());

TaskExecutionResult result = executeMockPublish(
Expand All @@ -258,6 +292,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
assertThat(jarPutRequest.getKey(), is("s3publish-0.1.31.jar"));
assertNull(jarPutRequest.getMetadata());
}

private TaskExecutionResult executeMockPublish(final AmazonS3Client mockClient, Config config, String[] files) {
return executeMockPublish(mockClient, config, files, mockEnvironmentVariables);
}
Expand Down
1 change: 1 addition & 0 deletions utils/src/main/java/com/indix/gocd/utils/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public class Constants {

public static final String SOURCEDESTINATIONS = "sourceDestinations";
public static final String DESTINATION_PREFIX = "destinationPrefix";
public static final String ARTIFACTS_BUCKET = "artifactsBucket";

public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
Expand Down

0 comments on commit ed4b365

Please sign in to comment.