Multiple bucket support

fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java

@@ -19,12 +19,11 @@ public class FetchExecutor {
     private static Logger logger = Logger.getLoggerFor(FetchTask.class);
 
     public TaskExecutionResult execute(Config config, final Context context) {
-        final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
-        if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
 
         try {
+            final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
             String artifactPathOnS3 = getArtifactsLocationTemplate(config, env);
-            final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+            final String bucket = getBucket(config, env);
             final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
 
             context.printMessage(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3)));
@@ -54,6 +53,21 @@ private void setupDestinationDirectory(String destination) {
         }
     }
 
+    private String getBucket(Config config, GoEnvironment env) {
+        String repoName = config.getRepo();
+        String packageName = config.getPkg();
+        String bucketFromMaterial = env.get(String.format("GO_REPO_%s_%s_S3_BUCKET", repoName, packageName));
+        if(bucketFromMaterial != null) {
+            return bucketFromMaterial;
+        }
+
+        if(env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) {
+            throw new RuntimeException("S3 bucket to fetch from should be from material plugin or GO_ARTIFACTS_S3_BUCKET env var.");
+        }
+
+        return env.get(GO_ARTIFACTS_S3_BUCKET);
+    }
+
     private String getArtifactsLocationTemplate(Config config, GoEnvironment env) {
         String repoName = config.getRepo();
         String packageName = config.getPkg();

fetch/src/main/resources/views/task.template.html

@@ -13,6 +13,3 @@
   <input type="text" ng-model="Destination" ng-required="true">
   <span class="form_error" ng-show="GOINPUTNAME[Destination].$error.server">{{ GOINPUTNAME[Destination].$error.server }}</span>
 </div>
-<div class="form_item_block">
-    <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET environment variables is present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
-</div>

fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java

@@ -36,7 +36,7 @@ public void setUp() throws Exception {
                 .with(GO_ARTIFACTS_S3_BUCKET, bucket)
                 .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1")
-                .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob");
@@ -85,9 +85,6 @@ public void shouldBeFailureIfUnableToFetchArtifacts() {
     @Test
     public void shouldBeSuccessResultOnSuccessfulFetch() {
         Map<String, String> mockVariables = mockEnvironmentVariables.build();
-        AmazonS3Client mockClient = mockClient();
-        S3ArtifactStore store = new S3ArtifactStore(mockClient, bucket);
-        doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
         S3ArtifactStore mockStore = mockStore();
 
         doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), any(String.class));
@@ -99,6 +96,23 @@ public void shouldBeSuccessResultOnSuccessfulFetch() {
 
     }
 
+    @Test
+    public void shouldGetBucketInfoFromMaterialEnvVars() {
+        Map<String, String> mockVariables = mockEnvironmentVariables
+                .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+                .with(GO_ARTIFACTS_S3_BUCKET, "")
+                .build();
+        S3ArtifactStore mockStore = mockStore();
+
+        doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
+        TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables));
+
+        assertTrue(result.isSuccessful());
+        assertThat(result.message(), is("Fetched all artifacts"));
+        verify(mockStore, times(1)).getPrefix("TestPublish/defaultStage/defaultJob/20.1", "here/artifacts");
+
+    }
+
     @Test
     public void shouldBeAbleToHandleTaskConfigEntriesWithDashesInTheName() {
         Map<String, String> mockVariables = mockEnvironmentVariables

publish/src/main/java/com/indix/gocd/s3publish/Config.java

@@ -9,18 +9,20 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 
 public class Config {
 
     public String sourceDestinationsJson;
     public String destinationPrefix;
-
+    public String artifactsBucket;
 
     public Config(Map config) {
         sourceDestinationsJson  = getValue(config, SOURCEDESTINATIONS);
         destinationPrefix  = getValue(config, DESTINATION_PREFIX);
+        artifactsBucket  = getValue(config, ARTIFACTS_BUCKET);
     }
 
     public List<SourceDestination> sourceDestinations() throws JsonSyntaxException {

publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java

@@ -33,10 +33,11 @@ public class PublishExecutor {
     public TaskExecutionResult execute(Config config, final Context context) {
         try {
             final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
-            if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
             if (env.isAbsent(GO_SERVER_DASHBOARD_URL)) return envNotFound(GO_SERVER_DASHBOARD_URL);
 
-            final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+            String bucket = getBucket(env, config);
+            if(bucket == null) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
+
             final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
             store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD));
 
@@ -69,6 +70,15 @@ public TaskExecutionResult execute(Config config, final Context context) {
         }
     }
 
+    private String getBucket(GoEnvironment env, Config config) {
+        if(StringUtils.isNotBlank(config.artifactsBucket)) {
+            return config.artifactsBucket;
+        } else if(env.has(GO_ARTIFACTS_S3_BUCKET)) {
+            return env.get(GO_ARTIFACTS_S3_BUCKET);
+        }
+        return null;
+    }
+
     protected S3ArtifactStore getS3ArtifactStore(GoEnvironment env, String bucket) {
         return new S3ArtifactStore(env, bucket);
     }

publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 import static com.indix.gocd.utils.utils.Lists.foreach;
@@ -110,6 +111,7 @@ public void execute(SourceDestination input) {
 
     private GoPluginApiResponse handleGetConfigRequest() {
         HashMap config = new HashMap();
+
         HashMap sourceDestinations = new HashMap();
         sourceDestinations.put("default-value", "");
         sourceDestinations.put("required", true);
@@ -120,6 +122,11 @@ private GoPluginApiResponse handleGetConfigRequest() {
         destinationPrefix.put("required", false);
         config.put(DESTINATION_PREFIX, destinationPrefix);
 
+        HashMap artifactsBucket = new HashMap();
+        artifactsBucket.put("default-value", "");
+        artifactsBucket.put("required", false);
+        config.put(ARTIFACTS_BUCKET, artifactsBucket);
+
         return createResponse(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE, config);
     }
 

publish/src/main/resources/views/task.template.html

@@ -37,7 +37,18 @@
     </p>
 </div>
 <div class="form_item_block">
-    <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
+    <label for="artifactsBucket">Artifacts Bucket</label>
+    <input id="artifactsBucket" type="text" ng-model="artifactsBucket" />
+    <span class="form_error" ng-show="GOINPUTNAME[artifactsBucket].$error.server">{{ GOINPUTNAME[artifactsBucket].$error.server }}</span>
+</div>
+<div class="form_item_block">
+    <p>
+        <span ng-show="artifactsBucket">The artifacts will be pushed to the bucket {{artifactsBucket}}</span>
+        <span ng-show="!artifactsBucket">GO_ARTIFACTS_S3_BUCKET env var must be set if the above is not configured.</span>
+    </p>
+</div>
+<div class="form_item_block">
+    <p class="required">Make sure GO_SERVER_DASHBOARD_URL environment variable is present with appropriate value on any of pipeline / Go Environments / on all agent machines. </p>
 </div>
 <script type="text/javascript">
     jQuery(document).ready(function() {

publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java

@@ -60,6 +60,7 @@ public void shouldFailIfNoFilesToUploadBasedOnSource() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -83,6 +84,7 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -100,18 +102,46 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
     public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
         Map<String, String> mockVariables = mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "").build();
 
+        Config config = new Config(Maps.builder()
+                .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+                .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
+                .build());
+
         TaskExecutionResult result = publishExecutor.execute(config, mockContext(mockVariables));
         assertFalse(result.isSuccessful());
         assertThat(result.message(), is("GO_ARTIFACTS_S3_BUCKET environment variable is not set"));
     }
 
+    @Test
+    public void shouldGetBucketFromConfig() {
+        AmazonS3Client mockClient = mockClient();
+
+        Config config = new Config(Maps.builder()
+                .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+                .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", testS3Bucket).build())
+                .build());
+
+        TaskExecutionResult result = executeMockPublish(
+                mockClient,
+                config,
+                new String[]{"README.md"},
+                mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "")
+        );
+
+        assertTrue(result.isSuccessful());
+        assertThat(result.message(), is("Published all artifacts to S3 successfully"));
+    }
+
     @Test
     public void shouldGetDisplayMessageAfterUpload() {
         AmazonS3Client mockClient = mockClient();
 
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -131,6 +161,7 @@ public void shouldUploadALocalFileToS3WithDefaultPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -173,6 +204,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "destinationPrefix").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -204,6 +236,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefixUsingEnvVariable() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "test/${GO_PIPELINE_COUNTER}/").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -235,6 +268,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "/").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -258,6 +292,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
         assertThat(jarPutRequest.getKey(), is("s3publish-0.1.31.jar"));
         assertNull(jarPutRequest.getMetadata());
     }
+
     private TaskExecutionResult executeMockPublish(final AmazonS3Client mockClient, Config config, String[] files) {
         return executeMockPublish(mockClient, config, files, mockEnvironmentVariables);
     }

utils/src/main/java/com/indix/gocd/utils/Constants.java

@@ -10,6 +10,7 @@ public class Constants {
 
     public static final String SOURCEDESTINATIONS = "sourceDestinations";
     public static final String DESTINATION_PREFIX = "destinationPrefix";
+    public static final String ARTIFACTS_BUCKET = "artifactsBucket";
 
     public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
     public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";