From f78879b43663cbe55573aceaf88ae05024f4065c Mon Sep 17 00:00:00 2001
From: Manoj <manojlds@gmail.com>
Date: Thu, 13 Jul 2017 17:27:21 +0530
Subject: [PATCH 1/2] Update publish plugin to support multiple buckets

---
 .../java/com/indix/gocd/s3publish/Config.java |  4 ++-
 .../indix/gocd/s3publish/PublishExecutor.java | 14 ++++++--
 .../com/indix/gocd/s3publish/PublishTask.java |  7 ++++
 .../main/resources/views/task.template.html   | 13 ++++++-
 .../gocd/s3publish/PublishExecutorTest.java   | 35 +++++++++++++++++++
 .../java/com/indix/gocd/utils/Constants.java  |  1 +
 6 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/publish/src/main/java/com/indix/gocd/s3publish/Config.java b/publish/src/main/java/com/indix/gocd/s3publish/Config.java
index bb6fc3a..906ab4a 100644
--- a/publish/src/main/java/com/indix/gocd/s3publish/Config.java
+++ b/publish/src/main/java/com/indix/gocd/s3publish/Config.java
@@ -9,6 +9,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 
@@ -16,11 +17,12 @@ public class Config {
 
     public String sourceDestinationsJson;
     public String destinationPrefix;
-
+    public String artifactsBucket;
 
     public Config(Map config) {
         sourceDestinationsJson  = getValue(config, SOURCEDESTINATIONS);
         destinationPrefix  = getValue(config, DESTINATION_PREFIX);
+        artifactsBucket  = getValue(config, ARTIFACTS_BUCKET);
     }
 
     public List<SourceDestination> sourceDestinations() throws JsonSyntaxException {
diff --git a/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java b/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java
index fdb7655..e603428 100644
--- a/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java
+++ b/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java
@@ -33,10 +33,11 @@ public class PublishExecutor {
     public TaskExecutionResult execute(Config config, final Context context) {
         try {
             final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
-            if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
             if (env.isAbsent(GO_SERVER_DASHBOARD_URL)) return envNotFound(GO_SERVER_DASHBOARD_URL);
 
-            final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+            String bucket = getBucket(env, config);
+            if(bucket == null) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
+
             final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
             store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD));
 
@@ -69,6 +70,15 @@ public TaskExecutionResult execute(Config config, final Context context) {
         }
     }
 
+    private String getBucket(GoEnvironment env, Config config) {
+        if(StringUtils.isNotBlank(config.artifactsBucket)) {
+            return config.artifactsBucket;
+        } else if(env.has(GO_ARTIFACTS_S3_BUCKET)) {
+            return env.get(GO_ARTIFACTS_S3_BUCKET);
+        }
+        return null;
+    }
+
     protected S3ArtifactStore getS3ArtifactStore(GoEnvironment env, String bucket) {
         return new S3ArtifactStore(env, bucket);
     }
diff --git a/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java b/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java
index e475b76..d0e88ff 100644
--- a/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java
+++ b/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET;
 import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
 import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
 import static com.indix.gocd.utils.utils.Lists.foreach;
@@ -110,6 +111,7 @@ public void execute(SourceDestination input) {
 
     private GoPluginApiResponse handleGetConfigRequest() {
         HashMap config = new HashMap();
+
         HashMap sourceDestinations = new HashMap();
         sourceDestinations.put("default-value", "");
         sourceDestinations.put("required", true);
@@ -120,6 +122,11 @@ private GoPluginApiResponse handleGetConfigRequest() {
         destinationPrefix.put("required", false);
         config.put(DESTINATION_PREFIX, destinationPrefix);
 
+        HashMap artifactsBucket = new HashMap();
+        artifactsBucket.put("default-value", "");
+        artifactsBucket.put("required", false);
+        config.put(ARTIFACTS_BUCKET, artifactsBucket);
+
         return createResponse(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE, config);
     }
 
diff --git a/publish/src/main/resources/views/task.template.html b/publish/src/main/resources/views/task.template.html
index bc551af..8428375 100644
--- a/publish/src/main/resources/views/task.template.html
+++ b/publish/src/main/resources/views/task.template.html
@@ -37,7 +37,18 @@
     </p>
 </div>
 <div class="form_item_block">
-    <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
+    <label for="artifactsBucket">Artifacts Bucket</label>
+    <input id="artifactsBucket" type="text" ng-model="artifactsBucket" />
+    <span class="form_error" ng-show="GOINPUTNAME[artifactsBucket].$error.server">{{ GOINPUTNAME[artifactsBucket].$error.server }}</span>
+</div>
+<div class="form_item_block">
+    <p>
+        <span ng-show="artifactsBucket">The artifacts will be pushed to the bucket {{artifactsBucket}}</span>
+        <span ng-show="!artifactsBucket">GO_ARTIFACTS_S3_BUCKET env var must be set if the above is not configured.</span>
+    </p>
+</div>
+<div class="form_item_block">
+    <p class="required">Make sure GO_SERVER_DASHBOARD_URL environment variable is present with appropriate value on any of pipeline / Go Environments / on all agent machines. </p>
 </div>
 <script type="text/javascript">
     jQuery(document).ready(function() {
diff --git a/publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java b/publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java
index 55a4ae3..b3c9d67 100644
--- a/publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java
+++ b/publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java
@@ -60,6 +60,7 @@ public void shouldFailIfNoFilesToUploadBasedOnSource() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -83,6 +84,7 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -100,11 +102,38 @@ public void shouldNotThrowIfAWSUseIAMRoleIsTrueAndAWS_SECRET_ACCESS_KEYNotPresen
     public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
         Map<String, String> mockVariables = mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "").build();
 
+        Config config = new Config(Maps.builder()
+                .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+                .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
+                .build());
+
         TaskExecutionResult result = publishExecutor.execute(config, mockContext(mockVariables));
         assertFalse(result.isSuccessful());
         assertThat(result.message(), is("GO_ARTIFACTS_S3_BUCKET environment variable is not set"));
     }
 
+    @Test
+    public void shouldGetBucketFromConfig() {
+        AmazonS3Client mockClient = mockClient();
+
+        Config config = new Config(Maps.builder()
+                .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
+                .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", testS3Bucket).build())
+                .build());
+
+        TaskExecutionResult result = executeMockPublish(
+                mockClient,
+                config,
+                new String[]{"README.md"},
+                mockEnvironmentVariables.with(GO_ARTIFACTS_S3_BUCKET, "")
+        );
+
+        assertTrue(result.isSuccessful());
+        assertThat(result.message(), is("Published all artifacts to S3 successfully"));
+    }
+
     @Test
     public void shouldGetDisplayMessageAfterUpload() {
         AmazonS3Client mockClient = mockClient();
@@ -112,6 +141,7 @@ public void shouldGetDisplayMessageAfterUpload() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -131,6 +161,7 @@ public void shouldUploadALocalFileToS3WithDefaultPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -173,6 +204,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "destinationPrefix").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -204,6 +236,7 @@ public void shouldUploadALocalFileToS3WithDestinationPrefixUsingEnvVariable() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "test/${GO_PIPELINE_COUNTER}/").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -235,6 +268,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
         Config config = new Config(Maps.builder()
                 .with(Constants.SOURCEDESTINATIONS, Maps.builder().with("value", "[{\"source\": \"target/*\", \"destination\": \"\"}]").build())
                 .with(Constants.DESTINATION_PREFIX, Maps.builder().with("value", "/").build())
+                .with(Constants.ARTIFACTS_BUCKET, Maps.builder().with("value", "").build())
                 .build());
 
         TaskExecutionResult result = executeMockPublish(
@@ -258,6 +292,7 @@ public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
         assertThat(jarPutRequest.getKey(), is("s3publish-0.1.31.jar"));
         assertNull(jarPutRequest.getMetadata());
     }
+
     private TaskExecutionResult executeMockPublish(final AmazonS3Client mockClient, Config config, String[] files) {
         return executeMockPublish(mockClient, config, files, mockEnvironmentVariables);
     }
diff --git a/utils/src/main/java/com/indix/gocd/utils/Constants.java b/utils/src/main/java/com/indix/gocd/utils/Constants.java
index 08916e3..b95e0cc 100644
--- a/utils/src/main/java/com/indix/gocd/utils/Constants.java
+++ b/utils/src/main/java/com/indix/gocd/utils/Constants.java
@@ -10,6 +10,7 @@ public class Constants {
 
     public static final String SOURCEDESTINATIONS = "sourceDestinations";
     public static final String DESTINATION_PREFIX = "destinationPrefix";
+    public static final String ARTIFACTS_BUCKET = "artifactsBucket";
 
     public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
     public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";

From 7dc98bd03945f23e9876ab8f03c0c2477acd5ede Mon Sep 17 00:00:00 2001
From: Manoj <manojlds@gmail.com>
Date: Thu, 13 Jul 2017 22:30:24 +0530
Subject: [PATCH 2/2] Fetch task to get bucket info from material plugin or env
 var

---
 .../com/indix/gocd/s3fetch/FetchExecutor.java | 20 ++++++++++++++---
 .../main/resources/views/task.template.html   |  3 ---
 .../indix/gocd/s3fetch/FetchExecutorTest.java | 22 +++++++++++++++----
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java
index f6b6261..f5d0c18 100644
--- a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java
+++ b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java
@@ -19,12 +19,11 @@ public class FetchExecutor {
     private static Logger logger = Logger.getLoggerFor(FetchTask.class);
 
     public TaskExecutionResult execute(Config config, final Context context) {
-        final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
-        if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET);
 
         try {
+            final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables());
             String artifactPathOnS3 = getArtifactsLocationTemplate(config, env);
-            final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
+            final String bucket = getBucket(config, env);
             final S3ArtifactStore store = getS3ArtifactStore(env, bucket);
 
             context.printMessage(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3)));
@@ -54,6 +53,21 @@ private void setupDestinationDirectory(String destination) {
         }
     }
 
+    private String getBucket(Config config, GoEnvironment env) {
+        String repoName = config.getRepo();
+        String packageName = config.getPkg();
+        String bucketFromMaterial = env.get(String.format("GO_REPO_%s_%s_S3_BUCKET", repoName, packageName));
+        if(bucketFromMaterial != null) {
+            return bucketFromMaterial;
+        }
+
+        if(env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) {
+            throw new RuntimeException("S3 bucket to fetch from should be from material plugin or GO_ARTIFACTS_S3_BUCKET env var.");
+        }
+
+        return env.get(GO_ARTIFACTS_S3_BUCKET);
+    }
+
     private String getArtifactsLocationTemplate(Config config, GoEnvironment env) {
         String repoName = config.getRepo();
         String packageName = config.getPkg();
diff --git a/fetch/src/main/resources/views/task.template.html b/fetch/src/main/resources/views/task.template.html
index ebe44ac..fda8ac7 100644
--- a/fetch/src/main/resources/views/task.template.html
+++ b/fetch/src/main/resources/views/task.template.html
@@ -13,6 +13,3 @@
   <input type="text" ng-model="Destination" ng-required="true">
   <span class="form_error" ng-show="GOINPUTNAME[Destination].$error.server">{{ GOINPUTNAME[Destination].$error.server }}</span>
 </div>
-<div class="form_item_block">
-    <p class="required">Make sure GO_ARTIFACTS_S3_BUCKET environment variables is present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
-</div>
diff --git a/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java b/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java
index 1fb196f..c42c9bb 100644
--- a/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java
+++ b/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java
@@ -36,7 +36,7 @@ public void setUp() throws Exception {
                 .with(GO_ARTIFACTS_S3_BUCKET, bucket)
                 .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1")
-                .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage")
                 .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob");
@@ -85,9 +85,6 @@ public void shouldBeFailureIfUnableToFetchArtifacts() {
     @Test
     public void shouldBeSuccessResultOnSuccessfulFetch() {
         Map<String, String> mockVariables = mockEnvironmentVariables.build();
-        AmazonS3Client mockClient = mockClient();
-        S3ArtifactStore store = new S3ArtifactStore(mockClient, bucket);
-        doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
         S3ArtifactStore mockStore = mockStore();
 
         doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), any(String.class));
@@ -99,6 +96,23 @@ public void shouldBeSuccessResultOnSuccessfulFetch() {
 
     }
 
+    @Test
+    public void shouldGetBucketInfoFromMaterialEnvVars() {
+        Map<String, String> mockVariables = mockEnvironmentVariables
+                .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket)
+                .with(GO_ARTIFACTS_S3_BUCKET, "")
+                .build();
+        S3ArtifactStore mockStore = mockStore();
+
+        doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket));
+        TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables));
+
+        assertTrue(result.isSuccessful());
+        assertThat(result.message(), is("Fetched all artifacts"));
+        verify(mockStore, times(1)).getPrefix("TestPublish/defaultStage/defaultJob/20.1", "here/artifacts");
+
+    }
+
     @Test
     public void shouldBeAbleToHandleTaskConfigEntriesWithDashesInTheName() {
         Map<String, String> mockVariables = mockEnvironmentVariables