diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java index f6b6261..f5d0c18 100644 --- a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java +++ b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchExecutor.java @@ -19,12 +19,11 @@ public class FetchExecutor { private static Logger logger = Logger.getLoggerFor(FetchTask.class); public TaskExecutionResult execute(Config config, final Context context) { - final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables()); - if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET); try { + final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables()); String artifactPathOnS3 = getArtifactsLocationTemplate(config, env); - final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET); + final String bucket = getBucket(config, env); final S3ArtifactStore store = getS3ArtifactStore(env, bucket); context.printMessage(String.format("Getting artifacts from %s", store.pathString(artifactPathOnS3))); @@ -54,6 +53,21 @@ private void setupDestinationDirectory(String destination) { } } + private String getBucket(Config config, GoEnvironment env) { + String repoName = config.getRepo(); + String packageName = config.getPkg(); + String bucketFromMaterial = env.get(String.format("GO_REPO_%s_%s_S3_BUCKET", repoName, packageName)); + if(bucketFromMaterial != null) { + return bucketFromMaterial; + } + + if(env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) { + throw new RuntimeException("S3 bucket to fetch from should be from material plugin or GO_ARTIFACTS_S3_BUCKET env var."); + } + + return env.get(GO_ARTIFACTS_S3_BUCKET); + } + private String getArtifactsLocationTemplate(Config config, GoEnvironment env) { String repoName = config.getRepo(); String packageName = config.getPkg(); diff --git a/fetch/src/main/resources/views/task.template.html b/fetch/src/main/resources/views/task.template.html index ebe44ac..fda8ac7 100644 --- a/fetch/src/main/resources/views/task.template.html +++ b/fetch/src/main/resources/views/task.template.html @@ -13,6 +13,3 @@ {{ GOINPUTNAME[Destination].$error.server }} -
-

Make sure GO_ARTIFACTS_S3_BUCKET environment variables is present with appropriate values on any of pipeline / Go Environments / on all agent machines.

-
diff --git a/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java b/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java index 1fb196f..c42c9bb 100644 --- a/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java +++ b/fetch/src/test/java/com/indix/gocd/s3fetch/FetchExecutorTest.java @@ -36,7 +36,7 @@ public void setUp() throws Exception { .with(GO_ARTIFACTS_S3_BUCKET, bucket) .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153") .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_LABEL", "20.1") - .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket) + .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_PIPELINE_NAME", "TestPublish") .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_STAGE_NAME", "defaultStage") .with("GO_PACKAGE_GOCD_TESTPUBLISHS3ARTIFACTS_JOB_NAME", "defaultJob"); @@ -85,9 +85,6 @@ public void shouldBeFailureIfUnableToFetchArtifacts() { @Test public void shouldBeSuccessResultOnSuccessfulFetch() { Map mockVariables = mockEnvironmentVariables.build(); - AmazonS3Client mockClient = mockClient(); - S3ArtifactStore store = new S3ArtifactStore(mockClient, bucket); - doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket)); S3ArtifactStore mockStore = mockStore(); doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), any(String.class)); @@ -99,6 +96,23 @@ public void shouldBeSuccessResultOnSuccessfulFetch() { } + @Test + public void shouldGetBucketInfoFromMaterialEnvVars() { + Map mockVariables = mockEnvironmentVariables + .with("GO_REPO_GOCD_TESTPUBLISHS3ARTIFACTS_S3_BUCKET", bucket) + .with(GO_ARTIFACTS_S3_BUCKET, "") + .build(); + S3ArtifactStore mockStore = mockStore(); + + doReturn(mockStore).when(fetchExecutor).getS3ArtifactStore(any(GoEnvironment.class), eq(bucket)); + TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables)); + + assertTrue(result.isSuccessful()); + assertThat(result.message(), is("Fetched all artifacts")); + verify(mockStore, times(1)).getPrefix("TestPublish/defaultStage/defaultJob/20.1", "here/artifacts"); + + } + @Test public void shouldBeAbleToHandleTaskConfigEntriesWithDashesInTheName() { Map mockVariables = mockEnvironmentVariables diff --git a/publish/src/main/java/com/indix/gocd/s3publish/Config.java b/publish/src/main/java/com/indix/gocd/s3publish/Config.java index bb6fc3a..906ab4a 100644 --- a/publish/src/main/java/com/indix/gocd/s3publish/Config.java +++ b/publish/src/main/java/com/indix/gocd/s3publish/Config.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; +import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET; import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX; import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS; @@ -16,11 +17,12 @@ public class Config { public String sourceDestinationsJson; public String destinationPrefix; - + public String artifactsBucket; public Config(Map config) { sourceDestinationsJson = getValue(config, SOURCEDESTINATIONS); destinationPrefix = getValue(config, DESTINATION_PREFIX); + artifactsBucket = getValue(config, ARTIFACTS_BUCKET); } public List sourceDestinations() throws JsonSyntaxException { diff --git a/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java b/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java index fdb7655..e603428 100644 --- a/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java +++ b/publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java @@ -33,10 +33,11 @@ public class PublishExecutor { public TaskExecutionResult execute(Config config, final Context context) { try { final GoEnvironment env = new GoEnvironment(context.getEnvironmentVariables()); - if (env.isAbsent(GO_ARTIFACTS_S3_BUCKET)) return envNotFound(GO_ARTIFACTS_S3_BUCKET); if (env.isAbsent(GO_SERVER_DASHBOARD_URL)) return envNotFound(GO_SERVER_DASHBOARD_URL); - final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET); + String bucket = getBucket(env, config); + if(bucket == null) return envNotFound(GO_ARTIFACTS_S3_BUCKET); + final S3ArtifactStore store = getS3ArtifactStore(env, bucket); store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD)); @@ -69,6 +70,15 @@ public TaskExecutionResult execute(Config config, final Context context) { } } + private String getBucket(GoEnvironment env, Config config) { + if(StringUtils.isNotBlank(config.artifactsBucket)) { + return config.artifactsBucket; + } else if(env.has(GO_ARTIFACTS_S3_BUCKET)) { + return env.get(GO_ARTIFACTS_S3_BUCKET); + } + return null; + } + protected S3ArtifactStore getS3ArtifactStore(GoEnvironment env, String bucket) { return new S3ArtifactStore(env, bucket); } diff --git a/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java b/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java index e475b76..d0e88ff 100644 --- a/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java +++ b/publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import static com.indix.gocd.utils.Constants.ARTIFACTS_BUCKET; import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX; import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS; import static com.indix.gocd.utils.utils.Lists.foreach; @@ -110,6 +111,7 @@ public void execute(SourceDestination input) { private GoPluginApiResponse handleGetConfigRequest() { HashMap config = new HashMap(); + HashMap sourceDestinations = new HashMap(); sourceDestinations.put("default-value", ""); sourceDestinations.put("required", true); @@ -120,6 +122,11 @@ private GoPluginApiResponse handleGetConfigRequest() { destinationPrefix.put("required", false); config.put(DESTINATION_PREFIX, destinationPrefix); + HashMap artifactsBucket = new HashMap(); + artifactsBucket.put("default-value", ""); + artifactsBucket.put("required", false); + config.put(ARTIFACTS_BUCKET, artifactsBucket); + return createResponse(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE, config); } diff --git a/publish/src/main/resources/views/task.template.html b/publish/src/main/resources/views/task.template.html index bc551af..8428375 100644 --- a/publish/src/main/resources/views/task.template.html +++ b/publish/src/main/resources/views/task.template.html @@ -37,7 +37,18 @@

-

Make sure GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines.

+ + + {{ GOINPUTNAME[artifactsBucket].$error.server }} +
+
+

+ The artifacts will be pushed to the bucket {{artifactsBucket}} + GO_ARTIFACTS_S3_BUCKET env var must be set if the above is not configured. +

+
+
+

Make sure GO_SERVER_DASHBOARD_URL environment variable is present with appropriate value on any of pipeline / Go Environments / on all agent machines.