diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/Config.java b/fetch/src/main/java/com/indix/gocd/s3fetch/Config.java index 0551e6e..0d02dd3 100644 --- a/fetch/src/main/java/com/indix/gocd/s3fetch/Config.java +++ b/fetch/src/main/java/com/indix/gocd/s3fetch/Config.java @@ -6,14 +6,19 @@ public class Config { - public String materialType; - public String repo; - public String pkg; - public String material; - public String job; - public String destination; + private final String materialType; + private final String repo; + private final String pkg; + private final String material; + private final String job; + private final String stage; + private final String source; + private final String sourcePrefix; + private final String destination; - public String getMaterialType() { return materialType; } + public String getMaterialType() { + return materialType; + } public String getRepo() { return escapeEnvironmentVariable(repo); @@ -23,9 +28,25 @@ public String getPkg() { return escapeEnvironmentVariable(pkg); } - public String getMaterial() { return escapeEnvironmentVariable(material); } + public String getMaterial() { + return escapeEnvironmentVariable(material); + } - public String getJob() { return job; } + public String getJob() { + return job; + } + + public String getStage() { + return stage; + } + + public String getSource() { + return source; + } + + public String getSourcePrefix() { + return sourcePrefix; + } public String getDestination() { return destination; @@ -37,6 +58,9 @@ public Config(Map config) { pkg = getValue(config, PACKAGE); material = getValue(config, MATERIAL); job = getValue(config, JOB); + stage = getValue(config, STAGE); + source = getValue(config, SOURCE); + sourcePrefix = getValue(config, SOURCE_PREFIX); destination = getValue(config, DESTINATION); } diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchTask.java b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchTask.java index 53dabce..efaa185 100644 --- a/fetch/src/main/java/com/indix/gocd/s3fetch/FetchTask.java +++ b/fetch/src/main/java/com/indix/gocd/s3fetch/FetchTask.java @@ -61,6 +61,8 @@ private FetchExecutor getFetchExecutor(Config config) { return new PackageFetchExecutor(); } else if (materialType.equals("Pipeline")) { return new PipelineFetchExecutor(); + } else if (materialType.equals("Self")) { + return new SelfFetchExecutor(); } else { throw new IllegalStateException("No such material type: " + materialType); } @@ -123,6 +125,21 @@ private GoPluginApiResponse handleGetConfigRequest() { job.put("required", false); config.put(Constants.JOB, job); + HashMap stage = new HashMap(); + stage.put("default-value", ""); + stage.put("required", false); + config.put(Constants.STAGE, stage); + + HashMap source = new HashMap(); + source.put("default-value", ""); + source.put("required", false); + config.put(Constants.SOURCE, source); + + HashMap sourcePrefix = new HashMap(); + sourcePrefix.put("default-value", ""); + sourcePrefix.put("required", false); + config.put(Constants.SOURCE_PREFIX, sourcePrefix); + HashMap destination = new HashMap(); destination.put("default-value", ""); destination.put("required", false); diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/PipelineFetchExecutor.java b/fetch/src/main/java/com/indix/gocd/s3fetch/PipelineFetchExecutor.java index 2d31039..b88f200 100644 --- a/fetch/src/main/java/com/indix/gocd/s3fetch/PipelineFetchExecutor.java +++ b/fetch/src/main/java/com/indix/gocd/s3fetch/PipelineFetchExecutor.java @@ -8,6 +8,7 @@ import java.util.Map; public class PipelineFetchExecutor extends FetchExecutor { + @Override protected String getArtifactsLocationTemplate(Config config, GoEnvironment env) { String materialLocator = env.get(String.format("GO_DEPENDENCY_LOCATOR_%s", config.getMaterial())); diff --git a/fetch/src/main/java/com/indix/gocd/s3fetch/SelfFetchExecutor.java b/fetch/src/main/java/com/indix/gocd/s3fetch/SelfFetchExecutor.java new file mode 100644 index 0000000..05ac6ad --- /dev/null +++ b/fetch/src/main/java/com/indix/gocd/s3fetch/SelfFetchExecutor.java @@ -0,0 +1,63 @@ +package com.indix.gocd.s3fetch; + +import com.indix.gocd.utils.Constants; +import com.indix.gocd.utils.GoEnvironment; +import com.indix.gocd.utils.store.S3ArtifactStore; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +public class SelfFetchExecutor extends FetchExecutor { + + @Override + protected String getArtifactsLocationTemplate(Config config, GoEnvironment env) { + + String prefix = config.getSourcePrefix(); + String source = config.getSource(); + + if (StringUtils.isBlank(prefix)) { + String pipeline = env.get("GO_PIPELINE_NAME"); + String pipelineCounter = env.get("GO_PIPELINE_COUNTER"); + String stage = config.getStage(); + String job = config.getJob(); + String bucket = getBucket(config, env); + + final S3ArtifactStore store = getS3ArtifactStore(env, bucket); + prefix = store.getLatestPrefix(pipeline, stage, job, pipelineCounter); + + if (StringUtils.isBlank(prefix)) { + throw new RuntimeException( + String.format("Could not determine stage counter on s3 with path: s3://%s/%s/%s/%s/%s.", + bucket, + pipeline, + stage, + job, + pipelineCounter)); + } + } + + return prefix + "/" + source; + } + + @Override + public Map validate(Config config) { + Map errors = new HashMap<>(); + + if (StringUtils.isBlank(config.getSourcePrefix())) { + + if (StringUtils.isBlank(config.getStage())) { + errors.put(Constants.STAGE, Constants.REQUIRED_FIELD_MESSAGE); + } + if (StringUtils.isBlank(config.getJob())) { + errors.put(Constants.JOB, Constants.REQUIRED_FIELD_MESSAGE); + } + } + + if (StringUtils.isBlank(config.getSource())) { + errors.put(Constants.SOURCE, Constants.REQUIRED_FIELD_MESSAGE); + } + + return errors; + } +} diff --git a/fetch/src/main/resources/views/task.template.html b/fetch/src/main/resources/views/task.template.html index 69009bf..5343c83 100644 --- a/fetch/src/main/resources/views/task.template.html +++ b/fetch/src/main/resources/views/task.template.html @@ -3,6 +3,7 @@ @@ -30,6 +31,36 @@ {{ GOINPUTNAME[Job].$error.server }} + +
+ + +
+
+ + + {{ GOINPUTNAME[Stage].$error.server }} +
+
+ + + {{ GOINPUTNAME[Job].$error.server }} +
+
+ + + {{ GOINPUTNAME[SourcePrefix].$error.server }} +
+
+ + + {{ GOINPUTNAME[Source].$error.server }} +
+ +
@@ -42,11 +73,19 @@ try { var $scope = angular.element(document.getElementById("task_angular_pluggable_task_indix_s3fetch")).scope(); $scope.$apply(function() { + + $scope.clearStageFields = function() { + this.Job = null; + this.Stage = null; + this.Source = null; + this.SourcePrefix = null; + } + $scope.clearFields = function() { this.Repo = null; this.Package = null; this.Material = null; - this.Job = null; + this.clearStageFields(); }; }); diff --git a/fetch/src/test/java/com/indix/gocd/s3fetch/PackageFetchExecutorTest.java b/fetch/src/test/java/com/indix/gocd/s3fetch/PackageFetchExecutorTest.java index e078f4f..0683835 100644 --- a/fetch/src/test/java/com/indix/gocd/s3fetch/PackageFetchExecutorTest.java +++ b/fetch/src/test/java/com/indix/gocd/s3fetch/PackageFetchExecutorTest.java @@ -197,4 +197,4 @@ private Context mockContext(final Map environmentMap) { private S3ArtifactStore mockStore() { return mock(S3ArtifactStore.class); } private AmazonS3Client mockClient() { return mock(AmazonS3Client.class); } -} \ No newline at end of file +} diff --git a/fetch/src/test/java/com/indix/gocd/s3fetch/SelfFetchExecutorTest.java b/fetch/src/test/java/com/indix/gocd/s3fetch/SelfFetchExecutorTest.java new file mode 100644 index 0000000..6f1389f --- /dev/null +++ b/fetch/src/test/java/com/indix/gocd/s3fetch/SelfFetchExecutorTest.java @@ -0,0 +1,97 @@ +package com.indix.gocd.s3fetch; + +import com.indix.gocd.utils.Constants; +import com.indix.gocd.utils.Context; +import com.indix.gocd.utils.TaskExecutionResult; +import com.indix.gocd.utils.mocks.MockContext; +import com.indix.gocd.utils.store.S3ArtifactStore; +import com.indix.gocd.utils.utils.Maps; +import org.junit.Before; +import org.junit.Test; + +import java.util.Map; + +import static com.indix.gocd.utils.Constants.*; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +public class SelfFetchExecutorTest { + + private Maps.MapBuilder mockEnvironmentVariables; + private FetchExecutor fetchExecutor; + private Config config; + private S3ArtifactStore store; + + private final String PIPELINE = "pipeline"; + private final String PIPELINE_COUNTER = "1"; + private final String STAGE = "stage"; + private final String JOB = "job"; + + @Before + public void setUp() throws Exception { + mockEnvironmentVariables = Maps.builder() + .with(AWS_SECRET_ACCESS_KEY, "secretKey") + .with(AWS_ACCESS_KEY_ID, "accessId") + .with(GO_ARTIFACTS_S3_BUCKET, "bucket") + .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153") + .with("GO_PIPELINE_NAME", PIPELINE) + .with("GO_PIPELINE_COUNTER", PIPELINE_COUNTER); + + config = new Config(Maps.builder() + .with(Constants.STAGE, Maps.builder().with("value", STAGE).build()) + .with(Constants.JOB, Maps.builder().with("value", JOB).build()) + .with(Constants.SOURCE, Maps.builder().with("value", "source").build()) + .with(Constants.DESTINATION, Maps.builder().with("value", "artifacts").build()) + .build()); + + store = mock(S3ArtifactStore.class); + fetchExecutor = spy(new SelfFetchExecutor()); + doReturn(store).when(fetchExecutor).getS3ArtifactStore(any(), any()); + } + + @Test + public void shouldBeFailureIfCouldntFindS3Path() { + Map mockVariables = mockEnvironmentVariables.build(); + doReturn(null).when(store).getLatestPrefix(PIPELINE, STAGE, JOB, PIPELINE_COUNTER); + TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables) ); + + assertFalse(result.isSuccessful()); + assertEquals("Failure while downloading artifacts - Could not determine stage counter on s3 with path: s3://bucket/pipeline/stage/job/1.", result.message()); + } + + @Test + public void shouldBeSuccessWhenAbleToFindSS3Path() { + Map mockVariables = mockEnvironmentVariables.build(); + doReturn("sourcePrefix").when(store).getLatestPrefix(PIPELINE, STAGE, JOB, PIPELINE_COUNTER); + TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables) ); + + assertTrue(result.isSuccessful()); + assertThat(result.message(), is("Fetched all artifacts")); + verify(store).getPrefix("sourcePrefix/source", "here/artifacts"); + } + + @Test + public void shouldBeSuccessWhenCustomPrefixProvided() { + Map mockVariables = mockEnvironmentVariables.build(); + config = new Config(Maps.builder() + .with(Constants.SOURCE, Maps.builder().with("value", "source").build()) + .with(Constants.SOURCE_PREFIX, Maps.builder().with("value", "sourcePrefix").build()) + .with(Constants.DESTINATION, Maps.builder().with("value", "artifacts").build()) + .build()); + TaskExecutionResult result = fetchExecutor.execute(config, mockContext(mockVariables) ); + + assertTrue(result.isSuccessful()); + assertThat(result.message(), is("Fetched all artifacts")); + verify(store).getPrefix("sourcePrefix/source", "here/artifacts"); + } + + private Context mockContext(final Map environmentMap) { + Map contextMap = Maps.builder() + .with("environmentVariables", environmentMap) + .with("workingDirectory", "here") + .build(); + return new MockContext(contextMap); + } +} diff --git a/utils/src/main/java/com/indix/gocd/utils/Constants.java b/utils/src/main/java/com/indix/gocd/utils/Constants.java index 4e64187..25ecd82 100644 --- a/utils/src/main/java/com/indix/gocd/utils/Constants.java +++ b/utils/src/main/java/com/indix/gocd/utils/Constants.java @@ -29,6 +29,9 @@ public class Constants { public static final String PACKAGE = "Package"; public static final String MATERIAL = "Material"; public static final String JOB = "Job"; + public static final String STAGE = "Stage"; + public static final String SOURCE = "Source"; + public static final String SOURCE_PREFIX = "SourcePrefix"; public static final String DESTINATION = "Destination"; public static final String REQUIRED_FIELD_MESSAGE = "This field is required"; diff --git a/utils/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java b/utils/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java index 497b646..ac8bbd7 100644 --- a/utils/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java +++ b/utils/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java @@ -197,10 +197,40 @@ public RevisionStatus getLatest(Artifact artifact) { return null; } + public String getLatestPrefix(String pipeline, String stage, String job, String pipelineCounter) { + String prefix = String.format("%s/%s/%s/%s.", pipeline, stage, job, pipelineCounter); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucket) + .withPrefix(prefix) + .withDelimiter("/"); + + ObjectListing listing = client.listObjects(listObjectsRequest); + + if (listing != null) { + List commonPrefixes = listing.getCommonPrefixes(); + List stageCounters = Lists.map(commonPrefixes, + input -> + input.replaceAll(prefix, "").replaceAll("/", "")); + if (stageCounters.size() > 0) { + int maxStageCounter = Integer.valueOf(stageCounters.get(0)); + + for (int i = 1; i < stageCounters.size(); i++) { + int stageCounter = Integer.valueOf(stageCounters.get(i)); + if (stageCounter > maxStageCounter) { + maxStageCounter = stageCounter; + } + } + + return prefix + maxStageCounter; + } + } + return null; + } + public static AmazonS3 getS3client(GoEnvironment env) { AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard(); - if(env.has(AWS_REGION)) { + if (env.has(AWS_REGION)) { amazonS3ClientBuilder.withRegion(env.get(AWS_REGION)); } if (env.hasAWSUseIamRole()) { diff --git a/utils/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java b/utils/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java index a383549..c85e5ca 100644 --- a/utils/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java +++ b/utils/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java @@ -4,18 +4,30 @@ import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.PutObjectRequest; +import com.indix.gocd.utils.GoEnvironment; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import static com.indix.gocd.utils.Constants.AWS_ACCESS_KEY_ID; +import static com.indix.gocd.utils.Constants.AWS_REGION; +import static com.indix.gocd.utils.Constants.AWS_SECRET_ACCESS_KEY; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.*; public class S3ArtifactStoreTest { + AmazonS3Client mockClient = mock(AmazonS3Client.class); ArgumentCaptor putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); + ArgumentCaptor listingCaptor = ArgumentCaptor.forClass(ListObjectsRequest.class); @Test public void shouldUseStandardStorageClassAsDefault() { @@ -70,4 +82,51 @@ public void shouldHandleBucketDoesNotExists() { assertThat(store.bucketExists(), is(false)); } -} \ No newline at end of file + @Test + public void verifyObjectListingRequestIsRight() { + doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.getLatestPrefix("pipeline", "stage", "job", "1"); + + verify(mockClient).listObjects(listingCaptor.capture()); + ListObjectsRequest request = listingCaptor.getValue(); + assertEquals("foo-bar", request.getBucketName()); + assertEquals("pipeline/stage/job/1.", request.getPrefix()); + assertEquals("/", request.getDelimiter()); + } + + @Test + public void shouldReturnNullWhenObjectListingIsNull() { + doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertNull(prefix); + } + + @Test + public void shouldReturnNullWhenObjectListingIsSize0() { + ObjectListing listing = new ObjectListing(); + doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertNull(prefix); + } + + @Test + public void shouldReturnTheLatestStageCounter() { + ObjectListing listing = new ObjectListing(); + List commonPrefixes = new ArrayList<>(); + commonPrefixes.add("pipeline/stage/job/1.2"); + commonPrefixes.add("pipeline/stage/job/1.1"); + commonPrefixes.add("pipeline/stage/job/1.7"); + listing.setCommonPrefixes(commonPrefixes); + + doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertEquals("pipeline/stage/job/1.7", prefix); + } +}