Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add destination prefix option to publish task #11

Merged
merged 3 commits into from
Dec 20, 2015
Merged

Add destination prefix option to publish task #11

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9a6482e
Add destination prefix option to publish task
johncmckim Jul 22, 2015
5ce02b1
Fix issue where destinationPrefix of / creates empty folder
johncmckim Oct 10, 2015
deaa25f
Don't set metadata if a custom destination prefix is being used (i.e.…
johncmckim Dec 19, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 40 additions & 8 deletions publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public ExecutionResult execute(TaskConfig config, final TaskExecutionContext con
final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
final S3ArtifactStore store = new S3ArtifactStore(s3Client(env), bucket);

final String destinationPrefix = getDestinationPrefix(config, env);

try {
List<Tuple2<String, String>> sourceDestinations = PublishTask.getSourceDestinations(config.getValue(SOURCEDESTINATIONS));
foreach(sourceDestinations, new VoidFunction<Tuple2<String, String>>() {
Expand All @@ -59,7 +61,8 @@ public void execute(Tuple2<String, String> input) {
@Override
public void execute(String includedFile) {
File localFileToUpload = new File(String.format("%s/%s", context.workingDir(), includedFile));
pushToS3(context, env, store, localFileToUpload, destination);

pushToS3(context, destinationPrefix, store, localFileToUpload, destination);
}
});
}
Expand All @@ -69,7 +72,7 @@ public void execute(String includedFile) {
log.error(message);
return ExecutionResult.failure(message, e);
}
setMetadata(env, bucket, store);
setMetadata(env, bucket, destinationPrefix, store);

return ExecutionResult.success("Published all artifacts to S3");
}
Expand All @@ -94,10 +97,10 @@ public AmazonS3Client s3Client(GoEnvironment env) {
return new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
}

private void pushToS3(final TaskExecutionContext context, final GoEnvironment env, final S3ArtifactStore store, File localFileToUpload, String destination) {
String templateSoFar = env.artifactsLocationTemplate();
private void pushToS3(final TaskExecutionContext context, final String destinationPrefix, final S3ArtifactStore store, File localFileToUpload, String destination) {
String templateSoFar = ensureKeySegmentValid(destinationPrefix);
if(!org.apache.commons.lang3.StringUtils.isBlank(destination)) {
templateSoFar += "/" + destination;
templateSoFar += destination;
}
List<FilePathToTemplate> filesToUpload = generateFilesToUpload(templateSoFar, localFileToUpload);
foreach(filesToUpload, new VoidFunction<FilePathToTemplate>() {
Expand All @@ -123,7 +126,8 @@ private ObjectMetadata metadata(GoEnvironment env) {
}

private List<FilePathToTemplate> generateFilesToUpload(final String templateSoFar, final File fileToUpload) {
final String templateWithFolder = String.format("%s/%s", templateSoFar, fileToUpload.getName());
final String templateWithFolder = ensureKeySegmentValid(templateSoFar) + fileToUpload.getName(); // ensure it ends with a slash and add filename

if (fileToUpload.isDirectory()) {
return flatMap(fileToUpload.listFiles(), new Function<File, List<FilePathToTemplate>>() {
@Override
Expand All @@ -142,17 +146,45 @@ private ExecutionResult envNotFound(String environmentVariable) {
return ExecutionResult.failure(message);
}

private void setMetadata(GoEnvironment env, String bucket, S3ArtifactStore store) {
private void setMetadata(GoEnvironment env, String bucket, String destinationPrefix, S3ArtifactStore store) {
ObjectMetadata metadata = metadata(env);
metadata.setContentLength(0);
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket,
env.artifactsLocationTemplate() + "/",
ensureKeySegmentValid(destinationPrefix),
emptyContent,
metadata);

store.put(putObjectRequest);
}

private String getDestinationPrefix(final TaskConfig config, final GoEnvironment env) {
String destinationPrefix = config.getValue(DESTINATION_PREFIX);

if(org.apache.commons.lang3.StringUtils.isBlank(destinationPrefix)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we could extract this, if check outside then we could choose to set / not set the metadata.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. I changed to to not set the metadata if a destinationPrefix is configured. I chose this instead of adding another setting as most using a destinationPrefix are deploying rather than storing artifacts.

return env.artifactsLocationTemplate();
}

destinationPrefix = env.replaceVariables(destinationPrefix);

if(destinationPrefix.endsWith("/")) {
destinationPrefix = destinationPrefix.substring(0, destinationPrefix.length() - 1);
}

return destinationPrefix;
}

private String ensureKeySegmentValid(String segment) {
if(org.apache.commons.lang3.StringUtils.isBlank(segment)) {
return segment;
}

if(!org.apache.commons.lang3.StringUtils.endsWith(segment, "/")) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please have them imported (even statically)? Or is there any other reason to use full namespace in here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I just used this as I saw it done this way elsewhere in the project. Most of my experience is with C#, I've only worked on a few small Java projects. As a result, I'm not too sure on the best practice in regards to static import vs import. I've just imported StringUtils (not statically) as that is how ArrayUtils is imported.

segment += "/";
}

return segment;
}
}

/**
Expand Down
2 changes: 2 additions & 0 deletions publish/src/main/java/com/indix/gocd/s3publish/PublishTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.List;

import static com.indix.gocd.utils.Constants.SOURCEDESTINATIONS;
import static com.indix.gocd.utils.Constants.DESTINATION_PREFIX;
import static com.indix.gocd.utils.utils.Lists.foreach;
import static org.apache.commons.lang3.StringUtils.trim;

Expand All @@ -30,6 +31,7 @@ public class PublishTask implements Task {
public TaskConfig config() {
TaskConfig taskConfig = new TaskConfig();
taskConfig.addProperty(SOURCEDESTINATIONS);
taskConfig.addProperty(DESTINATION_PREFIX);
return taskConfig;
}

Expand Down
11 changes: 11 additions & 0 deletions publish/src/main/resources/views/task.template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,17 @@
<a id="add_artifact" ng-click="addSource()" class="action_icon skip_dirty_stop add_icon" title="" href="#"><span class="icon"></span><span>Add</span></a>
<span class="form_error" ng-show="GOINPUTNAME[sourceDestinations].$error.server">{{ GOINPUTNAME[sourceDestinations].$error.server }}</span>
</div>
<div class="form_item_block">
<label for="destinationPrefix">Destination Prefix</label>
<input id="destinationPrefix" type="text" ng-model="destinationPrefix" />
<span class="form_error" ng-show="GOINPUTNAME[destinationPrefix].$error.server">{{ GOINPUTNAME[destinationPrefix].$error.server }}</span>
</div>
<div class="form_item_block">
<p>
<span ng-show="destinationPrefix">Your artifact destinations will be prefixed with {{destinationPrefix}}</span>
<span ng-show="!destinationPrefix">Your artifact destinations will be prefixed with pipeline/stage/job/pipelineCounter.stageCounter</span>
</p>
</div>
<div class="form_item_block">
<p class="required">Make sure AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, GO_ARTIFACTS_S3_BUCKET and GO_SERVER_DASHBOARD_URL environment variables are present with appropriate values on any of pipeline / Go Environments / on all agent machines. </p>
</div>
Expand Down
166 changes: 152 additions & 14 deletions publish/src/test/java/com/indix/gocd/s3publish/PublishExecutorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,25 +74,34 @@ public void shouldThrowIfGO_ARTIFACTS_S3_BUCKETNotPresent() {
}

@Test
public void shouldUploadALocalFileToS3() {
Map<String, String> mockVariables = mockEnvironmentVariables.build();
public void shouldGetDisplayMessageAfterUpload() {
AmazonS3Client mockClient = mockClient();
doReturn(mockClient).when(publishExecutor).s3Client(any(GoEnvironment.class));
when(config.getValue(SOURCEDESTINATIONS)).thenReturn("[{\"source\": \"target/*\", \"destination\": \"\"}]");
doReturn(new String[]{"README.md", "s3publish-0.1.31.jar"}).when(publishExecutor).parseSourcePath(anyString(), anyString());

ExecutionResult executionResult = publishExecutor.execute(config, mockContext(mockVariables));
ExecutionResult executionResult = executeMockPublish(
mockClient,
"[{\"source\": \"target/*\", \"destination\": \"\"}]",
"",
new String[]{"README.md"}
);

assertTrue(executionResult.isSuccessful());
assertThat(executionResult.getMessagesForDisplay(), is("Published all artifacts to S3"));
}

ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor = ArgumentCaptor.forClass(PutObjectRequest.class);
verify(mockClient, times(3)).putObject(putObjectRequestArgumentCaptor.capture());
List<PutObjectRequest> allPutObjectRequests = putObjectRequestArgumentCaptor.getAllValues();
@Test
public void shouldUploadALocalFileToS3WithDefaultPrefix() {
AmazonS3Client mockClient = mockClient();

ExecutionResult executionResult = executeMockPublish(
mockClient,
"[{\"source\": \"target/*\", \"destination\": \"\"}]",
"",
new String[]{"README.md", "s3publish-0.1.31.jar"}
);

assertTrue(executionResult.isSuccessful());

PutObjectRequest filePutRequest = allPutObjectRequests.get(0);
assertThat(filePutRequest.getBucketName(), is("testS3Bucket"));
assertThat(filePutRequest.getKey(), is("pipeline/stage/job/pipelineCounter.stageCounter/README.md"));
final List<PutObjectRequest> allPutObjectRequests = getPutObjectRequests(mockClient, 3);

PutObjectRequest metadataPutRequest = allPutObjectRequests.get(2);
Map<String, String> expectedUserMetadata = Maps.<String, String>builder()
Expand All @@ -101,8 +110,8 @@ public void shouldUploadALocalFileToS3() {
.with(COMPLETED, COMPLETED)
.build();
assertThat(metadataPutRequest.getMetadata().getUserMetadata(), is(expectedUserMetadata));
assertThat(metadataPutRequest.getKey(), is("pipeline/stage/job/pipelineCounter.stageCounter/"));

assertNull(filePutRequest.getMetadata());
PutObjectRequest readmePutRequest = allPutObjectRequests.get(0);
assertThat(readmePutRequest.getBucketName(), is("testS3Bucket"));
assertThat(readmePutRequest.getKey(), is("pipeline/stage/job/pipelineCounter.stageCounter/README.md"));
Expand All @@ -115,11 +124,140 @@ public void shouldUploadALocalFileToS3() {
assertNull(jarPutRequest.getMetadata());
}


@Test
public void shouldUploadALocalFileToS3WithDestinationPrefix() {
AmazonS3Client mockClient = mockClient();

ExecutionResult executionResult = executeMockPublish(
mockClient,
"[{\"source\": \"target/*\", \"destination\": \"\"}]",
"destinationPrefix",
new String[]{"README.md", "s3publish-0.1.31.jar"}
);

assertTrue(executionResult.isSuccessful());

final List<PutObjectRequest> allPutObjectRequests = getPutObjectRequests(mockClient, 3);

PutObjectRequest metadataPutRequest = allPutObjectRequests.get(2);
Map<String, String> expectedUserMetadata = Maps.<String, String>builder()
.with(METADATA_USER, "Krishna")
.with(METADATA_TRACEBACK_URL, "http://go.server:8153/go/tab/build/detail/pipeline/pipelineCounter/stage/stageCounter/job")
.with(COMPLETED, COMPLETED)
.build();
assertThat(metadataPutRequest.getMetadata().getUserMetadata(), is(expectedUserMetadata));
assertThat(metadataPutRequest.getKey(), is("destinationPrefix/"));

PutObjectRequest readmePutRequest = allPutObjectRequests.get(0);
assertThat(readmePutRequest.getBucketName(), is("testS3Bucket"));
assertThat(readmePutRequest.getKey(), is("destinationPrefix/README.md"));
assertNull(readmePutRequest.getMetadata());

PutObjectRequest jarPutRequest = allPutObjectRequests.get(1);
assertNull(jarPutRequest.getMetadata());
assertThat(jarPutRequest.getBucketName(), is("testS3Bucket"));
assertThat(jarPutRequest.getKey(), is("destinationPrefix/s3publish-0.1.31.jar"));
assertNull(jarPutRequest.getMetadata());
}

@Test
public void shouldUploadALocalFileToS3WithDestinationPrefixUsingEnvVariable() {
AmazonS3Client mockClient = mockClient();

ExecutionResult executionResult = executeMockPublish(
mockClient,
"[{\"source\": \"target/*\", \"destination\": \"\"}]",
"test/${GO_PIPELINE_COUNTER}/",
new String[]{"README.md", "s3publish-0.1.31.jar"}
);

assertTrue(executionResult.isSuccessful());

final List<PutObjectRequest> allPutObjectRequests = getPutObjectRequests(mockClient, 3);

PutObjectRequest metadataPutRequest = allPutObjectRequests.get(2);
Map<String, String> expectedUserMetadata = Maps.<String, String>builder()
.with(METADATA_USER, "Krishna")
.with(METADATA_TRACEBACK_URL, "http://go.server:8153/go/tab/build/detail/pipeline/pipelineCounter/stage/stageCounter/job")
.with(COMPLETED, COMPLETED)
.build();
assertThat(metadataPutRequest.getMetadata().getUserMetadata(), is(expectedUserMetadata));
assertThat(metadataPutRequest.getKey(), is("test/pipelineCounter/"));

PutObjectRequest readmePutRequest = allPutObjectRequests.get(0);
assertThat(readmePutRequest.getBucketName(), is("testS3Bucket"));
assertThat(readmePutRequest.getKey(), is("test/pipelineCounter/README.md"));
assertNull(readmePutRequest.getMetadata());

PutObjectRequest jarPutRequest = allPutObjectRequests.get(1);
assertNull(jarPutRequest.getMetadata());
assertThat(jarPutRequest.getBucketName(), is("testS3Bucket"));
assertThat(jarPutRequest.getKey(), is("test/pipelineCounter/s3publish-0.1.31.jar"));
assertNull(jarPutRequest.getMetadata());
}

@Test
public void shouldUploadALocalFileToS3WithSlashDestinationPrefix() {
AmazonS3Client mockClient = mockClient();

ExecutionResult executionResult = executeMockPublish(
mockClient,
"[{\"source\": \"target/*\", \"destination\": \"\"}]",
"/",
new String[]{"README.md", "s3publish-0.1.31.jar"}
);

assertTrue(executionResult.isSuccessful());

final List<PutObjectRequest> allPutObjectRequests = getPutObjectRequests(mockClient, 3);

PutObjectRequest metadataPutRequest = allPutObjectRequests.get(2);
Map<String, String> expectedUserMetadata = Maps.<String, String>builder()
.with(METADATA_USER, "Krishna")
.with(METADATA_TRACEBACK_URL, "http://go.server:8153/go/tab/build/detail/pipeline/pipelineCounter/stage/stageCounter/job")
.with(COMPLETED, COMPLETED)
.build();
assertThat(metadataPutRequest.getMetadata().getUserMetadata(), is(expectedUserMetadata));
assertThat(metadataPutRequest.getKey(), is(""));

PutObjectRequest readmePutRequest = allPutObjectRequests.get(0);
assertThat(readmePutRequest.getBucketName(), is("testS3Bucket"));
assertThat(readmePutRequest.getKey(), is("README.md"));
assertNull(readmePutRequest.getMetadata());

PutObjectRequest jarPutRequest = allPutObjectRequests.get(1);
assertNull(jarPutRequest.getMetadata());
assertThat(jarPutRequest.getBucketName(), is("testS3Bucket"));
assertThat(jarPutRequest.getKey(), is("s3publish-0.1.31.jar"));
assertNull(jarPutRequest.getMetadata());
}

private ExecutionResult executeMockPublish(final AmazonS3Client mockClient, String sourceDestinations, String destinationPrefix, String[] files) {
Map<String, String> mockVariables = mockEnvironmentVariables.build();

doReturn(mockClient).when(publishExecutor).s3Client(any(GoEnvironment.class));
when(config.getValue(SOURCEDESTINATIONS)).thenReturn(sourceDestinations);
when(config.getValue(DESTINATION_PREFIX)).thenReturn(destinationPrefix);
doReturn(files).when(publishExecutor).parseSourcePath(anyString(), anyString());

ExecutionResult executionResult = publishExecutor.execute(config, mockContext(mockVariables));

return executionResult;
}

private List<PutObjectRequest> getPutObjectRequests(AmazonS3Client mockClient, int expectedRequestsCount) {
ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor = ArgumentCaptor.forClass(PutObjectRequest.class);
verify(mockClient, times(expectedRequestsCount)).putObject(putObjectRequestArgumentCaptor.capture());
List<PutObjectRequest> allPutObjectRequests = putObjectRequestArgumentCaptor.getAllValues();

return allPutObjectRequests;
}
private TaskExecutionContext mockContext(final Map<String, String> environmentMap) {
return new MockTaskExecutionContext(environmentMap);
}

private AmazonS3Client mockClient() {
return mock(AmazonS3Client.class);
}
}
}
3 changes: 3 additions & 0 deletions utils/src/main/java/com/indix/gocd/utils/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ public class Constants {

public static final String GO_ARTIFACTS_S3_BUCKET = "GO_ARTIFACTS_S3_BUCKET";
public static final String GO_SERVER_DASHBOARD_URL = "GO_SERVER_DASHBOARD_URL";

public static final String SOURCEDESTINATIONS = "sourceDestinations";
public static final String DESTINATION_PREFIX = "destinationPrefix";

public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
}
22 changes: 22 additions & 0 deletions utils/src/main/java/com/indix/gocd/utils/GoEnvironment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
package com.indix.gocd.utils;

import java.lang.StringBuffer;

import java.util.HashMap;
import java.util.Map;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static com.indix.gocd.utils.Constants.GO_SERVER_DASHBOARD_URL;

/**
* Wrapper around Go's Environment variables
*/
public class GoEnvironment {
private Pattern envPat = Pattern.compile("\\$\\{(\\w+)\\}");
private Map<String, String> environment = new HashMap<String, String>();

public GoEnvironment() {
Expand Down Expand Up @@ -47,6 +53,22 @@ public String triggeredUser() {
return get("GO_TRIGGER_USER");
}

public String replaceVariables(String str) {
Matcher m = envPat.matcher(str);

StringBuffer sb = new StringBuffer();
while (m.find()) {
String replacement = get(m.group(1));
if(replacement != null) {
m.appendReplacement(sb, replacement);
}
}

m.appendTail(sb);

return sb.toString();
}

/**
* Version Format on S3 is <code>pipeline/stage/job/pipeline_counter.stage_counter</code>
*/
Expand Down
Loading