Making S3's storage class for artifacts configurable

build.sbt

@@ -38,7 +38,7 @@ lazy val utils = (project in file("utils")).
     libraryDependencies ++= Seq(
       junit, junitInterface, hamcrest
     ),
-    javacOptions ++= Seq("-source", "1.6", "-target", "1.6")
+    javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
   )
 
 lazy val publish = (project in file("publish")).
@@ -51,7 +51,7 @@ lazy val publish = (project in file("publish")).
     libraryDependencies ++= Seq(
       ant, junit, junitInterface, hamcrest
     ),
-    javacOptions ++= Seq("-source", "1.6", "-target", "1.6")
+    javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
   )
 
 lazy val material = (project in file("material")).
@@ -64,7 +64,7 @@ lazy val material = (project in file("material")).
     libraryDependencies ++= Seq(
       scalaTest
     ),
-    javacOptions ++= Seq("-source", "1.6", "-target", "1.6")
+    javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
   )
 
 lazy val fetch = (project in file("fetch")).
@@ -77,5 +77,5 @@ lazy val fetch = (project in file("fetch")).
     libraryDependencies ++= Seq(
       junit, hamcrest
     ),
-    javacOptions ++= Seq("-source", "1.6", "-target", "1.6")
+    javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
   )

publish/src/main/java/com/indix/gocd/s3publish/PublishExecutor.java

@@ -6,6 +6,7 @@
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 
+import com.amazonaws.services.s3.model.StorageClass;
 import com.amazonaws.util.json.JSONException;
 import com.indix.gocd.utils.GoEnvironment;
 import com.thoughtworks.go.plugin.api.logging.Logger;
@@ -50,6 +51,7 @@ public ExecutionResult execute(TaskConfig config, final TaskExecutionContext con
 
         final String bucket = env.get(GO_ARTIFACTS_S3_BUCKET);
         final S3ArtifactStore store = new S3ArtifactStore(s3Client(env), bucket);
+        store.setStorageClass(env.getOrElse(AWS_STORAGE_CLASS, STORAGE_CLASS_STANDARD));
 
         final String destinationPrefix = getDestinationPrefix(config, env);
 

utils/src/main/java/com/indix/gocd/utils/Constants.java

@@ -14,4 +14,10 @@ public class Constants {
     public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
     public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
     public static final String AWS_USE_IAM_ROLE = "AWS_USE_IAM_ROLE";
+    public static final String AWS_STORAGE_CLASS = "AWS_STORAGE_CLASS";
+    public static final String STORAGE_CLASS_STANDARD = "standard";
+    public static final String STORAGE_CLASS_STANDARD_IA = "standard-ia";
+    public static final String STORAGE_CLASS_RRS = "rrs";
+    public static final String STORAGE_CLASS_GLACIER = "glacier";
+
 }

utils/src/main/java/com/indix/gocd/utils/GoEnvironment.java

@@ -31,6 +31,11 @@ public String get(String name) {
         return environment.get(name);
     }
 
+    public String getOrElse(String name, String defaultValue) {
+        if(has(name)) return get(name);
+        else return defaultValue;
+    }
+
     public boolean has(String name) {
         return environment.containsKey(name) && isNotEmpty(get(name));
     }

utils/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java

@@ -2,28 +2,50 @@
 
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.*;
-import com.indix.gocd.models.ResponseMetadataConstants;
 import com.indix.gocd.models.Artifact;
+import com.indix.gocd.models.ResponseMetadataConstants;
 import com.indix.gocd.models.Revision;
 import com.indix.gocd.models.RevisionStatus;
 import com.indix.gocd.utils.utils.Function;
 import com.indix.gocd.utils.utils.Functions;
 import com.indix.gocd.utils.utils.Lists;
+import com.indix.gocd.utils.utils.Maps;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.File;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static com.indix.gocd.utils.Constants.*;
+
 public class S3ArtifactStore {
+
+    private static Map<String, StorageClass> STORAGE_CLASSES = Maps.<String, StorageClass>builder()
+            .with(STORAGE_CLASS_STANDARD, StorageClass.Standard)
+            .with(STORAGE_CLASS_STANDARD_IA, StorageClass.StandardInfrequentAccess)
+            .with(STORAGE_CLASS_RRS, StorageClass.ReducedRedundancy)
+            .with(STORAGE_CLASS_GLACIER, StorageClass.Glacier)
+            .build();
+
     private AmazonS3Client client;
     private String bucket;
+    private StorageClass storageClass = StorageClass.Standard;
 
     public S3ArtifactStore(AmazonS3Client client, String bucket) {
         this.client = client;
         this.bucket = bucket;
     }
 
+    public void setStorageClass(String storageClass) {
+        String key = StringUtils.lowerCase(storageClass);
+        if (STORAGE_CLASSES.containsKey(key)) {
+            this.storageClass = STORAGE_CLASSES.get(key);
+        } else {
+            throw new IllegalArgumentException("Invalid storage class specified for S3 - " + storageClass + ". Accepted values are standard, standard-ia, rrs and glacier");
+        }
+    }
+
     public void put(String from, String to) {
         put(new PutObjectRequest(bucket, to, new File(from)));
     }
@@ -34,7 +56,7 @@ public void put(String from, String to, ObjectMetadata metadata) {
     }
 
     public void put(PutObjectRequest putObjectRequest) {
-        putObjectRequest.setStorageClass(StorageClass.ReducedRedundancy);
+        putObjectRequest.setStorageClass(this.storageClass);
         client.putObject(putObjectRequest);
     }
 
@@ -64,7 +86,7 @@ public void getPrefix(String prefix, String to) {
             for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                 String destinationPath = to + "/" + objectSummary.getKey().replace(prefix + "/", "");
                 long size = objectSummary.getSize();
-                if(size > 0) {
+                if (size > 0) {
                     get(objectSummary.getKey(), destinationPath);
                 }
             }
@@ -82,7 +104,7 @@ public Boolean execute(Bucket input) {
                     return input.getName().equals(bucket);
                 }
             });
-        } catch(Exception ex) {
+        } catch (Exception ex) {
             return false;
         }
     }
@@ -95,14 +117,15 @@ public boolean exists(String bucket, String key) {
         try {
             ObjectListing objectListing = client.listObjects(listObjectsRequest);
             return objectListing != null && objectListing.getCommonPrefixes().size() > 0;
-        } catch(Exception ex) {
+        } catch (Exception ex) {
             return false;
         }
     }
 
     private Boolean isComplete(AmazonS3Client client, String prefix) {
         return client.getObjectMetadata(bucket, prefix).getUserMetadata().containsKey(ResponseMetadataConstants.COMPLETED);
     }
+
     private Revision mostRecentRevision(final AmazonS3Client client, ObjectListing listing) {
         List<String> prefixes = Lists.filter(listing.getCommonPrefixes(), new Functions.Predicate<String>() {
             @Override
@@ -115,24 +138,24 @@ public Boolean execute(String input) {
             @Override
             public Revision apply(String prefix) {
                 String[] parts = prefix.split("/");
-                String last = parts[parts.length-1];
+                String last = parts[parts.length - 1];
                 return new Revision(last);
             }
         });
 
-        if(revisions.size() > 0)
+        if (revisions.size() > 0)
             return Collections.max(revisions);
         else
             return Revision.base();
     }
 
     private Revision latestOfInternal(AmazonS3Client client, ObjectListing listing, Revision latestSoFar) {
-        if (! listing.isTruncated()){
+        if (!listing.isTruncated()) {
             return latestSoFar;
-        }else {
+        } else {
             ObjectListing objects = client.listNextBatchOfObjects(listing);
             Revision mostRecent = mostRecentRevision(client, objects);
-            if(latestSoFar.compareTo(mostRecent) > 0)
+            if (latestSoFar.compareTo(mostRecent) > 0)
                 mostRecent = latestSoFar;
             return latestOfInternal(client, objects, mostRecent);
         }
@@ -149,7 +172,7 @@ public RevisionStatus getLatest(AmazonS3Client client, Artifact artifact) {
                 .withDelimiter("/");
 
         ObjectListing listing = client.listObjects(listObjectsRequest);
-        if(listing != null){
+        if (listing != null) {
             Revision recent = latestOf(client, listing);
             Artifact artifactWithRevision = artifact.withRevision(recent);
             GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, artifactWithRevision.prefixWithRevision());

utils/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java

@@ -0,0 +1,57 @@
+package com.indix.gocd.utils.store;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.File;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.*;
+
+public class S3ArtifactStoreTest {
+    AmazonS3Client mockClient = mock(AmazonS3Client.class);
+    ArgumentCaptor<PutObjectRequest> putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class);
+
+    @Test
+    public void shouldUseStandardStorageClassAsDefault() {
+        S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");
+        store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz")));
+        verify(mockClient, times(1)).putObject(putCaptor.capture());
+        PutObjectRequest putRequest = putCaptor.getValue();
+        assertThat(putRequest.getStorageClass(), is("STANDARD"));
+    }
+
+    @Test
+    public void shouldUseStandardIAStorageClassAsDefault() {
+        S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");
+        store.setStorageClass("standard-ia");
+        store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz")));
+        verify(mockClient, times(1)).putObject(putCaptor.capture());
+        PutObjectRequest putRequest = putCaptor.getValue();
+        assertThat(putRequest.getStorageClass(), is("STANDARD_IA"));
+    }
+
+    @Test
+    public void shouldUseReducedRedundancyStorageClass() {
+        S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");
+        store.setStorageClass("rrs");
+        store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz")));
+        verify(mockClient, times(1)).putObject(putCaptor.capture());
+        PutObjectRequest putRequest = putCaptor.getValue();
+        assertThat(putRequest.getStorageClass(), is("REDUCED_REDUNDANCY"));
+    }
+
+    @Test
+    public void shouldUseGlacierStorageClass() {
+        S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar");
+        store.setStorageClass("glacier");
+        store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz")));
+        verify(mockClient, times(1)).putObject(putCaptor.capture());
+        PutObjectRequest putRequest = putCaptor.getValue();
+        assertThat(putRequest.getStorageClass(), is("GLACIER"));
+    }
+
+}