Merge pull request #334 from avimanyum/using_s3

Enable S3 backed image tag store

dockerfile-image-update/pom.xml

@@ -122,6 +122,24 @@
             <artifactId>github-api</artifactId>
             <version>1.303</version>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>1.12.192</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-core</artifactId>
+            <version>1.12.192</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.checkerframework</groupId>
+            <artifactId>checker-qual</artifactId>
+            <version>3.12.0</version>
+            <scope>compile</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -185,6 +203,7 @@
                             <ignoredUnusedDeclaredDependencies>
                                 <ignoredUnusedDeclaredDependency>org.slf4j</ignoredUnusedDeclaredDependency>
                             </ignoredUnusedDeclaredDependencies>
+
                         </configuration>
                     </execution>
                 </executions>

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/CommandLine.java

@@ -8,7 +8,6 @@
 
 package com.salesforce.dockerfileimageupdate;
 
-
 import com.google.common.reflect.ClassPath;
 import com.salesforce.dockerfileimageupdate.subcommands.ExecutableWithNamespace;
 import com.salesforce.dockerfileimageupdate.utils.Constants;
@@ -38,7 +37,7 @@ public class CommandLine {
     private CommandLine () { }
 
     public static void main(String[] args)
-            throws IOException, IllegalAccessException, InstantiationException, InterruptedException {
+            throws Exception {
         ArgumentParser parser = getArgumentParser();
 
         Set<ClassPath.ClassInfo> allClasses = findSubcommands(parser);

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/GitHubJsonStore.java

@@ -13,8 +13,9 @@
 import java.io.InputStreamReader;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.stream.Collectors;
 
-public class GitHubJsonStore {
+public class GitHubJsonStore implements ImageTagStore {
     private static final Logger log = LoggerFactory.getLogger(GitHubJsonStore.class);
     private final GitHubUtil gitHubUtil;
     private final String store;
@@ -115,4 +116,11 @@ public Set<Map.Entry<String, JsonElement>> parseStoreToImagesMap(DockerfileGitHu
         JsonElement imagesJson = json.getAsJsonObject().get("images");
         return imagesJson.getAsJsonObject().entrySet();
     }
+
+    public List<ImageTagStoreContent> getStoreContent(DockerfileGitHubUtil dockerfileGitHubUtil, String storeName) throws IOException, InterruptedException {
+        Set<Map.Entry<String, JsonElement>> imageToTagStore = parseStoreToImagesMap(dockerfileGitHubUtil, storeName);
+        return imageToTagStore.stream()
+                .map(entry -> new ImageTagStoreContent(entry.getKey(), entry.getValue().getAsString()))
+                .collect(Collectors.toList());
+    }
 }

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/ImageStoreType.java

@@ -0,0 +1,43 @@
+package com.salesforce.dockerfileimageupdate.storage;
+
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.salesforce.dockerfileimageupdate.utils.DockerfileGitHubUtil;
+
+import java.util.Optional;
+
+/**
+ * ImageStoreType is an enum that contains the different types of image tag stores that are supported
+ * @author amukhopadhyay
+ */
+public enum ImageStoreType {
+    S3{
+        @Override
+        public ImageTagStore getStore(DockerfileGitHubUtil dockerfileGitHubUtil, String store) {
+            AmazonS3 s3;
+            DefaultAWSCredentialsProviderChain awsCredentials = new DefaultAWSCredentialsProviderChain();
+            Optional<String> region = Optional.ofNullable(System.getenv("AWS_DEFAULT_REGION"));
+            if (!region.isPresent()){
+                s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).withCredentials(awsCredentials).build();
+            } else {
+                s3 = AmazonS3ClientBuilder.standard().withRegion(region.get()).withCredentials(awsCredentials).build();
+            }
+            return new S3BackedImageTagStore(s3, store);
+        }
+    },
+    GIT{
+        @Override
+        public ImageTagStore getStore(DockerfileGitHubUtil dockerfileGitHubUtil, String store) {
+            return dockerfileGitHubUtil.getGitHubJsonStore(store);
+        }
+    };
+
+    public ImageTagStore getStore(DockerfileGitHubUtil dockerfileGitHubUtil, String store) {
+        return this.getStore(dockerfileGitHubUtil, store);
+    }
+
+}
+
+

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/ImageTagStore.java

@@ -0,0 +1,30 @@
+package com.salesforce.dockerfileimageupdate.storage;
+
+import com.salesforce.dockerfileimageupdate.utils.DockerfileGitHubUtil;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This is an interface for the image tag store. The underlying image tag store can be a Git repo or an S3 bucket.
+ */
+
+public interface ImageTagStore {
+
+    /**
+     * This method updates the image tag store by updating the image version for the image name passed.
+     *
+     * @param img the name of the image that needs to be updated.
+     * @param tag the version of the image that it needs to update to.
+     */
+    void updateStore(String img, String tag) throws IOException;
+
+    /**
+     * This method gets the content of the image tag store.
+     *
+     * @param dockerfileGitHubUtil the dockerfileGitHubUtil object that is used to interact with an underlying Git repo.
+     * @param storeName the name of the store whose content needs to be fetched.
+     * @return A Map of image name to image version.
+     */
+    List<ImageTagStoreContent> getStoreContent(DockerfileGitHubUtil dockerfileGitHubUtil, String storeName) throws IOException, InterruptedException;
+}

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/ImageTagStoreContent.java

@@ -0,0 +1,40 @@
+package com.salesforce.dockerfileimageupdate.storage;
+
+import java.util.Objects;
+
+/**
+ * ImageTagStoreContent is the main entity we'll be using to get the content of the Image tag store
+ * @author amukhopadhyay
+ */
+public class ImageTagStoreContent {
+
+    private final String imageName;
+    private final String tag;
+
+    public ImageTagStoreContent(String imageName, String tag) {
+        this.imageName = imageName;
+        this.tag = tag;
+    }
+
+    public String getImageName() {
+        return this.imageName;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof ImageTagStoreContent)) return false;
+        ImageTagStoreContent that = (ImageTagStoreContent) o;
+        return Objects.equals(getImageName(), that.getImageName()) && Objects.equals(getTag(), that.getTag());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getImageName(), getTag());
+    }
+
+    public String getTag() {
+        return this.tag;
+    }
+
+}

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/S3BackedImageTagStore.java

@@ -0,0 +1,118 @@
+package com.salesforce.dockerfileimageupdate.storage;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.google.common.base.Charsets;
+import com.google.common.io.CharStreams;
+import com.salesforce.dockerfileimageupdate.utils.DockerfileGitHubUtil;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * S3BackedImageTagStore is the main entity we'll be using to interact with the image tag store stored in S3
+ * @author amukhopadhyay
+ */
+public class S3BackedImageTagStore implements ImageTagStore {
+    private static final Logger log = LoggerFactory.getLogger(S3BackedImageTagStore.class);
+    private static final Character S3_FILE_KEY_PATH_DELIMITER = '!';
+    public static final String s3Prefix = "s3";
+    private final AmazonS3 s3;
+    private final String store;
+
+    public S3BackedImageTagStore(AmazonS3 s3, @NonNull String store) {
+        this.s3 = s3;
+        this.store = store;
+    }
+
+    /**
+     * This method updates the image tag store backed by S3 by updating the image version for the image name passed.
+     *
+     * @param img the name of the image that needs to be updated.
+     * @param tag the version of the image that it needs to update to.
+     */
+    public void updateStore(String img, String tag) throws IOException {
+        log.info("Updating store: {} with image: {} tag: {}...", store, img, tag);
+        if (s3.doesBucketExistV2(store)) {
+            String key = convertImageStringToS3ObjectKey(img);
+            s3.putObject(store, key, tag);
+        } else {
+            throw new IOException(String.format("The S3 bucket with name %s does not exist. Cannot proceed.", store));
+        }
+
+    }
+
+    /**
+     * This method gets the content of the image tag store backed by S3.
+     *
+     * @param dockerfileGitHubUtil
+     * @param storeName The name of the store.
+     * @return List of ImageTagStoreContent objects that contain the image name and the image tag.
+     */
+    public List<ImageTagStoreContent> getStoreContent(DockerfileGitHubUtil dockerfileGitHubUtil, String storeName) throws InterruptedException {
+        List<ImageTagStoreContent> imageNamesWithTag;
+        Map<String, Date> imageNameWithAccessTime = new HashMap<>();
+        ListObjectsV2Result result = getS3Objects();
+        List<S3ObjectSummary> objects = result.getObjectSummaries();
+        for (S3ObjectSummary os : objects) {
+            Date lastModified = os.getLastModified();
+            String key = os.getKey();
+            imageNameWithAccessTime.put(key, lastModified);
+        }
+        imageNamesWithTag = getStoreContentSortedByAccessDate(imageNameWithAccessTime);
+        return imageNamesWithTag;
+    }
+
+    private List<ImageTagStoreContent> getStoreContentSortedByAccessDate(Map<String, Date> imageNameWithAccessTime) throws InterruptedException {
+        List<ImageTagStoreContent> imageNameWithTagSortedByAccessDate = new ArrayList<>();
+        LinkedHashMap<String, Date> sortedResult = new LinkedHashMap<>();
+        // Sort the content by the access date so that the file which was accessed most recently gets processed first
+        imageNameWithAccessTime.entrySet()
+                .stream()
+                .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
+                .forEachOrdered(x -> sortedResult.put(x.getKey(), x.getValue()));
+
+        for (Map.Entry<String, Date> set : sortedResult.entrySet()) {
+            String key = set.getKey();
+            try {
+                S3Object o = getS3Object(store, key);
+                String image = convertS3ObjectKeyToImageString(key);
+                String tag = getTagValueFromObject(o);
+                ImageTagStoreContent imageTagStoreContent = new ImageTagStoreContent(image, tag);
+                imageNameWithTagSortedByAccessDate.add(imageTagStoreContent);
+            } catch (Exception e){
+                log.error("Encountered issues reading S3 object with key {}. Exception: {}", key, e);
+            }
+        }
+        return imageNameWithTagSortedByAccessDate;
+    }
+
+    private String getTagValueFromObject(S3Object o) throws IOException {
+        String tag = "";
+        S3ObjectInputStream is = o.getObjectContent();
+        tag = CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
+        return tag;
+    }
+
+    private String convertImageStringToS3ObjectKey(String img) {
+        return img.replace('/', S3_FILE_KEY_PATH_DELIMITER);
+    }
+
+    private String convertS3ObjectKeyToImageString(String key) {
+        return key.replace(S3_FILE_KEY_PATH_DELIMITER, '/');
+    }
+
+    private ListObjectsV2Result getS3Objects() {
+        return s3.listObjectsV2(store);
+    }
+
+    private S3Object getS3Object(String store, String key) {
+        return s3.getObject(store, key);
+    }
+}

dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/subcommands/ExecutableWithNamespace.java

@@ -21,6 +21,6 @@
 public interface ExecutableWithNamespace {
 
     void execute(Namespace ns, DockerfileGitHubUtil dockerfileGitHubUtil)
-            throws IOException, IllegalAccessException, InstantiationException, InterruptedException;
+            throws Exception;
 
 }