implement ListObjectsV2 (#70)

Robothy · Aug 28, 2023 · ace8e82 · ace8e82
1 parent ca802ca
commit ace8e82
Show file tree

Hide file tree

Showing 27 changed files with 1,485 additions and 134 deletions.
diff --git a/README.md b/README.md
@@ -38,6 +38,7 @@ and without heavy dependencies, it starts up quickly and handles requests effici
 + HeadObject
 + ListBuckets
 + ListObjects
++ ListObjectsV2
 + ListObjectVersions
 + ListParts
 + PutBucketAcl

diff --git a/buildSrc/src/main/groovy/local-s3.java-conventions.gradle b/buildSrc/src/main/groovy/local-s3.java-conventions.gradle
@@ -1,6 +1,7 @@
 plugins {
     id 'java'
     id 'jacoco'
+    //id 'com.robothy.cn-repo'
 }
 
 repositories {

diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/model/ContinuationParameters.java b/local-s3-core/src/main/java/com/robothy/s3/core/model/ContinuationParameters.java
@@ -0,0 +1,132 @@
+package com.robothy.s3.core.model;
+
+import com.robothy.s3.core.exception.LocalS3InvalidArgumentException;
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Builder
+@Getter
+@EqualsAndHashCode
+public class ContinuationParameters {
+
+    private Character delimiter;
+
+    private String encodingType;
+
+    private boolean fetchOwner;
+
+    private int maxKeys;
+
+    private String prefix;
+
+    private String startAfter;
+
+
+    public String encode() {
+        String joined = Stream.of(encodeDelimiter(), encodeEncodingType(),
+                encodeFetchOwner(), encodeMaxKeys(),
+                encodePrefix(), encodeStartAfter())
+            .filter(StringUtils::isNotBlank)
+            .collect(Collectors.joining("/"));
+
+        String hashAppended = joined + "/" + joined.hashCode();
+        return Base64.getEncoder().encodeToString(hashAppended.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private String encodeDelimiter() {
+        return Objects.isNull(delimiter) ? "" : 1 + URLEncoder.encode(delimiter + "", StandardCharsets.UTF_8);
+    }
+
+    private String encodeEncodingType() {
+        return Objects.isNull(encodingType) ? "" : 2 + URLEncoder.encode(encodingType, StandardCharsets.UTF_8);
+    }
+
+    private String encodeFetchOwner() {
+        return fetchOwner ? "3" : "";
+    }
+
+    private String encodeMaxKeys() {
+        return 4 + "" + maxKeys;
+    }
+
+    private String encodePrefix() {
+        return Objects.isNull(prefix) ? "" : 5 + URLEncoder.encode(prefix, StandardCharsets.UTF_8);
+    }
+
+    private String encodeStartAfter() {
+        return Objects.isNull(startAfter) ? "" : 6 + URLEncoder.encode(startAfter, StandardCharsets.UTF_8);
+    }
+
+    public static ContinuationParameters decode(String continuationToken) {
+        String hashAppended = ensureContinuationTokenIsBase64Encoded(continuationToken);
+        String joined = verifyHash(hashAppended);
+        return decodeParameters(joined);
+    }
+
+    private static String ensureContinuationTokenIsBase64Encoded(String continuationToken) {
+        try {
+            return new String(Base64.getDecoder().decode(continuationToken), StandardCharsets.UTF_8);
+        } catch (IllegalArgumentException e) {
+            throw new LocalS3InvalidArgumentException("continuation-token", continuationToken, "The continuation token provided is incorrect.");
+        }
+    }
+
+    private static String verifyHash(String hashAppended) {
+        int lastSlash;
+        if (-1 == (lastSlash = hashAppended.lastIndexOf("/"))) {
+            throw new LocalS3InvalidArgumentException("continuation-token", hashAppended, "The continuation token provided is incorrect.");
+        }
+        int hash = Integer.parseInt(hashAppended.substring(lastSlash + 1));
+        String joined = hashAppended.substring(0, lastSlash);
+        if (hash != joined.hashCode()) {
+            throw new LocalS3InvalidArgumentException("continuation-token", hashAppended, "The continuation token provided is incorrect.");
+        }
+        return joined;
+    }
+
+    private static ContinuationParameters decodeParameters(String joined) {
+        String[] parts = joined.split("/");
+        if (parts.length == 0) {
+            return ContinuationParameters.builder().build();
+        }
+        ContinuationParametersBuilder builder = ContinuationParameters.builder();
+        Arrays.stream(parts).forEach(part -> {
+            if (StringUtils.isBlank(part)) return;
+            switch (part.charAt(0)) {
+                case '1':
+                    builder.delimiter(URLDecoder.decode(part.substring(1), StandardCharsets.UTF_8).charAt(0));
+                    break;
+                case '2':
+                    builder.encodingType(URLDecoder.decode(part.substring(1), StandardCharsets.UTF_8));
+                    break;
+                case '3':
+                    builder.fetchOwner(true);
+                    break;
+                case '4':
+                    builder.maxKeys(Integer.parseInt(part.substring(1)));
+                    break;
+                case '5':
+                    builder.prefix(URLDecoder.decode(part.substring(1), StandardCharsets.UTF_8));
+                    break;
+                case '6':
+                    builder.startAfter(URLDecoder.decode(part.substring(1), StandardCharsets.UTF_8));
+                    break;
+                default:
+                    throw new LocalS3InvalidArgumentException("continuation-token", joined, "The continuation token provided is incorrect.");
+            }
+        });
+        return builder.build();
+    }
+
+}
diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/model/answers/ListObjectsAns.java b/local-s3-core/src/main/java/com/robothy/s3/core/model/answers/ListObjectsAns.java
@@ -1,9 +1,7 @@
 package com.robothy.s3.core.model.answers;
 
 import com.robothy.s3.datatypes.response.S3Object;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
 import lombok.AllArgsConstructor;
@@ -20,12 +18,33 @@
 @AllArgsConstructor
 public class ListObjectsAns {
 
+  /**
+   * The delimiter returned in the response. The delimiter will be encoded if the encoding type is not empty.
+   */
+  private String delimiter;
+
+  private String encodingType;
+
+  /**
+   * Maker returned in the response. The marker will be encoded if the encoding type is not empty.
+   */
+  private String marker;
+
+  private int maxKeys;
+
   /**
    * Represents the first key of subsequent objects if the result is truncated.
    * Its value is {@code null} if the result is not truncated.
    */
   private String nextMarker;
 
+  private boolean isTruncated;
+
+  /**
+   * The prefix returned in the response. The prefix will be encoded if the encoding type is not empty.
+   */
+  private String prefix;
+
   @Builder.Default
   private List<S3Object> objects = Collections.emptyList();
 

diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/model/answers/ListObjectsV2Ans.java b/local-s3-core/src/main/java/com/robothy/s3/core/model/answers/ListObjectsV2Ans.java
@@ -1,5 +1,44 @@
 package com.robothy.s3.core.model.answers;
 
+import com.robothy.s3.datatypes.response.S3Object;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+@Builder
+@Data
 public class ListObjectsV2Ans {
 
+    private String continuationToken;
+
+    private String delimiter;
+
+    private String encodingType;
+
+    private boolean isTruncated;
+
+    private int keyCount;
+
+    private int maxKeys;
+
+    private String nextContinuationToken;
+
+    private String prefix;
+
+    private String startAfter;
+
+    @Builder.Default
+    private List<S3Object> objects = Collections.emptyList();
+
+    @Builder.Default
+    private List<String> commonPrefixes = Collections.emptyList();
+
+
+    public Optional<String> getNextContinuationToken() {
+        return Optional.ofNullable(nextContinuationToken);
+    }
+
 }
diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/service/ListObjectVersionsService.java b/local-s3-core/src/main/java/com/robothy/s3/core/service/ListObjectVersionsService.java
@@ -24,7 +24,7 @@
 public interface ListObjectVersionsService extends LocalS3MetadataApplicable {
 
 
-  default ListObjectVersionsAns listObjectVersions(String bucket, Character delimiter, String keyMarker, int maxKeys, String prefix, String versionIdMarker) {
+  default ListObjectVersionsAns listObjectVersions(String bucket, String delimiter, String keyMarker, int maxKeys, String prefix, String versionIdMarker) {
     BucketMetadata bucketMetadata = BucketAssertions.assertBucketExists(localS3Metadata(), bucket);
 
     if (Objects.nonNull(versionIdMarker) && Objects.isNull(keyMarker)) {