Merge pull request #129 from ptirador/issue-95

Issue 95: Use multipart upload API

pom.xml

@@ -260,6 +260,14 @@
             <artifactId>mockito-inline</artifactId>
             <version>${version.mockito}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>${version.mockito}</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <profiles>

src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java

@@ -264,7 +264,6 @@ protected void sync()
     {
         try (InputStream stream = new BufferedInputStream(Files.newInputStream(tempFile)))
         {
-            //TODO: If the temp file is larger than 5 GB then, instead of a putObject, a multi-part upload is needed.
             final PutObjectRequest.Builder builder = PutObjectRequest.builder();
             final long length = Files.size(tempFile);
             builder.bucket(path.getFileStore().name())

src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java

@@ -11,6 +11,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
 import java.lang.reflect.InvocationTargetException;
 import java.net.URI;
@@ -34,6 +35,7 @@
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributeView;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileAttribute;
@@ -45,6 +47,7 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -58,6 +61,7 @@
 import com.google.common.collect.Sets;
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.internal.util.Mimetype;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.Bucket;
@@ -72,6 +76,7 @@
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.utils.StringUtils;
 import static com.google.common.collect.Sets.difference;
 import static java.lang.String.format;
 import static org.carlspring.cloud.storage.s3fs.S3Factory.ACCESS_KEY;
@@ -94,6 +99,7 @@
 import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_SEND_BUFFER_SIZE_HINT;
 import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_TIMEOUT;
 import static org.carlspring.cloud.storage.s3fs.S3Factory.USER_AGENT;
+import static software.amazon.awssdk.http.Header.CONTENT_TYPE;
 import static software.amazon.awssdk.http.HttpStatusCode.NOT_FOUND;
 
 /**
@@ -528,6 +534,84 @@ public InputStream newInputStream(Path path,
         }
     }
 
+    @Override
+    public OutputStream newOutputStream(final Path path,
+                                        final OpenOption... options)
+            throws IOException
+    {
+        final S3Path s3Path = toS3Path(path);
+
+        // validate options
+        if (options.length > 0)
+        {
+            final Set<OpenOption> opts = new LinkedHashSet<>(Arrays.asList(options));
+
+            // cannot handle APPEND here -> use newByteChannel() implementation
+            if (opts.contains(StandardOpenOption.APPEND))
+            {
+                return super.newOutputStream(path, options);
+            }
+
+            if (opts.contains(StandardOpenOption.READ))
+            {
+                throw new IllegalArgumentException("READ not allowed");
+            }
+
+            final boolean create = opts.remove(StandardOpenOption.CREATE);
+            final boolean createNew = opts.remove(StandardOpenOption.CREATE_NEW);
+            final boolean truncateExisting = opts.remove(StandardOpenOption.TRUNCATE_EXISTING);
+
+            // remove irrelevant/ignored options
+            opts.remove(StandardOpenOption.WRITE);
+            opts.remove(StandardOpenOption.SPARSE);
+
+            if (!opts.isEmpty())
+            {
+                throw new UnsupportedOperationException(opts.iterator().next() + " not supported");
+            }
+
+            validateCreateAndTruncateOptions(path, s3Path, create, createNew, truncateExisting);
+        }
+
+
+        final Map<String, String> metadata = buildMetadataFromPath(path);
+        return new S3OutputStream(s3Path.getFileSystem().getClient(), s3Path.toS3ObjectId(), metadata);
+    }
+
+    private void validateCreateAndTruncateOptions(final Path path,
+                                                  final S3Path s3Path,
+                                                  final boolean create,
+                                                  final boolean createNew,
+                                                  final boolean truncateExisting)
+            throws FileAlreadyExistsException, NoSuchFileException
+    {
+        if (!(create && truncateExisting))
+        {
+            if (s3Path.getFileSystem().provider().exists(s3Path))
+            {
+                if (createNew || !truncateExisting)
+                {
+                    throw new FileAlreadyExistsException(path.toString());
+                }
+            }
+            else if (!createNew && !create)
+            {
+                throw new NoSuchFileException(path.toString());
+            }
+        }
+    }
+
+    private Map<String, String> buildMetadataFromPath(final Path path)
+    {
+        final Map<String, String> metadata = new HashMap<>();
+        final String contentType = Mimetype.getInstance().getMimetype(path);
+        if (!StringUtils.isEmpty(contentType))
+        {
+            metadata.put(CONTENT_TYPE, contentType);
+        }
+        return metadata;
+    }
+
     @Override
     public SeekableByteChannel newByteChannel(Path path,
                                               Set<? extends OpenOption> options,
@@ -586,7 +670,6 @@ public void createDirectory(Path dir,
 
         // create the object as directory
         final String directoryKey = s3Path.getKey().endsWith("/") ? s3Path.getKey() : s3Path.getKey() + "/";
-        //TODO: If the temp file is larger than 5 GB then, instead of a putObject, a multi-part upload is needed.
         final PutObjectRequest request = PutObjectRequest.builder()
                                                          .bucket(bucketName)
                                                          .key(directoryKey)
@@ -660,7 +743,6 @@ public void copy(Path source, Path target, CopyOption... options)
 
         final String encodedUrl = encodeUrl(bucketNameOrigin, keySource);
 
-        //TODO: If the temp file is larger than 5 GB then, instead of a copyObject, a multi-part copy is needed.
         final CopyObjectRequest request = CopyObjectRequest.builder()
                                                            .copySource(encodedUrl)
                                                            .destinationBucket(bucketNameTarget)

src/main/java/org/carlspring/cloud/storage/s3fs/S3ObjectId.java

@@ -0,0 +1,130 @@
+package org.carlspring.cloud.storage.s3fs;
+
+import java.io.Serializable;
+
+/**
+ * An Immutable S3 object identifier. Used to uniquely identify an S3 object.
+ * Can be instantiated via the convenient builder {@link Builder}.
+ */
+public class S3ObjectId
+        implements Serializable
+{
+
+    private final String bucket;
+    private final String key;
+
+    /**
+     * @param builder must not be null.
+     */
+    private S3ObjectId(final Builder builder)
+    {
+        this.bucket = builder.getBucket();
+        this.key = builder.getKey();
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    public Builder cloneBuilder()
+    {
+        return new Builder(this);
+    }
+
+    public String getBucket()
+    {
+        return bucket;
+    }
+
+    public String getKey()
+    {
+        return key;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        S3ObjectId that = (S3ObjectId) o;
+
+        if (getBucket() != null ? !getBucket().equals(that.getBucket()) : that.getBucket() != null) return false;
+        return getKey() != null ? getKey().equals(that.getKey()) : that.getKey() == null;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int result = getBucket() != null ? getBucket().hashCode() : 0;
+        result = 31 * result + (getKey() != null ? getKey().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "bucket: " + bucket + ", key: " + key;
+    }
+
+    static final class Builder
+    {
+
+        private String bucket;
+        private String key;
+
+        public Builder()
+        {
+            super();
+        }
+
+        /**
+         * @param src S3 object id, which must not be null.
+         */
+        public Builder(final S3ObjectId src)
+        {
+            super();
+            this.bucket(src.getBucket());
+            this.key(src.getKey());
+        }
+
+        public String getBucket()
+        {
+            return bucket;
+        }
+
+        public String getKey()
+        {
+            return key;
+        }
+
+        public void setBucket(final String bucket)
+        {
+            this.bucket = bucket;
+        }
+
+        public void setKey(final String key)
+        {
+            this.key = key;
+        }
+
+        public Builder bucket(final String bucket)
+        {
+            this.bucket = bucket;
+            return this;
+        }
+
+        public Builder key(final String key)
+        {
+            this.key = key;
+            return this;
+        }
+
+        public S3ObjectId build()
+        {
+            return new S3ObjectId(this);
+        }
+
+    }
+}