add support for AWS S3 external file storage

README.md

@@ -113,6 +113,22 @@ If you also would like to test download count via Azure Blob, follow these steps
   * `AZURE_LOGS_SAS_TOKEN` with the shared access token for the `insights-logs-storageread` container.
   * If you change the variables in a running workspace, run `scripts/generate-properties.sh` in the `server` directory to update the application properties.
 
+### Amazon S3 Setup
+
+If you would like to test file storage via Amazon S3, follow these steps:
+
+* Login to the AWS Console and create an S3 [storage bucket](https://s3.console.aws.amazon.com/s3/home?refid=ft_card)
+* Follow the steps for [programmatic access](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) to create your access key id and secret access key
+* Configure the following environment variables on your server environment
+  * `AWS_ACCESS_KEY_ID` with your access key id
+  * `AWS_SECRET_ACCESS_KEY` with your secret access key
+  * `AWS_REGION` with your bucket region name
+  * `AWS_SERVICE_ENDPOINT` with the url of your S3 provider if not using AWS (for AWS do not set)
+  * `AWS_BUCKET` with your bucket name
+  * `AWS_PATH_STYLE_ACCESS` whether or not to use path style access, (defaults to `false`)
+    * Path-style access: `https://s3.<region>.amazonaws.com/<bucket-name>/<resource-key>`
+    * Virtual-style access: `https://<bucket-name>.s3.<region>.amazonaws.com/<resource-key>`
+
 ## License
 
 [Eclipse Public License 2.0](https://www.eclipse.org/legal/epl-2.0/)

server/build.gradle

@@ -25,6 +25,7 @@ def versions = [
     spdx: '2.2.4',
     gcloud: '1.113.4',
     azure: '12.9.0',
+    aws: '1.12.247',
     guava: '28.2-jre',
     junit: '5.7.1',
     testcontainers: '1.15.2',
@@ -70,6 +71,7 @@ dependencies {
     implementation "org.flywaydb:flyway-core:${versions.flyway}"
     implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}"
     implementation "com.azure:azure-storage-blob:${versions.azure}"
+    implementation "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
     implementation "io.springfox:springfox-boot-starter:${versions.springfox}"
     implementation "net.javacrumbs.shedlock:shedlock-spring:${versions.shedlock}"
     implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-template:${versions.shedlock}"

server/scripts/generate-properties.sh

@@ -60,3 +60,15 @@ then
     echo "ovsx.logs.azure.sas-token=$AZURE_LOGS_SAS_TOKEN" >> $OVSX_APP_PROFILE
     echo "Using Azure Logs Storage: $AZURE_LOGS_SERVICE_ENDPOINT"
 fi
+
+# Set the AWS Storage service access key id, secret access key, region and endpoint
+if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ] && [ -n "$AWS_REGION" ] && [ -n "$AWS_BUCKET" ]
+then
+  echo "ovsx.storage.aws.access-key-id=$AWS_ACCESS_KEY_ID" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.secret-access-key=$AWS_SECRET_ACCESS_KEY" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.region=$AWS_REGION" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.service-endpoint=$AWS_SERVICE_ENDPOINT" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.bucket=$AWS_BUCKET" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.path-style-access=$AWS_PATH_STYLE_ACCESS" >> $OVSX_APP_PROFILE
+  echo "Using AWS S3 Storage: $AWS_SERVICE_ENDPOINT"
+fi

server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java

@@ -16,7 +16,6 @@
 
 import java.io.InputStream;
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import javax.persistence.EntityManager;
@@ -36,6 +35,7 @@
 import org.eclipse.openvsx.search.ExtensionSearch;
 import org.eclipse.openvsx.search.ISearchService;
 import org.eclipse.openvsx.search.SearchUtilService;
+import org.eclipse.openvsx.storage.FileCacheDurationConfig;
 import org.eclipse.openvsx.storage.StorageUtilService;
 import org.eclipse.openvsx.util.*;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -79,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry {
     @Autowired
     CacheService cache;
 
+    @Autowired
+    FileCacheDurationConfig fileCacheDurationConfig;
+
     @Override
     public NamespaceJson getNamespace(String namespaceName) {
         var namespace = repositories.findNamespace(namespaceName);
@@ -170,7 +173,7 @@ public ResponseEntity<byte[]> getFile(String namespace, String extensionName, St
         } else {
             return ResponseEntity.status(HttpStatus.FOUND)
                     .location(storageUtil.getLocation(resource))
-                    .cacheControl(CacheControl.maxAge(7, TimeUnit.DAYS).cachePublic())
+                    .cacheControl(CacheControl.maxAge(fileCacheDurationConfig.getCacheDuration()).cachePublic())
                     .build();
         }
     }
@@ -720,4 +723,4 @@ private boolean isVerified(ExtensionVersionDTO extVersion, Map<Long, List<Namesp
         return memberships.stream().anyMatch(m -> m.getRole().equalsIgnoreCase(NamespaceMembership.ROLE_OWNER))
                 && memberships.stream().anyMatch(m -> m.getUserId() == user.getId());
     }
-}
+}

server/src/main/java/org/eclipse/openvsx/entities/FileResource.java

@@ -31,6 +31,7 @@ public class FileResource {
     public static final String STORAGE_DB = "database";
     public static final String STORAGE_GOOGLE = "google-cloud";
     public static final String STORAGE_AZURE = "azure-blob";
+    public static final String STORAGE_AWS = "aws";
 
     @Id
     @GeneratedValue
@@ -97,4 +98,4 @@ public String getStorageType() {
     public void setStorageType(String storageType) {
         this.storageType = storageType;
     }
-}
+}

server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java

@@ -13,6 +13,7 @@
 import org.eclipse.openvsx.entities.ExtractResourcesMigrationItem;
 import org.eclipse.openvsx.entities.FileResource;
 import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.storage.AwsStorageService;
 import org.eclipse.openvsx.storage.AzureBlobStorageService;
 import org.eclipse.openvsx.storage.GoogleCloudStorageService;
 import org.eclipse.openvsx.storage.IStorageService;
@@ -52,6 +53,9 @@ public class ExtractResourcesJobRequestHandler implements JobRequestHandler<Extr
     @Autowired
     GoogleCloudStorageService googleStorage;
 
+    @Autowired
+    AwsStorageService awsStorage;
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     @Job(name = "Extract resources from published extension version", retries = 3)
@@ -102,7 +106,8 @@ private void uploadResource(FileResource resource) {
     private IStorageService getStorage(FileResource resource) {
         var storages = Map.of(
                 FileResource.STORAGE_AZURE, azureStorage,
-                FileResource.STORAGE_GOOGLE, googleStorage
+                FileResource.STORAGE_GOOGLE, googleStorage,
+                FileResource.STORAGE_AWS, awsStorage
         );
 
         return storages.get(resource.getStorageType());

server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java

@@ -0,0 +1,141 @@
+/********************************************************************************
+ * Copyright (c) 2022 Marshall Walker and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ ********************************************************************************/
+
+package org.eclipse.openvsx.storage;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.google.common.base.Strings;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+import org.apache.commons.lang3.ArrayUtils;
+import org.eclipse.openvsx.entities.FileResource;
+import org.eclipse.openvsx.util.TargetPlatform;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AwsStorageService implements IStorageService {
+
+    @Autowired
+    FileCacheDurationConfig fileCacheDurationConfig;
+
+    @Value("${ovsx.storage.aws.access-key-id:}")
+    String accessKeyId;
+
+    @Value("${ovsx.storage.aws.secret-access-key:}")
+    String secretAccessKey;
+
+    @Value("${ovsx.storage.aws.region:}")
+    String region;
+
+    @Value("${ovsx.storage.aws.service-endpoint:}")
+    String serviceEndpoint;
+
+    @Value("${ovsx.storage.aws.bucket:}")
+    String bucket;
+
+    @Value("${ovsx.storage.aws.path-style-access:false}")
+    boolean pathStyleAccess;
+
+    private AmazonS3 s3Client;
+
+    protected AmazonS3 getS3Client() {
+        if (s3Client == null) {
+            var credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
+            var s3ClientBuilder = AmazonS3ClientBuilder.standard()
+                .withPathStyleAccessEnabled(pathStyleAccess)
+                .withCredentials(new AWSStaticCredentialsProvider(credentials));
+
+            if (Strings.isNullOrEmpty(serviceEndpoint)) {
+                s3ClientBuilder.withRegion(region);
+            } else {
+                s3ClientBuilder.withEndpointConfiguration(
+                    new EndpointConfiguration(serviceEndpoint, region));
+            }
+            s3Client = s3ClientBuilder.build();
+        }
+        return s3Client;
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return !Strings.isNullOrEmpty(accessKeyId);
+    }
+
+    @Override
+    public void uploadFile(FileResource resource) {
+        var client = getS3Client();
+        var key = getObjectKey(resource);
+        var resourceName = resource.getName();
+
+        var metadata = new ObjectMetadata();
+        metadata.setContentLength(resource.getContent().length);
+        metadata.setContentType(StorageUtil.getFileType(resourceName).toString());
+
+        if (resourceName.endsWith(".vsix")) {
+            metadata.setContentDisposition("attachment; filename=\"" + resourceName + "\"");
+        } else {
+            metadata.setCacheControl(StorageUtil.getCacheControl(resourceName).getHeaderValue());
+        }
+
+        try(var stream = new ByteArrayInputStream(resource.getContent())) {
+            client.putObject(bucket, key, stream, metadata);
+        } catch(IOException exc) {
+            throw new RuntimeException(exc);
+        }
+    }
+
+    @Override
+    public void removeFile(FileResource resource) {
+        getS3Client().deleteObject(bucket, getObjectKey(resource));
+    }
+
+    @Override
+    public URI getLocation(FileResource resource) {
+        var client = getS3Client();
+        var objectKey = getObjectKey(resource);
+
+        var instant = LocalDateTime.now().toInstant(ZoneOffset.UTC);
+        instant.plus(fileCacheDurationConfig.getCacheDuration());
+        var date = Date.from(instant);
+
+        try {
+            return client.generatePresignedUrl(bucket, objectKey, date).toURI();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected String getObjectKey(FileResource resource) {
+        var extensionVersion = resource.getExtension();
+        var extension = extensionVersion.getExtension();
+        var namespace = extension.getNamespace();
+        var segments = new String[] {namespace.getName(), extension.getName()};
+        if (!TargetPlatform.isUniversal(extensionVersion)) {
+            segments = ArrayUtils.add(segments, extensionVersion.getTargetPlatform());
+        }
+
+        segments = ArrayUtils.add(segments, extensionVersion.getVersion());
+        segments = ArrayUtils.addAll(segments, resource.getName().split("/"));
+        return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
+    }
+}

server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java

@@ -104,7 +104,7 @@ public void removeFile(FileResource resource) {
         }
 	}
 
-	@Override
+    @Override
 	public URI getLocation(FileResource resource) {
         var blobName = getBlobName(resource);
         if (Strings.isNullOrEmpty(serviceEndpoint)) {
@@ -131,4 +131,4 @@ protected String getBlobName(FileResource resource) {
         return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
     }
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/FileCacheDurationConfig.java

@@ -0,0 +1,26 @@
+/********************************************************************************
+ * Copyright (c) 2022 Marshall Walker and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ ********************************************************************************/
+
+package org.eclipse.openvsx.storage;
+
+import java.time.Duration;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class FileCacheDurationConfig {
+
+    @Value("#{T(java.time.Duration).parse('${ovsx.storage.file-cache-duration:P7D}')}")
+    private Duration cacheDuration;
+
+    public Duration getCacheDuration() {
+        return cacheDuration;
+    }
+}

server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java

@@ -112,4 +112,4 @@ protected String getObjectId(FileResource resource) {
         return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
     }
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java

@@ -35,4 +35,4 @@ public interface IStorageService {
      */
     URI getLocation(FileResource resource);
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java

@@ -67,7 +67,7 @@ public void findResources(ApplicationStartedEvent event) {
             return;
         }
 
-        var migrations = Lists.newArrayList(STORAGE_DB, STORAGE_GOOGLE, STORAGE_AZURE);
+        var migrations = Lists.newArrayList(STORAGE_DB, STORAGE_GOOGLE, STORAGE_AZURE, STORAGE_AWS);
         migrations.remove(storageType);
         var migrationCount = new int[migrations.size()];
         for (var i = 0; i < migrations.size(); i++) {
@@ -124,4 +124,4 @@ private byte[] downloadFile(FileResource resource) {
         return restTemplate.getForObject(location, byte[].class);
     }
 
-}
+}