add support for AWS S3 external file storage

server/build.gradle

@@ -25,6 +25,7 @@ def versions = [
     spdx: '2.2.4',
     gcloud: '1.113.4',
     azure: '12.9.0',
+    aws: '1.12.247',
     guava: '28.2-jre',
     junit: '5.7.1',
     testcontainers: '1.15.2',
@@ -70,6 +71,7 @@ dependencies {
     implementation "org.flywaydb:flyway-core:${versions.flyway}"
     implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}"
     implementation "com.azure:azure-storage-blob:${versions.azure}"
+    implementation "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
     implementation "io.springfox:springfox-boot-starter:${versions.springfox}"
     implementation "net.javacrumbs.shedlock:shedlock-spring:${versions.shedlock}"
     implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-template:${versions.shedlock}"

server/scripts/generate-properties.sh

@@ -60,3 +60,15 @@ then
     echo "ovsx.logs.azure.sas-token=$AZURE_LOGS_SAS_TOKEN" >> $OVSX_APP_PROFILE
     echo "Using Azure Logs Storage: $AZURE_LOGS_SERVICE_ENDPOINT"
 fi
+
+# Set the AWS Storage service access key id, secret access key, region and endpoint
+if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY"] && [ -n "$AWS_REGION"] && [ -n "$AWS_SERVICE_ENDPOINT"] && [ -n "$AWS_BUCKET"]
+then
+  echo "ovsx.storage.aws.access-key-id=$AWS_ACCESS_KEY_ID" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.secret-access-key=$AWS_SECRET_ACCESS_KEY" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.region=$AWS_REGION" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.service-endpoint=$AWS_SERVICE_ENDPOINT" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.bucket=$AWS_BUCKET" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.path-style-access=$AWS_PATH_STYLE_ACCESS" >> $OVSX_APP_PROFILE
+  echo "ovsx.storage.aws.presign-expiry-minutes=$AWS_CACHE_DURATION_MINUTES" >> $OVSX_APP_PROFILE
+  echo "Using AWS S3 Storage: $AWS_SERVICE_ENDPOINT"

server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java

@@ -36,6 +36,7 @@
 import org.eclipse.openvsx.search.ExtensionSearch;
 import org.eclipse.openvsx.search.ISearchService;
 import org.eclipse.openvsx.search.SearchUtilService;
+import org.eclipse.openvsx.storage.AwsStorageService;
 import org.eclipse.openvsx.storage.StorageUtilService;
 import org.eclipse.openvsx.util.*;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -170,7 +171,7 @@ public ResponseEntity<byte[]> getFile(String namespace, String extensionName, St
         } else {
             return ResponseEntity.status(HttpStatus.FOUND)
                     .location(storageUtil.getLocation(resource))
-                    .cacheControl(CacheControl.maxAge(7, TimeUnit.DAYS).cachePublic())
+                    .cacheControl(CacheControl.maxAge(storageUtil.getCacheDuration(resource)).cachePublic())
                     .build();
         }
     }
@@ -720,4 +721,4 @@ private boolean isVerified(ExtensionVersionDTO extVersion, Map<Long, List<Namesp
         return memberships.stream().anyMatch(m -> m.getRole().equalsIgnoreCase(NamespaceMembership.ROLE_OWNER))
                 && memberships.stream().anyMatch(m -> m.getUserId() == user.getId());
     }
-}
+}

server/src/main/java/org/eclipse/openvsx/entities/FileResource.java

@@ -31,6 +31,7 @@ public class FileResource {
     public static final String STORAGE_DB = "database";
     public static final String STORAGE_GOOGLE = "google-cloud";
     public static final String STORAGE_AZURE = "azure-blob";
+    public static final String STORAGE_AWS = "aws";
 
     @Id
     @GeneratedValue
@@ -97,4 +98,4 @@ public String getStorageType() {
     public void setStorageType(String storageType) {
         this.storageType = storageType;
     }
-}
+}

server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java

@@ -13,6 +13,7 @@
 import org.eclipse.openvsx.entities.ExtractResourcesMigrationItem;
 import org.eclipse.openvsx.entities.FileResource;
 import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.storage.AwsStorageService;
 import org.eclipse.openvsx.storage.AzureBlobStorageService;
 import org.eclipse.openvsx.storage.GoogleCloudStorageService;
 import org.eclipse.openvsx.storage.IStorageService;
@@ -52,6 +53,9 @@ public class ExtractResourcesJobRequestHandler implements JobRequestHandler<Extr
     @Autowired
     GoogleCloudStorageService googleStorage;
 
+    @Autowired
+    AwsStorageService awsStorage;
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     @Job(name = "Extract resources from published extension version", retries = 3)
@@ -102,7 +106,8 @@ private void uploadResource(FileResource resource) {
     private IStorageService getStorage(FileResource resource) {
         var storages = Map.of(
                 FileResource.STORAGE_AZURE, azureStorage,
-                FileResource.STORAGE_GOOGLE, googleStorage
+                FileResource.STORAGE_GOOGLE, googleStorage,
+                FileResource.STORAGE_AWS, awsStorage
         );
 
         return storages.get(resource.getStorageType());

server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java

@@ -0,0 +1,133 @@
+/********************************************************************************
+ * Copyright (c) 2022 TypeFox and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ ********************************************************************************/
+
+package org.eclipse.openvsx.storage;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.google.common.base.Strings;
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.util.Calendar;
+import org.apache.commons.lang3.ArrayUtils;
+import org.eclipse.openvsx.entities.FileResource;
+import org.eclipse.openvsx.util.TargetPlatform;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AwsStorageService implements IStorageService {
+
+    @Value("${ovsx.storage.aws.access-key-id:}")
+    String accessKeyId;
+
+    @Value("${ovsx.storage.aws.secret-access-key:}")
+    String secretAccessKey;
+
+    @Value("${ovsx.storage.aws.region:}")
+    String region;
+
+    @Value("${ovsx.storage.aws.service-endpoint:}")
+    String serviceEndpoint;
+
+    @Value("${ovsx.storage.aws.bucket:}")
+    String bucket;
+
+    @Value("${ovsx.storage.aws.path-style-access:false}")
+    boolean pathStyleAccess;
+
+    @Value("${ovsx.storage.aws.presign-expiry-minutes:60}")
+    int presignExpiryMinutes;
+
+    private AmazonS3 s3Client;
+
+    protected AmazonS3 getS3Client() {
+        if (s3Client == null) {
+            var credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
+            this.s3Client = AmazonS3ClientBuilder.standard()
+                .withPathStyleAccessEnabled(pathStyleAccess)
+                .withEndpointConfiguration(new EndpointConfiguration(serviceEndpoint, region))
+                .withCredentials(new AWSStaticCredentialsProvider(credentials))
+                .build();
+        }
+        return s3Client;
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return !Strings.isNullOrEmpty(serviceEndpoint);
+    }
+
+    @Override
+    public void uploadFile(FileResource resource) {
+        var client = getS3Client();
+        var key = getObjectKey(resource);
+        var stream = new ByteArrayInputStream(resource.getContent());
+        var resourceName = resource.getName();
+
+        var metadata = new ObjectMetadata();
+        metadata.setContentLength(resource.getContent().length);
+        metadata.setContentType(StorageUtil.getFileType(resourceName).toString());
+
+        if (resourceName.endsWith(".vsix")) {
+            metadata.setContentDisposition("attachment; filename=\"" + resourceName + "\"");
+        } else {
+            metadata.setCacheControl(StorageUtil.getCacheControl(resourceName).getHeaderValue());
+        }
+        client.putObject(bucket, key, stream, metadata);
+    }
+
+    @Override
+    public void removeFile(FileResource resource) {
+        var client = getS3Client();
+        client.deleteObject(bucket, getObjectKey(resource));
+    }
+
+    @Override
+    public Duration getCacheDuration(FileResource resource) {
+        return Duration.ofMinutes(presignExpiryMinutes);
+    }
+
+    @Override
+    public URI getLocation(FileResource resource) {
+        var client = getS3Client();
+        var objectKey = getObjectKey(resource);
+
+        var calendar = Calendar.getInstance();
+        calendar.add(Calendar.MINUTE, presignExpiryMinutes);
+
+        try {
+            return client.generatePresignedUrl(bucket, objectKey, calendar.getTime()).toURI();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected String getObjectKey(FileResource resource) {
+        var extensionVersion = resource.getExtension();
+        var extension = extensionVersion.getExtension();
+        var namespace = extension.getNamespace();
+        var segments = new String[] {namespace.getName(), extension.getName()};
+        if (!TargetPlatform.isUniversal(extensionVersion)) {
+            segments = ArrayUtils.add(segments, extensionVersion.getTargetPlatform());
+        }
+
+        segments = ArrayUtils.add(segments, extensionVersion.getVersion());
+        segments = ArrayUtils.addAll(segments, resource.getName().split("/"));
+        return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
+    }
+}

server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java

@@ -14,6 +14,7 @@
 import com.azure.storage.blob.models.BlobHttpHeaders;
 import com.azure.storage.blob.models.BlobStorageException;
 import com.google.common.base.Strings;
+import java.time.Duration;
 import org.apache.commons.lang3.ArrayUtils;
 import org.eclipse.openvsx.entities.FileResource;
 import org.eclipse.openvsx.util.TargetPlatform;
@@ -104,7 +105,12 @@ public void removeFile(FileResource resource) {
         }
 	}
 
-	@Override
+    @Override
+    public Duration getCacheDuration(FileResource resource) {
+        return Duration.ofDays(7);
+    }
+
+    @Override
 	public URI getLocation(FileResource resource) {
         var blobName = getBlobName(resource);
         if (Strings.isNullOrEmpty(serviceEndpoint)) {
@@ -131,4 +137,4 @@ protected String getBlobName(FileResource resource) {
         return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
     }
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java

@@ -14,6 +14,7 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.base.Strings;
+import java.time.Duration;
 import org.apache.commons.lang3.ArrayUtils;
 import org.eclipse.openvsx.entities.FileResource;
 import org.eclipse.openvsx.util.TargetPlatform;
@@ -89,6 +90,11 @@ public void removeFile(FileResource resource) {
         getStorage().delete(BlobId.of(bucketId, objectId));
     }
 
+    @Override
+    public Duration getCacheDuration(FileResource resource) {
+        return Duration.ofDays(7);
+    }
+
     @Override
     public URI getLocation(FileResource resource) {
         if (Strings.isNullOrEmpty(bucketId)) {
@@ -112,4 +118,4 @@ protected String getObjectId(FileResource resource) {
         return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/'
     }
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java

@@ -11,7 +11,9 @@
 
 import java.net.URI;
 
+import java.time.Duration;
 import org.eclipse.openvsx.entities.FileResource;
+import org.springframework.http.CacheControl;
 
 public interface IStorageService {
 
@@ -30,9 +32,14 @@ public interface IStorageService {
      */
     void removeFile(FileResource resource);
 
+    /**
+     * Returns the cache control duration of a resource.
+     */
+    Duration getCacheDuration(FileResource resource);
+
     /**
      * Returns the public access location of a resource.
      */
     URI getLocation(FileResource resource);
 
-}
+}

server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java

@@ -67,7 +67,7 @@ public void findResources(ApplicationStartedEvent event) {
             return;
         }
 
-        var migrations = Lists.newArrayList(STORAGE_DB, STORAGE_GOOGLE, STORAGE_AZURE);
+        var migrations = Lists.newArrayList(STORAGE_DB, STORAGE_GOOGLE, STORAGE_AZURE, STORAGE_AWS);
         migrations.remove(storageType);
         var migrationCount = new int[migrations.size()];
         for (var i = 0; i < migrations.size(); i++) {
@@ -124,4 +124,4 @@ private byte[] downloadFile(FileResource resource) {
         return restTemplate.getForObject(location, byte[].class);
     }
 
-}
+}