diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000..b74f3478 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1 @@ +This product includes software developed by Minio. (https://github.com/minio/minio-java) diff --git a/pom.xml b/pom.xml index 81585780..70b32bb1 100644 --- a/pom.xml +++ b/pom.xml @@ -40,11 +40,13 @@ 22.0.0 ${project.basedir} - 8 true true + 8 + true 245 + 2.25.32 @@ -56,6 +58,32 @@ pom import + + + software.amazon.awssdk + bom + ${dep.aws-sdk.version} + pom + import + + + + + + com.mycila + license-maven-plugin + + + + + **/io/trino/s3/proxy/server/credentials/Signer.java + + + + + + + diff --git a/trino-s3-proxy/pom.xml b/trino-s3-proxy/pom.xml index 28d9ab81..ecae50e8 100644 --- a/trino-s3-proxy/pom.xml +++ b/trino-s3-proxy/pom.xml @@ -64,6 +64,12 @@ jakarta.ws.rs-api + + org.assertj + assertj-core + test + + org.junit.jupiter junit-jupiter-api diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServer.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServer.java index 1c2ce982..642bf526 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServer.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServer.java @@ -14,10 +14,12 @@ package io.trino.s3.proxy.server; import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.event.client.EventModule; import io.airlift.http.server.HttpServerModule; +import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.jaxrs.JaxrsModule; import io.airlift.json.JsonModule; import io.airlift.log.Logger; @@ -39,8 +41,13 @@ public static void main(String[] args) .add(new JaxrsModule()); Bootstrap app = new Bootstrap(modules.build()); - app.initialize(); + Injector injector = app.initialize(); log.info("======== SERVER STARTED ========"); + + TestingHttpServer httpServer = injector.getInstance(TestingHttpServer.class); + log.info(""); + log.info("Endpoint: %s", httpServer.getBaseUrl()); + log.info(""); } } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Credentials.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Credentials.java new file mode 100644 index 00000000..ed1533a4 --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Credentials.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.credentials; + +import static java.util.Objects.requireNonNull; + +public record Credentials(Credential emulated, Credential real) +{ + public Credentials + { + requireNonNull(emulated, "emulated is null"); + requireNonNull(real, "real is null"); + } + + public record Credential(String accessKey, String secretKey) + { + public Credential + { + requireNonNull(accessKey, "accessKey is null"); + requireNonNull(secretKey, "secretKey is null"); + } + } +} diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/DummyTest.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java similarity index 76% rename from trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/DummyTest.java rename to trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java index 00ca9ac6..96f7852b 100644 --- a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/DummyTest.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java @@ -11,15 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.s3.proxy.server; +package io.trino.s3.proxy.server.credentials; -import org.junit.jupiter.api.Test; +import java.util.Optional; -public class DummyTest +public interface CredentialsController { - @Test - public void testDummy() - { - // stub test for now - } + Optional credentials(String emulatedAccessKey); } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Signer.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Signer.java new file mode 100644 index 00000000..97b738f7 --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Signer.java @@ -0,0 +1,197 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.s3.proxy.server.credentials; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.common.collect.MultimapBuilder; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; + +import javax.crypto.spec.SecretKeySpec; + +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static com.google.common.hash.Hashing.sha256; + +/** + * Amazon AWS S3 signature V4 signer. + */ +class Signer +{ + // + // Excerpts from @legal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 + // + // * User-Agent + // This is ignored from signing because signing this causes problems with generating pre-signed + // URLs (that are executed by other agents) or when customers pass requests through proxies, which + // may modify the user-agent. + // + // * Authorization + // Is skipped for obvious reasons. + // + // * Accept-Encoding + // Some S3 servers like Hitachi Content Platform do not honour this header for signature + // calculation. + // + private static final Set IGNORED_HEADERS = ImmutableSet.of("accept-encoding", "authorization", "user-agent"); + + private static final ZoneId UTC = ZoneId.of("Z"); + + private static final DateTimeFormatter AMZ_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.US).withZone(UTC); + + // Formatted string is convertible to LocalDate only, not to LocalDateTime or ZonedDateTime. + // Below example shows how to use this to get ZonedDateTime. + // LocalDate.parse("20200225", SIGNER_DATE_FORMAT).atStartOfDay(UTC); + private static final DateTimeFormatter SIGNER_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.US).withZone(UTC); + + private Signer() {} + + /** + * Returns signed request of given request for S3 service. + */ + static String signV4S3(SignerMetadata metadata, String region, String accessKey, String secretKey, String contentSha256) + { + ZonedDateTime date = ZonedDateTime.parse(metadata.headerValue("x-amz-date").orElseThrow(), AMZ_DATE_FORMAT); + + String scope = date.format(SIGNER_DATE_FORMAT) + "/" + region + "/s3/aws4_request"; + Map canonicalHeaders = buildCanonicalHeaders(metadata); + String signedHeaders = Joiner.on(";").join(canonicalHeaders.keySet()); + String canonicalRequestHash = buildCanonicalRequestHash(metadata, canonicalHeaders, signedHeaders, contentSha256); + String stringToSign = "AWS4-HMAC-SHA256" + "\n" + date.format(AMZ_DATE_FORMAT) + "\n" + scope + "\n" + canonicalRequestHash; + byte[] signingKey = buildSigningKey(secretKey, date, region); + byte[] digest = sumHmac(signingKey, stringToSign.getBytes(StandardCharsets.UTF_8)); + String signature = BaseEncoding.base16().encode(digest).toLowerCase(Locale.US); + + return "AWS4-HMAC-SHA256 Credential=" + + accessKey + + "/" + + scope + + ", SignedHeaders=" + + signedHeaders + + ", Signature=" + + signature; + } + + /** + * Returns HMacSHA256 digest of given key and data. + */ + private static byte[] sumHmac(byte[] key, byte[] data) + { + return Hashing.hmacSha256(new SecretKeySpec(key, "HmacSHA256")).hashBytes(data).asBytes(); + } + + private static Map buildCanonicalHeaders(SignerMetadata metadata) + { + Map canonicalHeaders = new TreeMap<>(); + + for (String name : metadata.headerNames()) { + String signedHeader = name.toLowerCase(Locale.US); + if (!IGNORED_HEADERS.contains(signedHeader)) { + // Convert and add header values as per + // https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + // * Header having multiple values should be converted to comma separated values. + // * Multi-spaced value of header should be trimmed to single spaced value. + canonicalHeaders.put( + signedHeader, + metadata.headerValues(name).stream() + .map(value -> value.replaceAll("( +)", " ")) + .collect(Collectors.joining(","))); + } + } + + return canonicalHeaders; + } + + private static String buildCanonicalQueryString(SignerMetadata metadata) + { + if (metadata.encodedQuery().isEmpty()) { + return ""; + } + + // Building a multimap which only order keys, ordering values is not performed + // until MinIO server supports it. + Multimap signedQueryParams = MultimapBuilder.treeKeys().arrayListValues().build(); + + for (String queryParam : metadata.encodedQuery().split("&")) { + String[] tokens = queryParam.split("="); + if (tokens.length > 1) { + signedQueryParams.put(tokens[0], tokens[1]); + } + else { + signedQueryParams.put(tokens[0], ""); + } + } + + return Joiner.on("&").withKeyValueSeparator("=").join(signedQueryParams.entries()); + } + + private static String buildCanonicalRequestHash(SignerMetadata metadata, Map canonicalHeaders, String signedHeaders, String contentSha256) + { + String canonicalQueryString = buildCanonicalQueryString(metadata); + + // CanonicalRequest = + // HTTPRequestMethod + '\n' + + // CanonicalURI + '\n' + + // CanonicalQueryString + '\n' + + // CanonicalHeaders + '\n' + + // SignedHeaders + '\n' + + // HexEncode(Hash(RequestPayload)) + String canonicalRequest = metadata.httpMethod() + + "\n" + + metadata.encodedPath() + + "\n" + + canonicalQueryString + + "\n" + + Joiner.on("\n").withKeyValueSeparator(":").join(canonicalHeaders) + + "\n\n" + + signedHeaders + + "\n" + + contentSha256; + + return sha256Hash(canonicalRequest); + } + + private static byte[] buildSigningKey(String secretKey, ZonedDateTime date, String region) + { + String aws4SecretKey = "AWS4" + secretKey; + + byte[] dateKey = sumHmac( + aws4SecretKey.getBytes(StandardCharsets.UTF_8), + date.format(SIGNER_DATE_FORMAT).getBytes(StandardCharsets.UTF_8)); + + byte[] dateRegionKey = sumHmac(dateKey, region.getBytes(StandardCharsets.UTF_8)); + + byte[] dateRegionServiceKey = sumHmac(dateRegionKey, "s3".getBytes(StandardCharsets.UTF_8)); + + return sumHmac(dateRegionServiceKey, "aws4_request".getBytes(StandardCharsets.UTF_8)); + } + + private static String sha256Hash(String str) + { + return sha256().hashString(str, StandardCharsets.UTF_8).toString(); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SignerMetadata.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SignerMetadata.java new file mode 100644 index 00000000..fc1008cd --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SignerMetadata.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.credentials; + +import com.google.common.collect.ImmutableList; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +record SignerMetadata(MultivaluedMap headers, String httpMethod, String encodedPath, String encodedQuery) +{ + SignerMetadata + { + requireNonNull(headers, "headers is null"); + requireNonNull(httpMethod, "httpMethod is null"); + requireNonNull(encodedPath, "encodedPath is null"); + requireNonNull(encodedQuery, "encodedQuery is null"); + + MultivaluedMap headersCopy = new MultivaluedHashMap<>(); + headers.forEach((key, values) -> headersCopy.put(key.toLowerCase(Locale.ROOT), values)); + headers = headersCopy; + } + + Collection headerNames() + { + return headers.keySet(); + } + + List headerValues(String name) + { + return headers.getOrDefault(name, ImmutableList.of()); + } + + Optional headerValue(String name) + { + return Optional.ofNullable(headers.getFirst(name)); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java new file mode 100644 index 00000000..78139702 --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.credentials; + +import com.google.inject.Inject; +import jakarta.ws.rs.core.MultivaluedMap; + +import static io.trino.s3.proxy.server.credentials.Signer.signV4S3; +import static java.util.Objects.requireNonNull; + +public class SigningController +{ + private final CredentialsController credentialsController; + + @Inject + public SigningController(CredentialsController credentialsController) + { + this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); + } + + public String signRequest(String method, MultivaluedMap requestHeaders, String encodedPath, String encodedQuery, String region, String accessKey) + { + // TODO + Credentials credentials = credentialsController.credentials(accessKey).orElseThrow(); + + SignerMetadata signerMetadata = new SignerMetadata(requestHeaders, method, encodedPath, encodedQuery); + + // TODO + String sha256 = signerMetadata.headerValue("x-amz-content-sha256").orElseThrow(); + + return signV4S3(signerMetadata, region, accessKey, credentials.emulated().secretKey(), sha256); + } +} diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java new file mode 100644 index 00000000..47420284 --- /dev/null +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import io.trino.s3.proxy.server.credentials.Credentials; +import io.trino.s3.proxy.server.credentials.Credentials.Credential; +import io.trino.s3.proxy.server.credentials.CredentialsController; +import io.trino.s3.proxy.server.credentials.SigningController; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestSigningController +{ + private static final Credentials CREDENTIALS = new Credentials(new Credential("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY"), new Credential("dummy", "dummy")); + + @Test + public void testRootLs() + { + CredentialsController credentialsController = accessKey -> Optional.of(CREDENTIALS); + SigningController signingController = new SigningController(credentialsController); + + // values discovered from an AWS CLI request sent to a dummy local HTTP server + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle("X-Amz-Date", "20240516T024511Z"); + requestHeaders.putSingle("X-Amz-Content-SHA256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + requestHeaders.putSingle("X-Amz-Security-Token", "FwoGZXIvYXdzEP3//////////wEaDG79rlcAjsgKPP9N3SKIAu7/Zvngne5Ov6kGrDcIIPUZYkGpwNbj8zNnbWgOhiqmOCM3hrk4NuH17mP5n3nC7urlXZxaTCywKpAHpO3YsvLXcwjlfaYFA0Au4oejwSbU9ybIlzPzrqz7lVesgCfJOV+rj5F5UAh19d7RpRpA6Vy4nxGBTTlCNIVbkW9fp2Esql2/vsdh77rAG+j+BQegtegDCKBfen4gHMdvEOF6hyc4ne43eLXjpvUKxBgpI9MjOHtNHrDbOOBFXDDyknoESgE9Hsm12nDuVQhwrI/hhA4YB/MSIpl4FTgVs2sQP3K+v65tmyvIlpL6O78S6spMM9Tv/F4JLtksTzb90w46uZk9sxKC/RBkRijisM6tBjIrr/0znxnW3i5ggGAX4H/Z3aWlxSdzNs2UGWtqig9Plp3Xa9gG+zCKcXmDAA=="); + requestHeaders.putSingle("Host", "localhost:10064"); + requestHeaders.putSingle("User-Agent", "aws-cli/2.15.16 Python/3.11.7 Darwin/22.6.0 source/x86_64 prompt/off command/s3.ls"); + requestHeaders.putSingle("Accept-Encoding", "identity"); + + String signature = signingController.signRequest("GET", requestHeaders, "/", "", "us-east-1", "THIS_IS_AN_ACCESS_KEY"); + + assertThat(signature).isEqualTo("AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=9a19c251bf4e1533174e80da59fa57c65b3149b611ec9a4104f6944767c25704"); + } + + @Test + public void testBucketLs() + { + CredentialsController credentialsController = accessKey -> Optional.of(CREDENTIALS); + SigningController signingController = new SigningController(credentialsController); + + // values discovered from an AWS CLI request sent to a dummy local HTTP server + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle("X-Amz-Date", "20240516T034003Z"); + requestHeaders.putSingle("X-Amz-Content-SHA256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + requestHeaders.putSingle("X-Amz-Security-Token", "FwoGZXIvYXdzEP3//////////wEaDG79rlcAjsgKPP9N3SKIAu7/Zvngne5Ov6kGrDcIIPUZYkGpwNbj8zNnbWgOhiqmOCM3hrk4NuH17mP5n3nC7urlXZxaTCywKpAHpO3YsvLXcwjlfaYFA0Au4oejwSbU9ybIlzPzrqz7lVesgCfJOV+rj5F5UAh19d7RpRpA6Vy4nxGBTTlCNIVbkW9fp2Esql2/vsdh77rAG+j+BQegtegDCKBfen4gHMdvEOF6hyc4ne43eLXjpvUKxBgpI9MjOHtNHrDbOOBFXDDyknoESgE9Hsm12nDuVQhwrI/hhA4YB/MSIpl4FTgVs2sQP3K+v65tmyvIlpL6O78S6spMM9Tv/F4JLtksTzb90w46uZk9sxKC/RBkRijisM6tBjIrr/0znxnW3i5ggGAX4H/Z3aWlxSdzNs2UGWtqig9Plp3Xa9gG+zCKcXmDAA=="); + requestHeaders.putSingle("Host", "localhost:10064"); + requestHeaders.putSingle("User-Agent", "aws-cli/2.15.16 Python/3.11.7 Darwin/22.6.0 source/x86_64 prompt/off command/s3.ls"); + requestHeaders.putSingle("Accept-Encoding", "identity"); + + String signature = signingController.signRequest("GET", requestHeaders, "/mybucket", "list-type=2&prefix=foo%2Fbar&delimiter=%2F&encoding-type=url", "us-east-1", "THIS_IS_AN_ACCESS_KEY"); + + assertThat(signature).isEqualTo("AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=222d7b7fcd4d5560c944e8fecd9424ee3915d131c3ad9e000d65db93e87946c4"); + } +}