Skip to content

Commit

Permalink
Import Minio signing classes
Browse files Browse the repository at this point in the history
- Copied signing classes from Minio: https://github.com/minio/minio-java
- Added Notice file for Minio
- Replaced OkHttp, etc. classes with emulated equivalents
- Added an initial test for a simple `aws s3 ls` based on this doc: https://min.io/docs/minio/linux/integrations/aws-cli-with-minio.html

Closes #5
  • Loading branch information
Randgalt committed May 17, 2024
1 parent 4b59e9d commit 35f1f81
Show file tree
Hide file tree
Showing 10 changed files with 449 additions and 10 deletions.
1 change: 1 addition & 0 deletions NOTICE.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This product includes software developed by Minio. (https://github.com/minio/minio-java)
30 changes: 29 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,13 @@

<air.java.version>22.0.0</air.java.version>
<air.main.basedir>${project.basedir}</air.main.basedir>
<air.modernizer.java-version>8</air.modernizer.java-version>
<air.check.skip-spotbugs>true</air.check.skip-spotbugs>
<air.check.skip-pmd>true</air.check.skip-pmd>
<air.modernizer.java-version>8</air.modernizer.java-version>
<air.check.skip-modernizer>true</air.check.skip-modernizer>

<dep.airlift.version>245</dep.airlift.version>
<dep.aws-sdk.version>2.25.32</dep.aws-sdk.version>
</properties>

<dependencyManagement>
Expand All @@ -56,6 +58,32 @@
<type>pom</type>
<scope>import</scope>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${dep.aws-sdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<build>
<plugins>
<plugin>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<configuration>
<licenseSets>
<licenseSet>
<excludes>
<exclude>**/io/trino/s3/proxy/server/credentials/Signer.java</exclude>
</excludes>
</licenseSet>
</licenseSets>
</configuration>
</plugin>
</plugins>
</build>
</project>
6 changes: 6 additions & 0 deletions trino-s3-proxy/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@
<artifactId>jakarta.ws.rs-api</artifactId>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
package io.trino.s3.proxy.server;

import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import com.google.inject.Module;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.event.client.EventModule;
import io.airlift.http.server.HttpServerModule;
import io.airlift.http.server.testing.TestingHttpServer;
import io.airlift.jaxrs.JaxrsModule;
import io.airlift.json.JsonModule;
import io.airlift.log.Logger;
Expand All @@ -39,8 +41,13 @@ public static void main(String[] args)
.add(new JaxrsModule());

Bootstrap app = new Bootstrap(modules.build());
app.initialize();
Injector injector = app.initialize();

log.info("======== SERVER STARTED ========");

TestingHttpServer httpServer = injector.getInstance(TestingHttpServer.class);
log.info("");
log.info("Endpoint: %s", httpServer.getBaseUrl());
log.info("");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.s3.proxy.server.credentials;

import static java.util.Objects.requireNonNull;

public record Credentials(Credential emulated, Credential real)
{
public Credentials
{
requireNonNull(emulated, "emulated is null");
requireNonNull(real, "real is null");
}

public record Credential(String accessKey, String secretKey)
{
public Credential
{
requireNonNull(accessKey, "accessKey is null");
requireNonNull(secretKey, "secretKey is null");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.s3.proxy.server;
package io.trino.s3.proxy.server.credentials;

import org.junit.jupiter.api.Test;
import java.util.Optional;

public class DummyTest
public interface CredentialsController
{
@Test
public void testDummy()
{
// stub test for now
}
Optional<Credentials> credentials(String emulatedAccessKey);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
/*
* MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.trino.s3.proxy.server.credentials;

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;

import javax.crypto.spec.SecretKeySpec;

import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import static com.google.common.hash.Hashing.sha256;

/**
* Amazon AWS S3 signature V4 signer.
*/
class Signer
{
//
// Excerpts from @legal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
//
// * User-Agent
// This is ignored from signing because signing this causes problems with generating pre-signed
// URLs (that are executed by other agents) or when customers pass requests through proxies, which
// may modify the user-agent.
//
// * Authorization
// Is skipped for obvious reasons.
//
// * Accept-Encoding
// Some S3 servers like Hitachi Content Platform do not honour this header for signature
// calculation.
//
private static final Set<String> IGNORED_HEADERS = ImmutableSet.of("accept-encoding", "authorization", "user-agent");

private static final ZoneId UTC = ZoneId.of("Z");

private static final DateTimeFormatter AMZ_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.US).withZone(UTC);

// Formatted string is convertible to LocalDate only, not to LocalDateTime or ZonedDateTime.
// Below example shows how to use this to get ZonedDateTime.
// LocalDate.parse("20200225", SIGNER_DATE_FORMAT).atStartOfDay(UTC);
private static final DateTimeFormatter SIGNER_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.US).withZone(UTC);

private Signer() {}

/**
* Returns signed request of given request for S3 service.
*/
static String signV4S3(SignerMetadata metadata, String region, String accessKey, String secretKey, String contentSha256)
{
ZonedDateTime date = ZonedDateTime.parse(metadata.headerValue("x-amz-date").orElseThrow(), AMZ_DATE_FORMAT);

String scope = date.format(SIGNER_DATE_FORMAT) + "/" + region + "/s3/aws4_request";
Map<String, String> canonicalHeaders = buildCanonicalHeaders(metadata);
String signedHeaders = Joiner.on(";").join(canonicalHeaders.keySet());
String canonicalRequestHash = buildCanonicalRequestHash(metadata, canonicalHeaders, signedHeaders, contentSha256);
String stringToSign = "AWS4-HMAC-SHA256" + "\n" + date.format(AMZ_DATE_FORMAT) + "\n" + scope + "\n" + canonicalRequestHash;
byte[] signingKey = buildSigningKey(secretKey, date, region);
byte[] digest = sumHmac(signingKey, stringToSign.getBytes(StandardCharsets.UTF_8));
String signature = BaseEncoding.base16().encode(digest).toLowerCase(Locale.US);

return "AWS4-HMAC-SHA256 Credential="
+ accessKey
+ "/"
+ scope
+ ", SignedHeaders="
+ signedHeaders
+ ", Signature="
+ signature;
}

/**
* Returns HMacSHA256 digest of given key and data.
*/
private static byte[] sumHmac(byte[] key, byte[] data)
{
return Hashing.hmacSha256(new SecretKeySpec(key, "HmacSHA256")).hashBytes(data).asBytes();
}

private static Map<String, String> buildCanonicalHeaders(SignerMetadata metadata)
{
Map<String, String> canonicalHeaders = new TreeMap<>();

for (String name : metadata.headerNames()) {
String signedHeader = name.toLowerCase(Locale.US);
if (!IGNORED_HEADERS.contains(signedHeader)) {
// Convert and add header values as per
// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// * Header having multiple values should be converted to comma separated values.
// * Multi-spaced value of header should be trimmed to single spaced value.
canonicalHeaders.put(
signedHeader,
metadata.headerValues(name).stream()
.map(value -> value.replaceAll("( +)", " "))
.collect(Collectors.joining(",")));
}
}

return canonicalHeaders;
}

private static String buildCanonicalQueryString(SignerMetadata metadata)
{
if (metadata.encodedQuery().isEmpty()) {
return "";
}

// Building a multimap which only order keys, ordering values is not performed
// until MinIO server supports it.
Multimap<String, String> signedQueryParams = MultimapBuilder.treeKeys().arrayListValues().build();

for (String queryParam : metadata.encodedQuery().split("&")) {
String[] tokens = queryParam.split("=");
if (tokens.length > 1) {
signedQueryParams.put(tokens[0], tokens[1]);
}
else {
signedQueryParams.put(tokens[0], "");
}
}

return Joiner.on("&").withKeyValueSeparator("=").join(signedQueryParams.entries());
}

private static String buildCanonicalRequestHash(SignerMetadata metadata, Map<String, String> canonicalHeaders, String signedHeaders, String contentSha256)
{
String canonicalQueryString = buildCanonicalQueryString(metadata);

// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
String canonicalRequest = metadata.httpMethod()
+ "\n"
+ metadata.encodedPath()
+ "\n"
+ canonicalQueryString
+ "\n"
+ Joiner.on("\n").withKeyValueSeparator(":").join(canonicalHeaders)
+ "\n\n"
+ signedHeaders
+ "\n"
+ contentSha256;

return sha256Hash(canonicalRequest);
}

private static byte[] buildSigningKey(String secretKey, ZonedDateTime date, String region)
{
String aws4SecretKey = "AWS4" + secretKey;

byte[] dateKey = sumHmac(
aws4SecretKey.getBytes(StandardCharsets.UTF_8),
date.format(SIGNER_DATE_FORMAT).getBytes(StandardCharsets.UTF_8));

byte[] dateRegionKey = sumHmac(dateKey, region.getBytes(StandardCharsets.UTF_8));

byte[] dateRegionServiceKey = sumHmac(dateRegionKey, "s3".getBytes(StandardCharsets.UTF_8));

return sumHmac(dateRegionServiceKey, "aws4_request".getBytes(StandardCharsets.UTF_8));
}

private static String sha256Hash(String str)
{
return sha256().hashString(str, StandardCharsets.UTF_8).toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.s3.proxy.server.credentials;

import com.google.common.collect.ImmutableList;
import jakarta.ws.rs.core.MultivaluedHashMap;
import jakarta.ws.rs.core.MultivaluedMap;

import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Optional;

import static java.util.Objects.requireNonNull;

record SignerMetadata(MultivaluedMap<String, String> headers, String httpMethod, String encodedPath, String encodedQuery)
{
SignerMetadata
{
requireNonNull(headers, "headers is null");
requireNonNull(httpMethod, "httpMethod is null");
requireNonNull(encodedPath, "encodedPath is null");
requireNonNull(encodedQuery, "encodedQuery is null");

MultivaluedMap<String, String> headersCopy = new MultivaluedHashMap<>();
headers.forEach((key, values) -> headersCopy.put(key.toLowerCase(Locale.ROOT), values));
headers = headersCopy;
}

Collection<String> headerNames()
{
return headers.keySet();
}

List<String> headerValues(String name)
{
return headers.getOrDefault(name, ImmutableList.of());
}

Optional<String> headerValue(String name)
{
return Optional.ofNullable(headers.getFirst(name));
}
}
Loading

0 comments on commit 35f1f81

Please sign in to comment.