-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Copied signing classes from Minio: https://github.com/minio/minio-java - Added Notice file for Minio - Replaced OkHttp, etc. classes with emulated equivalents - Added an initial test for a simple `aws s3 ls` based on this doc: https://min.io/docs/minio/linux/integrations/aws-cli-with-minio.html Closes #5
- Loading branch information
Showing
10 changed files
with
449 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
This product includes software developed by Minio. (https://github.com/minio/minio-java) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Credentials.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
/* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.trino.s3.proxy.server.credentials; | ||
|
||
import static java.util.Objects.requireNonNull; | ||
|
||
public record Credentials(Credential emulated, Credential real) | ||
{ | ||
public Credentials | ||
{ | ||
requireNonNull(emulated, "emulated is null"); | ||
requireNonNull(real, "real is null"); | ||
} | ||
|
||
public record Credential(String accessKey, String secretKey) | ||
{ | ||
public Credential | ||
{ | ||
requireNonNull(accessKey, "accessKey is null"); | ||
requireNonNull(secretKey, "secretKey is null"); | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
197 changes: 197 additions & 0 deletions
197
trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/Signer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,197 @@ | ||
/* | ||
* MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.trino.s3.proxy.server.credentials; | ||
|
||
import com.google.common.base.Joiner; | ||
import com.google.common.collect.ImmutableSet; | ||
import com.google.common.collect.Multimap; | ||
import com.google.common.collect.MultimapBuilder; | ||
import com.google.common.hash.Hashing; | ||
import com.google.common.io.BaseEncoding; | ||
|
||
import javax.crypto.spec.SecretKeySpec; | ||
|
||
import java.nio.charset.StandardCharsets; | ||
import java.time.ZoneId; | ||
import java.time.ZonedDateTime; | ||
import java.time.format.DateTimeFormatter; | ||
import java.util.Locale; | ||
import java.util.Map; | ||
import java.util.Set; | ||
import java.util.TreeMap; | ||
import java.util.stream.Collectors; | ||
|
||
import static com.google.common.hash.Hashing.sha256; | ||
|
||
/** | ||
* Amazon AWS S3 signature V4 signer. | ||
*/ | ||
class Signer | ||
{ | ||
// | ||
// Excerpts from @legal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 | ||
// | ||
// * User-Agent | ||
// This is ignored from signing because signing this causes problems with generating pre-signed | ||
// URLs (that are executed by other agents) or when customers pass requests through proxies, which | ||
// may modify the user-agent. | ||
// | ||
// * Authorization | ||
// Is skipped for obvious reasons. | ||
// | ||
// * Accept-Encoding | ||
// Some S3 servers like Hitachi Content Platform do not honour this header for signature | ||
// calculation. | ||
// | ||
private static final Set<String> IGNORED_HEADERS = ImmutableSet.of("accept-encoding", "authorization", "user-agent"); | ||
|
||
private static final ZoneId UTC = ZoneId.of("Z"); | ||
|
||
private static final DateTimeFormatter AMZ_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.US).withZone(UTC); | ||
|
||
// Formatted string is convertible to LocalDate only, not to LocalDateTime or ZonedDateTime. | ||
// Below example shows how to use this to get ZonedDateTime. | ||
// LocalDate.parse("20200225", SIGNER_DATE_FORMAT).atStartOfDay(UTC); | ||
private static final DateTimeFormatter SIGNER_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.US).withZone(UTC); | ||
|
||
private Signer() {} | ||
|
||
/** | ||
* Returns signed request of given request for S3 service. | ||
*/ | ||
static String signV4S3(SignerMetadata metadata, String region, String accessKey, String secretKey, String contentSha256) | ||
{ | ||
ZonedDateTime date = ZonedDateTime.parse(metadata.headerValue("x-amz-date").orElseThrow(), AMZ_DATE_FORMAT); | ||
|
||
String scope = date.format(SIGNER_DATE_FORMAT) + "/" + region + "/s3/aws4_request"; | ||
Map<String, String> canonicalHeaders = buildCanonicalHeaders(metadata); | ||
String signedHeaders = Joiner.on(";").join(canonicalHeaders.keySet()); | ||
String canonicalRequestHash = buildCanonicalRequestHash(metadata, canonicalHeaders, signedHeaders, contentSha256); | ||
String stringToSign = "AWS4-HMAC-SHA256" + "\n" + date.format(AMZ_DATE_FORMAT) + "\n" + scope + "\n" + canonicalRequestHash; | ||
byte[] signingKey = buildSigningKey(secretKey, date, region); | ||
byte[] digest = sumHmac(signingKey, stringToSign.getBytes(StandardCharsets.UTF_8)); | ||
String signature = BaseEncoding.base16().encode(digest).toLowerCase(Locale.US); | ||
|
||
return "AWS4-HMAC-SHA256 Credential=" | ||
+ accessKey | ||
+ "/" | ||
+ scope | ||
+ ", SignedHeaders=" | ||
+ signedHeaders | ||
+ ", Signature=" | ||
+ signature; | ||
} | ||
|
||
/** | ||
* Returns HMacSHA256 digest of given key and data. | ||
*/ | ||
private static byte[] sumHmac(byte[] key, byte[] data) | ||
{ | ||
return Hashing.hmacSha256(new SecretKeySpec(key, "HmacSHA256")).hashBytes(data).asBytes(); | ||
} | ||
|
||
private static Map<String, String> buildCanonicalHeaders(SignerMetadata metadata) | ||
{ | ||
Map<String, String> canonicalHeaders = new TreeMap<>(); | ||
|
||
for (String name : metadata.headerNames()) { | ||
String signedHeader = name.toLowerCase(Locale.US); | ||
if (!IGNORED_HEADERS.contains(signedHeader)) { | ||
// Convert and add header values as per | ||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html | ||
// * Header having multiple values should be converted to comma separated values. | ||
// * Multi-spaced value of header should be trimmed to single spaced value. | ||
canonicalHeaders.put( | ||
signedHeader, | ||
metadata.headerValues(name).stream() | ||
.map(value -> value.replaceAll("( +)", " ")) | ||
.collect(Collectors.joining(","))); | ||
} | ||
} | ||
|
||
return canonicalHeaders; | ||
} | ||
|
||
private static String buildCanonicalQueryString(SignerMetadata metadata) | ||
{ | ||
if (metadata.encodedQuery().isEmpty()) { | ||
return ""; | ||
} | ||
|
||
// Building a multimap which only order keys, ordering values is not performed | ||
// until MinIO server supports it. | ||
Multimap<String, String> signedQueryParams = MultimapBuilder.treeKeys().arrayListValues().build(); | ||
|
||
for (String queryParam : metadata.encodedQuery().split("&")) { | ||
String[] tokens = queryParam.split("="); | ||
if (tokens.length > 1) { | ||
signedQueryParams.put(tokens[0], tokens[1]); | ||
} | ||
else { | ||
signedQueryParams.put(tokens[0], ""); | ||
} | ||
} | ||
|
||
return Joiner.on("&").withKeyValueSeparator("=").join(signedQueryParams.entries()); | ||
} | ||
|
||
private static String buildCanonicalRequestHash(SignerMetadata metadata, Map<String, String> canonicalHeaders, String signedHeaders, String contentSha256) | ||
{ | ||
String canonicalQueryString = buildCanonicalQueryString(metadata); | ||
|
||
// CanonicalRequest = | ||
// HTTPRequestMethod + '\n' + | ||
// CanonicalURI + '\n' + | ||
// CanonicalQueryString + '\n' + | ||
// CanonicalHeaders + '\n' + | ||
// SignedHeaders + '\n' + | ||
// HexEncode(Hash(RequestPayload)) | ||
String canonicalRequest = metadata.httpMethod() | ||
+ "\n" | ||
+ metadata.encodedPath() | ||
+ "\n" | ||
+ canonicalQueryString | ||
+ "\n" | ||
+ Joiner.on("\n").withKeyValueSeparator(":").join(canonicalHeaders) | ||
+ "\n\n" | ||
+ signedHeaders | ||
+ "\n" | ||
+ contentSha256; | ||
|
||
return sha256Hash(canonicalRequest); | ||
} | ||
|
||
private static byte[] buildSigningKey(String secretKey, ZonedDateTime date, String region) | ||
{ | ||
String aws4SecretKey = "AWS4" + secretKey; | ||
|
||
byte[] dateKey = sumHmac( | ||
aws4SecretKey.getBytes(StandardCharsets.UTF_8), | ||
date.format(SIGNER_DATE_FORMAT).getBytes(StandardCharsets.UTF_8)); | ||
|
||
byte[] dateRegionKey = sumHmac(dateKey, region.getBytes(StandardCharsets.UTF_8)); | ||
|
||
byte[] dateRegionServiceKey = sumHmac(dateRegionKey, "s3".getBytes(StandardCharsets.UTF_8)); | ||
|
||
return sumHmac(dateRegionServiceKey, "aws4_request".getBytes(StandardCharsets.UTF_8)); | ||
} | ||
|
||
private static String sha256Hash(String str) | ||
{ | ||
return sha256().hashString(str, StandardCharsets.UTF_8).toString(); | ||
} | ||
} |
55 changes: 55 additions & 0 deletions
55
trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SignerMetadata.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
/* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.trino.s3.proxy.server.credentials; | ||
|
||
import com.google.common.collect.ImmutableList; | ||
import jakarta.ws.rs.core.MultivaluedHashMap; | ||
import jakarta.ws.rs.core.MultivaluedMap; | ||
|
||
import java.util.Collection; | ||
import java.util.List; | ||
import java.util.Locale; | ||
import java.util.Optional; | ||
|
||
import static java.util.Objects.requireNonNull; | ||
|
||
record SignerMetadata(MultivaluedMap<String, String> headers, String httpMethod, String encodedPath, String encodedQuery) | ||
{ | ||
SignerMetadata | ||
{ | ||
requireNonNull(headers, "headers is null"); | ||
requireNonNull(httpMethod, "httpMethod is null"); | ||
requireNonNull(encodedPath, "encodedPath is null"); | ||
requireNonNull(encodedQuery, "encodedQuery is null"); | ||
|
||
MultivaluedMap<String, String> headersCopy = new MultivaluedHashMap<>(); | ||
headers.forEach((key, values) -> headersCopy.put(key.toLowerCase(Locale.ROOT), values)); | ||
headers = headersCopy; | ||
} | ||
|
||
Collection<String> headerNames() | ||
{ | ||
return headers.keySet(); | ||
} | ||
|
||
List<String> headerValues(String name) | ||
{ | ||
return headers.getOrDefault(name, ImmutableList.of()); | ||
} | ||
|
||
Optional<String> headerValue(String name) | ||
{ | ||
return Optional.ofNullable(headers.getFirst(name)); | ||
} | ||
} |
Oops, something went wrong.