diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java index ba5235ce39..08e589e22f 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java @@ -16,12 +16,7 @@ package com.dremio.plugins.s3.store; import static com.dremio.exec.ExecConstants.S3_NATIVE_ASYNC_CLIENT; -import static com.dremio.plugins.s3.store.S3StoragePlugin.ACCESS_KEY_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.ASSUME_ROLE_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.AWS_PROFILE_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.DREMIO_ASSUME_ROLE_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.EC2_METADATA_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.NONE_PROVIDER; +import static com.dremio.plugins.s3.store.S3StoragePlugin.*; import static org.apache.hadoop.fs.s3a.Constants.ALLOW_REQUESTER_PAYS; import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS; @@ -124,7 +119,8 @@ public class S3FileSystem extends ContainerFileSystem implements MayProvideAsync .setWaitStrategy(Retryer.WaitStrategy.EXPONENTIAL, 250, 2500) .setMaxRetries(10).build(); - private final LoadingCache> syncClientCache = CacheBuilder + private final LoadingCache> + syncClientCache = CacheBuilder .newBuilder() .expireAfterAccess(1, TimeUnit.HOURS) .removalListener(notification -> @@ -225,7 +221,8 @@ protected void verifyCredentials(Configuration conf) throws RuntimeException { final StsClientBuilder stsClientBuilder = StsClient.builder() // Note that AWS SDKv2 client will close the credentials provider if needed when the client is closed .credentialsProvider(awsCredentialsProvider) - .region(getAWSRegionFromConfigurationOrDefault(conf)); + .region(getAWSRegionFromConfigurationOrDefault(conf)) + .endpointOverride(URI.create(conf.get(Constants.ASSUMED_ROLE_STS_ENDPOINT,"https://sts.amazonaws.com"))); try (StsClient stsClient = stsClientBuilder.build()) { retryer.call(() -> { GetCallerIdentityRequest request = GetCallerIdentityRequest.builder().build(); @@ -400,6 +397,8 @@ protected AwsCredentialsProvider getAsync2Provider(Configuration config) { return new DremioAssumeRoleCredentialsProviderV2(); case AWS_PROFILE_PROVIDER: return new AWSProfileCredentialsProviderV2(config); + case WEB_IDENTITY_TOKEN_PROVIDER: + return new WebIdentityCredentialsProviderV2(config); default: throw new IllegalStateException("Invalid AWSCredentialsProvider provided: " + config.get(Constants.AWS_CREDENTIALS_PROVIDER)); } diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3PluginConfig.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3PluginConfig.java index bb95b52735..c59d8cd7ca 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3PluginConfig.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3PluginConfig.java @@ -15,9 +15,7 @@ */ package com.dremio.plugins.s3.store; -import static com.dremio.plugins.s3.store.S3StoragePlugin.AWS_PROFILE_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.EC2_METADATA_PROVIDER; -import static com.dremio.plugins.s3.store.S3StoragePlugin.NONE_PROVIDER; +import static com.dremio.plugins.s3.store.S3StoragePlugin.*; import javax.inject.Provider; @@ -71,6 +69,8 @@ private AWSCredentialsConfigurator getS3CredentialsProvider() { }; case EC2_METADATA: return properties -> EC2_METADATA_PROVIDER; + case WEB_IDENTITY_TOKEN: + return properties -> WEB_IDENTITY_TOKEN_PROVIDER; case NONE: return properties -> NONE_PROVIDER; default: diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java index 60089df551..590ab32196 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java @@ -81,6 +81,7 @@ public class S3StoragePlugin extends FileSystemPlugin { public static final String DREMIO_ASSUME_ROLE_PROVIDER = "com.dremio.service.coordinator" + ".DremioAssumeRoleCredentialsProviderV1"; public static final String AWS_PROFILE_PROVIDER = "com.dremio.plugins.s3.store.AWSProfileCredentialsProviderV1"; + public static final String WEB_IDENTITY_TOKEN_PROVIDER = "com.dremio.plugins.s3.store.WebIdentityCredentialsProviderV1"; private final AWSCredentialsConfigurator awsCredentialsConfigurator; private final boolean isAuthTypeNone; diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV1.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV1.java new file mode 100644 index 0000000000..6099347959 --- /dev/null +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV1.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2017-2019 Dremio Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dremio.plugins.s3.store; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider; + +import software.amazon.awssdk.core.SdkSystemSetting; + +public class WebIdentityCredentialsProviderV1 implements AWSCredentialsProvider, Closeable { + private final STSAssumeRoleWithWebIdentitySessionCredentialsProvider stsAssumeRoleWithWebIdentitySessionCredentialsProvider; + private static final Logger logger = LoggerFactory.getLogger(WebIdentityCredentialsProviderV1.class); + private final static String DREMIO_SESSION_NAME = "dremio-session"; + + public WebIdentityCredentialsProviderV1(Configuration conf) throws IOException { + String webIdentityTokenFile = System.getenv(SdkSystemSetting.AWS_WEB_IDENTITY_TOKEN_FILE.name()); + String assumeRole = System.getenv(SdkSystemSetting.AWS_ROLE_ARN.name()); + String sessionName = System.getenv(SdkSystemSetting.AWS_ROLE_SESSION_NAME.name()); + if (sessionName == null) { + sessionName = DREMIO_SESSION_NAME; + } + logger.debug(assumeRole, sessionName, webIdentityTokenFile); + this.stsAssumeRoleWithWebIdentitySessionCredentialsProvider = new STSAssumeRoleWithWebIdentitySessionCredentialsProvider.Builder(assumeRole, sessionName, webIdentityTokenFile).build(); + } + + @Override + public AWSCredentials getCredentials() { + return stsAssumeRoleWithWebIdentitySessionCredentialsProvider.getCredentials(); + } + + @Override + public void refresh() { + this.stsAssumeRoleWithWebIdentitySessionCredentialsProvider.refresh(); + } + + @Override + public void close() throws IOException { + stsAssumeRoleWithWebIdentitySessionCredentialsProvider.close(); + } +} diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV2.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV2.java new file mode 100644 index 0000000000..6985a17aea --- /dev/null +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV2.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2017-2019 Dremio Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dremio.plugins.s3.store; + +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; +import software.amazon.awssdk.utils.IoUtils; +import software.amazon.awssdk.utils.SdkAutoCloseable; + +public class WebIdentityCredentialsProviderV2 implements AwsCredentialsProvider, SdkAutoCloseable { + private final WebIdentityTokenFileCredentialsProvider webIdentityTokenCredentialsProvider; + private static final Logger logger = LoggerFactory.getLogger(WebIdentityCredentialsProviderV2.class); + + public WebIdentityCredentialsProviderV2(Configuration conf) { + this.webIdentityTokenCredentialsProvider = WebIdentityTokenFileCredentialsProvider.create(); + } + + @Override + public AwsCredentials resolveCredentials() { + return webIdentityTokenCredentialsProvider.resolveCredentials(); + } + + @Override + public void close() { + IoUtils.closeIfCloseable(webIdentityTokenCredentialsProvider, logger); + } +} diff --git a/plugins/s3/src/main/resources/s3-layout.json b/plugins/s3/src/main/resources/s3-layout.json index b8d33cdd06..76e2e7fa11 100644 --- a/plugins/s3/src/main/resources/s3-layout.json +++ b/plugins/s3/src/main/resources/s3-layout.json @@ -91,6 +91,15 @@ "text": "Only the buckets provided in Public Buckets will be available." } } + }, + { + "value": "WEB_IDENTITY_TOKEN", + "container": { + "help": { + "position": "top", + "text": "Use this in EKS service Account" + } + } } ] }, diff --git a/sabot/kernel/src/main/java/com/dremio/exec/catalog/conf/AWSAuthenticationType.java b/sabot/kernel/src/main/java/com/dremio/exec/catalog/conf/AWSAuthenticationType.java index a58a24463a..7137fa506a 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/catalog/conf/AWSAuthenticationType.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/catalog/conf/AWSAuthenticationType.java @@ -39,5 +39,10 @@ public enum AWSAuthenticationType { /** * Use files from a AWS named profile */ - @Tag(4) @DisplayMetadata(label = "AWS Profile") AWS_PROFILE; + @Tag(4) @DisplayMetadata(label = "AWS Profile") AWS_PROFILE, + + /** + * Access from web Identity Token (EKS service Account) + */ + @Tag(5) @DisplayMetadata(label = "Web Identity Token") WEB_IDENTITY_TOKEN; }