allow webIdentityToken Provider AWS Authentication for eks

plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java

@@ -16,12 +16,7 @@
 package com.dremio.plugins.s3.store;
 
 import static com.dremio.exec.ExecConstants.S3_NATIVE_ASYNC_CLIENT;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.ACCESS_KEY_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.ASSUME_ROLE_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.AWS_PROFILE_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.DREMIO_ASSUME_ROLE_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.EC2_METADATA_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.NONE_PROVIDER;
+import static com.dremio.plugins.s3.store.S3StoragePlugin.*;
 import static org.apache.hadoop.fs.s3a.Constants.ALLOW_REQUESTER_PAYS;
 import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
@@ -124,7 +119,8 @@ public class S3FileSystem extends ContainerFileSystem implements MayProvideAsync
     .setWaitStrategy(Retryer.WaitStrategy.EXPONENTIAL, 250, 2500)
     .setMaxRetries(10).build();
 
-  private final LoadingCache<String, CloseableRef<S3Client>> syncClientCache = CacheBuilder
+  private final LoadingCache<String, CloseableRef<S3Client>>
+    syncClientCache = CacheBuilder
           .newBuilder()
           .expireAfterAccess(1, TimeUnit.HOURS)
           .removalListener(notification ->
@@ -225,7 +221,8 @@ protected void verifyCredentials(Configuration conf) throws RuntimeException {
       final StsClientBuilder stsClientBuilder = StsClient.builder()
         // Note that AWS SDKv2 client will close the credentials provider if needed when the client is closed
         .credentialsProvider(awsCredentialsProvider)
-        .region(getAWSRegionFromConfigurationOrDefault(conf));
+        .region(getAWSRegionFromConfigurationOrDefault(conf))
+        .endpointOverride(URI.create(conf.get(Constants.ASSUMED_ROLE_STS_ENDPOINT,"https://sts.amazonaws.com")));
       try (StsClient stsClient = stsClientBuilder.build()) {
         retryer.call(() -> {
           GetCallerIdentityRequest request = GetCallerIdentityRequest.builder().build();
@@ -400,6 +397,8 @@ protected AwsCredentialsProvider getAsync2Provider(Configuration config) {
         return new DremioAssumeRoleCredentialsProviderV2();
       case AWS_PROFILE_PROVIDER:
         return new AWSProfileCredentialsProviderV2(config);
+      case WEB_IDENTITY_TOKEN_PROVIDER:
+        return new WebIdentityCredentialsProviderV2(config);
       default:
         throw new IllegalStateException("Invalid AWSCredentialsProvider provided: " + config.get(Constants.AWS_CREDENTIALS_PROVIDER));
     }

plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3PluginConfig.java

@@ -15,9 +15,7 @@
  */
 package com.dremio.plugins.s3.store;
 
-import static com.dremio.plugins.s3.store.S3StoragePlugin.AWS_PROFILE_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.EC2_METADATA_PROVIDER;
-import static com.dremio.plugins.s3.store.S3StoragePlugin.NONE_PROVIDER;
+import static com.dremio.plugins.s3.store.S3StoragePlugin.*;
 
 import javax.inject.Provider;
 
@@ -71,6 +69,8 @@ private AWSCredentialsConfigurator getS3CredentialsProvider() {
         };
       case EC2_METADATA:
         return properties -> EC2_METADATA_PROVIDER;
+      case WEB_IDENTITY_TOKEN:
+        return properties -> WEB_IDENTITY_TOKEN_PROVIDER;
       case NONE:
         return properties -> NONE_PROVIDER;
       default:

plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java

@@ -81,6 +81,7 @@ public class S3StoragePlugin extends FileSystemPlugin<AbstractS3PluginConfig> {
   public static final String DREMIO_ASSUME_ROLE_PROVIDER = "com.dremio.service.coordinator" +
     ".DremioAssumeRoleCredentialsProviderV1";
   public static final String AWS_PROFILE_PROVIDER = "com.dremio.plugins.s3.store.AWSProfileCredentialsProviderV1";
+  public static final String WEB_IDENTITY_TOKEN_PROVIDER = "com.dremio.plugins.s3.store.WebIdentityCredentialsProviderV1";
 
   private final AWSCredentialsConfigurator awsCredentialsConfigurator;
   private final boolean isAuthTypeNone;

plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV1.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.plugins.s3.store;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider;
+
+import software.amazon.awssdk.core.SdkSystemSetting;
+
+public class WebIdentityCredentialsProviderV1 implements AWSCredentialsProvider, Closeable {
+  private final STSAssumeRoleWithWebIdentitySessionCredentialsProvider stsAssumeRoleWithWebIdentitySessionCredentialsProvider;
+  private static final Logger logger = LoggerFactory.getLogger(WebIdentityCredentialsProviderV1.class);
+  private final static String DREMIO_SESSION_NAME = "dremio-session";
+
+  public WebIdentityCredentialsProviderV1(Configuration conf) throws IOException {
+    String webIdentityTokenFile = System.getenv(SdkSystemSetting.AWS_WEB_IDENTITY_TOKEN_FILE.name());
+    String assumeRole = System.getenv(SdkSystemSetting.AWS_ROLE_ARN.name());
+    String sessionName = System.getenv(SdkSystemSetting.AWS_ROLE_SESSION_NAME.name());
+    if (sessionName == null) {
+      sessionName = DREMIO_SESSION_NAME;
+    }
+    logger.debug(assumeRole, sessionName, webIdentityTokenFile);
+    this.stsAssumeRoleWithWebIdentitySessionCredentialsProvider = new STSAssumeRoleWithWebIdentitySessionCredentialsProvider.Builder(assumeRole, sessionName, webIdentityTokenFile).build();
+  }
+
+  @Override
+  public AWSCredentials getCredentials() {
+    return stsAssumeRoleWithWebIdentitySessionCredentialsProvider.getCredentials();
+  }
+
+  @Override
+  public void refresh() {
+    this.stsAssumeRoleWithWebIdentitySessionCredentialsProvider.refresh();
+  }
+
+  @Override
+  public void close() throws IOException {
+    stsAssumeRoleWithWebIdentitySessionCredentialsProvider.close();
+  }
+}

plugins/s3/src/main/java/com/dremio/plugins/s3/store/WebIdentityCredentialsProviderV2.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.plugins.s3.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
+import software.amazon.awssdk.utils.IoUtils;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
+
+public class WebIdentityCredentialsProviderV2 implements AwsCredentialsProvider, SdkAutoCloseable {
+  private final WebIdentityTokenFileCredentialsProvider webIdentityTokenCredentialsProvider;
+  private static final Logger logger = LoggerFactory.getLogger(WebIdentityCredentialsProviderV2.class);
+
+  public WebIdentityCredentialsProviderV2(Configuration conf) {
+    this.webIdentityTokenCredentialsProvider = WebIdentityTokenFileCredentialsProvider.create();
+  }
+
+  @Override
+  public AwsCredentials resolveCredentials() {
+    return webIdentityTokenCredentialsProvider.resolveCredentials();
+  }
+
+  @Override
+  public void close() {
+    IoUtils.closeIfCloseable(webIdentityTokenCredentialsProvider, logger);
+  }
+}

plugins/s3/src/main/resources/s3-layout.json

@@ -91,6 +91,15 @@
                         "text": "Only the buckets provided in Public Buckets will be available."
                       }
                     }
+                  },
+                  {
+                    "value": "WEB_IDENTITY_TOKEN",
+                    "container": {
+                      "help": {
+                        "position": "top",
+                        "text": "Use this in EKS service Account"
+                      }
+                    }
                   }
                 ]
               },

sabot/kernel/src/main/java/com/dremio/exec/catalog/conf/AWSAuthenticationType.java

@@ -39,5 +39,10 @@ public enum AWSAuthenticationType {
   /**
    * Use files from a AWS named profile
    */
-  @Tag(4) @DisplayMetadata(label = "AWS Profile") AWS_PROFILE;
+  @Tag(4) @DisplayMetadata(label = "AWS Profile") AWS_PROFILE,
+
+  /**
+   * Access from web Identity Token (EKS service Account)
+   */
+  @Tag(5) @DisplayMetadata(label = "Web Identity Token") WEB_IDENTITY_TOKEN;
 }