Remove s3:listAllMyBuckets permission

athena-aws-cmdb/athena-aws-cmdb.yaml

@@ -70,7 +70,6 @@ Resources:
                 - rds:Describe*
                 - rds:ListTagsForResource
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
                 - s3:ListBucket
                 - athena:GetQueryExecution
               Effect: Allow

athena-clickhouse/athena-clickhouse.yaml

@@ -128,7 +128,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
             - athena:GetQueryExecution
-            - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-cloudera-hive/athena-cloudera-hive.yaml

@@ -95,7 +95,6 @@ Resources:
         - Statement:
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           Version: '2012-10-17'

athena-cloudera-impala/athena-cloudera-impala.yaml

@@ -100,7 +100,6 @@ Resources:
         - Statement:
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           Version: '2012-10-17'

athena-cloudwatch-metrics/athena-cloudwatch-metrics.yaml

@@ -66,7 +66,6 @@ Resources:
                 - cloudwatch:Get*
                 - cloudwatch:List*
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-cloudwatch/athena-cloudwatch.yaml

@@ -108,7 +108,6 @@ Resources:
               - logs:TestMetricFilter
               - logs:FilterLogEvents
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-datalakegen2/athena-datalakegen2.yaml

@@ -101,7 +101,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-db2-as400/athena-db2-as400.yaml

@@ -98,7 +98,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-db2/athena-db2.yaml

@@ -98,7 +98,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-docdb/athena-docdb.yaml

@@ -91,7 +91,6 @@ Resources:
                 - glue:GetPartition
                 - glue:GetDatabase
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-dynamodb/athena-dynamodb.yaml

@@ -115,7 +115,6 @@ Resources:
              - glue:GetPartition
              - glue:GetDatabase
              - athena:GetQueryExecution
-             - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-elasticsearch/athena-elasticsearch.yaml

@@ -130,7 +130,6 @@ Resources:
                 - glue:GetPartition
                 - glue:GetDatabase
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-example/athena-example.yaml

@@ -64,7 +64,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-federation-integ-test/src/main/java/com/amazonaws/athena/connector/integ/stacks/ConnectorStack.java

@@ -189,7 +189,6 @@ private PolicyDocument getGlueAthenaS3AccessPolicy()
         statementActionsPolicy.add("glue:GetPartition");
         statementActionsPolicy.add("glue:GetDatabase");
         statementActionsPolicy.add("athena:GetQueryExecution");
-        statementActionsPolicy.add("s3:ListAllMyBuckets");
 
         return PolicyDocument.Builder.create()
                 .statements(ImmutableList.of(PolicyStatement.Builder.create()

athena-federation-sdk/athena-federation-sdk.yaml

@@ -53,12 +53,6 @@ Resources:
       Timeout: !Ref LambdaTimeout
       MemorySize: !Ref LambdaMemory
       Policies:
-        - Statement:
-            - Action:
-                - s3:ListAllMyBuckets
-              Effect: Allow
-              Resource: '*'
-          Version: '2012-10-17'
         #S3CrudPolicy allows our connector to spill large responses to S3. You can optionally replace this pre-made policy
         #with one that is more restrictive and can only 'put' but not read,delete, or overwrite files.
         - S3CrudPolicy:

athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/spill/SpillLocationVerifier.java

@@ -1,5 +1,3 @@
-package com.amazonaws.athena.connector.lambda.domain.spill;
-
 /*-
  * #%L
  * Amazon Athena Query Federation SDK
@@ -20,15 +18,15 @@
  * #L%
  */
 
+package com.amazonaws.athena.connector.lambda.domain.spill;
+
+import com.amazonaws.AmazonServiceException;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.HeadBucketRequest;
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Set;
-import java.util.stream.Collectors;
-
 /**
  * This class is used to track the bucket and its state, and check its validity
  */
@@ -85,19 +83,21 @@ public void checkBucketAuthZ(String spillBucket)
     void updateBucketState()
     {
         try {
-            Set<String> buckets = amazons3.listBuckets().stream().map(b -> b.getName()).collect(Collectors.toSet());
-
-            if (!buckets.contains(bucket)) {
+            amazons3.headBucket(new HeadBucketRequest(bucket));
+            state = BucketState.VALID;
+        }
+        catch (AmazonServiceException ex) {
+            int statusCode = ex.getStatusCode();
+            // returns 404 if bucket was not found, 403 if bucket access is forbidden
+            if (statusCode == 404 || statusCode == 403) {
                 state = BucketState.INVALID;
             }
             else {
-                state = BucketState.VALID;
+                throw new RuntimeException("Error while checking bucket ownership for " + bucket, ex);
             }
-
-            logger.info("The state of bucket {} has been updated to {} from {}", bucket, state, BucketState.UNCHECKED);
         }
-        catch (AmazonS3Exception ex) {
-            throw new RuntimeException("Error while checking bucket ownership for " + bucket, ex);
+        finally {
+            logger.info("The state of bucket {} has been updated to {} from {}", bucket, state, BucketState.UNCHECKED);
         }
     }
 

athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/domain/spill/SpillLocationVerifierTest.java

@@ -1,5 +1,7 @@
 package com.amazonaws.athena.connector.lambda.domain.spill;
 
+import com.amazonaws.AmazonServiceException;
+
 /*-
  * #%L
  * Amazon Athena Query Federation SDK
@@ -22,10 +24,15 @@
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.HeadBucketRequest;
+import com.amazonaws.services.s3.model.HeadBucketResult;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Spy;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,7 +42,10 @@
 import java.util.Random;
 
 import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
@@ -56,8 +66,7 @@ public void setup()
         logger.info("setUpBefore - enter");
 
         bucketNames = Arrays.asList("bucket1", "bucket2", "bucket3");
-        List<Bucket> buckets = createBuckets(bucketNames);
-        AmazonS3 mockS3 = createMockS3(buckets);
+        AmazonS3 mockS3 = createMockS3(bucketNames);
         spyVerifier = spy(new SpillLocationVerifier(mockS3));
 
         logger.info("setUpBefore - exit");
@@ -112,7 +121,7 @@ public void checkBucketAuthZFail()
     {
         logger.info("checkBucketAuthZFail - enter");
 
-        String bucketNotOwn = "spill-bucket";
+        String bucketNotOwn = "forbidden";
 
         try {
             spyVerifier.checkBucketAuthZ(bucketNotOwn);
@@ -137,22 +146,26 @@ public void checkBucketAuthZFail()
         logger.info("checkBucketAuthZFail - exit");
     }
 
-    private AmazonS3 createMockS3(List<Bucket> buckets)
+    private AmazonS3 createMockS3(List<String> buckets)
     {
         AmazonS3 s3mock = mock(AmazonS3.class);
-        when(s3mock.listBuckets()).thenReturn(buckets);
+        when(s3mock.headBucket(any(HeadBucketRequest.class)))
+                .thenAnswer((Answer<HeadBucketResult>) invocationOnMock -> {
+                    String bucketName = ((HeadBucketRequest) invocationOnMock.getArguments()[0]).getBucketName();
+                    if (buckets.contains(bucketName)) {
+                        return null;
+                    }
+                    AmazonServiceException exception;
+                    if (bucketName.equals("forbidden")) {
+                        exception = new AmazonServiceException("Forbidden");
+                        exception.setStatusCode(403);
+                    }
+                    else {
+                        exception = new AmazonServiceException("Not Found");
+                        exception.setStatusCode(404);
+                    }
+                    throw exception;
+                });
         return s3mock;
     }
-
-    private List<Bucket> createBuckets(List<String> names)
-    {
-        List<Bucket> buckets = new ArrayList();
-        for (String name : names) {
-            Bucket bucket = mock(Bucket.class);
-            when(bucket.getName()).thenReturn(name);
-            buckets.add(bucket);
-        }
-
-        return buckets;
-    }
 }

athena-gcs/athena-gcs.yaml

@@ -70,7 +70,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
                 - glue:GetTableVersions
                 - glue:GetPartitions
                 - glue:GetTables

athena-google-bigquery/athena-google-bigquery.yaml

@@ -109,7 +109,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-hbase/athena-hbase.yaml

@@ -110,7 +110,6 @@ Resources:
                 - glue:GetPartition
                 - glue:GetDatabase
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
                 - s3:ListBucket
                 - s3:GetObject
                 - s3:GetBucketLocation

athena-hortonworks-hive/athena-hortonworks-hive.yaml

@@ -99,7 +99,6 @@ Resources:
         - Statement:
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           Version: '2012-10-17'

athena-kafka/athena-kafka.yaml

@@ -152,7 +152,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
               - s3:ListBucket
               - s3:GetObject
               - s3:GetBucketLocation

athena-msk/athena-msk.yaml

@@ -147,7 +147,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
               - s3:ListBucket
               - s3:GetObject
               - s3:GetBucketLocation

athena-mysql/athena-mysql.yaml

@@ -121,7 +121,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
             - athena:GetQueryExecution
-            - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-neptune/athena-neptune.yaml

@@ -115,7 +115,6 @@ Resources:
                 - glue:GetPartition
                 - glue:GetDatabase              
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
             - Action:

athena-oracle/athena-oracle.yaml

@@ -134,7 +134,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-postgresql/athena-postgresql.yaml

@@ -134,7 +134,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
             - athena:GetQueryExecution
-            - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action:

athena-redis/athena-redis.yaml

@@ -106,7 +106,6 @@ Resources:
                 - glue:GetPartition
                 - glue:GetDatabase
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-redshift/athena-redshift.yaml

@@ -123,7 +123,6 @@ Resources:
           - Effect: Allow
             Action:
               - athena:GetQueryExecution
-              - s3:ListAllMyBuckets
             Resource: '*'
           - Effect: Allow
             Action:

athena-saphana/athena-saphana.yaml

@@ -99,7 +99,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-snowflake/athena-snowflake.yaml

@@ -99,7 +99,6 @@ Resources:
         - Statement:
             - Action:
                 - athena:GetQueryExecution
-                - s3:ListAllMyBuckets
               Effect: Allow
               Resource: '*'
           Version: '2012-10-17'

athena-sqlserver/athena-sqlserver.yaml

@@ -127,7 +127,6 @@ Resources:
             Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${LambdaFunctionName}:*'
           - Action:
             - athena:GetQueryExecution
-            - s3:ListAllMyBuckets
             Effect: Allow
             Resource: '*'
           - Action: