From 51a9010f4a3a7e51d9014064a0680318f09f5a22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:38:41 -0500 Subject: [PATCH 1/7] build(deps): bump software.amazon.jsii:jsii-runtime from 1.105.0 to 1.106.0 (#2494) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 363e67d55f..c76aeca3f2 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 1.2.2 1.6.0 1.204.0 - 1.105.0 + 1.106.0 2.0.16 4.11.0 From 4eea8e210705176ef39ba12d9fc1f5b450c1cc47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:39:55 +0000 Subject: [PATCH 2/7] build(deps): bump net.java.dev.jna:jna-platform from 5.15.0 to 5.16.0 Bumps [net.java.dev.jna:jna-platform](https://github.com/java-native-access/jna) from 5.15.0 to 5.16.0. - [Changelog](https://github.com/java-native-access/jna/blob/master/CHANGES.md) - [Commits](https://github.com/java-native-access/jna/compare/5.15.0...5.16.0) --- updated-dependencies: - dependency-name: net.java.dev.jna:jna-platform dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- athena-gcs/pom.xml | 2 +- athena-google-bigquery/pom.xml | 2 +- athena-vertica/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/athena-gcs/pom.xml b/athena-gcs/pom.xml index 214e45b1fe..99f2833a58 100644 --- a/athena-gcs/pom.xml +++ b/athena-gcs/pom.xml @@ -24,7 +24,7 @@ net.java.dev.jna jna-platform - 5.15.0 + 5.16.0 org.slf4j diff --git a/athena-google-bigquery/pom.xml b/athena-google-bigquery/pom.xml index ac5ae171a8..710bef7d82 100644 --- a/athena-google-bigquery/pom.xml +++ b/athena-google-bigquery/pom.xml @@ -23,7 +23,7 @@ net.java.dev.jna jna-platform - 5.15.0 + 5.16.0 diff --git a/athena-vertica/pom.xml b/athena-vertica/pom.xml index 1b319fa6a2..c098ed633b 100644 --- a/athena-vertica/pom.xml +++ b/athena-vertica/pom.xml @@ -25,7 +25,7 @@ net.java.dev.jna jna-platform - 5.15.0 + 5.16.0 org.slf4j From 36a6ee01f4e386471b0ecd70db982136deef2dfa Mon Sep 17 00:00:00 2001 From: Nitin Singla Date: Tue, 14 Jan 2025 21:41:08 +0530 Subject: [PATCH 3/7] Bug fix: Enhanced SQL statement validation with word boundary matching (#2324) Co-authored-by: AbdulRehman Faraj --- .../connectors/dynamodb/qpt/DDBQueryPassthrough.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/athena-dynamodb/src/main/java/com/amazonaws/athena/connectors/dynamodb/qpt/DDBQueryPassthrough.java b/athena-dynamodb/src/main/java/com/amazonaws/athena/connectors/dynamodb/qpt/DDBQueryPassthrough.java index 68a6d70403..ec52d81714 100644 --- a/athena-dynamodb/src/main/java/com/amazonaws/athena/connectors/dynamodb/qpt/DDBQueryPassthrough.java +++ b/athena-dynamodb/src/main/java/com/amazonaws/athena/connectors/dynamodb/qpt/DDBQueryPassthrough.java @@ -21,7 +21,6 @@ import com.amazonaws.athena.connector.lambda.exceptions.AthenaConnectorException; import com.amazonaws.athena.connector.lambda.metadata.optimizations.querypassthrough.QueryPassthroughSignature; -import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.glue.model.ErrorDetails; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; public class DDBQueryPassthrough implements QueryPassthroughSignature { @@ -82,15 +80,5 @@ public void customConnectorVerifications(Map engineQptArguments) if (!upperCaseStatement.startsWith("SELECT")) { throw new AthenaConnectorException("Statement does not start with SELECT.", ErrorDetails.builder().errorCode(FederationSourceErrorCode.OPERATION_NOT_SUPPORTED_EXCEPTION.toString()).build()); } - - // List of disallowed keywords - Set disallowedKeywords = ImmutableSet.of("INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER"); - - // Check if the statement contains any disallowed keywords - for (String keyword : disallowedKeywords) { - if (upperCaseStatement.contains(keyword)) { - throw new AthenaConnectorException("Unaccepted operation; only SELECT statements are allowed. Found: " + keyword, ErrorDetails.builder().errorCode(FederationSourceErrorCode.OPERATION_NOT_SUPPORTED_EXCEPTION.toString()).build()); - } - } } } From a1a4242254290e3f119c9844de0956869a8f5a71 Mon Sep 17 00:00:00 2001 From: VenkatasivareddyTR <110587813+VenkatasivareddyTR@users.noreply.github.com> Date: Wed, 15 Jan 2025 22:03:43 +0530 Subject: [PATCH 4/7] DB2/DB2as400 glue get table api issue fix (#2506) --- .../db2as400/Db2As400MetadataHandler.java | 2 +- .../db2as400/Db2As400MetadataHandlerTest.java | 13 +++++++------ .../db2as400/Db2As400QueryStringBuilderTest.java | 2 +- .../athena/connectors/db2/Db2Constants.java | 2 +- .../athena/connectors/db2/Db2MetadataHandler.java | 2 +- .../connectors/db2/Db2QueryStringBuilder.java | 4 +++- .../connectors/db2/Db2MetadataHandlerTest.java | 13 +++++++------ .../connectors/db2/Db2QueryStringBuilderTest.java | 2 +- .../athena/connectors/db2/Db2RecordHandlerTest.java | 3 ++- 9 files changed, 24 insertions(+), 19 deletions(-) diff --git a/athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandler.java b/athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandler.java index a589bbc33d..286eb0452c 100644 --- a/athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandler.java +++ b/athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandler.java @@ -77,7 +77,7 @@ public class Db2As400MetadataHandler extends JdbcMetadataHandler { private static final Logger LOGGER = LoggerFactory.getLogger(Db2As400MetadataHandler.class); - static final String PARTITION_NUMBER = "PARTITION_NUMBER"; + static final String PARTITION_NUMBER = "partition_number"; static final String PARTITIONING_COLUMN = "PARTITIONING_COLUMN"; /** * DB2 has max number of partition 32,000 diff --git a/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandlerTest.java b/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandlerTest.java index ce35bab8e8..75ec44bd15 100644 --- a/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandlerTest.java +++ b/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400MetadataHandlerTest.java @@ -69,11 +69,12 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.amazonaws.athena.connectors.db2as400.Db2As400MetadataHandler.PARTITION_NUMBER; import static org.mockito.ArgumentMatchers.nullable; public class Db2As400MetadataHandlerTest extends TestBase { private static final Logger logger = LoggerFactory.getLogger(Db2As400MetadataHandlerTest.class); - private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField("PARTITION_NUMBER", org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); + private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField(PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); private DatabaseConnectionConfig databaseConnectionConfig = new DatabaseConnectionConfig("testCatalog", Db2As400Constants.NAME, "db2as400://jdbc:as400://testhost;user=dummy;password=dummy;"); private Db2As400MetadataHandler db2As400MetadataHandler; @@ -103,7 +104,7 @@ public void setup() throws Exception { public void getPartitionSchema() { Assert.assertEquals(SchemaBuilder.newBuilder() - .addField(Db2As400MetadataHandler.PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), + .addField(PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), this.db2As400MetadataHandler.getPartitionSchema("testCatalogName")); } @@ -130,7 +131,7 @@ public void doGetSplitsWithNoPartition() GetSplitsResponse getSplitsResponse = this.db2As400MetadataHandler.doGetSplits(splitBlockAllocator, getSplitsRequest); Set> expectedSplits = new HashSet<>(); - expectedSplits.add(Collections.singletonMap(db2As400MetadataHandler.PARTITION_NUMBER, "0")); + expectedSplits.add(Collections.singletonMap(PARTITION_NUMBER, "0")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); Set> actualSplits = getSplitsResponse.getSplits().stream().map(Split::getProperties).collect(Collectors.toSet()); Assert.assertEquals(expectedSplits, actualSplits); @@ -167,13 +168,13 @@ public void doGetSplits() Set> expectedSplits = com.google.common.collect.ImmutableSet.of( com.google.common.collect.ImmutableMap.of( - db2As400MetadataHandler.PARTITION_NUMBER, "0", + PARTITION_NUMBER, "0", db2As400MetadataHandler.PARTITIONING_COLUMN, "PC"), com.google.common.collect.ImmutableMap.of( - db2As400MetadataHandler.PARTITION_NUMBER, "1", + PARTITION_NUMBER, "1", db2As400MetadataHandler.PARTITIONING_COLUMN, "PC"), com.google.common.collect.ImmutableMap.of( - db2As400MetadataHandler.PARTITION_NUMBER, "2", + PARTITION_NUMBER, "2", db2As400MetadataHandler.PARTITIONING_COLUMN, "PC")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); diff --git a/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400QueryStringBuilderTest.java b/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400QueryStringBuilderTest.java index ec8132341e..b6a4d35874 100644 --- a/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400QueryStringBuilderTest.java +++ b/athena-db2-as400/src/test/java/com/amazonaws/athena/connectors/db2as400/Db2As400QueryStringBuilderTest.java @@ -45,7 +45,7 @@ public void testGetPartitionWhereClauses() { Db2As400QueryStringBuilder builder = new Db2As400QueryStringBuilder("'"); Split split = Mockito.mock(Split.class); - Mockito.when(split.getProperty(Mockito.eq("PARTITION_NUMBER"))).thenReturn("0"); + Mockito.when(split.getProperty(Mockito.eq("partition_number"))).thenReturn("0"); Mockito.when(split.getProperty(Mockito.eq("PARTITIONING_COLUMN"))).thenReturn("PC"); Assert.assertEquals(Arrays.asList(" DATAPARTITIONNUM(PC) = 0"), builder.getPartitionWhereClauses(split)); } diff --git a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2Constants.java b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2Constants.java index a1bc23c988..907ecaa1fb 100644 --- a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2Constants.java +++ b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2Constants.java @@ -25,7 +25,7 @@ public class Db2Constants public static final String DRIVER_CLASS = "com.ibm.db2.jcc.DB2Driver"; public static final int DEFAULT_PORT = 50001; public static final String QUOTE_CHARACTER = "\""; - + static final String PARTITION_NUMBER = "partition_number"; public static final String QRY_TO_LIST_SCHEMAS = "select schemaname as name " + "from syscat.schemata " + "where schemaname not like 'SYS%' " + diff --git a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandler.java b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandler.java index 965197ff0a..bc0b0d0049 100644 --- a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandler.java +++ b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandler.java @@ -82,11 +82,11 @@ import java.util.stream.Collectors; import static com.amazonaws.athena.connector.lambda.domain.predicate.functions.StandardFunctions.NULLIF_FUNCTION_NAME; +import static com.amazonaws.athena.connectors.db2.Db2Constants.PARTITION_NUMBER; public class Db2MetadataHandler extends JdbcMetadataHandler { private static final Logger LOGGER = LoggerFactory.getLogger(Db2MetadataHandler.class); - static final String PARTITION_NUMBER = "PARTITION_NUMBER"; static final String PARTITIONING_COLUMN = "PARTITIONING_COLUMN"; /** * DB2 has max number of partition 32,000 diff --git a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilder.java b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilder.java index e31970b7b7..79d939bac4 100644 --- a/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilder.java +++ b/athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilder.java @@ -29,6 +29,8 @@ import java.util.Collections; import java.util.List; +import static com.amazonaws.athena.connectors.db2.Db2Constants.PARTITION_NUMBER; + public class Db2QueryStringBuilder extends JdbcSplitQueryBuilder { private static final Logger LOGGER = LoggerFactory.getLogger(Db2QueryStringBuilder.class); @@ -74,7 +76,7 @@ protected List getPartitionWhereClauses(Split split) if (column != null) { LOGGER.debug("Fetching data using Partition"); //example query: select * from EMP_TABLE WHERE DATAPARTITIONNUM(EMP_NO) = 0 - return Collections.singletonList(" DATAPARTITIONNUM(" + column + ") = " + split.getProperty(Db2MetadataHandler.PARTITION_NUMBER)); + return Collections.singletonList(" DATAPARTITIONNUM(" + column + ") = " + split.getProperty(PARTITION_NUMBER)); } else { LOGGER.debug("Fetching data without Partition"); diff --git a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandlerTest.java b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandlerTest.java index 81a1ebb474..4c62da12b2 100644 --- a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandlerTest.java +++ b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2MetadataHandlerTest.java @@ -69,12 +69,13 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.amazonaws.athena.connectors.db2.Db2Constants.PARTITION_NUMBER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; public class Db2MetadataHandlerTest extends TestBase { private static final Logger logger = LoggerFactory.getLogger(Db2MetadataHandlerTest.class); - private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField("PARTITION_NUMBER", org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); + private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField(PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); private DatabaseConnectionConfig databaseConnectionConfig = new DatabaseConnectionConfig("testCatalog", Db2Constants.NAME, "dbtwo://jdbc:db2://hostname:50001/dummydatabase:user=dummyuser;password=dummypwd"); private Db2MetadataHandler db2MetadataHandler; @@ -104,7 +105,7 @@ public void setup() throws Exception { public void getPartitionSchema() { Assert.assertEquals(SchemaBuilder.newBuilder() - .addField(Db2MetadataHandler.PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), + .addField(PARTITION_NUMBER, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), this.db2MetadataHandler.getPartitionSchema("testCatalogName")); } @@ -131,7 +132,7 @@ public void doGetSplitsWithNoPartition() GetSplitsResponse getSplitsResponse = this.db2MetadataHandler.doGetSplits(splitBlockAllocator, getSplitsRequest); Set> expectedSplits = new HashSet<>(); - expectedSplits.add(Collections.singletonMap(db2MetadataHandler.PARTITION_NUMBER, "0")); + expectedSplits.add(Collections.singletonMap(PARTITION_NUMBER, "0")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); Set> actualSplits = getSplitsResponse.getSplits().stream().map(Split::getProperties).collect(Collectors.toSet()); Assert.assertEquals(expectedSplits, actualSplits); @@ -168,13 +169,13 @@ public void doGetSplits() Set> expectedSplits = com.google.common.collect.ImmutableSet.of( com.google.common.collect.ImmutableMap.of( - db2MetadataHandler.PARTITION_NUMBER, "0", + PARTITION_NUMBER, "0", db2MetadataHandler.PARTITIONING_COLUMN, "PC"), com.google.common.collect.ImmutableMap.of( - db2MetadataHandler.PARTITION_NUMBER, "1", + PARTITION_NUMBER, "1", db2MetadataHandler.PARTITIONING_COLUMN, "PC"), com.google.common.collect.ImmutableMap.of( - db2MetadataHandler.PARTITION_NUMBER, "2", + PARTITION_NUMBER, "2", db2MetadataHandler.PARTITIONING_COLUMN, "PC")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); diff --git a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilderTest.java b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilderTest.java index 859a8415b9..607cfcf17b 100644 --- a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilderTest.java +++ b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2QueryStringBuilderTest.java @@ -47,7 +47,7 @@ public void testGetPartitionWhereClauses() { Db2QueryStringBuilder builder = new Db2QueryStringBuilder(QUOTE_CHARACTER, new Db2FederationExpressionParser(QUOTE_CHARACTER)); Split split = Mockito.mock(Split.class); - Mockito.when(split.getProperty(Mockito.eq("PARTITION_NUMBER"))).thenReturn("0"); + Mockito.when(split.getProperty(Mockito.eq("partition_number"))).thenReturn("0"); Mockito.when(split.getProperty(Mockito.eq("PARTITIONING_COLUMN"))).thenReturn("PC"); Assert.assertEquals(Arrays.asList(" DATAPARTITIONNUM(PC) = 0"), builder.getPartitionWhereClauses(split)); } diff --git a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2RecordHandlerTest.java b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2RecordHandlerTest.java index b7de058f8d..f4c37b3e97 100644 --- a/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2RecordHandlerTest.java +++ b/athena-db2/src/test/java/com/amazonaws/athena/connectors/db2/Db2RecordHandlerTest.java @@ -47,6 +47,7 @@ import java.sql.SQLException; import java.util.Collections; +import static com.amazonaws.athena.connectors.db2.Db2Constants.PARTITION_NUMBER; import static com.amazonaws.athena.connectors.db2.Db2Constants.QUOTE_CHARACTER; import static org.mockito.ArgumentMatchers.nullable; @@ -97,7 +98,7 @@ public void buildSplitSqlNew() Schema schema = schemaBuilder.build(); Split split = Mockito.mock(Split.class); - Mockito.when(split.getProperty(Db2MetadataHandler.PARTITION_NUMBER)).thenReturn("0"); + Mockito.when(split.getProperty(PARTITION_NUMBER)).thenReturn("0"); ValueSet valueSet = getSingleValueSet("varcharTest"); Constraints constraints = Mockito.mock(Constraints.class); From 01bd504f4ee023fcbe152cedd6a8ceee692c7493 Mon Sep 17 00:00:00 2001 From: VenkatasivareddyTR <110587813+VenkatasivareddyTR@users.noreply.github.com> Date: Wed, 15 Jan 2025 22:09:39 +0530 Subject: [PATCH 5/7] Saphana glue get table api issue fix (#2505) Co-authored-by: akshay.kachore --- .../connectors/saphana/SaphanaConstants.java | 4 +-- .../saphana/SaphanaMetadataHandlerTest.java | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaConstants.java b/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaConstants.java index 19c221895d..c4633fc675 100644 --- a/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaConstants.java +++ b/athena-saphana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaConstants.java @@ -32,7 +32,7 @@ public final class SaphanaConstants public static final int SAPHANA_DEFAULT_PORT = 1025; static final Map JDBC_PROPERTIES = ImmutableMap.of("databaseTerm", "SCHEMA"); static final String ALL_PARTITIONS = "0"; - static final String BLOCK_PARTITION_COLUMN_NAME = "PART_ID"; + static final String BLOCK_PARTITION_COLUMN_NAME = "part_id"; static final String COLUMN_NAME = "COLUMN_NAME"; static final String CASE_UPPER = "upper"; static final String CASE_LOWER = "lower"; @@ -40,7 +40,7 @@ public final class SaphanaConstants /** * partition query for saphana */ - static final String GET_PARTITIONS_QUERY = "SELECT DISTINCT PART_ID FROM SYS.TABLE_PARTITIONS " + + static final String GET_PARTITIONS_QUERY = "SELECT DISTINCT PART_ID as \"part_id\" FROM SYS.TABLE_PARTITIONS " + "WHERE TABLE_NAME = ? AND SCHEMA_NAME = ? AND PART_ID IS NOT NULL"; /** diff --git a/athena-saphana/src/test/java/com/amazonaws/athena/connectors/saphana/SaphanaMetadataHandlerTest.java b/athena-saphana/src/test/java/com/amazonaws/athena/connectors/saphana/SaphanaMetadataHandlerTest.java index 59d9973b8e..44ec23d5fa 100644 --- a/athena-saphana/src/test/java/com/amazonaws/athena/connectors/saphana/SaphanaMetadataHandlerTest.java +++ b/athena-saphana/src/test/java/com/amazonaws/athena/connectors/saphana/SaphanaMetadataHandlerTest.java @@ -53,6 +53,7 @@ import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import static com.amazonaws.athena.connectors.saphana.SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; @@ -68,7 +69,7 @@ public class SaphanaMetadataHandlerTest private SecretsManagerClient secretsManager; private AthenaClient athena; private BlockAllocator blockAllocator; - private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField("PART_ID", org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); + private static final Schema PARTITION_SCHEMA = SchemaBuilder.newBuilder().addField(BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(); @Before @@ -90,7 +91,7 @@ public void setup() public void getPartitionSchema() { Assert.assertEquals(SchemaBuilder.newBuilder() - .addField(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), + .addField(BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build(), this.saphanaMetadataHandler.getPartitionSchema("testCatalogName")); } @@ -102,14 +103,14 @@ public void doGetTableLayout() Constraints constraints = Mockito.mock(Constraints.class); TableName tableName = new TableName("testSchema", "testTable"); Schema partitionSchema = this.saphanaMetadataHandler.getPartitionSchema("testCatalogName"); - Set partitionCols = new HashSet<>(Arrays.asList("PART_ID")); //partitionSchema.getFields().stream().map(Field::getName).collect(Collectors.toSet()); + Set partitionCols = new HashSet<>(Arrays.asList(BLOCK_PARTITION_COLUMN_NAME)); //partitionSchema.getFields().stream().map(Field::getName).collect(Collectors.toSet()); GetTableLayoutRequest getTableLayoutRequest = new GetTableLayoutRequest(this.federatedIdentity, "testQueryId", "testCatalogName", tableName, constraints, partitionSchema, partitionCols); PreparedStatement preparedStatement = Mockito.mock(PreparedStatement.class); Mockito.when(this.connection.prepareStatement(SaphanaConstants.GET_PARTITIONS_QUERY)).thenReturn(preparedStatement); - String[] columns = {"PART_ID"}; + String[] columns = {BLOCK_PARTITION_COLUMN_NAME}; int[] types = {Types.VARCHAR}; Object[][] values = {{"p0"}, {"p1"}}; ResultSet resultSet = mockResultSet(columns, types, values, new AtomicInteger(-1)); @@ -123,10 +124,10 @@ public void doGetTableLayout() for (int i = 0; i < getTableLayoutResponse.getPartitions().getRowCount(); i++) { expectedValues.add(BlockUtils.rowToString(getTableLayoutResponse.getPartitions(), i)); } - Assert.assertEquals(expectedValues, Arrays.asList("[PART_ID : p0]", "[PART_ID : p1]")); + Assert.assertEquals(expectedValues, Arrays.asList("[part_id : p0]", "[part_id : p1]")); SchemaBuilder expectedSchemaBuilder = SchemaBuilder.newBuilder(); - expectedSchemaBuilder.addField(FieldBuilder.newBuilder(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build()); + expectedSchemaBuilder.addField(FieldBuilder.newBuilder(BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build()); Schema expectedSchema = expectedSchemaBuilder.build(); Assert.assertEquals(expectedSchema, getTableLayoutResponse.getPartitions().getSchema()); Assert.assertEquals(tableName, getTableLayoutResponse.getTableName()); @@ -149,7 +150,7 @@ public void doGetTableLayoutWithNoPartitions() PreparedStatement preparedStatement = Mockito.mock(PreparedStatement.class); Mockito.when(this.connection.prepareStatement(SaphanaConstants.GET_PARTITIONS_QUERY)).thenReturn(preparedStatement); - String[] columns = {"PART_ID"}; + String[] columns = {BLOCK_PARTITION_COLUMN_NAME}; int[] types = {Types.VARCHAR}; Object[][] values = {{}}; ResultSet resultSet = mockResultSet(columns, types, values, new AtomicInteger(-1)); @@ -165,10 +166,10 @@ public void doGetTableLayoutWithNoPartitions() for (int i = 0; i < getTableLayoutResponse.getPartitions().getRowCount(); i++) { expectedValues.add(BlockUtils.rowToString(getTableLayoutResponse.getPartitions(), i)); } - Assert.assertEquals(expectedValues, Collections.singletonList("[PART_ID : 0]")); + Assert.assertEquals(expectedValues, Collections.singletonList("[part_id : 0]")); SchemaBuilder expectedSchemaBuilder = SchemaBuilder.newBuilder(); - expectedSchemaBuilder.addField(FieldBuilder.newBuilder(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build()); + expectedSchemaBuilder.addField(FieldBuilder.newBuilder(BLOCK_PARTITION_COLUMN_NAME, org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()).build()); Assert.assertEquals(tableName, getTableLayoutResponse.getTableName()); Mockito.verify(preparedStatement, Mockito.times(1)).setString(1, tableName.getTableName()); @@ -203,7 +204,7 @@ public void doGetSplits() PreparedStatement preparedStatement = Mockito.mock(PreparedStatement.class); - String[] columns = {SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME}; + String[] columns = {BLOCK_PARTITION_COLUMN_NAME}; int[] types = {Types.VARCHAR}; Object[][] values = {{"p0"}, {"p1"}}; ResultSet resultSet = mockResultSet(columns, types, values, new AtomicInteger(-1)); @@ -224,8 +225,8 @@ public void doGetSplits() GetSplitsResponse getSplitsResponse = this.saphanaMetadataHandler.doGetSplits(splitBlockAllocator, getSplitsRequest); Set> expectedSplits = new HashSet<>(); - expectedSplits.add(Collections.singletonMap(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, "p0")); - expectedSplits.add(Collections.singletonMap(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, "p1")); + expectedSplits.add(Collections.singletonMap(BLOCK_PARTITION_COLUMN_NAME, "p0")); + expectedSplits.add(Collections.singletonMap(BLOCK_PARTITION_COLUMN_NAME, "p1")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); Set> actualSplits = getSplitsResponse.getSplits().stream().map(Split::getProperties).collect(Collectors.toSet()); Assert.assertEquals(expectedSplits, actualSplits); @@ -245,7 +246,7 @@ public void doGetSplitsContinuation() PreparedStatement preparedStatement = Mockito.mock(PreparedStatement.class); Mockito.when(this.connection.prepareStatement(SaphanaConstants.GET_PARTITIONS_QUERY)).thenReturn(preparedStatement); - String[] columns = {"PART_ID"}; + String[] columns = {BLOCK_PARTITION_COLUMN_NAME}; int[] types = {Types.VARCHAR}; Object[][] values = {{"p0"}, {"p1"}}; ResultSet resultSet = mockResultSet(columns, types, values, new AtomicInteger(-1)); @@ -260,7 +261,7 @@ public void doGetSplitsContinuation() GetSplitsResponse getSplitsResponse = this.saphanaMetadataHandler.doGetSplits(splitBlockAllocator, getSplitsRequest); Set> expectedSplits = new HashSet<>(); - expectedSplits.add(Collections.singletonMap("PART_ID", "p1")); + expectedSplits.add(Collections.singletonMap(BLOCK_PARTITION_COLUMN_NAME, "p1")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); Set> actualSplits = getSplitsResponse.getSplits().stream().map(Split::getProperties).collect(Collectors.toSet()); Assert.assertEquals(expectedSplits, actualSplits); @@ -339,7 +340,7 @@ public void doGetSplitsForView() GetSplitsResponse getSplitsResponse = this.saphanaMetadataHandler.doGetSplits(splitBlockAllocator, getSplitsRequest); Set> expectedSplits = new HashSet<>(); - expectedSplits.add(Collections.singletonMap(SaphanaConstants.BLOCK_PARTITION_COLUMN_NAME, "0")); + expectedSplits.add(Collections.singletonMap(BLOCK_PARTITION_COLUMN_NAME, "0")); Assert.assertEquals(expectedSplits.size(), getSplitsResponse.getSplits().size()); From 2d1bea3ff59074224530292117d21e32a4e86d92 Mon Sep 17 00:00:00 2001 From: Jithendar Trianz <106380520+Jithendar12@users.noreply.github.com> Date: Thu, 16 Jan 2025 00:19:44 +0530 Subject: [PATCH 6/7] Add port number as a DBS_PORT parameter in Teradata JDBC Connection String (#2452) --- .../TeradataEnvironmentProperties.java | 17 +++++ .../TeradataEnvironmentPropertiesTest.java | 70 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 athena-teradata/src/test/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentPropertiesTest.java diff --git a/athena-teradata/src/main/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentProperties.java b/athena-teradata/src/main/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentProperties.java index 0b41bbcb0f..d5ddbee34f 100644 --- a/athena-teradata/src/main/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentProperties.java +++ b/athena-teradata/src/main/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentProperties.java @@ -21,12 +21,29 @@ import com.amazonaws.athena.connectors.jdbc.JdbcEnvironmentProperties; +import java.util.HashMap; import java.util.Map; import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DATABASE; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DEFAULT; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HOST; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.PORT; public class TeradataEnvironmentProperties extends JdbcEnvironmentProperties { + @Override + public Map connectionPropertiesToEnvironment(Map connectionProperties) + { + HashMap environment = new HashMap<>(); + + // Construct the JDBC connection string and include the port as a DBS_PORT parameter + String connectionString = getConnectionStringPrefix(connectionProperties) + connectionProperties.get(HOST) + + getDatabase(connectionProperties) + ",DBS_PORT=" + connectionProperties.getOrDefault(PORT, String.valueOf(TeradataConstants.TERADATA_DEFAULT_PORT)) + getJdbcParameters(connectionProperties); + + environment.put(DEFAULT, connectionString); + return environment; + } + @Override protected String getConnectionStringPrefix(Map connectionProperties) { diff --git a/athena-teradata/src/test/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentPropertiesTest.java b/athena-teradata/src/test/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentPropertiesTest.java new file mode 100644 index 0000000000..64f2ecb252 --- /dev/null +++ b/athena-teradata/src/test/java/com/amazonaws/athena/connectors/teradata/TeradataEnvironmentPropertiesTest.java @@ -0,0 +1,70 @@ +/*- + * #%L + * athena-teradata + * %% + * Copyright (C) 2019 Amazon Web Services + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.amazonaws.athena.connectors.teradata; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DATABASE; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DEFAULT; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HOST; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.PORT; +import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.SECRET_NAME; +import static org.junit.Assert.assertEquals; + +public class TeradataEnvironmentPropertiesTest +{ + Map connectionProperties; + TeradataEnvironmentProperties teradataEnvironmentProperties; + + @Before + public void setUp() + { + connectionProperties = new HashMap<>(); + connectionProperties.put(HOST, "test.teradata.com"); + connectionProperties.put(DATABASE, "testdb"); + connectionProperties.put(SECRET_NAME, "testSecret"); + teradataEnvironmentProperties = new TeradataEnvironmentProperties(); + } + + @Test + public void connectionPropertiesWithCustomPort() + { + // adding custom port + connectionProperties.put(PORT, "1234"); + + Map teradataConnectionProperties = teradataEnvironmentProperties.connectionPropertiesToEnvironment(connectionProperties); + + String expectedConnectionString = "teradata://jdbc:teradata://test.teradata.com/TMODE=ANSI,CHARSET=UTF8,DATABASE=testdb,DBS_PORT=1234,${testSecret}"; + assertEquals(expectedConnectionString, teradataConnectionProperties.get(DEFAULT)); + } + + @Test + public void connectionPropertiesWithDefaultPort() + { + Map teradataConnectionProperties = teradataEnvironmentProperties.connectionPropertiesToEnvironment(connectionProperties); + + String expectedConnectionString = "teradata://jdbc:teradata://test.teradata.com/TMODE=ANSI,CHARSET=UTF8,DATABASE=testdb,DBS_PORT=1025,${testSecret}"; + assertEquals(expectedConnectionString, teradataConnectionProperties.get(DEFAULT)); + } +} From 66736eabbef411fb074bf970d15f1c5a8aa43c4b Mon Sep 17 00:00:00 2001 From: AbdulRehman Date: Thu, 16 Jan 2025 11:44:17 -0500 Subject: [PATCH 7/7] Added clearification comment regarding Oracle Cipher Suit (#2521) Co-authored-by: AbdulRehman Faraj --- .../athena/connectors/oracle/OracleJdbcConnectionFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/athena-oracle/src/main/java/com/amazonaws/athena/connectors/oracle/OracleJdbcConnectionFactory.java b/athena-oracle/src/main/java/com/amazonaws/athena/connectors/oracle/OracleJdbcConnectionFactory.java index 22efdd5ebb..25b5f904ea 100644 --- a/athena-oracle/src/main/java/com/amazonaws/athena/connectors/oracle/OracleJdbcConnectionFactory.java +++ b/athena-oracle/src/main/java/com/amazonaws/athena/connectors/oracle/OracleJdbcConnectionFactory.java @@ -68,6 +68,9 @@ public Connection getConnection(final JdbcCredentialProvider jdbcCredentialProvi properties.put("javax.net.ssl.trustStore", "rds-truststore.jks"); properties.put("javax.net.ssl.trustStorePassword", "federationStorePass"); properties.put("oracle.net.ssl_server_dn_match", "true"); + // By default; Oracle RDS uses SSL_RSA_WITH_AES_256_CBC_SHA + // Adding the following cipher suits to support others listed in Doc + // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.Oracle.Options.SSL.html#Appendix.Oracle.Options.SSL.CipherSuites if (System.getenv().getOrDefault(IS_FIPS_ENABLED, "false").equalsIgnoreCase("true") || System.getenv().getOrDefault(IS_FIPS_ENABLED_LEGACY, "false").equalsIgnoreCase("true")) { properties.put("oracle.net.ssl_cipher_suites", "(TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA)"); }