Skip to content

Commit

Permalink
Add extra_properties to hive table properties
Browse files Browse the repository at this point in the history
  • Loading branch information
@posulliv
posulliv committed Feb 28, 2023
1 parent ac40973 commit 2ff3b99
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 2 deletions.
10 changes: 10 additions & 0 deletions core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import io.trino.sql.tree.Explain;
import io.trino.sql.tree.ExplainAnalyze;
import io.trino.sql.tree.Expression;
import io.trino.sql.tree.FunctionCall;
import io.trino.sql.tree.Identifier;
import io.trino.sql.tree.LikePredicate;
import io.trino.sql.tree.LongLiteral;
Expand All @@ -75,6 +76,7 @@
import io.trino.sql.tree.QualifiedName;
import io.trino.sql.tree.Query;
import io.trino.sql.tree.Relation;
import io.trino.sql.tree.Row;
import io.trino.sql.tree.ShowCatalogs;
import io.trino.sql.tree.ShowColumns;
import io.trino.sql.tree.ShowCreate;
Expand Down Expand Up @@ -154,6 +156,7 @@
import static io.trino.sql.tree.ShowCreate.Type.TABLE;
import static io.trino.sql.tree.ShowCreate.Type.VIEW;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
Expand Down Expand Up @@ -563,6 +566,13 @@ private static Expression toExpression(Object value)
.collect(toList()));
}

if (value instanceof Map) {
Map<?, ?> map = (Map<?, ?>) value;
return new FunctionCall(QualifiedName.of("map_from_entries"), ImmutableList.of(new Array(map.entrySet().stream()
.map(entry -> new Row(asList(toExpression(entry.getKey()), toExpression(entry.getValue()))))
.collect(toImmutableList()))));
}

throw new TrinoException(INVALID_TABLE_PROPERTY, format("Failed to convert object of type %s to expression: %s", value.getClass().getName(), value));
}

Expand Down
17 changes: 17 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.getOnlyElement;
import static io.airlift.json.JsonCodec.mapJsonCodec;
import static io.trino.hdfs.ConfigurationUtils.toJobConf;
import static io.trino.plugin.hive.HiveAnalyzeProperties.getColumnNames;
import static io.trino.plugin.hive.HiveAnalyzeProperties.getPartitionList;
Expand Down Expand Up @@ -213,6 +214,7 @@
import static io.trino.plugin.hive.HiveTableProperties.CSV_QUOTE;
import static io.trino.plugin.hive.HiveTableProperties.CSV_SEPARATOR;
import static io.trino.plugin.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY;
import static io.trino.plugin.hive.HiveTableProperties.EXTRA_PROPERTIES;
import static io.trino.plugin.hive.HiveTableProperties.NULL_FORMAT_PROPERTY;
import static io.trino.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_COLUMNS;
import static io.trino.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP;
Expand All @@ -227,6 +229,7 @@
import static io.trino.plugin.hive.HiveTableProperties.getAvroSchemaUrl;
import static io.trino.plugin.hive.HiveTableProperties.getBucketProperty;
import static io.trino.plugin.hive.HiveTableProperties.getExternalLocation;
import static io.trino.plugin.hive.HiveTableProperties.getExtraProperties;
import static io.trino.plugin.hive.HiveTableProperties.getFooterSkipCount;
import static io.trino.plugin.hive.HiveTableProperties.getHeaderSkipCount;
import static io.trino.plugin.hive.HiveTableProperties.getHiveStorageFormat;
Expand Down Expand Up @@ -685,6 +688,11 @@ private ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, Sche
// Partition Projection specific properties
properties.putAll(partitionProjectionService.getPartitionProjectionTrinoTableProperties(table));

String extraProperties = table.getParameters().get(EXTRA_PROPERTIES);
if (extraProperties != null) {
properties.put(EXTRA_PROPERTIES, mapJsonCodec(String.class, String.class).fromJson(extraProperties));
}

return new ConnectorTableMetadata(tableName, columns.build(), properties.buildOrThrow(), comment);
}

Expand Down Expand Up @@ -1090,6 +1098,15 @@ else if (avroSchemaLiteral != null) {
tableProperties.put("numFiles", "-1");
tableProperties.put("totalSize", "-1");

// Extra properties
Map<String, String> extraProperties = getExtraProperties(tableMetadata.getProperties());
if (extraProperties != null) {
tableProperties.put(EXTRA_PROPERTIES, mapJsonCodec(String.class, String.class).toJson(extraProperties));
for (Map.Entry<String, String> extraProperty : extraProperties.entrySet()) {
tableProperties.put(extraProperty.getKey(), extraProperty.getValue());
}
}

// Table comment property
tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value));

Expand Down
22 changes: 20 additions & 2 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import io.trino.spi.TrinoException;
import io.trino.spi.session.PropertyMetadata;
import io.trino.spi.type.ArrayType;
import io.trino.spi.type.MapType;
import io.trino.spi.type.TypeManager;

import javax.inject.Inject;

Expand Down Expand Up @@ -67,13 +69,15 @@ public class HiveTableProperties
public static final String CSV_ESCAPE = "csv_escape";
public static final String TRANSACTIONAL = "transactional";
public static final String AUTO_PURGE = "auto_purge";
public static final String EXTRA_PROPERTIES = "extra_properties";

private final List<PropertyMetadata<?>> tableProperties;

@Inject
public HiveTableProperties(
HiveConfig config,
OrcWriterConfig orcWriterConfig)
OrcWriterConfig orcWriterConfig,
TypeManager typeManager)
{
tableProperties = ImmutableList.of(
stringProperty(
Expand Down Expand Up @@ -169,7 +173,16 @@ public HiveTableProperties(
PARTITION_PROJECTION_LOCATION_TEMPLATE,
"Partition projection location template",
null,
false));
false),
new PropertyMetadata<>(
EXTRA_PROPERTIES,
"Extra table properties",
new MapType(VARCHAR, VARCHAR, typeManager.getTypeOperators()),
Map.class,
null,
false,
value -> ((Map<String, String>) value),
value -> value));
}

public List<PropertyMetadata<?>> getTableProperties()
Expand Down Expand Up @@ -297,4 +310,9 @@ public static Optional<Boolean> isAutoPurge(Map<String, Object> tableProperties)
{
return Optional.ofNullable((Boolean) tableProperties.get(AUTO_PURGE));
}

public static Map<String, String> getExtraProperties(Map<String, Object> tableProperties)
{
return (Map<String, String>) tableProperties.get(EXTRA_PROPERTIES);
}
}
94 changes: 94 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import io.trino.metadata.QualifiedObjectName;
import io.trino.metadata.TableHandle;
import io.trino.metadata.TableMetadata;
import io.trino.spi.QueryId;
import io.trino.spi.connector.CatalogSchemaTableName;
import io.trino.spi.connector.ColumnHandle;
import io.trino.spi.connector.ColumnMetadata;
Expand All @@ -54,6 +55,7 @@
import io.trino.testing.MaterializedResult;
import io.trino.testing.MaterializedResultWithQueryId;
import io.trino.testing.MaterializedRow;
import io.trino.testing.QueryFailedException;
import io.trino.testing.QueryRunner;
import io.trino.testing.TestingConnectorBehavior;
import io.trino.testing.sql.TestTable;
Expand Down Expand Up @@ -8453,6 +8455,98 @@ public void testCreateAcidTableUnsupported()
assertQueryFails("CREATE TABLE acid_unsupported WITH (transactional = true) AS SELECT 123 x", "FileHiveMetastore does not support ACID tables");
}

@Test
public void testExtraProperties()
{
String tableName = format("%s.%s.test_extra_properties", getSession().getCatalog().get(), getSession().getSchema().get());
@Language("SQL") String createTableSql = format("""
CREATE TABLE %s (
c1 integer)
WITH (
extra_properties = MAP(ARRAY['extra.property'], ARRAY['true']),
format = 'ORC'
)""",
tableName);
MaterializedResultWithQueryId result = getDistributedQueryRunner().executeWithQueryId(getSession(), createTableSql);
QueryId queryId = result.getQueryId();
String nodeVersion = (String) computeScalar("SELECT node_version FROM system.runtime.nodes WHERE coordinator");
assertQuery(
"SELECT * FROM \"test_extra_properties$properties\"",
"SELECT 'workaround for potential lack of HIVE-12730', 'false', 'true', '{\n \"extra.property\" : \"true\"\n}', '0', '0', '" + queryId + "', '" + nodeVersion + "', '0', '0', 'false'");
MaterializedResult actualResult = computeActual("SHOW CREATE TABLE " + tableName);
String expectedShowCreateTableSql = "CREATE TABLE hive.tpch.test_extra_properties (\n" +
" c1 integer\n" +
")\n" +
"WITH (\n" +
" extra_properties = map_from_entries(ARRAY[ROW('extra.property', 'true')]),\n" +
" format = 'ORC'\n" +
")";
assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), expectedShowCreateTableSql);
assertUpdate("DROP TABLE " + tableName);
}

@Test
public void testMultipleExtraProperties()
{
String tableName = format("%s.%s.test_multiple_extra_properties", getSession().getCatalog().get(), getSession().getSchema().get());
@Language("SQL") String createTableSql = format("""
CREATE TABLE %s (
c1 integer)
WITH (
extra_properties = MAP(ARRAY['extra.property.one', 'extra.property.two'], ARRAY['one', 'two']),
format = 'ORC'
)""",
tableName);
MaterializedResultWithQueryId result = getDistributedQueryRunner().executeWithQueryId(getSession(), createTableSql);
QueryId queryId = result.getQueryId();
String nodeVersion = (String) computeScalar("SELECT node_version FROM system.runtime.nodes WHERE coordinator");
assertQuery(
"SELECT * FROM \"test_multiple_extra_properties$properties\"",
"SELECT 'workaround for potential lack of HIVE-12730', 'false', 'one', 'two', '{\n \"extra.property.one\" : \"one\",\n \"extra.property.two\" : \"two\"\n}', '0', '0', '" + queryId + "', '" + nodeVersion + "', '0', '0', 'false'");
MaterializedResult actualResult = computeActual("SHOW CREATE TABLE " + tableName);
String expectedShowCreateTableSql = "CREATE TABLE hive.tpch.test_multiple_extra_properties (\n" +
" c1 integer\n" +
")\n" +
"WITH (\n" +
" extra_properties = map_from_entries(ARRAY[ROW('extra.property.one', 'one'),ROW('extra.property.two', 'two')]),\n" +
" format = 'ORC'\n" +
")";
assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), expectedShowCreateTableSql);
assertUpdate("DROP TABLE " + tableName);
}

@Test
public void testDuplicateExtraProperties()
{
String tableName = format("%s.%s.test_duplicate_extra_properties", getSession().getCatalog().get(), getSession().getSchema().get());
@Language("SQL") String createTableSql = format("""
CREATE TABLE %s (
c1 integer)
WITH (
extra_properties = MAP(ARRAY['extra.property', 'extra.property'], ARRAY['true', 'false']),
format = 'ORC'
)""",
tableName);
assertQueryFails(createTableSql, "Invalid value for catalog 'hive' table property 'extra_properties': Cannot convert.*");
}

@Test
public void testOverwriteExistingPropertyWithExtraProperties()
{
String tableName = format("%s.%s.test_overwrite_extra_properties", getSession().getCatalog().get(), getSession().getSchema().get());
@Language("SQL") String createTableSql = format("""
CREATE TABLE %s (
c1 integer)
WITH (
extra_properties = MAP(ARRAY['transactional'], ARRAY['true']),
format = 'ORC'
)""",
tableName);
assertThatThrownBy(() -> assertUpdate(createTableSql))
.isInstanceOf(QueryFailedException.class)
.hasMessage("Multiple entries with same key: transactional=true and transactional=false");
}

private static final Set<HiveStorageFormat> NAMED_COLUMN_ONLY_FORMATS = ImmutableSet.of(HiveStorageFormat.AVRO, HiveStorageFormat.JSON);

@DataProvider
Expand Down

0 comments on commit 2ff3b99

Please sign in to comment.