diff --git a/core/trino-main/src/main/java/io/trino/connector/ConnectorManager.java b/core/trino-main/src/main/java/io/trino/connector/ConnectorManager.java index d480474486c3..d4675cc044cd 100644 --- a/core/trino-main/src/main/java/io/trino/connector/ConnectorManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/ConnectorManager.java @@ -47,6 +47,7 @@ import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorRecordSetProvider; import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.connector.SystemTable; import io.trino.spi.eventlistener.EventListener; import io.trino.spi.procedure.Procedure; @@ -303,7 +304,7 @@ private synchronized void addConnectorInternal(MaterializedConnector connector) connector.getAccessControl() .ifPresent(accessControl -> accessControlManager.addCatalogAccessControl(catalogName, accessControl)); - metadataManager.getTablePropertyManager().addProperties(catalogName, connector.getTableProperties()); + metadataManager.getTablePropertyManager().addProperties(catalogName, connector.getTablePropertyProvider()); metadataManager.getMaterializedViewPropertyManager().addProperties(catalogName, connector.getMaterializedViewProperties()); metadataManager.getColumnPropertyManager().addProperties(catalogName, connector.getColumnProperties()); metadataManager.getSchemaPropertyManager().addProperties(catalogName, connector.getSchemaProperties()); @@ -409,7 +410,7 @@ private static class MaterializedConnector private final Optional accessControl; private final List eventListeners; private final List> sessionProperties; - private final List> tableProperties; + private final PropertyProvider tablePropertyProvider; private final List> materializedViewProperties; private final List> schemaProperties; private final List> columnProperties; @@ -497,9 +498,9 @@ public MaterializedConnector(CatalogName catalogName, Connector connector) requireNonNull(sessionProperties, format("Connector '%s' returned a null system properties set", catalogName)); this.sessionProperties = ImmutableList.copyOf(sessionProperties); - List> tableProperties = connector.getTableProperties(); - requireNonNull(tableProperties, format("Connector '%s' returned a null table properties set", catalogName)); - this.tableProperties = ImmutableList.copyOf(tableProperties); + PropertyProvider tablePropertyProvider = connector.getTablePropertyProvider(); + requireNonNull(tablePropertyProvider, format("Connector '%s' returned a null table properties provider", catalogName)); + this.tablePropertyProvider = tablePropertyProvider; List> materializedViewProperties = connector.getMaterializedViewProperties(); requireNonNull(materializedViewProperties, format("Connector '%s' returned a null materialized view properties set", catalogName)); @@ -578,9 +579,9 @@ public List> getSessionProperties() return sessionProperties; } - public List> getTableProperties() + public PropertyProvider getTablePropertyProvider() { - return tableProperties; + return tablePropertyProvider; } public List> getMaterializedViewProperties() diff --git a/core/trino-main/src/main/java/io/trino/connector/system/AbstractPropertiesSystemTable.java b/core/trino-main/src/main/java/io/trino/connector/system/AbstractPropertiesSystemTable.java index 993a18f495e8..4be004f5df10 100644 --- a/core/trino-main/src/main/java/io/trino/connector/system/AbstractPropertiesSystemTable.java +++ b/core/trino-main/src/main/java/io/trino/connector/system/AbstractPropertiesSystemTable.java @@ -13,22 +13,22 @@ */ package io.trino.connector.system; -import com.google.common.collect.ImmutableMap; import io.trino.connector.CatalogName; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.InMemoryRecordSet; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.connector.RecordCursor; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SystemTable; import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.session.PropertyMetadata; import io.trino.transaction.TransactionId; import io.trino.transaction.TransactionManager; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.TreeMap; import java.util.function.Supplier; @@ -43,9 +43,9 @@ abstract class AbstractPropertiesSystemTable { private final ConnectorTableMetadata tableMetadata; private final TransactionManager transactionManager; - private final Supplier>>> propertySupplier; + private final Supplier> propertySupplier; - protected AbstractPropertiesSystemTable(String tableName, TransactionManager transactionManager, Supplier>>> propertySupplier) + protected AbstractPropertiesSystemTable(String tableName, TransactionManager transactionManager, Supplier> propertySupplier) { this.tableMetadata = tableMetadataBuilder(new SchemaTableName("metadata", tableName)) .column("catalog_name", createUnboundedVarcharType()) @@ -76,18 +76,20 @@ public final RecordCursor cursor(ConnectorTransactionHandle transactionHandle, C TransactionId transactionId = ((GlobalSystemTransactionHandle) transactionHandle).getTransactionId(); InMemoryRecordSet.Builder table = InMemoryRecordSet.builder(tableMetadata); - Map>> connectorProperties = propertySupplier.get(); + Map connectorProperties = propertySupplier.get(); for (Entry entry : new TreeMap<>(transactionManager.getCatalogNames(transactionId)).entrySet()) { - String catalog = entry.getKey(); - Map> properties = new TreeMap<>(connectorProperties.getOrDefault(entry.getValue(), ImmutableMap.of())); - for (PropertyMetadata propertyMetadata : properties.values()) { - table.addRow( - catalog, - propertyMetadata.getName(), - firstNonNull(propertyMetadata.getDefaultValue(), "").toString(), - propertyMetadata.getSqlType().toString(), - propertyMetadata.getDescription()); - } + PropertyProvider propertyProvider = connectorProperties.get(entry.getValue()); + propertyProvider.getKnownPropertyNames().stream() + .sorted() + .map(propertyProvider::getProperty) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(propertyMetadata -> table.addRow( + entry.getKey(), + propertyMetadata.getName(), + firstNonNull(propertyMetadata.getDefaultValue(), "").toString(), + propertyMetadata.getSqlType().toString(), + propertyMetadata.getDescription())); } return table.build().cursor(); } diff --git a/core/trino-main/src/main/java/io/trino/metadata/AbstractPropertyManager.java b/core/trino-main/src/main/java/io/trino/metadata/AbstractPropertyManager.java index 5acbd549a2b7..6a4845f0c116 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/AbstractPropertyManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/AbstractPropertyManager.java @@ -14,13 +14,14 @@ package io.trino.metadata; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import io.trino.Session; import io.trino.connector.CatalogName; import io.trino.security.AccessControl; import io.trino.spi.ErrorCodeSupplier; import io.trino.spi.TrinoException; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.connector.FixedPropertyProvider; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.type.Type; import io.trino.sql.planner.ParameterRewriter; @@ -44,7 +45,7 @@ abstract class AbstractPropertyManager { - private final ConcurrentMap>> connectorProperties = new ConcurrentHashMap<>(); + private final ConcurrentMap connectorProperties = new ConcurrentHashMap<>(); private final String propertyType; private final ErrorCodeSupplier propertyError; @@ -57,12 +58,15 @@ protected AbstractPropertyManager(String propertyType, ErrorCodeSupplier propert public final void addProperties(CatalogName catalogName, List> properties) { - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(properties, "properties is null"); + addProperties(catalogName, new FixedPropertyProvider(properties)); + } - Map> propertiesByName = Maps.uniqueIndex(properties, PropertyMetadata::getName); + public final void addProperties(CatalogName catalogName, PropertyProvider propertyProvider) + { + requireNonNull(catalogName, "catalogName is null"); + requireNonNull(propertyProvider, "propertyProvider is null"); - checkState(connectorProperties.putIfAbsent(catalogName, propertiesByName) == null, "Properties for connector '%s' are already registered", catalogName); + checkState(connectorProperties.putIfAbsent(catalogName, propertyProvider) == null, "Properties for connector '%s' are already registered", catalogName); } public final void removeProperties(CatalogName catalogName) @@ -79,8 +83,8 @@ public final Map getProperties( AccessControl accessControl, Map, Expression> parameters) { - Map> supportedProperties = connectorProperties.get(catalogName); - if (supportedProperties == null) { + PropertyProvider propertyProvider = connectorProperties.get(catalogName); + if (propertyProvider == null) { throw new TrinoException(NOT_FOUND, "Catalog not found: " + catalog); } @@ -89,15 +93,14 @@ public final Map getProperties( // Fill in user-specified properties for (Map.Entry sqlProperty : sqlPropertyValues.entrySet()) { String propertyName = sqlProperty.getKey().toLowerCase(ENGLISH); - PropertyMetadata property = supportedProperties.get(propertyName); - if (property == null) { - throw new TrinoException( - propertyError, - format("Catalog '%s' does not support %s property '%s'", - catalog, - propertyType, - propertyName)); - } + PropertyMetadata property = propertyProvider.getProperty(propertyName) + .orElseThrow(() -> new TrinoException( + propertyError, + format( + "Catalog '%s' does not support %s property '%s'", + catalog, + propertyType, + propertyName))); Object sqlObjectValue; try { @@ -135,7 +138,9 @@ public final Map getProperties( Map userSpecifiedProperties = properties.build(); // Fill in the remaining properties with non-null defaults - for (PropertyMetadata propertyMetadata : supportedProperties.values()) { + for (String propertyName : propertyProvider.getKnownPropertyNames()) { + PropertyMetadata propertyMetadata = propertyProvider.getProperty(propertyName) + .orElseThrow(() -> new TrinoException(propertyError, "Could not get load default value property: " + propertyName)); if (!userSpecifiedProperties.containsKey(propertyMetadata.getName())) { Object value = propertyMetadata.getDefaultValue(); if (value != null) { @@ -146,7 +151,7 @@ public final Map getProperties( return properties.build(); } - public Map>> getAllProperties() + public Map getAllProperties() { return ImmutableMap.copyOf(connectorProperties); } diff --git a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java index 6cd5b6d88426..95436dddb0f9 100644 --- a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java +++ b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java @@ -38,6 +38,7 @@ import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorViewDefinition; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.GroupProvider; import io.trino.spi.security.PrincipalType; @@ -521,10 +522,10 @@ protected Node visitShowCreate(ShowCreate node, Void context) accessControl.checkCanShowCreateTable(session.toSecurityContext(), new QualifiedObjectName(catalogName.getValue(), schemaName.getValue(), tableName.getValue())); Map properties = viewDefinition.get().getProperties(); - Map> allMaterializedViewProperties = metadata.getMaterializedViewPropertyManager() + PropertyProvider materializedViewPropertyProvider = metadata.getMaterializedViewPropertyManager() .getAllProperties() .get(new CatalogName(catalogName.getValue())); - List propertyNodes = buildProperties(objectName, Optional.empty(), INVALID_MATERIALIZED_VIEW_PROPERTY, properties, allMaterializedViewProperties); + List propertyNodes = buildProperties(objectName, Optional.empty(), INVALID_MATERIALIZED_VIEW_PROPERTY, properties, materializedViewPropertyProvider); String sql = formatSql(new CreateMaterializedView(Optional.empty(), QualifiedName.of(ImmutableList.of(catalogName, schemaName, tableName)), query, false, false, propertyNodes, viewDefinition.get().getComment())).trim(); @@ -587,12 +588,12 @@ protected Node visitShowCreate(ShowCreate node, Void context) accessControl.checkCanShowCreateTable(session.toSecurityContext(), targetTableName); ConnectorTableMetadata connectorTableMetadata = metadata.getTableMetadata(session, tableHandle.get()).getMetadata(); - Map> allColumnProperties = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getCatalogName()); + PropertyProvider columnPropertyProvider = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getCatalogName()); List columns = connectorTableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(column -> { - List propertyNodes = buildProperties(targetTableName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties); + List propertyNodes = buildProperties(targetTableName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), columnPropertyProvider); return new ColumnDefinition( new Identifier(column.getName()), toSqlType(column.getType()), @@ -603,8 +604,8 @@ protected Node visitShowCreate(ShowCreate node, Void context) .collect(toImmutableList()); Map properties = connectorTableMetadata.getProperties(); - Map> allTableProperties = metadata.getTablePropertyManager().getAllProperties().get(tableHandle.get().getCatalogName()); - List propertyNodes = buildProperties(targetTableName, Optional.empty(), INVALID_TABLE_PROPERTY, properties, allTableProperties); + PropertyProvider tablePropertyProvider = metadata.getTablePropertyManager().getAllProperties().get(tableHandle.get().getCatalogName()); + List propertyNodes = buildProperties(targetTableName, Optional.empty(), INVALID_TABLE_PROPERTY, properties, tablePropertyProvider); CreateTable createTable = new CreateTable( QualifiedName.of(objectName.getCatalogName(), objectName.getSchemaName(), objectName.getObjectName()), @@ -625,9 +626,9 @@ protected Node visitShowCreate(ShowCreate node, Void context) accessControl.checkCanShowCreateSchema(session.toSecurityContext(), schemaName); Map properties = metadata.getSchemaProperties(session, schemaName); - Map> allTableProperties = metadata.getSchemaPropertyManager().getAllProperties().get(new CatalogName(schemaName.getCatalogName())); + PropertyProvider tablePropertyProvider = metadata.getSchemaPropertyManager().getAllProperties().get(new CatalogName(schemaName.getCatalogName())); QualifiedName qualifiedSchemaName = QualifiedName.of(schemaName.getCatalogName(), schemaName.getSchemaName()); - List propertyNodes = buildProperties(qualifiedSchemaName, Optional.empty(), INVALID_SCHEMA_PROPERTY, properties, allTableProperties); + List propertyNodes = buildProperties(qualifiedSchemaName, Optional.empty(), INVALID_SCHEMA_PROPERTY, properties, tablePropertyProvider); Optional owner = metadata.getSchemaOwner(session, schemaName).map(MetadataUtil::createPrincipal); @@ -647,7 +648,7 @@ private List buildProperties( Optional columnName, StandardErrorCode errorCode, Map properties, - Map> allProperties) + PropertyProvider propertyProvider) { if (properties.isEmpty()) { return Collections.emptyList(); @@ -662,10 +663,8 @@ private List buildProperties( throw new TrinoException(errorCode, format("Property %s for %s cannot have a null value", propertyName, toQualifiedName(objectName, columnName))); } - PropertyMetadata property = allProperties.get(propertyName); - if (property == null) { - throw new TrinoException(errorCode, "No PropertyMetadata for property: " + propertyName); - } + PropertyMetadata property = propertyProvider.getProperty(propertyName) + .orElseThrow(() -> new TrinoException(errorCode, "No PropertyMetadata for property: " + propertyName)); if (!Primitives.wrap(property.getJavaType()).isInstance(value)) { throw new TrinoException(errorCode, format( "Property %s for %s should have value of type %s, not %s", diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java b/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java index 795f05f77ca5..e15a2c5e1064 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/Connector.java @@ -134,12 +134,22 @@ default List> getAnalyzeProperties() /** * @return the table properties for this connector + * @deprecated use getTablePropertyProvider */ + @Deprecated default List> getTableProperties() { return emptyList(); } + /** + * @return the table properties for this connector + */ + default PropertyProvider getTablePropertyProvider() + { + return new FixedPropertyProvider(getTableProperties()); + } + /** * @return the materialized view properties for this connector */ diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPropertyProvider.java b/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPropertyProvider.java new file mode 100644 index 000000000000..6d44dda2669e --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/FixedPropertyProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.connector; + +import io.trino.spi.session.PropertyMetadata; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +public class FixedPropertyProvider + implements PropertyProvider +{ + private final Map> properties; + + public FixedPropertyProvider(Collection> properties) + { + requireNonNull(properties, "properties is null"); + this.properties = properties.stream() + .collect(Collectors.toUnmodifiableMap(PropertyMetadata::getName, identity())); + } + + @Override + public Set getKnownPropertyNames() + { + return properties.keySet(); + } + + @Override + public Optional> getProperty(String name) + { + return Optional.ofNullable(properties.get(name)); + } +} diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/PropertyProvider.java b/core/trino-spi/src/main/java/io/trino/spi/connector/PropertyProvider.java new file mode 100644 index 000000000000..5ec30ce5792a --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/PropertyProvider.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.connector; + +import io.trino.spi.session.PropertyMetadata; + +import java.util.Optional; +import java.util.Set; + +public interface PropertyProvider +{ + Set getKnownPropertyNames(); + + Optional> getProperty(String name); +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java index c11dce46aeff..83a6c4b0ff1e 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnector.java @@ -28,6 +28,7 @@ import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.connector.SystemTable; import io.trino.spi.eventlistener.EventListener; import io.trino.spi.procedure.Procedure; @@ -58,7 +59,7 @@ public class HiveConnector private final Set eventListeners; private final List> sessionProperties; private final List> schemaProperties; - private final List> tableProperties; + private final HiveTableProperties tableProperties; private final List> analyzeProperties; private final List> materializedViewProperties; @@ -80,7 +81,7 @@ public HiveConnector( Set eventListeners, Set sessionPropertiesProviders, List> schemaProperties, - List> tableProperties, + HiveTableProperties tableProperties, List> analyzeProperties, List> materializedViewProperties, ConnectorAccessControl accessControl, @@ -100,7 +101,7 @@ public HiveConnector( .flatMap(sessionPropertiesProvider -> sessionPropertiesProvider.getSessionProperties().stream()) .collect(toImmutableList()); this.schemaProperties = ImmutableList.copyOf(requireNonNull(schemaProperties, "schemaProperties is null")); - this.tableProperties = ImmutableList.copyOf(requireNonNull(tableProperties, "tableProperties is null")); + this.tableProperties = requireNonNull(tableProperties, "tableProperties is null"); this.analyzeProperties = ImmutableList.copyOf(requireNonNull(analyzeProperties, "analyzeProperties is null")); this.materializedViewProperties = requireNonNull(materializedViewProperties, "materializedViewProperties is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); @@ -176,7 +177,7 @@ public List> getAnalyzeProperties() } @Override - public List> getTableProperties() + public PropertyProvider getTablePropertyProvider() { return tableProperties; } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index fb55ebf130cd..ad5d27bb0125 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -125,6 +125,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalInt; @@ -196,6 +197,8 @@ import static io.trino.plugin.hive.HiveTableProperties.CSV_QUOTE; import static io.trino.plugin.hive.HiveTableProperties.CSV_SEPARATOR; import static io.trino.plugin.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY; +import static io.trino.plugin.hive.HiveTableProperties.EXTRA_PROPERTY_PATTERN; +import static io.trino.plugin.hive.HiveTableProperties.EXTRA_PROPERTY_PREFIX; import static io.trino.plugin.hive.HiveTableProperties.NULL_FORMAT_PROPERTY; import static io.trino.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_COLUMNS; import static io.trino.plugin.hive.HiveTableProperties.ORC_BLOOM_FILTER_FPP; @@ -316,6 +319,29 @@ public class HiveMetadata private static final String CSV_QUOTE_KEY = OpenCSVSerde.QUOTECHAR; private static final String CSV_ESCAPE_KEY = OpenCSVSerde.ESCAPECHAR; + private static final Set MANAGED_HIVE_TABLE_PROPERTIES = ImmutableSet.builder() + .add(PRESTO_VERSION_NAME) + .add(TRINO_CREATED_BY) + .add(PRESTO_QUERY_ID_NAME) + .add(BUCKETING_VERSION) + .add(TABLE_COMMENT) + .add(STORAGE_TABLE) + .add(TRANSACTIONAL) + .add(PRESTO_VIEW_COMMENT) + .add(PRESTO_VIEW_EXPANDED_TEXT_MARKER) + .add(ORC_BLOOM_FILTER_COLUMNS_KEY) + .add(ORC_BLOOM_FILTER_FPP_KEY) + .add(SKIP_HEADER_COUNT_KEY) + .add(SKIP_FOOTER_COUNT_KEY) + .add(TEXT_FIELD_SEPARATOR_KEY) + .add(TEXT_FIELD_SEPARATOR_ESCAPE_KEY) + .add(NULL_FORMAT_KEY) + .add(AVRO_SCHEMA_URL_KEY) + .add(CSV_SEPARATOR_KEY) + .add(CSV_QUOTE_KEY) + .add(CSV_ESCAPE_KEY) + .build(); + private final CatalogName catalogName; private final SemiTransactionalHiveMetastore metastore; private final HdfsEnvironment hdfsEnvironment; @@ -606,6 +632,15 @@ private ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, Sche Optional comment = Optional.ofNullable(table.getParameters().get(TABLE_COMMENT)); + // add all other table properties that are not already mapped to explicitly managed properties + for (Entry entry : table.getParameters().entrySet()) { + if (!MANAGED_HIVE_TABLE_PROPERTIES.contains(entry.getKey())) { + String extraPropertiesName = EXTRA_PROPERTY_PREFIX + entry.getKey(); + if (EXTRA_PROPERTY_PATTERN.matcher(extraPropertiesName).matches()) { + properties.put(extraPropertiesName, entry.getValue()); + } + } + } return new ConnectorTableMetadata(tableName, columns.build(), properties.build(), comment); } @@ -965,6 +1000,16 @@ private Map getEmptyTableProperties(ConnectorTableMetadata table // Table comment property tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value)); + // add "extra" table properties + for (Entry entry : tableMetadata.getProperties().entrySet()) { + if (EXTRA_PROPERTY_PATTERN.matcher(entry.getKey()).matches()) { + String hivePropertyName = entry.getKey().substring(EXTRA_PROPERTY_PREFIX.length()); + if (MANAGED_HIVE_TABLE_PROPERTIES.contains(hivePropertyName)) { + throw new TrinoException(INVALID_TABLE_PROPERTY, format("Extra property is a managed table property: %s", entry.getKey())); + } + tableProperties.put(hivePropertyName, (String) entry.getValue()); + } + } return tableProperties.build(); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java index 4eaadac52e5f..5a83587b47e1 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveTableProperties.java @@ -14,11 +14,14 @@ package io.trino.plugin.hive; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import io.trino.plugin.hive.metastore.SortingColumn; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.util.HiveBucketing.BucketingVersion; import io.trino.plugin.hive.util.HiveUtil; import io.trino.spi.TrinoException; +import io.trino.spi.connector.PropertyProvider; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.type.ArrayType; @@ -28,6 +31,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V1; @@ -43,7 +47,10 @@ import static java.util.Locale.ENGLISH; public class HiveTableProperties + implements PropertyProvider { + static final String EXTRA_PROPERTY_PREFIX = "extra_property_"; + static final Pattern EXTRA_PROPERTY_PATTERN = Pattern.compile("^extra_property_[a-z_]+$"); public static final String EXTERNAL_LOCATION_PROPERTY = "external_location"; public static final String STORAGE_FORMAT_PROPERTY = "format"; public static final String PARTITIONED_BY_PROPERTY = "partitioned_by"; @@ -70,6 +77,7 @@ public class HiveTableProperties public static final String TRANSACTIONAL = "transactional"; private final List> tableProperties; + private final ImmutableMap> knownTableProperties; @Inject public HiveTableProperties( @@ -154,6 +162,8 @@ public HiveTableProperties( stringProperty(CSV_QUOTE, "CSV quote character", null, false), stringProperty(CSV_ESCAPE, "CSV escape character", null, false), booleanProperty(TRANSACTIONAL, "Table is transactional", null, false)); + + knownTableProperties = Maps.uniqueIndex(tableProperties, PropertyMetadata::getName); } public List> getTableProperties() @@ -161,6 +171,21 @@ public List> getTableProperties() return tableProperties; } + @Override + public Set getKnownPropertyNames() + { + return knownTableProperties.keySet(); + } + + @Override + public Optional> getProperty(String name) + { + if (EXTRA_PROPERTY_PATTERN.matcher(name).matches()) { + return Optional.of(stringProperty(name, "Extra property", null, false)); + } + return Optional.ofNullable(knownTableProperties.get(name)); + } + public static String getExternalLocation(Map tableProperties) { return (String) tableProperties.get(EXTERNAL_LOCATION_PROPERTY); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java index e5c13b97d918..afcda26403c6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java @@ -152,7 +152,7 @@ public static Connector createConnector(String catalogName, Map eventListeners, sessionPropertiesProviders, HiveSchemaProperties.SCHEMA_PROPERTIES, - hiveTableProperties.getTableProperties(), + hiveTableProperties, hiveAnalyzeProperties.getAnalyzeProperties(), hiveMaterializedViewPropertiesProvider.getMaterializedViewProperties(), accessControl,