From da5fe7605d831b2391ea4f45431faf225cc363d8 Mon Sep 17 00:00:00 2001 From: pengzhiwei Date: Fri, 29 Mar 2019 00:17:38 +0800 Subject: [PATCH] [CALCITE-1581] Support UDTF like Hive Close apache/calcite#1138 --- core/src/main/codegen/templates/Parser.jj | 42 ++++-- .../calcite/runtime/CalciteResource.java | 12 ++ .../org/apache/calcite/sql/SqlAsOperator.java | 25 +++- .../java/org/apache/calcite/sql/SqlUtil.java | 15 ++ .../calcite/sql/validate/AliasNamespace.java | 8 +- .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 13 ++ .../sql/validate/SqlConformanceEnum.java | 15 +- .../validate/SqlDelegatingConformance.java | 3 + .../calcite/sql/validate/SqlValidator.java | 9 ++ .../sql/validate/SqlValidatorImpl.java | 129 ++++++++++++++++++ .../calcite/sql2rel/SqlToRelConverter.java | 27 ++++ .../runtime/CalciteResource.properties | 4 + .../calcite/sql/parser/SqlParserTest.java | 22 ++- .../calcite/test/MockSqlOperatorTable.java | 59 ++++++++ .../calcite/test/SqlToRelConverterTest.java | 53 ++++++- .../apache/calcite/test/SqlValidatorTest.java | 30 ++++ .../calcite/test/TableFunctionTest.java | 19 ++- .../calcite/test/SqlToRelConverterTest.xml | 128 ++++++++++++++++- site/_docs/reference.md | 4 + 20 files changed, 601 insertions(+), 20 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 279fa386369..f52b3aacce6 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1822,23 +1822,47 @@ List SelectList() : SqlNode SelectItem() : { SqlNode e; + SqlIdentifier columnAlias; + final List columnAliases = new ArrayList(); + final Span s = span(); final SqlIdentifier id; } { e = SelectExpression() [ - [ ] ( - id = SimpleIdentifier() + LOOKAHEAD(2) + ( + [ ] + ( + id = SimpleIdentifier() + | + // Mute the warning about ambiguity between alias and continued + // string literal. + LOOKAHEAD(1) + id = SimpleIdentifierFromStringLiteral() + ) + { + e = SqlStdOperatorTable.AS.createCall(span().end(e), e, id); + } + ) | - // Mute the warning about ambiguity between alias and continued - // string literal. - LOOKAHEAD(1) - id = SimpleIdentifierFromStringLiteral() + ( + + columnAlias = SimpleIdentifier() { + columnAliases.add(columnAlias); + } + ( columnAlias = SimpleIdentifier() { columnAliases.add(columnAlias);} )* + { + if (!this.conformance.allowSelectTableFunction()) { + throw SqlUtil.newContextException(getPos(), + RESOURCE.notAllowTableFunctionInSelect()); + } + e = SqlStdOperatorTable.AS.createCall(s.end(e), e, + new SqlNodeList(columnAliases, s.end(e))); + } + ) ) - { - e = SqlStdOperatorTable.AS.createCall(span().end(e), e, id); - } ] { return e; diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 7987546e540..dfdca96bf5b 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -910,6 +910,18 @@ ExInst invalidTypesForComparison(String clazzName0, String op, @BaseMessage("Not a valid input for REGEXP_REPLACE: ''{0}''") ExInst invalidInputForRegexpReplace(String value); + @BaseMessage("Table function is not allowed in select list in current SQL conformance level") + ExInst notAllowTableFunctionInSelect(); + + @BaseMessage("''{0}'' should be a table function") + ExInst exceptTableFunction(String name); + + @BaseMessage("Only one table function is allowed in select list") + ExInst onlyOneTableFunctionAllowedInSelect(); + + @BaseMessage("Table function is not allowed in aggregate statement") + ExInst notAllowTableFunctionInAggregate(); + @BaseMessage("Illegal xslt specified : ''{0}''") ExInst illegalXslt(String xslt); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java index 514c29aac67..cb17e66f32e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java @@ -30,6 +30,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Util; +import java.util.ArrayList; import java.util.List; import static org.apache.calcite.util.Static.RESOURCE; @@ -102,12 +103,26 @@ public void validateCall( // we don't want to validate the identifier. final List operands = call.getOperandList(); assert operands.size() == 2; - assert operands.get(1) instanceof SqlIdentifier; operands.get(0).validateExpr(validator, scope); - SqlIdentifier id = (SqlIdentifier) operands.get(1); - if (!id.isSimple()) { - throw validator.newValidationError(id, - RESOURCE.aliasMustBeSimpleIdentifier()); + + SqlNode asIdentifier = operands.get(1); + assert asIdentifier instanceof SqlIdentifier + || (asIdentifier instanceof SqlNodeList + && validator.config().sqlConformance().allowSelectTableFunction()); + + List ids = new ArrayList<>(); + if (asIdentifier instanceof SqlIdentifier) { + ids.add(operands.get(1)); + } else { + ids.addAll(((SqlNodeList) operands.get(1)).getList()); + } + for (int i = 0; i < ids.size(); i++) { + assert ids.get(i) instanceof SqlIdentifier; + SqlIdentifier id = (SqlIdentifier) ids.get(i); + if (!id.isSimple()) { + throw validator.newValidationError(id, + RESOURCE.aliasMustBeSimpleIdentifier()); + } } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index 9bfdd92dcba..ee9ebdac403 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -1189,4 +1189,19 @@ private void visitChild(SqlNode node) { return check(type); } } + + /** + * Whether the selectItem is a table function node in the select. + * eg. "select table_func(1) as (f0,f1)" + * + * @param selectItem select item + * @return true if this selectItem is a table function call + */ + public static boolean isTableFunctionInSelect(SqlNode selectItem) { + if (selectItem.getKind() == SqlKind.AS) { + SqlBasicCall call = (SqlBasicCall) selectItem; + return call.getOperands()[1] instanceof SqlNodeList; + } + return false; + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java index 4abd72edbf8..32003a42129 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java @@ -23,6 +23,7 @@ import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.Util; @@ -70,7 +71,12 @@ protected RelDataType validateImpl(RelDataType targetRowType) { final SqlValidatorNamespace childNs = validator.getNamespace(operands.get(0)); final RelDataType rowType = childNs.getRowTypeSansSystemColumns(); - final List columnNames = Util.skip(operands, 2); + List columnNames; + if (SqlUtil.isTableFunctionInSelect(call)) { + columnNames = ((SqlNodeList) operands.get(1)).getList(); + } else { + columnNames = Util.skip(operands, 2); + } for (final SqlNode operand : columnNames) { String name = ((SqlIdentifier) operand).getSimple(); if (nameList.contains(name)) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 531543c20c9..702dbf1de5e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -111,6 +111,10 @@ public boolean allowPluralTimeUnits() { return SqlConformanceEnum.DEFAULT.allowPluralTimeUnits(); } + public boolean allowSelectTableFunction() { + return SqlConformanceEnum.DEFAULT.allowSelectTableFunction(); + } + public boolean allowQualifyingCommonColumn() { return SqlConformanceEnum.DEFAULT.allowQualifyingCommonColumn(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index 78b7db32068..58c9d3f61ac 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -454,6 +454,19 @@ public interface SqlConformance { */ boolean allowPluralTimeUnits(); + /** + * Whether SELECT can contain a table function. + * + * For example, consider the query + * + * 
 SELECT SPLIT(col) AS (F0, F1) FROM A
+ * + *

Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#HIVE}; + * false otherwise. + */ + boolean allowSelectTableFunction(); + /** * Whether to allow a qualified common column in a query that has a * NATURAL join or a join with a USING clause. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index ffccfd7eb24..f829b2e7cff 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -77,7 +77,11 @@ public enum SqlConformanceEnum implements SqlConformance { /** Conformance value that instructs Calcite to use SQL semantics * consistent with Microsoft SQL Server version 2008. */ - SQL_SERVER_2008; + SQL_SERVER_2008, + + /** Conformance value that instructs Calcite to use SQL semantics + * consistent with Hive. */ + HIVE; public boolean isLiberal() { switch (this) { @@ -362,6 +366,15 @@ public boolean allowExtendedTrim() { } } + @Override public boolean allowSelectTableFunction() { + switch (this) { + case HIVE: + return true; + default: + return false; + } + } + @Override public boolean allowAliasUnnestItems() { switch (this) { case PRESTO: diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 3a166fdf165..c3db1e30a55 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -77,4 +77,7 @@ protected SqlDelegatingConformance(SqlConformance delegate) { return delegate.allowAliasUnnestItems(); } + @Override public boolean allowSelectTableFunction() { + return delegate.allowSelectTableFunction(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index cd563cbac26..da553691eda 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -23,6 +23,7 @@ import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.runtime.Resources; +import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDelete; @@ -721,6 +722,14 @@ CalciteException handleUnresolvedFunction(SqlCall call, */ SqlValidatorScope getOverScope(SqlNode node); + /** + * Returns the table function SqlBasicCall in SqlSelect. + * + * @param select The select node + * @return The table function node associate with the select node + */ + SqlBasicCall getTableFunctionInSelect(SqlSelect select); + /** * Validates that a query is capable of producing a return of given modality * (relational or streaming). diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 0919ee232d8..452e5af7d6e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -280,6 +280,13 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { // TypeCoercion instance used for implicit type coercion. private TypeCoercion typeCoercion; + // Flag saying if we enable the implicit type coercion. + private boolean enableTypeCoercion; + + // Mapping the table function and the select node. + private final Map selectTableFunctions = + new IdentityHashMap<>(); + //~ Constructors ----------------------------------------------------------- /** @@ -1148,6 +1155,10 @@ public SqlValidatorScope getOverScope(SqlNode node) { return scopes.get(node); } + public SqlBasicCall getTableFunctionInSelect(SqlSelect select) { + return selectTableFunctions.get(select); + } + private SqlValidatorNamespace getNamespace(SqlNode node, SqlValidatorScope scope) { if (node instanceof SqlIdentifier && scope instanceof DelegatingScope) { @@ -4179,6 +4190,14 @@ protected RelDataType validateSelectList( expandedSelectItems, aliases, fieldList); + } else if (SqlUtil.isTableFunctionInSelect(selectItem)) { + handleTableFunctionInSelect( + select, + (SqlBasicCall) selectItem, + expandedSelectItems, + aliases, + fieldList + ); } else { // Use the field list size to record the field index // because the select item may be a STAR(*), which could have been expanded. @@ -4297,6 +4316,116 @@ private void handleScalarSubQuery( fieldList.add(Pair.of(alias, nodeType)); } + /** + * Processes table function found in select list.Checks that is + * actually a table function and validate the table function + * count in Select list. + * + * @param parentSelect Base SqlSelect + * @param selectItem Child select items from select list + * @param expandedSelectItems Select items after processing + * @param aliases Built from user or system values + * @param fields Built up entries for each select list entry + */ + private void handleTableFunctionInSelect( + SqlSelect parentSelect, + SqlBasicCall selectItem, + List expandedSelectItems, + Set aliases, + List> fields) { + SqlBasicCall functionCall = (SqlBasicCall) selectItem.getOperands()[0]; + SqlFunction function = (SqlFunction) functionCall.getOperator(); + // Check whether there are more than one table function in select list. + for (SqlNode item : parentSelect.getSelectList()) { + if (SqlUtil.isTableFunctionInSelect(item) + && item != selectItem) { + throw newValidationError(parentSelect.getSelectList(), + RESOURCE.onlyOneTableFunctionAllowedInSelect()); + } + } + + // Change the function category to USER_DEFINED_TABLE_FUNCTION. + // It is because that in sql-select list, the SqlFunctionCategory is USER_DEFINED_FUNCTION + // for a SqlUnresolvedFunction,so we should do this change. + if (function instanceof SqlUnresolvedFunction) { + if (!function.getFunctionType().isTableFunction()) { + SqlFunction newFunction = + new SqlUnresolvedFunction(function.getNameAsId(), + function.getReturnTypeInference(), + function.getOperandTypeInference(), + function.getOperandTypeChecker(), + function.getParamTypes(), + SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION); + functionCall.setOperator(newFunction); + function = newFunction; + } + } + // Check functionCall whether is a table function + List overloads = new ArrayList<>(); + opTab.lookupOperatorOverloads(function.getNameAsId(), + function.getFunctionType(), + function.getSyntax(), overloads, catalogReader.nameMatcher()); + if (overloads.size() == 0) { + throw newValidationError(functionCall, + RESOURCE.exceptTableFunction(function.getName())); + } + // Check the parent select whether is a aggregate statement + if (isAggregate(parentSelect)) { + throw newValidationError(functionCall, + RESOURCE.notAllowTableFunctionInAggregate()); + } + SqlNodeList aliasNodes + = (SqlNodeList) selectItem.getOperands()[1]; + List aliasList = new ArrayList<>(); + for (SqlNode aliasNode : aliasNodes) { + aliasList.add(deriveAlias(aliasNode, aliasList.size())); + } + aliases.addAll(aliasList); + + String tableAlias = functionCall.getOperator().getName(); + // Expand the table function alias + for (String alias : aliasList) { + List names = new ArrayList<>(2); + names.add(tableAlias); + names.add(alias); + SqlIdentifier id = new SqlIdentifier(names, SqlParserPos.ZERO); + expandedSelectItems.add(id); + } + + // Register namespace for table function + SqlValidatorScope fromScope = getFromScope(parentSelect); + ProcedureNamespace tableNs = new ProcedureNamespace(this, + fromScope, functionCall, selectItem); + tableNs.validateImpl(unknownType); + registerNamespace(null, null, + tableNs, false); + AliasNamespace aliasNs = new AliasNamespace(this, + selectItem, parentSelect); + aliasNs.validateImpl(unknownType); + registerNamespace(getSelectScope(parentSelect), + tableAlias, aliasNs, false); + + // Create a table scope for table function + TableScope tableScope = new TableScope(fromScope, parentSelect); + if (fromScope instanceof ListScope) { + for (ScopeChild child : ((ListScope) fromScope).children) { + tableScope.addChild(child.namespace, child.name, child.nullable); + } + } + scopes.put(functionCall, tableScope); + // Associate the select with the table function + selectTableFunctions.put(parentSelect, functionCall); + + RelDataType type = aliasNs.getRowType(); + setValidatedNodeType(selectItem, type); + for (int i = 0; i < aliasList.size(); i++) { + fields.add( + Pair.of( + aliasList.get(i), type.getFieldList() + .get(i).getType())); + } + } + /** * Derives a row-type for INSERT and UPDATE operations. * diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 6483786fde6..cef1fa24a61 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4120,6 +4120,8 @@ private void convertSelectList( SqlNodeList selectList = select.getSelectList(); selectList = validator.expandStar(selectList, select, false); + convertTableFunctionInSelect(bb, select); + replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); List fieldNames = new ArrayList<>(); @@ -4168,6 +4170,31 @@ private void convertSelectList( } } + /** + * Converts the table function in select to Join or TableFunctionScan. + * + * @param bb Blackboard + * @param select SqlSelect + * */ + private void convertTableFunctionInSelect(Blackboard bb, SqlSelect select) { + SqlBasicCall tableFunction = validator.getTableFunctionInSelect(select); + // rewrite the table function if select list contain one + if (tableFunction != null) { + SqlValidatorScope tableScope = validator.getJoinScope(tableFunction); + final Blackboard rightBb = + createBlackboard(tableScope, null, false); + convertCollectionTable(rightBb, tableFunction); + + if (select.getFrom() != null) { + RelNode join = createJoin(bb, bb.root, rightBb.root, + rexBuilder.makeLiteral(true), JoinRelType.INNER); + bb.setRoot(join, false); + } else { + bb.setRoot(rightBb.root, false); + } + } + } + /** * Adds extra select items. The default implementation adds nothing; derived * classes may add columns to exprList, nameList, aliasList and diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 3a42da30d49..ba455c4a3dc 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -303,4 +303,8 @@ InvalidInputForXmlTransform=Invalid input for XMLTRANSFORM xml: ''{0}'' InvalidInputForExtractValue=Invalid input for EXTRACTVALUE: xml: ''{0}'', xpath expression: ''{1}'' InvalidInputForExtractXml=Invalid input for EXTRACT xpath: ''{0}'', namespace: ''{1}'' InvalidInputForExistsNode=Invalid input for EXISTSNODE xpath: ''{0}'', namespace: ''{1}'' +NotAllowTableFunctionInSelect=Table function is not allowed in select list in current SQL conformance level +ExceptTableFunction=''{0}'' should be a table function +OnlyOneTableFunctionAllowedInSelect=Only one table function is allowed in select list +NotAllowTableFunctionInAggregate=Table function is not allowed in aggregate statement # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java index 3c4bf76290b..81f46107e27 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -3657,7 +3657,27 @@ void checkPeriodPredicate(Checker checker) { sql(sql4).ok(expected4); } - @Test void testCollectionTableWithLateral() { + @Test public void testTableFunctionInSelect() { + conformance = SqlConformanceEnum.HIVE; + sql("select n,c from (select TABLE_FUNC(1) as (n,c) from emp)") + .ok("SELECT `N`, `C`\n" + + "FROM (SELECT `TABLE_FUNC`(1) AS (`N`, `C`)\n" + + "FROM `EMP`)"); + sql("select TABLE_FUNC(1) as (c) from emp") + .ok("SELECT `TABLE_FUNC`(1) AS (`C`)\n" + + "FROM `EMP`"); + sql("select char_length(c)*2 from" + + " (select TABLE_FUNC(1) as (n,c) from emp) where char_length(c) > 1") + .ok("SELECT (CHAR_LENGTH(`C`) * 2)\n" + + "FROM (SELECT `TABLE_FUNC`(1) AS (`N`, `C`)\n" + + "FROM `EMP`)\n" + + "WHERE (CHAR_LENGTH(`C`) > 1)"); + conformance = SqlConformanceEnum.DEFAULT; + sql("select TABLE_FUNC(1) as (n,c^)^ from emp") + .fails("(.*)Table function is not allowed in select list(.*)"); + } + + @Test public void testCollectionTableWithLateral() { final String sql = "select * from dept, lateral table(ramp(dept.deptno))"; final String expected = "SELECT *\n" + "FROM `DEPT`,\n" diff --git a/core/src/test/java/org/apache/calcite/test/MockSqlOperatorTable.java b/core/src/test/java/org/apache/calcite/test/MockSqlOperatorTable.java index 8ebcd19feaa..495d414c169 100644 --- a/core/src/test/java/org/apache/calcite/test/MockSqlOperatorTable.java +++ b/core/src/test/java/org/apache/calcite/test/MockSqlOperatorTable.java @@ -18,6 +18,8 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.FunctionParameter; +import org.apache.calcite.schema.TableFunction; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; @@ -35,10 +37,13 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.util.ChainedSqlOperatorTable; import org.apache.calcite.sql.util.ListSqlOperatorTable; +import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction; import org.apache.calcite.util.Optionality; import com.google.common.collect.ImmutableList; +import java.lang.reflect.Type; +import java.util.List; /** * Mock operator table for testing purposes. Contains the standard SQL operator @@ -70,6 +75,7 @@ public static void addRamp(MockSqlOperatorTable opTab) { opTab.addOperator(new NotATableFunction()); opTab.addOperator(new BadTableFunction()); opTab.addOperator(new StructuredFunction()); + opTab.addOperator(new MyTableFunction()); } /** "RAMP" user-defined table function. */ @@ -212,6 +218,59 @@ public MyAvgAggFunction() { } } + /** "TABLE_FUNC" User-Defined table function. */ + public static class MyTableFunction extends SqlUserDefinedTableFunction { + public MyTableFunction() { + super( + new SqlIdentifier("TABLE_FUNC", SqlParserPos.ZERO), + ReturnTypes.CURSOR, + null, + OperandTypes.NUMERIC, + null, + new MyTableFunctionImpl()); + } + } + + /** User-Defined table function implementation. */ + private static class MyTableFunctionImpl implements TableFunction { + @Override public List getParameters() { + FunctionParameter parameter = new FunctionParameter() { + @Override public int getOrdinal() { + return 0; + } + + @Override public String getName() { + return "i0"; + } + + @Override public RelDataType getType(RelDataTypeFactory typeFactory) { + return typeFactory.createSqlType(SqlTypeName.BIGINT); + } + + @Override public boolean isOptional() { + return false; + } + }; + return ImmutableList.of(parameter); + } + + + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory, + List arguments) { + RelDataType type0 = typeFactory.createSqlType(SqlTypeName.BIGINT); + RelDataType type1 = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + return typeFactory.createStructType( + ImmutableList.of(type0, type1), + ImmutableList.of("f0", "f1")); + } + + + @Override public Type getElementType(List arguments) { + return Object[].class; + } + } + /** "ROW_FUNC" user-defined table function whose return type is * row type with nullable and non-nullable fields. */ public static class RowFunction extends SqlFunction diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index d560dc12ca0..cf2933ef4dd 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1121,7 +1121,58 @@ public final Sql sql(String sql) { sql("select * from dept, lateral table(ramp(dept.deptno))").ok(); } - @Test void testCollectionTableWithLateral2() { + @Test public void testTableFunctionInSelect() { + sql("select table_func(dept.deptno) as (f0, f1) from dept") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithUnion() { + sql("select * from (" + + "select a.deptno, table_func(a.deptno) as (f0, f1)" + + " from (select * from dept) a " + + " union all " + + " select 1, table_func(b.deptno) as (f0, f1) from dept b" + + ")") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithoutFrom() { + sql("select table_func(1) as (f0,f1)") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithOrderBy() { + sql("select table_func(dept.deptno) as (f0, f1) from" + + " dept order by deptno") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithJoin() { + sql("select table_func(a.deptno) as (f0, f1) from " + + "dept a join dept b on a.deptno = b.deptno") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithInQuery() { + sql("select * from dept where deptno in " + + "(select f0 from (select table_func(a.deptno) as (f0, f1) from dept a))") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testTableFunctionInSelectWithUnnest() { + sql("select table_func(e.empno) as (f0, f1) " + + "from dept_nested as d, UNNEST(d.employees) e") + .conformance(SqlConformanceEnum.HIVE) + .ok(); + } + + @Test public void testCollectionTableWithLateral2() { sql("select * from dept, lateral table(ramp(deptno))").ok(); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 32cc6786232..98a63c0612e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -11589,4 +11589,34 @@ private void checkCustomColumnResolving(String table) { .withOperatorTable(operatorTable) .type("RecordType(BIGINT NOT NULL A, BIGINT B) NOT NULL"); } + + @Test public void testTableFunction() { + tester = tester.withConformance(SqlConformanceEnum.HIVE); + sql("select n,c from (select TABLE_FUNC(1) as (n,c) from emp)") + .ok(); + sql("select char_length(c)*2 from" + + " (select TABLE_FUNC(1) as (n,c) from emp) where char_length(c) > 1") + .ok(); + sql("select char_length(c)*2, f0 from" + + " (select 1 as f0, TABLE_FUNC(1) as (n,c) from emp) where char_length(c) > 1") + .ok(); + sql("select TABLE_FUNC(1) as (n,c)") + .ok(); + + sql("select ^test_udf('1')^ as (n,c) from emp") + .fails("'TEST_UDF' should be a table function"); + + sql("^select TABLE_FUNC(1) as (n1,c1), TABLE_FUNC(2)^ as (n2,c2) from emp") + .fails("Only one table function is allowed in select list"); + + sql("select ^TABLE_FUNC(1)^ as (n1,c1) , count(1) from emp having count(1) > 0") + .fails("Table function is not allowed in aggregate statement"); + + sql("select ^TABLE_FUNC(1)^ as (n1,c1) from emp group by empno") + .fails("Table function is not allowed in aggregate statement"); + + tester = tester.withConformance(SqlConformanceEnum.DEFAULT); + sql("select TABLE_FUNC(1) as (n,c) from emp") + .fails("(.*)Table function is not allowed in select list(.*)"); + } } diff --git a/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java b/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java index f0f014863f3..1931538fd24 100644 --- a/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java +++ b/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java @@ -462,7 +462,24 @@ private Connection getConnectionWithMultiplyFunction() throws SQLException { with().query(q).returnsUnordered("C=2\nC=3\nC=4"); } - @Test void testCrossApply() { + @Test public void testTableFunctionInSelect() { + final String q = "select * from (" + + "select f0, \"s\".\"fibonacci2\"(20) as (c)" + + " from (select 1 as f0)" + + ")"; + with() + .with(CalciteConnectionProperty.CONFORMANCE, + SqlConformanceEnum.HIVE) + .query(q).returnsUnordered("F0=1; C=1", + "F0=1; C=1", + "F0=1; C=13", + "F0=1; C=2", + "F0=1; C=3", + "F0=1; C=5", + "F0=1; C=8"); + } + + @Test public void testCrossApply() { final String q1 = "select *\n" + "from (values 2, 5) as t (c)\n" + "cross apply table(\"s\".\"fibonacci2\"(c))"; diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 3b45f503c20..398edd28b1f 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -323,7 +323,121 @@ LogicalProject(DEPTNO=[$0], NAME=[$1], I=[$2]) ]]> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -335,6 +449,18 @@ LogicalProject(PRODUCTID=[$0], NAME=[$1], SUPPLIERID=[$2], SYS_START=[$3], SYS_E ]]> + + + + + + + + diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 3bf384f594c..54362ee0e16 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -213,6 +213,7 @@ selectWithoutFrom: projectItem: expression [ [ AS ] columnAlias ] + | expression AS '(' columnAlias [, columnAlias ]* ')' | tableAlias . * tableExpression: @@ -305,6 +306,9 @@ the nth item in the SELECT clause. In *query*, *count* and *start* may each be either an unsigned integer literal or a dynamic parameter whose value is an integer. +In *projectItem*, *expression* followed by a *As* column alias list is only allowed in +certain [conformance levels] ({{ site.apiRoot }}/org/apache/calcite/sql/validate/SqlConformance.html#allowSelectTableFunction--); +in those same conformance levels, table function can be used in the select list. An aggregate query is a query that contains a GROUP BY or a HAVING clause, or aggregate functions in the SELECT clause. In the SELECT, HAVING and ORDER BY clauses of an aggregate query, all expressions