diff --git a/pom.xml b/pom.xml index 2176ef6c..6fc6eb65 100644 --- a/pom.xml +++ b/pom.xml @@ -168,6 +168,18 @@ limitations under the License. + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + org/apache/calcite/sql/parser/parserextensiontesting/*.java + + ${project.build.directory}/generated-test-sources/javacc + + org.apache.maven.plugins maven-surefire-plugin @@ -195,6 +207,22 @@ limitations under the License. false + + javacc-test + generate-test-sources + + javacc + + + ${project.build.directory}/generated-test-sources/fmpp + ${project.build.directory}/generated-test-sources/javacc + + **/Parser.jj + + 2 + false + + @@ -444,18 +472,30 @@ limitations under the License. com.googlecode.fmpp-maven-plugin fmpp-maven-plugin - - src/main/codegen/config.fmpp - src/main/codegen/templates - + + src/main/codegen/config.fmpp + src/main/codegen/templates + generate-fmpp-sources validate generate + + + src/test/codegen/config.fmpp + src/main/codegen/templates + ${project.build.directory}/generated-test-sources/fmpp + + generate-fmpp-test-sources + validate + + generate + + diff --git a/src/main/codegen/config.fmpp b/src/main/codegen/config.fmpp index 87ed18f2..9f7a1185 100644 --- a/src/main/codegen/config.fmpp +++ b/src/main/codegen/config.fmpp @@ -70,6 +70,12 @@ data: { dataTypeParserMethods: [ ] + # List of methods for parsing extensions to ALTER SYSTEM calls. + # Each must accept arguments "(SqlParserPos pos, String scope)". + # Example: "SqlUploadJarNode" + alterStatementParserMethods: [ + ] + # List of files in @includes directory that have parser method # implementations for parsing custom SQL statements, literals or types # given as part of "statementParserMethods", "literalParserMethods" or diff --git a/src/main/codegen/templates/Parser.jj b/src/main/codegen/templates/Parser.jj index aac7ab30..d6bb06a1 100644 --- a/src/main/codegen/templates/Parser.jj +++ b/src/main/codegen/templates/Parser.jj @@ -41,6 +41,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; +import org.apache.calcite.sql.SqlAlter; import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCharStringLiteral; @@ -923,7 +924,9 @@ SqlNode SqlStmt() : } { ( - stmt = SqlSetOption() + stmt = SqlSetOption(null, null) + | + stmt = SqlAlter() | stmt = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) | @@ -2965,21 +2968,15 @@ SqlCall SequenceExpression() : } /** - * Parses an expression for setting or resetting an option in SQL, such as QUOTED_IDENTIFIERS, - * or explain plan level (physical/logical). + * Parses "SET <NAME> = VALUE" or "RESET <NAME>", without a leading + * "ALTER <SCOPE>". */ -SqlSetOption SqlSetOption() : +SqlSetOption SqlSetOption(SqlParserPos pos, String scope) : { - SqlParserPos pos = null; - String scope = null; SqlIdentifier name; - SqlNode val = null; + final SqlNode val; } { - ( - { pos = getPos(); } - scope = Scope() - )? ( { pos = pos == null ? getPos() : pos; @@ -2996,20 +2993,52 @@ SqlSetOption SqlSetOption() : val = new SqlIdentifier(token.image.toUpperCase(), getPos()); } ) + { + return new SqlSetOption(pos.plus(getPos()), scope, name, val); + } | { pos = pos == null ? getPos() : pos; } ( name = CompoundIdentifier() - | + | { name = new SqlIdentifier(token.image.toUpperCase(), getPos()); } ) + { + return new SqlSetOption(pos.plus(getPos()), scope, name, null); + } + ) +} + +/** + * Parses an expression for setting or resetting an option in SQL, such as QUOTED_IDENTIFIERS, + * or explain plan level (physical/logical). + */ +SqlAlter SqlAlter() : +{ + final SqlParserPos pos; + final String scope; + final SqlAlter alterNode; +} +{ + ( + { pos = getPos(); } + scope = Scope() + ) + ( + alterNode = SqlSetOption(pos, scope) + + <#-- additional literal parser methods are included here --> + <#list parser.alterStatementParserMethods as method> + | + alterNode = ${method}(pos, scope) + ) { - return new SqlSetOption(pos.plus(getPos()), scope, name, val); + return alterNode; } } diff --git a/src/main/java/org/apache/calcite/sql/SqlAlter.java b/src/main/java/org/apache/calcite/sql/SqlAlter.java new file mode 100644 index 00000000..960c7ef3 --- /dev/null +++ b/src/main/java/org/apache/calcite/sql/SqlAlter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.sql.parser.SqlParserPos; + +/** + * Base class for an ALTER statements parse tree nodes. The portion of the + * statement covered by this class is "ALTER <SCOPE>. Subclasses handle + * whatever comes after the scope. + */ +public abstract class SqlAlter extends SqlCall { + + /** Scope of the operation. Values "SYSTEM" and "SESSION" are typical. */ + String scope; + + public SqlAlter(SqlParserPos pos) { + this(pos, null); + } + + public SqlAlter(SqlParserPos pos, String scope) { + super(pos); + this.scope = scope; + } + + @Override public final void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + if (scope != null) { + writer.keyword("ALTER"); + writer.keyword(scope); + } + unparseAlterOperation(writer, leftPrec, rightPrec); + } + + protected abstract void unparseAlterOperation(SqlWriter writer, int leftPrec, int rightPrec); + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + +} + +// End SqlAlter.java diff --git a/src/main/java/org/apache/calcite/sql/SqlSetOption.java b/src/main/java/org/apache/calcite/sql/SqlSetOption.java index 0d20d96a..8538ffc3 100644 --- a/src/main/java/org/apache/calcite/sql/SqlSetOption.java +++ b/src/main/java/org/apache/calcite/sql/SqlSetOption.java @@ -58,7 +58,7 @@ *
  • ALTER SESSION RESET ALL
    • * */ -public class SqlSetOption extends SqlCall { +public class SqlSetOption extends SqlAlter { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("SET_OPTION", SqlKind.SET_OPTION) { @Override public SqlCall createCall(SqlLiteral functionQualifier, @@ -70,9 +70,6 @@ public class SqlSetOption extends SqlCall { } }; - /** Scope of the assignment. Values "SYSTEM" and "SESSION" are typical. */ - String scope; - /** Name of the option as an {@link org.apache.calcite.sql.SqlIdentifier} * with one or more parts.*/ SqlIdentifier name; @@ -94,7 +91,7 @@ public class SqlSetOption extends SqlCall { */ public SqlSetOption(SqlParserPos pos, String scope, SqlIdentifier name, SqlNode value) { - super(pos); + super(pos, scope); this.scope = scope; this.name = name; this.value = value; @@ -141,11 +138,7 @@ public SqlSetOption(SqlParserPos pos, String scope, SqlIdentifier name, } } - @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - if (scope != null) { - writer.keyword("ALTER"); - writer.keyword(scope); - } + @Override protected void unparseAlterOperation(SqlWriter writer, int leftPrec, int rightPrec) { if (value != null) { writer.keyword("SET"); } else { @@ -174,14 +167,6 @@ public void setName(SqlIdentifier name) { this.name = name; } - public String getScope() { - return scope; - } - - public void setScope(String scope) { - this.scope = scope; - } - public SqlNode getValue() { return value; } diff --git a/src/test/codegen/config.fmpp b/src/test/codegen/config.fmpp new file mode 100644 index 00000000..7b8b8e23 --- /dev/null +++ b/src/test/codegen/config.fmpp @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data: { + parser: { + # Generated parser implementation class package and name + package: "org.apache.calcite.sql.parser.parserextensiontesting", + class: "ExtensionSqlParserImpl", + + # List of import statements. + imports: [ + "org.apache.calcite.sql.parser.parserextensiontesting.SqlUploadJarNode" + ] + + # List of keywords. + keywords: [ + "UPLOAD" + "JAR" + ] + + # List of keywords from "keywords" section that are not reserved. + nonReservedKeywords: [ + ] + + # List of methods for parsing custom SQL statements. + statementParserMethods: [ + ] + + # List of methods for parsing custom literals. + # Example: ParseJsonLiteral(). + literalParserMethods: [ + ] + + # List of methods for parsing custom data types. + dataTypeParserMethods: [ + ] + + # List of methods for parsing extensions to ALTER SYSTEM calls. + # Each must accept arguments "(SqlParserPos pos, String scope)". + alterStatementParserMethods: [ + "SqlUploadJarNode" + ] + + # List of files in @includes directory that have parser method + # implementations for custom SQL statements, literals or types + # given as part of "statementParserMethods", "literalParserMethods" or + # "dataTypeParserMethods". + implementationFiles: [ + "parserImpls.ftl" + ] + + includeCompoundIdentifier: true + includeBraces: true + includeAdditionalDeclarations: false + + } +} +freemarkerLinks: { + includes: includes/ +} diff --git a/src/test/codegen/includes/compoundIdentifier.ftl b/src/test/codegen/includes/compoundIdentifier.ftl new file mode 100644 index 00000000..70db3c2e --- /dev/null +++ b/src/test/codegen/includes/compoundIdentifier.ftl @@ -0,0 +1,34 @@ +<#-- +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +--> + +<#-- + Add implementations of additional parser statements, literals or + data types. + + Example of SqlShowTables() implementation: + SqlNode SqlShowTables() + { + ...local variables... + } + { + + ... + { + return SqlShowTables(...) + } + } +--> diff --git a/src/test/codegen/includes/parserImpls.ftl b/src/test/codegen/includes/parserImpls.ftl new file mode 100644 index 00000000..2305fc1f --- /dev/null +++ b/src/test/codegen/includes/parserImpls.ftl @@ -0,0 +1,37 @@ +<#-- +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +--> + + +SqlAlter SqlUploadJarNode(SqlParserPos pos, String scope) : +{ + SqlNode jarPath; + final List jarPathsList; +} +{ + + jarPath = StringLiteral() { + jarPathsList = startList(jarPath); + } + ( + jarPath = StringLiteral() { + jarPathsList.add(jarPath); + } + )* + { + return new SqlUploadJarNode(pos.plus(getPos()), scope, jarPathsList); + } +} diff --git a/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java index b5ce24bb..71443833 100644 --- a/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -22,6 +22,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlSetOption; +import org.apache.calcite.sql.parser.impl.SqlParserImpl; import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; @@ -64,10 +65,15 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * A SqlParserTest is a unit-test for * {@link SqlParser the SQL parser}. + * + *

    To reuse this test for an extension parser, implement the + * {@link #parserImplFactory()} method to return the extension parser + * implementation. */ public class SqlParserTest { //~ Static fields/initializers --------------------------------------------- @@ -543,9 +549,18 @@ protected Sql sql(String sql) { return new Sql(sql); } - private SqlParser getSqlParser(String sql) { + /** + * Implementors of custom parsing logic who want to reuse this test should + * override this method with the factory for their extension parser. + */ + protected SqlParserImplFactory parserImplFactory() { + return SqlParserImpl.FACTORY; + } + + protected SqlParser getSqlParser(String sql) { return SqlParser.create(sql, SqlParser.configBuilder() + .setParserFactory(parserImplFactory()) .setQuoting(quoting) .setUnquotedCasing(unquotedCasing) .setQuotedCasing(quotedCasing) @@ -6681,6 +6696,7 @@ public void subTestIntervalSecondFailsValidation() { * non-reserved keyword list in the parser. */ @Test public void testNoUnintendedNewReservedKeywords() { + assumeTrue("don't run this test for sub-classes", isNotSubclass()); final SqlAbstractParserImpl.Metadata metadata = getSqlParser("").getMetadata(); @@ -6706,6 +6722,7 @@ public void subTestIntervalSecondFailsValidation() { /** Generates a copy of {@code reference.md} with the current set of key * words. Fails if the copy is different from the original. */ @Test public void testGenerateKeyWords() throws IOException { + assumeTrue("don't run this test for sub-classes", isNotSubclass()); // inUrl = "file:/home/x/calcite/core/target/test-classes/hsqldb-model.json" String path = "hsqldb-model.json"; final URL inUrl = SqlParserTest.class.getResource("/" + path); @@ -6908,8 +6925,7 @@ public void subTestIntervalSecondFailsValidation() { } @Test public void testSqlOptions() throws SqlParseException { - SqlNode node = - SqlParser.create("alter system set schema = true").parseStmt(); + SqlNode node = getSqlParser("alter system set schema = true").parseStmt(); SqlSetOption opt = (SqlSetOption) node; assertThat(opt.getScope(), equalTo("SYSTEM")); SqlPrettyWriter writer = new SqlPrettyWriter(SqlDialect.CALCITE); @@ -6941,7 +6957,7 @@ public void subTestIntervalSecondFailsValidation() { .ok("SET `APPROX` = -12.3450") .node(isDdl()); - node = SqlParser.create("reset schema").parseStmt(); + node = getSqlParser("reset schema").parseStmt(); opt = (SqlSetOption) node; assertThat(opt.getScope(), equalTo(null)); writer = new SqlPrettyWriter(SqlDialect.CALCITE); @@ -7115,6 +7131,10 @@ public void checkExpFails( } } + private boolean isNotSubclass() { + return this.getClass().equals(SqlParserTest.class); + } + /** * Implementation of {@link Tester} which makes sure that the results of * unparsing a query are consistent with the original query. diff --git a/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/ExtensionSqlParserTest.java b/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/ExtensionSqlParserTest.java new file mode 100644 index 00000000..51e24bd2 --- /dev/null +++ b/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/ExtensionSqlParserTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.parser.parserextensiontesting; + +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParserImplFactory; +import org.apache.calcite.sql.parser.SqlParserTest; + +import org.junit.Test; + +/** + * Testing for extension functionality of the base SQL parser impl. + * + *

    This test runs all test cases of the base {@link SqlParserTest}, as well + * as verifying specific extension points. + */ +public class ExtensionSqlParserTest extends SqlParserTest { + + @Override protected SqlParserImplFactory parserImplFactory() { + return ExtensionSqlParserImpl.FACTORY; + } + + @Test public void testAlterSystemExtension() throws SqlParseException { + check("alter system upload jar '/path/to/jar'", + "ALTER SYSTEM UPLOAD JAR '/path/to/jar'"); + } + + @Test public void testAlterSystemExtensionWithoutAlter() throws SqlParseException { + // We need to include the scope for custom alter operations + checkFails("^upload^ jar '/path/to/jar'", + "(?s).*Encountered \"upload\" at .*"); + } +} + +// End ExtensionSqlParserTest.java diff --git a/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/SqlUploadJarNode.java b/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/SqlUploadJarNode.java new file mode 100644 index 00000000..78184d0e --- /dev/null +++ b/src/test/java/org/apache/calcite/sql/parser/parserextensiontesting/SqlUploadJarNode.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.parser.parserextensiontesting; + +import org.apache.calcite.sql.SqlAlter; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; + +import java.util.List; + +/** + * Simple test example of a custom alter system call. + */ +public class SqlUploadJarNode extends SqlAlter { + public static final SqlOperator OPERATOR = + new SqlSpecialOperator("UPLOAD JAR", SqlKind.OTHER_DDL); + + private final List jarPaths; + + /** Creates a SqlUploadJarNode. */ + public SqlUploadJarNode(SqlParserPos pos, String scope, List jarPaths) { + super(pos, scope); + this.jarPaths = jarPaths; + } + + @Override public SqlOperator getOperator() { + return OPERATOR; + } + + @Override public List getOperandList() { + return jarPaths; + } + + @Override protected void unparseAlterOperation(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword("UPLOAD"); + writer.keyword("JAR"); + SqlWriter.Frame frame = writer.startList("", ""); + for (SqlNode jarPath : jarPaths) { + jarPath.unparse(writer, leftPrec, rightPrec); + } + writer.endList(frame); + } +} + +// End SqlUploadJarNode.java diff --git a/src/test/java/org/apache/calcite/test/CalciteSuite.java b/src/test/java/org/apache/calcite/test/CalciteSuite.java index dc59a8e3..df0167d2 100644 --- a/src/test/java/org/apache/calcite/test/CalciteSuite.java +++ b/src/test/java/org/apache/calcite/test/CalciteSuite.java @@ -38,6 +38,7 @@ import org.apache.calcite.sql.SqlSetOptionOperatorTest; import org.apache.calcite.sql.parser.SqlParserTest; import org.apache.calcite.sql.parser.SqlUnParserTest; +import org.apache.calcite.sql.parser.parserextensiontesting.ExtensionSqlParserTest; import org.apache.calcite.sql.test.SqlAdvisorTest; import org.apache.calcite.sql.test.SqlOperatorTest; import org.apache.calcite.sql.test.SqlPrettyWriterTest; @@ -112,6 +113,7 @@ // medium tests (above 0.1s) SqlParserTest.class, SqlUnParserTest.class, + ExtensionSqlParserTest.class, SqlSetOptionOperatorTest.class, SqlPrettyWriterTest.class, SqlValidatorTest.class,