This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git
commit ed692fd4f2f00261348ecac779cb1a6d544d052e Author: Ryan Fu <[email protected]> AuthorDate: Mon Sep 23 16:30:15 2019 -0700 [CALCITE-3323] Add mode to SqlValidator that treats statements as valid if they contain unknown functions (Ryan Fu) The mode is controlled by new method SqlValidator.isLenientOperatorLookup(). If it encounters an unknown function, or a known function with invalid number or types of arguments, the validator now creates a SqlUnresolvedFunction. The return type is inferred to be UNKNOWN, so that an enclosing function call can be validated. Removed .factorypath files and updated gitignore files to reject these. Close apache/calcite#1471 --- .gitignore | 1 + .../java/org/apache/calcite/sql/SqlFunction.java | 34 +++++++++++++++------- .../apache/calcite/sql/validate/SqlValidator.java | 21 +++++++++++++ .../calcite/sql/validate/SqlValidatorImpl.java | 12 ++++++++ .../apache/calcite/sql/test/AbstractSqlTester.java | 4 +++ .../apache/calcite/sql/test/SqlTestFactory.java | 6 +++- .../org/apache/calcite/sql/test/SqlTester.java | 4 +++ .../org/apache/calcite/test/SqlValidatorTest.java | 21 +++++++++++++ 8 files changed, 92 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 65e3686..75badb2 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ settings.xml .project .buildpath .classpath +.factorypath .settings .checkstyle diff --git a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java index b830a08..0a0c40b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java @@ -18,6 +18,8 @@ package org.apache.calcite.sql; import org.apache.calcite.linq4j.function.Functions; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; @@ -271,25 +273,37 @@ public class SqlFunction extends SqlOperator { return validator.deriveConstructorType(scope, call, this, function, argTypes); } + + validCoercionType: if (function == null) { - boolean changed = false; if (validator.isTypeCoercionEnabled()) { // try again if implicit type coercion is allowed. - function = (SqlFunction) SqlUtil.lookupRoutine(validator.getOperatorTable(), - getNameAsId(), argTypes, argNames, getFunctionType(), SqlSyntax.FUNCTION, getKind(), - validator.getCatalogReader().nameMatcher(), - true); + function = (SqlFunction) + SqlUtil.lookupRoutine(validator.getOperatorTable(), getNameAsId(), + argTypes, argNames, getFunctionType(), SqlSyntax.FUNCTION, + getKind(), validator.getCatalogReader().nameMatcher(), true); // try to coerce the function arguments to the declared sql type name. // if we succeed, the arguments would be wrapped with CAST operator. if (function != null) { TypeCoercion typeCoercion = validator.getTypeCoercion(); - changed = typeCoercion.userDefinedFunctionCoercion(scope, call, function); + if (typeCoercion.userDefinedFunctionCoercion(scope, call, function)) { + break validCoercionType; + } + } + // if function doesn't exist within operator table and known function + // handling is turned off then create a more permissive function + if (function == null && validator.isLenientOperatorLookup()) { + final SqlFunction x = (SqlFunction) call.getOperator(); + final SqlIdentifier identifier = + Util.first(x.getSqlIdentifier(), + new SqlIdentifier(x.getName(), SqlParserPos.ZERO)); + function = new SqlUnresolvedFunction(identifier, null, + null, OperandTypes.VARIADIC, null, x.getFunctionType()); + break validCoercionType; } } - if (!changed) { - throw validator.handleUnresolvedFunction(call, this, argTypes, - argNames); - } + throw validator.handleUnresolvedFunction(call, this, argTypes, + argNames); } // REVIEW jvs 25-Mar-2005: This is, in a sense, expanding diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index 1f47788..981a870 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -771,6 +771,27 @@ public interface SqlValidator { SqlValidatorScope getWithScope(SqlNode withItem); /** + * Sets whether this validator should be lenient upon encountering an unknown + * function. + * + * @param lenient Whether to be lenient when encountering an unknown function + */ + SqlValidator setLenientOperatorLookup(boolean lenient); + + /** Returns whether this validator should be lenient upon encountering an + * unknown function. + * + * <p>If true, if a statement contains a call to a function that is not + * present in the operator table, or if the call does not have the required + * number or types of operands, the validator nevertheless regards the + * statement as valid. The type of the function call will be + * {@link #getUnknownType() UNKNOWN}. + * + * <p>If false (the default behavior), an unknown function call causes a + * validation error to be thrown. */ + boolean isLenientOperatorLookup(); + + /** * Set if implicit type coercion is allowed when the validator does validation. * See {@link org.apache.calcite.sql.validate.implicit.TypeCoercionImpl} for the details. * @param enabled default as true. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index b00137e..1553646 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -273,6 +273,8 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { protected boolean expandColumnReferences; + protected boolean lenientOperatorLookup; + private boolean rewriteCalls; private NullCollation nullCollation = NullCollation.HIGH; @@ -326,6 +328,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { groupFinder = new AggFinder(opTab, false, false, true, null, nameMatcher); aggOrOverOrGroupFinder = new AggFinder(opTab, true, true, true, null, nameMatcher); + this.lenientOperatorLookup = false; this.enableTypeCoercion = catalogReader.getConfig() == null || catalogReader.getConfig().typeCoercion(); this.typeCoercion = TypeCoercions.getTypeCoercion(this, conformance); @@ -3832,6 +3835,15 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { return scopes.get(withItem); } + public SqlValidator setLenientOperatorLookup(boolean lenient) { + this.lenientOperatorLookup = lenient; + return this; + } + + public boolean isLenientOperatorLookup() { + return this.lenientOperatorLookup; + } + public SqlValidator setEnableTypeCoercion(boolean enabled) { this.enableTypeCoercion = enabled; return this; diff --git a/core/src/test/java/org/apache/calcite/sql/test/AbstractSqlTester.java b/core/src/test/java/org/apache/calcite/sql/test/AbstractSqlTester.java index 6a94d01..c79e106 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/AbstractSqlTester.java +++ b/core/src/test/java/org/apache/calcite/sql/test/AbstractSqlTester.java @@ -294,6 +294,10 @@ public abstract class AbstractSqlTester implements SqlTester, AutoCloseable { return with("caseSensitive", sensitive); } + public SqlTester withLenientOperatorLookup(boolean lenient) { + return with("lenientOperatorLookup", lenient); + } + public SqlTester withLex(Lex lex) { return withQuoting(lex.quoting) .withCaseSensitive(lex.caseSensitive) diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlTestFactory.java b/core/src/test/java/org/apache/calcite/sql/test/SqlTestFactory.java index c5b3259..0dabd83 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlTestFactory.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlTestFactory.java @@ -60,6 +60,7 @@ public class SqlTestFactory { .put("quotedCasing", Casing.UNCHANGED) .put("unquotedCasing", Casing.TO_UPPER) .put("caseSensitive", true) + .put("lenientOperatorLookup", false) .put("enableTypeCoercion", true) .put("conformance", SqlConformanceEnum.DEFAULT) .put("operatorTable", SqlStdOperatorTable.instance()) @@ -130,12 +131,15 @@ public class SqlTestFactory { public SqlValidator getValidator() { final SqlConformance conformance = (SqlConformance) options.get("conformance"); + final boolean lenientOperatorLookup = + (boolean) options.get("lenientOperatorLookup"); final boolean enableTypeCoercion = (boolean) options.get("enableTypeCoercion"); return validatorFactory.create(operatorTable.get(), catalogReader.get(), typeFactory.get(), conformance) - .setEnableTypeCoercion(enableTypeCoercion); + .setEnableTypeCoercion(enableTypeCoercion) + .setLenientOperatorLookup(lenientOperatorLookup); } public SqlAdvisor createAdvisor() { diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlTester.java b/core/src/test/java/org/apache/calcite/sql/test/SqlTester.java index 25b1758..d3bdd87 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlTester.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlTester.java @@ -83,6 +83,10 @@ public interface SqlTester extends AutoCloseable, SqlValidatorTestCase.Tester { /** Returns a tester that tests with implicit type coercion on/off. */ SqlTester enableTypeCoercion(boolean enabled); + /** Returns a tester that does not fail validation if it encounters an + * unknown function. */ + SqlTester withLenientOperatorLookup(boolean lenient); + /** Returns a tester that gets connections from a given factory. */ SqlTester withConnectionFactory( CalciteAssert.ConnectionFactory connectionFactory); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index a3d33ba..ee42e28 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1480,6 +1480,25 @@ public class SqlValidatorTest extends SqlValidatorTestCase { .fails("No match found for function signature FOO.."); } + @Test public void testUnknownFunctionHandling() { + final Sql s = sql("?").withTester(t -> t.withLenientOperatorLookup(true)); + s.expr("concat('a', 2)").ok(); + s.expr("foo('2001-12-21')").ok(); + s.expr("\"foo\"('b')").ok(); + s.expr("foo()").ok(); + s.expr("'a' || foo(bar('2001-12-21'))").ok(); + s.expr("cast(foo(5, 2) as DECIMAL)").ok(); + s.expr("select ascii('xyz')").ok(); + s.expr("select get_bit(CAST('FFFF' as BINARY), 1)").ok(); + s.expr("select now()").ok(); + s.expr("^TIMESTAMP_CMP_TIMESTAMPTZ^").fails("(?s).*"); + s.expr("atan(0)").ok(); + s.expr("select row_number() over () from emp").ok(); + s.expr("select coalesce(1, 2, 3)").ok(); + s.sql("select count() from emp").ok(); // too few args + s.sql("select sum(1, 2) from emp").ok(); // too many args + } + @Test public void testJdbcFunctionCall() { expr("{fn log10(1)}").ok(); expr("{fn locate('','')}").ok(); @@ -9893,6 +9912,8 @@ public class SqlValidatorTest extends SqlValidatorTestCase { @Test public void testDummy() { // (To debug individual statements, paste them into this method.) + final Sql s = sql("?").withTester(t -> t.withLenientOperatorLookup(true)); + s.sql("select count() from emp").ok(); } @Test public void testCustomColumnResolving() {
