This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 40747a780209cd138f28baa2d63d1bbe5d1ae51a Author: Steve Carlin <[email protected]> AuthorDate: Wed Nov 6 05:24:55 2024 -0800 IMPALA-13524: Calcite planner: support for functions in exprs.test Specifically, this commit supports translate, the not operator (!), the null not equals operator and some "not regexp" functions. A little extra code had to be added for the not regexp functions because Calcite treats the "not" part as an attribute to the operator. Also, added support for "is distinct from" and "is not distinct from" operators which required a convertlet change to handle the conversion from SqlNodes to RexNodes. Change-Id: Ib8c5d5719a409a32ddb6946d1a87c77773f20820 Reviewed-on: http://gerrit.cloudera.org:8080/22104 Reviewed-by: Aman Sinha <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../src/main/codegen/templates/Parser.jj | 22 +++--------- .../impala/calcite/functions/FunctionResolver.java | 2 ++ .../impala/calcite/functions/RexCallConverter.java | 13 +++++++ .../calcite/operators/ImpalaConvertletTable.java | 40 +++++++++++++++++++++- .../calcite/operators/ImpalaOperatorTable.java | 1 + 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/java/calcite-planner/src/main/codegen/templates/Parser.jj b/java/calcite-planner/src/main/codegen/templates/Parser.jj index 439ff2446..e41d4eb2f 100644 --- a/java/calcite-planner/src/main/codegen/templates/Parser.jj +++ b/java/calcite-planner/src/main/codegen/templates/Parser.jj @@ -6233,23 +6233,6 @@ SqlNode BuiltinFunctionCall() : return SqlLibraryOperators.MSSQL_CONVERT.createCall(s.end(this), args); } ) - | - <TRANSLATE> { s = span(); } - <LPAREN> - AddExpression(args, ExprContext.ACCEPT_SUB_QUERY) - ( - <USING> name = SimpleIdentifier() { args.add(name); } - <RPAREN> { - return SqlStdOperatorTable.TRANSLATE.createCall(s.end(this), - args); - } - | - ( <COMMA> AddExpression(args, ExprContext.ACCEPT_SUB_QUERY) )* - <RPAREN> { - return SqlLibraryOperators.TRANSLATE3.createCall(s.end(this), - args); - } - ) | <OVERLAY> { s = span(); } <LPAREN> AddExpression(args, ExprContext.ACCEPT_SUB_QUERY) @@ -7733,6 +7716,7 @@ SqlBinaryOperator BinaryRowOperator() : | <LE> { return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; } | <GE> { return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL; } | <NE> { return SqlStdOperatorTable.NOT_EQUALS; } +| <NULL_SAFE_EQUAL> { return SqlStdOperatorTable.IS_NOT_DISTINCT_FROM; } | <REGEXP> { return SqlStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE; } | LOOKAHEAD(2) <NOT> <REGEXP> { return SqlStdOperatorTable.NEGATED_POSIX_REGEX_CASE_SENSITIVE; } | <IREGEXP> { return SqlStdOperatorTable.POSIX_REGEX_CASE_INSENSITIVE; } @@ -7780,6 +7764,7 @@ SqlPrefixOperator PrefixRowOperator() : <PLUS> { return SqlStdOperatorTable.UNARY_PLUS; } | <MINUS> { return SqlStdOperatorTable.UNARY_MINUS; } | <NOT> { return SqlStdOperatorTable.NOT; } +| <BANG_NOT> { return SqlStdOperatorTable.NOT; } | <EXISTS> { return SqlStdOperatorTable.EXISTS; } | <UNIQUE> { return SqlStdOperatorTable.UNIQUE; } } @@ -7871,6 +7856,7 @@ SqlPostfixOperator PostfixRowOperator() : | < ATTRIBUTES: "ATTRIBUTES" > | < AUTHORIZATION: "AUTHORIZATION" > | < AVG: "AVG" > +| < BANG_NOT: "!" > | < BEFORE: "BEFORE" > | < BEGIN: "BEGIN" > | < BEGIN_FRAME: "BEGIN_FRAME" > @@ -8483,7 +8469,6 @@ SqlPostfixOperator PostfixRowOperator() : | < TRANSACTIONS_ROLLED_BACK: "TRANSACTIONS_ROLLED_BACK" > | < TRANSFORM: "TRANSFORM" > | < TRANSFORMS: "TRANSFORMS" > -| < TRANSLATE: "TRANSLATE" > | < TRANSLATE_REGEX: "TRANSLATE_REGEX" > | < TRANSLATION: "TRANSLATION" > | < TREAT: "TREAT" > @@ -8751,6 +8736,7 @@ void NonReservedKeyWord2of3() : | < VERTICAL_BAR: "|" > | < CARET: "^" > | < DOLLAR: "$" > +| < NULL_SAFE_EQUAL: "<=>" > | < REGEXP : "REGEXP" > | < IREGEXP : "IREGEXP" > <#list (parser.binaryOperatorsTokens!default.parser.binaryOperatorsTokens) as operator> diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java index b812a283a..d5544c1be 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java @@ -76,6 +76,8 @@ public class FunctionResolver { .put(SqlKind.SUM0, "sum_init_zero") .put(SqlKind.POSIX_REGEX_CASE_SENSITIVE, "regexp") .put(SqlKind.POSIX_REGEX_CASE_INSENSITIVE, "iregexp") + .put(SqlKind.IS_NOT_DISTINCT_FROM, "notdistinct") + .put(SqlKind.IS_DISTINCT_FROM, "distinctfrom") .build(); // Map of Calcite names to an Impala function name when the names are different diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java index 586ffe238..ffa30ba60 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java @@ -25,6 +25,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlPosixRegexOperator; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.impala.analysis.Analyzer; @@ -118,6 +119,9 @@ public class RexCallConverter { switch (rexCall.getOperator().getKind()) { case CASE: return createCaseExpr(fn, params, impalaRetType); + case POSIX_REGEX_CASE_SENSITIVE: + case POSIX_REGEX_CASE_INSENSITIVE: + return createRegexExpr(fn, params, impalaRetType, rexCall); default: return new AnalyzedFunctionCallExpr(fn, params, impalaRetType); } @@ -209,6 +213,15 @@ public class RexCallConverter { return new AnalyzedBinaryCompExpr(fn, op, params.get(0), params.get(1)); } + private static Expr createRegexExpr(Function fn, List<Expr> params, + Type impalaRetType, RexCall rexCall) throws ImpalaException { + SqlPosixRegexOperator op = (SqlPosixRegexOperator) rexCall.getOperator(); + Expr regexpExpr = new AnalyzedFunctionCallExpr(fn, params, impalaRetType); + return op.isNegated() + ? new CompoundPredicate(CompoundPredicate.Operator.NOT, regexpExpr, null) + : regexpExpr; + } + private static Expr createTimestampExpr(RexCall rexCall, List<Expr> params) { // one of the timestamp expressions should be a date time. The other should // be some number to add or subtract. timestampParamIndex contains the diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaConvertletTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaConvertletTable.java index ff3bee875..fa2e2ebcc 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaConvertletTable.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaConvertletTable.java @@ -21,6 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql2rel.ReflectiveConvertletTable; @@ -30,8 +31,12 @@ import org.apache.calcite.sql2rel.StandardConvertletTable; import org.apache.impala.calcite.operators.ImpalaCustomOperatorTable; import java.util.List; + /** - * + * ImpalaConvertletTable adds the ability to override any converlets in the + * StandardConvertlet table provided by Calcite. The convertlets are executed in + * the step where Calcite converts the SqlNode tree into a RelNode tree and creating + * RexNodes from SqlNodes. */ public class ImpalaConvertletTable extends ReflectiveConvertletTable { public static final ImpalaConvertletTable INSTANCE = @@ -40,6 +45,8 @@ public class ImpalaConvertletTable extends ReflectiveConvertletTable { public ImpalaConvertletTable() { addAlias(ImpalaCustomOperatorTable.PERCENT_REMAINDER, SqlStdOperatorTable.MOD); registerOp(ImpalaCastFunction.INSTANCE, this::convertExplicitCast); + registerOp(SqlStdOperatorTable.IS_DISTINCT_FROM, this::convertIsDistinctFrom); + registerOp(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, this::convertIsNotDistinctFrom); } @Override @@ -52,6 +59,11 @@ public class ImpalaConvertletTable extends ReflectiveConvertletTable { return super.get(call); } + if (call.getOperator().getKind().equals(SqlKind.IS_DISTINCT_FROM) || + call.getOperator().getKind().equals(SqlKind.IS_NOT_DISTINCT_FROM)) { + return super.get(call); + } + // EXPLICIT_CAST convertlet has to be handled by our convertlet. Operation // was registered in the constructor and it will call convertExplicitCast if (call.getOperator().getName().equals("EXPLICIT_CAST")) { @@ -61,6 +73,32 @@ public class ImpalaConvertletTable extends ReflectiveConvertletTable { return StandardConvertletTable.INSTANCE.get(call); } + protected RexNode convertIsDistinctFrom( + SqlRexContext cx, SqlCall call) { + final SqlNode expr1 = call.operand(0); + final SqlNode expr2 = call.operand(1); + final RexBuilder rexBuilder = cx.getRexBuilder(); + RelDataType returnType = + cx.getValidator().getValidatedNodeTypeIfKnown(call); + List<RexNode> operands = Lists.newArrayList(cx.convertExpression(expr1), + cx.convertExpression(expr2)); + return rexBuilder.makeCall(returnType, SqlStdOperatorTable.IS_DISTINCT_FROM, + operands); + } + + protected RexNode convertIsNotDistinctFrom( + SqlRexContext cx, SqlCall call) { + final SqlNode expr1 = call.operand(0); + final SqlNode expr2 = call.operand(1); + final RexBuilder rexBuilder = cx.getRexBuilder(); + RelDataType returnType = + cx.getValidator().getValidatedNodeTypeIfKnown(call); + List<RexNode> operands = Lists.newArrayList(cx.convertExpression(expr1), + cx.convertExpression(expr2)); + return rexBuilder.makeCall(returnType, SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, + operands); + } + protected RexNode convertExplicitCast( SqlRexContext cx, SqlCall call) { final SqlNode expr = call.operand(0); diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java index e13b43948..cb198582f 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java @@ -83,6 +83,7 @@ public class ImpalaOperatorTable extends ReflectiveSqlOperatorTable { .add("extract") .add("regr_count") .add("localtime") + .add("translate") .build(); private static ImpalaOperatorTable INSTANCE;
