This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3a450014475 [fix](Nerids) fix error when the view has lambda functions
(#25067)
3a450014475 is described below
commit 3a4500144751927cb57bbe1ea7512d1a22e59644
Author: 谢健 <[email protected]>
AuthorDate: Sun Oct 8 15:45:24 2023 +0800
[fix](Nerids) fix error when the view has lambda functions (#25067)
1. To ensure compatibility with the original optimizer, expose the
non-lambda signature of highorder function externally.
2. fix some bugs in toSql function in the original optimizer
---
.../doris/analysis/LambdaFunctionCallExpr.java | 17 ++-
.../expressions/functions/ComputeSignature.java | 1 +
.../expressions/functions/scalar/ArrayCount.java | 7 +-
.../expressions/functions/scalar/ArrayExists.java | 7 +-
.../expressions/functions/scalar/ArrayFilter.java | 4 +
.../functions/scalar/ArrayFirstIndex.java | 7 +-
.../functions/scalar/ArrayLastIndex.java | 7 +-
.../expressions/functions/scalar/ArraySortBy.java | 12 +-
.../doris/nereids/util/TypeCoercionUtils.java | 5 +-
.../nereids_function_p0/scalar_function/Array.out | 135 +++++++++++++++++++++
.../scalar_function/Array.groovy | 36 ++++++
11 files changed, 207 insertions(+), 31 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java
index c247b071a02..e1a5cc6cf7c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java
@@ -273,7 +273,14 @@ public class LambdaFunctionCallExpr extends
FunctionCallExpr {
@Override
public String toSqlImpl() {
StringBuilder sb = new StringBuilder();
- sb.append(getFnName().getFunction());
+
+ String fnName = getFnName().getFunction();
+ if (fn != null) {
+ // `array_last` will be replaced with `element_at` function after
analysis.
+ // At this moment, using the name `array_last` would generate
invalid SQL.
+ fnName = fn.getFunctionName().getFunction();
+ }
+ sb.append(fnName);
sb.append("(");
int childSize = children.size();
Expr lastExpr = getChild(childSize - 1);
@@ -295,8 +302,12 @@ public class LambdaFunctionCallExpr extends
FunctionCallExpr {
// and some functions is only implement as a normal array function;
// but also want use as lambda function, select
array_sortby(x->x,['b','a','c']);
// so we convert to: array_sortby(array('b', 'a', 'c'), array_map(x ->
`x`, array('b', 'a', 'c')))
- if (lastIsLambdaExpr == false) {
- sb.append(", ");
+ if (!lastIsLambdaExpr) {
+ if (childSize > 1) {
+ // some functions don't have lambda expr, so don't need to add
","
+ // such as array_exists(array_map(x->x>3, [1,2,3,6,34,3,11]))
+ sb.append(", ");
+ }
sb.append(lastExpr.toSql());
}
sb.append(")");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java
index a755e3c517f..5e83a3b4b4f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java
@@ -119,6 +119,7 @@ public interface ComputeSignature extends FunctionTrait,
ImplicitCastInputTypes
/** use processor to process computeSignature */
static boolean processComplexType(DataType signatureType, DataType
realType,
BiFunction<DataType, DataType, Boolean> processor) {
+
if (signatureType instanceof ArrayType && realType instanceof
ArrayType) {
return processor.apply(((ArrayType) signatureType).getItemType(),
((ArrayType) realType).getItemType());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCount.java
index a25c1a3943b..59cf50db987 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCount.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -49,11 +48,7 @@ public class ArrayCount extends ScalarFunction
* array_count(lambda, a1, ...) = array_count(array_map(lambda, a1, ...))
*/
public ArrayCount(Expression arg) {
- super("array_count", new ArrayMap(arg));
- if (!(arg instanceof Lambda)) {
- throw new AnalysisException(
- String.format("The 1st arg of %s must be lambda but is
%s", getName(), arg));
- }
+ super("array_count", arg instanceof Lambda ? new ArrayMap(arg) : arg);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExists.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExists.java
index deaf8a14960..72f45845d86 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExists.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExists.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -48,11 +47,7 @@ public class ArrayExists extends ScalarFunction
* array_exists(lambda, a1, ...) = array_exists(array_map(lambda, a1, ...))
*/
public ArrayExists(Expression arg) {
- super("array_exists", new ArrayMap(arg));
- if (!(arg instanceof Lambda)) {
- throw new AnalysisException(
- String.format("The 1st arg of %s must be lambda but is
%s", getName(), arg));
- }
+ super("array_exists", arg instanceof Lambda ? new ArrayMap(arg) : arg);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFilter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFilter.java
index 1e8f1c5acb9..386449116c0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFilter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFilter.java
@@ -56,6 +56,10 @@ public class ArrayFilter extends ScalarFunction
}
}
+ public ArrayFilter(Expression arg1, Expression arg2) {
+ super("array_filter", arg1, arg2);
+ }
+
@Override
public ArrayFilter withChildren(List<Expression> children) {
return new ArrayFilter(children);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirstIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirstIndex.java
index fef17fe8a10..ca6544da027 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirstIndex.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirstIndex.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -49,11 +48,7 @@ public class ArrayFirstIndex extends ScalarFunction
* array_first_index(lambda, a1, ...) =
array_first_index(array_map(lambda, a1, ...))
*/
public ArrayFirstIndex(Expression arg) {
- super("array_first_index", new ArrayMap(arg));
- if (!(arg instanceof Lambda)) {
- throw new AnalysisException(
- String.format("The 1st arg of %s must be lambda but is
%s", getName(), arg));
- }
+ super("array_first_index", arg instanceof Lambda ? new ArrayMap(arg) :
arg);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLastIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLastIndex.java
index 6ae19884ae4..ec7b5c776fc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLastIndex.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLastIndex.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -49,11 +48,7 @@ public class ArrayLastIndex extends ScalarFunction
* array_last_index(lambda, a1, ...) = array_last_index(array_map(lambda,
a1, ...))
*/
public ArrayLastIndex(Expression arg) {
- super("array_last_index", new ArrayMap(arg));
- if (!(arg instanceof Lambda)) {
- throw new AnalysisException(
- String.format("The 1st arg of %s must be lambda but is
%s", getName(), arg));
- }
+ super("array_last_index", arg instanceof Lambda ? new ArrayMap(arg) :
arg);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java
index ea4ac04aca0..c2c90717cdc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -33,7 +32,7 @@ import java.util.List;
* ScalarFunction 'array_sortby'.
*/
public class ArraySortBy extends ScalarFunction
- implements HighOrderFunction, PropagateNullable {
+ implements HighOrderFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.retArgType(0).args(ArrayType.of(AnyDataType.INSTANCE_WITHOUT_INDEX),
@@ -56,6 +55,10 @@ public class ArraySortBy extends ScalarFunction
}
}
+ public ArraySortBy(Expression arg1, Expression arg2) {
+ super("array_sortby", arg1, arg2);
+ }
+
@Override
public ArraySortBy withChildren(List<Expression> children) {
return new ArraySortBy(children);
@@ -70,4 +73,9 @@ public class ArraySortBy extends ScalarFunction
public List<FunctionSignature> getImplSignature() {
return SIGNATURES;
}
+
+ @Override
+ public boolean nullable() {
+ return child(0).nullable();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index b8c4f1366b8..c79f51dcf2e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -146,9 +146,10 @@ public class TypeCoercionUtils {
* Return Optional.empty() if we cannot do implicit cast.
*/
public static Optional<DataType> implicitCast(DataType input, DataType
expected) {
- if (input instanceof ArrayType && expected instanceof ArrayType) {
+ if ((input instanceof ArrayType || input instanceof NullType) &&
expected instanceof ArrayType) {
Optional<DataType> itemType = implicitCast(
- ((ArrayType) input).getItemType(), ((ArrayType)
expected).getItemType());
+ input instanceof ArrayType ? ((ArrayType)
input).getItemType() : input,
+ ((ArrayType) expected).getItemType());
return itemType.map(ArrayType::of);
} else if (input instanceof MapType && expected instanceof MapType) {
Optional<DataType> keyType = implicitCast(
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out
b/regression-test/data/nereids_function_p0/scalar_function/Array.out
index 21da869c4c8..df91f9f1770 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Array.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out
@@ -12034,3 +12034,138 @@ true
1.100000000
1.200000000
+-- !sql_view_array_first_index_Double --
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+1
+1
+
+-- !sql_view_array_count_Double --
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+1
+1
+
+-- !sql_view_array_first_Double --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+1.1
+1.2
+
+-- !sql_view_array_sortby_Double --
+\N
+[0.1]
+[0.2]
+[0.3]
+[0.4]
+[0.5]
+[0.6]
+[0.7]
+[0.8]
+[0.9]
+[1.1]
+[1.2]
+[1]
+
+-- !sql_view_array_filter_Double --
+\N
+[1.1]
+[1.2]
+[]
+[]
+[]
+[]
+[]
+[]
+[]
+[]
+[]
+[]
+
+-- !sql_view_array_exists_Double --
+\N
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[1]
+[1]
+
+-- !sql_view_array_last_index_Double --
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+1
+1
+
+-- !sql_view_array_last_Double --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+1.1
+1.2
+
+-- !sql_view_array_map_Double --
+\N
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[0]
+[1]
+[1]
+
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
index 3b99b2dd18c..8bd097bf56d 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
@@ -996,6 +996,42 @@ suite("nereids_scalar_fn_Array") {
order_qt_sql_array_last_DecimalV3 "select array_last(x -> x > 1, kadcml)
from fn_test"
order_qt_sql_array_last_DecimalV3_notnull "select array_last(x -> x > 1,
kadcml) from fn_test_not_nullable"
+ // test array_first_index
+ sql "create view v as select array_first_index(x -> x > 1, kadbl) from
fn_test;"
+ order_qt_sql_view_array_first_index_Double "select * from v;"
+ sql "drop view v"
+ // test array_count
+ sql "create view v as select array_count(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_count_Double "select * from v;"
+ sql "drop view v"
+ // test array_first
+ sql "create view v as select array_first(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_first_Double "select * from v;"
+ sql "drop view v"
+ // test array_sortby
+ sql "create view v as select array_sortby(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_sortby_Double "select * from v;"
+ sql "drop view v"
+ // test array_filter
+ sql "create view v as select array_filter(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_filter_Double "select * from v;"
+ sql "drop view v"
+ // test array_exists
+ sql "create view v as select array_exists(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_exists_Double "select * from v;"
+ sql "drop view v"
+ // test array_last_index
+ sql "create view v as select array_last_index(x -> x > 1, kadbl) from
fn_test;"
+ order_qt_sql_view_array_last_index_Double "select * from v;"
+ sql "drop view v"
+ // test array_last
+ sql "create view v as select array_last(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_last_Double "select * from v;"
+ sql "drop view v"
+ // test array_map
+ sql "create view v as select array_map(x -> x > 1, kadbl) from fn_test;"
+ order_qt_sql_view_array_map_Double "select * from v;"
+ sql "drop view v"
test {
sql "select tokenize('arg1','xxx = yyy,zzz');"
check{result, exception, startTime, endTime ->
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
