This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22fce33fb2 [fix](nereids) fix bitmap function nullable trait and
dphyper bugs (#18041)
22fce33fb2 is described below
commit 22fce33fb21daf50de51627eaf6794b0d1e1f656
Author: starocean999 <[email protected]>
AuthorDate: Fri Mar 24 10:53:45 2023 +0800
[fix](nereids) fix bitmap function nullable trait and dphyper bugs (#18041)
1. some bitmap functions like bitmap_or, bitmap_and_count, bitmap_or_count
etc shouldn't follow constant fold rule for PropagateNullable functions. So
remove PropagateNullable property and these functions would use their own
constant fold logic correctly
2. dphyper's PlanReceiver class shouldn't change hyperGraph's complex
project info. So make PlanReceiver use its own copy of complex project info now.
---
.../joinorder/hypergraph/receiver/PlanReceiver.java | 12 +++++++-----
.../expressions/functions/scalar/BitmapAndCount.java | 8 ++++++--
.../functions/scalar/BitmapAndNotCount.java | 8 ++++++--
.../trees/expressions/functions/scalar/BitmapOr.java | 8 ++++++--
.../expressions/functions/scalar/BitmapOrCount.java | 7 +++++--
.../expressions/functions/scalar/BitmapXorCount.java | 8 ++++++--
.../bitmap_functions/test_bitmap_function.groovy | 18 ++++++------------
7 files changed, 42 insertions(+), 27 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index 1079f4a6d2..6a6828f5b1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -56,7 +56,6 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -69,6 +68,7 @@ public class PlanReceiver implements AbstractReceiver {
HashMap<Long, Group> planTable = new HashMap<>();
HashMap<Long, BitSet> usdEdges = new HashMap<>();
HashMap<Long, List<NamedExpression>> projectsOnSubgraph = new HashMap<>();
+ HashMap<Long, List<NamedExpression>> complexProjectMap = new HashMap<>();
int limit;
int emitCount = 0;
@@ -293,10 +293,13 @@ public class PlanReceiver implements AbstractReceiver {
@Override
public void reset() {
+ Preconditions.checkArgument(complexProjectMap.isEmpty(),
+ "complexProjectMap should be empty when call reset()");
planTable.clear();
projectsOnSubgraph.clear();
usdEdges.clear();
emitCount = 0;
+ complexProjectMap.putAll(hyperGraph.getComplexProject());
}
@Override
@@ -360,13 +363,12 @@ public class PlanReceiver implements AbstractReceiver {
if (!projectsOnSubgraph.containsKey(fullKey)) {
List<NamedExpression> projects = new ArrayList<>();
// Calculate complex expression
- Map<Long, List<NamedExpression>> complexExpressionMap =
hyperGraph.getComplexProject();
- List<Long> bitmaps = complexExpressionMap.keySet().stream()
+ List<Long> bitmaps = complexProjectMap.keySet().stream()
.filter(bitmap -> LongBitmap.isSubset(bitmap,
fullKey)).collect(Collectors.toList());
for (long bitmap : bitmaps) {
- projects.addAll(complexExpressionMap.get(bitmap));
- complexExpressionMap.remove(bitmap);
+ projects.addAll(complexProjectMap.get(bitmap));
+ complexProjectMap.remove(bitmap);
}
// calculate required columns
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
index 24f1333fb7..dd9b43f76f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
@@ -35,7 +34,7 @@ import java.util.List;
* ScalarFunction 'bitmap_and_count'. This class is generated by
GenerateFunction.
*/
public class BitmapAndCount extends ScalarFunction
- implements ExplicitlyCastableSignature, PropagateNullable {
+ implements ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -67,4 +66,9 @@ public class BitmapAndCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapAndCount(this, context);
}
+
+ @Override
+ public boolean nullable() {
+ return children().stream().anyMatch(Expression::nullable);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
index c4968f0e07..95a03407b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@@ -35,7 +34,7 @@ import java.util.List;
* ScalarFunction 'bitmap_and_not_count'. This class is generated by
GenerateFunction.
*/
public class BitmapAndNotCount extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -66,4 +65,9 @@ public class BitmapAndNotCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapAndNotCount(this, context);
}
+
+ @Override
+ public boolean nullable() {
+ return children().stream().anyMatch(Expression::nullable);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOr.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOr.java
index aea7e7b497..883a0ffbfb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOr.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -34,7 +33,7 @@ import java.util.List;
* ScalarFunction 'bitmap_or'. This class is generated by GenerateFunction.
*/
public class BitmapOr extends ScalarFunction
- implements ExplicitlyCastableSignature, PropagateNullable {
+ implements ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BitmapType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -66,4 +65,9 @@ public class BitmapOr extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapOr(this, context);
}
+
+ @Override
+ public boolean nullable() {
+ return children().stream().anyMatch(Expression::nullable);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
index 352538644a..a7b6e716ea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
@@ -35,7 +34,7 @@ import java.util.List;
* ScalarFunction 'bitmap_or_count'. This class is generated by
GenerateFunction.
*/
public class BitmapOrCount extends ScalarFunction
- implements ExplicitlyCastableSignature, PropagateNullable {
+ implements ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -67,4 +66,8 @@ public class BitmapOrCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapOrCount(this, context);
}
+
+ public boolean nullable() {
+ return children().stream().anyMatch(Expression::nullable);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
index 4cbe9cde9a..93d8d43045 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
@@ -35,7 +34,7 @@ import java.util.List;
* ScalarFunction 'bitmap_xor_count'. This class is generated by
GenerateFunction.
*/
public class BitmapXorCount extends ScalarFunction
- implements ExplicitlyCastableSignature, PropagateNullable {
+ implements ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -67,4 +66,9 @@ public class BitmapXorCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapXorCount(this, context);
}
+
+ @Override
+ public boolean nullable() {
+ return children().stream().anyMatch(Expression::nullable);
+ }
}
diff --git
a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index 4f8d6416dd..53f08d6ce5 100644
---
a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++
b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -60,12 +60,10 @@ suite("test_bitmap_function") {
qt_sql_bitmap_or1 """ select bitmap_count(bitmap_or(to_bitmap(1),
to_bitmap(2))) cnt """
qt_sql_bitmap_or2 """ select bitmap_count(bitmap_or(to_bitmap(1),
to_bitmap(1))) cnt """
qt_sql_bitmap_or3 """ select bitmap_to_string(bitmap_or(to_bitmap(1),
to_bitmap(2))) """
- // TODO: fix constant fold of bitmap_or and enable this case
- // qt_sql_bitmap_or4 """ select bitmap_to_string(bitmap_or(to_bitmap(1),
to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """
+ qt_sql_bitmap_or4 """ select bitmap_to_string(bitmap_or(to_bitmap(1),
to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """
qt_sql_bitmap_or5 """ select bitmap_to_string(bitmap_or(to_bitmap(1),
to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """
qt_sql_bitmap_or6 """ select bitmap_to_string(bitmap_or(to_bitmap(10),
bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """
- // TODO: fix constant fold of bitmap_or and enable this case
- // qt_sql_bitmap_or7 """ select bitmap_count(bitmap_or(to_bitmap(1),
null)) cnt """
+ qt_sql_bitmap_or7 """ select bitmap_count(bitmap_or(to_bitmap(1), null))
cnt """
// bitmap_or of all nullable column
sql """ DROP TABLE IF EXISTS test_bitmap1 """
@@ -297,16 +295,14 @@ suite("test_bitmap_function") {
qt_sql_bitmap_and_count3 """ select
bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
qt_sql_bitmap_and_count4 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5')) """
qt_sql_bitmap_and_count5 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5'),bitmap_empty()) """
- // TODO: fix constant fold and enable this case
- // qt_sql_bitmap_and_count6 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5'), NULL) """
+ qt_sql_bitmap_and_count6 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5'), NULL) """
// bitmap_or_count
qt_sql_bitmap_or_count1 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """
qt_sql_bitmap_or_count2 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))"""
qt_sql_bitmap_or_count3 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
qt_sql_bitmap_or_count4 """ select
bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'),
to_bitmap(100), bitmap_empty()) """
- // TODO: fix constant fold of bitmap_or and enable this case
- // qt_sql_bitmap_or_count5 """ select
bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'),
to_bitmap(100), NULL) """
+ qt_sql_bitmap_or_count5 """ select
bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'),
to_bitmap(100), NULL) """
// BITMAP_XOR
qt_sql """ select
bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4')))
cnt """
@@ -321,8 +317,7 @@ suite("test_bitmap_function") {
qt_sql_bitmap_xor_count3 """ select
bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('4,5,6')) """
qt_sql_bitmap_xor_count4 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5')))
"""
qt_sql_bitmap_xor_count5 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty()))
"""
- // TODO: fix constant fold and enable this case
- // qt_sql_bitmap_xor_count6 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL))
"""
+ qt_sql_bitmap_xor_count6 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL))
"""
// bitmap_and_count, bitmap_xor_count, bitmap_and_not_count of all
nullable column
sql """ DROP TABLE IF EXISTS test_bitmap1 """
@@ -529,8 +524,7 @@ suite("test_bitmap_function") {
// BITMAP_AND_NOT_COUNT
qt_sql_bitmap_and_not_count1 """ select
bitmap_and_not_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5'))
cnt """
- // TODO: fix constant fold and enable this case
- // qt_sql_bitmap_and_not_count2 """ select
bitmap_and_not_count(bitmap_from_string('1,2,3'),null) cnt """
+ qt_sql_bitmap_and_not_count2 """ select
bitmap_and_not_count(bitmap_from_string('1,2,3'),null) cnt """
// BITMAP_SUBSET_IN_RANGE
qt_sql """ select
bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('1,2,3,4,5'), 0, 9))
value """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
