This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 47ba99526be [Pick](nereids) Pick: partition prune fails in case of NOT
expression (#27047) (#27507)
47ba99526be is described below
commit 47ba99526be536b2e2a8889d1354386919c79428
Author: minghong <[email protected]>
AuthorDate: Fri Nov 24 17:40:46 2023 +0800
[Pick](nereids) Pick: partition prune fails in case of NOT expression
(#27047) (#27507)
---
.../rules/OneRangePartitionEvaluator.java | 67 +++++-
.../org/apache/doris/planner/OlapScanNode.java | 7 +-
.../data/performance_p0/redundant_conjuncts.out | 4 +-
.../test_multi_range_partition.groovy | 260 +++++++++++++++++++++
4 files changed, 324 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 13ecddd150a..b4fb931ec1e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -222,8 +222,12 @@ public class OneRangePartitionEvaluator
if (expr.getDataType() instanceof BooleanType && !(expr instanceof
Literal)
&& result.childrenResult.stream().anyMatch(childResult ->
childResult.columnRanges.values().stream().anyMatch(ColumnRange::isEmptyRange)))
{
+ // this assumes that for expression: func(A)
+ // if A reject partition, then func(A) reject partition.
+ // implement visitFunc for Func if Func does not satisfy the above
assumption.
return new EvaluateRangeResult(BooleanLiteral.FALSE,
result.columnRanges, result.childrenResult);
}
+ // assumption: for func(A), if A accept range (n, m), then func(A)
accept range (n, m).
return result;
}
@@ -342,13 +346,16 @@ public class OneRangePartitionEvaluator
if (!(result.result instanceof EqualTo)) {
return result;
}
- equalTo = (EqualTo) result.result;
+ boolean isRejectNot = false;
if (equalTo.left() instanceof Slot && equalTo.right() instanceof
Literal) {
Slot slot = (Slot) equalTo.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges =
result.childrenResult.get(0).columnRanges;
ColumnRange atLeastRange = ColumnRange.singleton((Literal)
equalTo.right());
result = intersectSlotRange(result, leftColumnRanges, slot,
atLeastRange);
+ if (leftColumnRanges.get(slot).isSingleton()) {
+ isRejectNot = true;
+ }
}
} else if (equalTo.left() instanceof Literal && equalTo.right()
instanceof Slot) {
Slot slot = (Slot) equalTo.right();
@@ -356,7 +363,15 @@ public class OneRangePartitionEvaluator
Map<Slot, ColumnRange> rightColumnRanges =
result.childrenResult.get(1).columnRanges;
ColumnRange atMostRange = ColumnRange.singleton((Literal)
equalTo.left());
result = intersectSlotRange(result, rightColumnRanges, slot,
atMostRange);
+ if (rightColumnRanges.get(slot).isSingleton()) {
+ isRejectNot = true;
+ }
}
+ } else {
+ isRejectNot = false;
+ }
+ if (!isRejectNot) {
+ result = result.withRejectNot(false);
}
return result;
}
@@ -379,6 +394,7 @@ public class OneRangePartitionEvaluator
Map<Slot, ColumnRange> slotRanges =
result.childrenResult.get(0).columnRanges;
result = intersectSlotRange(result, slotRanges, slot,
unionLiteralRange);
}
+ result = result.withRejectNot(false);
return result;
}
@@ -388,14 +404,15 @@ public class OneRangePartitionEvaluator
if (!(result.result instanceof IsNull)) {
return result;
}
-
+ result = result.withRejectNot(false);
Expression child = isNull.child();
if (!(child instanceof Slot) || !isPartitionSlot((Slot) child)) {
return result;
}
if (!partitionSlotContainsNull.get((Slot) child)) {
- return new EvaluateRangeResult(BooleanLiteral.FALSE,
result.columnRanges, result.childrenResult);
+ return new EvaluateRangeResult(BooleanLiteral.FALSE,
+ result.columnRanges, result.childrenResult, false);
}
return result;
}
@@ -430,12 +447,16 @@ public class OneRangePartitionEvaluator
@Override
public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(not, context);
-
- Map<Slot, ColumnRange> newRanges =
result.childrenResult.get(0).columnRanges.entrySet()
- .stream()
- .map(slotToRange -> Pair.of(slotToRange.getKey(),
slotToRange.getValue().complete()))
- .collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value));
- result = new EvaluateRangeResult(result.result, newRanges,
result.childrenResult);
+ if (result.isRejectNot()) {
+ Map<Slot, ColumnRange> newRanges = Maps.newHashMap();
+ for (Map.Entry<Slot, ColumnRange> entry :
result.childrenResult.get(0).columnRanges.entrySet()) {
+ Slot slot = entry.getKey();
+ ColumnRange childRange = entry.getValue();
+ ColumnRange partitionRange = result.columnRanges.get(slot);
+ newRanges.put(slot,
partitionRange.intersect(childRange.complete()));
+ }
+ result = new EvaluateRangeResult(result.result, newRanges,
result.childrenResult);
+ }
return returnFalseIfExistEmptyRange(result);
}
@@ -658,11 +679,37 @@ public class OneRangePartitionEvaluator
private final Map<Slot, ColumnRange> columnRanges;
private final List<EvaluateRangeResult> childrenResult;
+ // rejectNot = true, if \exist e \in R, pred(e)=true, then we have
\forAll e \in R, !pred(e)=false
+ // that is, if pred holds true over R, then !pred does not hold true
over R.
+ // example 1. rejectNot=false
+ // R=(1,10), pred: k = 5. "k = 5" holds true over R, and "NOT k =
5" holds true over R.
+ // example 2. rejectNot=false
+ // R=(1,10), pred: k = 11. "k=10" dose not holds over R
+ // example 3. rejectNot=false
+ // R=(1,10), pred: k in (4, 5). "k in (4, 5)" holds true over R,
and "NOT k in (4, 5)" holds over R
+ // example 3. rejectNot=true
+ // R=(1,10), pred: k < 11. "k<11" holds true over R, and "NOT
k<11" dose not hold over R
+ private final boolean rejectNot;
+
public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange>
columnRanges,
- List<EvaluateRangeResult> childrenResult) {
+ List<EvaluateRangeResult> childrenResult,
boolean rejectNot) {
this.result = result;
this.columnRanges = columnRanges;
this.childrenResult = childrenResult;
+ this.rejectNot = rejectNot;
+ }
+
+ public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange>
columnRanges,
+ List<EvaluateRangeResult> childrenResult) {
+ this(result, columnRanges, childrenResult,
childrenResult.stream().allMatch(r -> r.isRejectNot()));
+ }
+
+ public EvaluateRangeResult withRejectNot(boolean rejectNot) {
+ return new EvaluateRangeResult(result, columnRanges,
childrenResult, rejectNot);
+ }
+
+ public boolean isRejectNot() {
+ return rejectNot;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index afee004507a..d7c869943ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1218,8 +1218,11 @@ public class OlapScanNode extends ScanNode {
output.append(getRuntimeFilterExplainString(false));
}
- output.append(prefix).append(String.format("partitions=%s/%s,
tablets=%s/%s", selectedPartitionNum,
- olapTable.getPartitions().size(), selectedTabletsNum,
totalTabletsNum));
+ String selectedPartitions = getSelectedPartitionIds().stream().sorted()
+ .map(id -> olapTable.getPartition(id).getName())
+ .collect(Collectors.joining(","));
+ output.append(prefix).append(String.format("partitions=%s/%s (%s),
tablets=%s/%s", selectedPartitionNum,
+ olapTable.getPartitions().size(), selectedPartitions,
selectedTabletsNum, totalTabletsNum));
// We print up to 3 tablet, and we print "..." if the number is more
than 3
if (scanTabletIds.size() > 3) {
List<Long> firstTenTabletIds = scanTabletIds.subList(0, 3);
diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out
b/regression-test/data/performance_p0/redundant_conjuncts.out
index 4c432cf6d2b..06003d12d2f 100644
--- a/regression-test/data/performance_p0/redundant_conjuncts.out
+++ b/regression-test/data/performance_p0/redundant_conjuncts.out
@@ -12,7 +12,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE:
default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts),
PREAGGREGATION: OFF. Reason: No AggregateInfo
PREDICATES: `k1` = 1
- partitions=0/1, tablets=0/0, tabletList=
+ partitions=0/1 (), tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
pushAggOp=NONE
@@ -29,7 +29,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE:
default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts),
PREAGGREGATION: OFF. Reason: No AggregateInfo
PREDICATES: `k1` = 1 OR `k1` = 2
- partitions=0/1, tablets=0/0, tabletList=
+ partitions=0/1 (), tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
pushAggOp=NONE
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
new file mode 100644
index 00000000000..d24972c6b5f
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_multi_range_partition") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "set partition_pruning_expand_threshold=10;"
+ sql "drop table if exists pt"
+ sql """
+ CREATE TABLE `pt` (
+ `k1` int(11) NULL COMMENT "",
+ `k2` int(11) NULL COMMENT "",
+ `k3` int(11) NULL COMMENT ""
+ )
+ PARTITION BY RANGE(`k1`, `k2`)
+ (PARTITION p1 VALUES LESS THAN ("3", "1"),
+ PARTITION p2 VALUES [("3", "1"), ("7", "10")),
+ PARTITION p3 VALUES [("7", "10"), ("10", "15")))
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+ PROPERTIES ('replication_num' = '1');
+ """
+ sql "insert into pt values (7, 0, 0);"
+ sql "insert into pt(k1) values (7);"
+ sql "insert into pt(k1) values (0);"
+ sql "insert into pt values (7, 11, 0);"
+
+ // basic test
+ explain{
+ sql "select * from pt where k1=7;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain{
+ sql "select * from pt where k1=10;"
+ contains "partitions=1/3 (p3)"
+ }
+
+ explain{
+ sql "select * from pt where k1=-1;"
+ contains "partitions=1/3 (p1)"
+ }
+
+ // ===============function(part_key)===================
+ explain{
+ sql "select * from pt where 2*k1=20; --漏裁 p1"
+ contains "partitions=2/3 (p1,p3)"
+ }
+ explain{
+ sql "select * from pt where 2*(k1+1)=22; --漏裁 p1"
+ contains "partitions=2/3 (p1,p3)"
+ }
+ explain{
+ sql "select * from pt where sin(k1)=0"
+ contains "artitions=3/3 (p1,p2,p3)"
+ }
+
+ // fix BUG: p1 missed
+ explain{
+ sql "select * from pt where sin(k1)<>0"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ // ============= in predicate ======================
+ explain{
+ sql "select * from pt where k1 in (7, 8);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ // =========== is null ===================
+ explain{
+ sql "select * from pt where k1 is null"
+ contains "partitions=1/3 (p1)"
+ }
+
+ // fix BUG: p1 missed
+ explain{
+ sql "select * from pt where k1 is not null"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ //======== the second partition key =========
+ explain{
+ sql "select * from pt where k1=7 and (k1>k2);"
+ contains "partitions=1/3 (p2)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and not (k1>k2);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ // p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and (k1<k2);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1=k2"
+ contains "partitions=1/3 (p2)"
+ }
+
+ //fix BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and k1<>k2"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and (k1 > cast(k2 as bigint));"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //fix BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and not (k2 is null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and not (k2 is not null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //fix BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (1, 2);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (1, 12);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //fix BUG: p2,p3 pruned
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (1, 12)"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql" select * from pt where k1=7 and k2 in (0);"
+ contains "partitions=1/3 (p2)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (11)"
+ contains "partitions=1/3 (p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1 > k3;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1 <> k3;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 in (null);"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ // p1/p2/p3 NOT pruned
+ explain {
+ sql "select * from pt where k2 not in (null)"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 in (0)"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 > 100"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 < -1"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and (k3 is null)"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and not (k3 is null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+
+ // test if a range is not sliced to multiple single point
+ // for example: range [3,7) is sliced to [3,3], [4,4],[5,5],[6,6]
+ sql "set partition_pruning_expand_threshold=1;"
+
+ explain {
+ sql "select * from pt where k1 < 5;"
+ contains "partitions=2/3 (p1,p2)"
+ }
+
+ explain {
+ sql "select * from pt where not k1 < 5;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 < 5;"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where not k2 < 5;"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k3 < 5;"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where not k3 < 5;"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
