This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f1d90ffc4e3 [regression](nereids) add test case for partition prune
(#26849)
f1d90ffc4e3 is described below
commit f1d90ffc4e3523bc752a2fd8765fc0f6359fbec6
Author: minghong <[email protected]>
AuthorDate: Tue Nov 14 11:51:32 2023 +0800
[regression](nereids) add test case for partition prune (#26849)
* list selected partition name in explain
* add prune partition test case （multi-range key）
---
.../rules/OneRangePartitionEvaluator.java | 4 +
.../org/apache/doris/planner/OlapScanNode.java | 7 +-
.../data/performance_p0/redundant_conjuncts.out | 4 +-
.../test_multi_range_partition.groovy | 241 +++++++++++++++++++++
4 files changed, 252 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 13ecddd150a..f4aa327647a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -222,8 +222,12 @@ public class OneRangePartitionEvaluator
if (expr.getDataType() instanceof BooleanType && !(expr instanceof
Literal)
&& result.childrenResult.stream().anyMatch(childResult ->
childResult.columnRanges.values().stream().anyMatch(ColumnRange::isEmptyRange)))
{
+ // this assumes that for expression: func(A)
+ // if A reject partition, then func(A) reject partition.
+ // implement visitFunc for Func if Func does not satisfy the above
assumption.
return new EvaluateRangeResult(BooleanLiteral.FALSE,
result.columnRanges, result.childrenResult);
}
+ // assumption: for func(A), if A accept range (n, m), then func(A)
accept range (n, m).
return result;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 45d5910f83d..1164e84f7ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1239,8 +1239,11 @@ public class OlapScanNode extends ScanNode {
output.append(getRuntimeFilterExplainString(false));
}
- output.append(prefix).append(String.format("partitions=%s/%s,
tablets=%s/%s", selectedPartitionNum,
- olapTable.getPartitions().size(), selectedTabletsNum,
totalTabletsNum));
+ String selectedPartitions = getSelectedPartitionIds().stream().sorted()
+ .map(id -> olapTable.getPartition(id).getName())
+ .collect(Collectors.joining(","));
+ output.append(prefix).append(String.format("partitions=%s/%s (%s),
tablets=%s/%s", selectedPartitionNum,
+ olapTable.getPartitions().size(), selectedPartitions,
selectedTabletsNum, totalTabletsNum));
// We print up to 3 tablet, and we print "..." if the number is more
than 3
if (scanTabletIds.size() > 3) {
List<Long> firstTenTabletIds = scanTabletIds.subList(0, 3);
diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out
b/regression-test/data/performance_p0/redundant_conjuncts.out
index e0ce1f3f75c..fbe84dbd083 100644
--- a/regression-test/data/performance_p0/redundant_conjuncts.out
+++ b/regression-test/data/performance_p0/redundant_conjuncts.out
@@ -13,7 +13,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE:
default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts),
PREAGGREGATION: OFF. Reason: No AggregateInfo
PREDICATES: `k1` = 1
- partitions=0/1, tablets=0/0, tabletList=
+ partitions=0/1 (), tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
pushAggOp=NONE
@@ -31,7 +31,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE:
default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts),
PREAGGREGATION: OFF. Reason: No AggregateInfo
PREDICATES: `k1` = 1 OR `k1` = 2
- partitions=0/1, tablets=0/0, tabletList=
+ partitions=0/1 (), tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
pushAggOp=NONE
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
new file mode 100644
index 00000000000..12304b4a71e
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
@@ -0,0 +1,241 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_multi_range_partition") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "set partition_pruning_expand_threshold=10;"
+ sql "drop table if exists pt"
+ sql """
+ CREATE TABLE `pt` (
+ `k1` int(11) NULL COMMENT "",
+ `k2` int(11) NULL COMMENT "",
+ `k3` int(11) NULL COMMENT ""
+ )
+ PARTITION BY RANGE(`k1`, `k2`)
+ (PARTITION p1 VALUES LESS THAN ("3", "1"),
+ PARTITION p2 VALUES [("3", "1"), ("7", "10")),
+ PARTITION p3 VALUES [("7", "10"), ("10", "15")))
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+ PROPERTIES ('replication_num' = '1');
+ """
+ sql "insert into pt values (7, 0, 0);"
+ sql "insert into pt(k1) values (7);"
+ sql "insert into pt(k1) values (0);"
+ sql "insert into pt values (7, 11, 0);"
+
+ // basic test
+ explain{
+ sql "select * from pt where k1=7;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain{
+ sql "select * from pt where k1=10;"
+ contains "partitions=1/3 (p3)"
+ }
+
+ explain{
+ sql "select * from pt where k1=11;"
+ contains "VEMPTYSET"
+ }
+
+ explain{
+ sql "select * from pt where k1=-1;"
+ contains "partitions=1/3 (p1)"
+ }
+
+ // ===============function(part_key)===================
+ explain{
+ sql "select * from pt where 2*k1=20; --漏裁 p1"
+ contains "partitions=2/3 (p1,p3)"
+ }
+ explain{
+ sql "select * from pt where 2*(k1+1)=22; --漏裁 p1"
+ contains "partitions=2/3 (p1,p3)"
+ }
+ explain{
+ sql "select * from pt where sin(k1)=0"
+ contains "artitions=3/3 (p1,p2,p3)"
+ }
+
+ // BUG: p1 missed
+ explain{
+ sql "select * from pt where sin(k1)<>0"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ // ============= in predicate ======================
+ explain{
+ sql "select * from pt where k1 in (7, 8);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain{
+ sql "select * from pt where k1 in (15, 18);"
+ contains "VEMPTYSET"
+ }
+
+ // =========== is null ===================
+ explain{
+ sql "select * from pt where k1 is null"
+ contains "partitions=1/3 (p1)"
+ }
+
+ // BUG: p1 missed
+ explain{
+ sql "select * from pt where k1 is not null"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //======== the second partition key =========
+ explain{
+ sql "select * from pt where k1=7 and (k1>k2);"
+ contains "partitions=1/3 (p2)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and not (k1>k2);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ // p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and (k1<k2);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1=k2"
+ contains "partitions=1/3 (p2)"
+ }
+
+ //BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and k1<>k2"
+ contains "partitions=1/3 (p3)"
+ }
+
+ //p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and (k1 > cast(k2 as bigint));"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and not (k2 is null);"
+ contains "VEMPTYSET"
+ }
+
+ //p3 NOT pruned
+ explain {
+ sql "select * from pt where k1=7 and not (k2 is not null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //BUG: p2 missed
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (1, 2);"
+ contains "partitions=1/3 (p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (1, 12);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ //BUG: p2,p3 pruned
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (1, 12)"
+ contains "VEMPTYSET"
+ }
+
+ explain {
+ sql" select * from pt where k1=7 and k2 in (0);"
+ contains "partitions=1/3 (p2)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (11)"
+ contains "partitions=1/3 (p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 in (null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k2 not in (null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=10 and k2 in (30);"
+ contains "VEMPTYSET"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1 > k3;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and k1 <> k3;"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 in (null);"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ // p1/p2/p3 NOT pruned
+ explain {
+ sql "select * from pt where k2 not in (null)"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 in (0)"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 > 100"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k2 < -1"
+ contains "partitions=3/3 (p1,p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and (k3 is null)"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+ explain {
+ sql "select * from pt where k1=7 and not (k3 is null);"
+ contains "partitions=2/3 (p2,p3)"
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
