This is an automated email from the ASF dual-hosted git repository. englefly pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new f1d90ffc4e3 [regression](nereids) add test case for partition prune (#26849) f1d90ffc4e3 is described below commit f1d90ffc4e3523bc752a2fd8765fc0f6359fbec6 Author: minghong <engle...@gmail.com> AuthorDate: Tue Nov 14 11:51:32 2023 +0800 [regression](nereids) add test case for partition prune (#26849) * list selected partition name in explain * add prune partition test case (multi-range key) --- .../rules/OneRangePartitionEvaluator.java | 4 + .../org/apache/doris/planner/OlapScanNode.java | 7 +- .../data/performance_p0/redundant_conjuncts.out | 4 +- .../test_multi_range_partition.groovy | 241 +++++++++++++++++++++ 4 files changed, 252 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index 13ecddd150a..f4aa327647a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -222,8 +222,12 @@ public class OneRangePartitionEvaluator if (expr.getDataType() instanceof BooleanType && !(expr instanceof Literal) && result.childrenResult.stream().anyMatch(childResult -> childResult.columnRanges.values().stream().anyMatch(ColumnRange::isEmptyRange))) { + // this assumes that for expression: func(A) + // if A reject partition, then func(A) reject partition. + // implement visitFunc for Func if Func does not satisfy the above assumption. return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult); } + // assumption: for func(A), if A accept range (n, m), then func(A) accept range (n, m). return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 45d5910f83d..1164e84f7ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1239,8 +1239,11 @@ public class OlapScanNode extends ScanNode { output.append(getRuntimeFilterExplainString(false)); } - output.append(prefix).append(String.format("partitions=%s/%s, tablets=%s/%s", selectedPartitionNum, - olapTable.getPartitions().size(), selectedTabletsNum, totalTabletsNum)); + String selectedPartitions = getSelectedPartitionIds().stream().sorted() + .map(id -> olapTable.getPartition(id).getName()) + .collect(Collectors.joining(",")); + output.append(prefix).append(String.format("partitions=%s/%s (%s), tablets=%s/%s", selectedPartitionNum, + olapTable.getPartitions().size(), selectedPartitions, selectedTabletsNum, totalTabletsNum)); // We print up to 3 tablet, and we print "..." if the number is more than 3 if (scanTabletIds.size() > 3) { List<Long> firstTenTabletIds = scanTabletIds.subList(0, 3); diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out b/regression-test/data/performance_p0/redundant_conjuncts.out index e0ce1f3f75c..fbe84dbd083 100644 --- a/regression-test/data/performance_p0/redundant_conjuncts.out +++ b/regression-test/data/performance_p0/redundant_conjuncts.out @@ -13,7 +13,7 @@ PLAN FRAGMENT 0 0:VOlapScanNode TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: `k1` = 1 - partitions=0/1, tablets=0/0, tabletList= + partitions=0/1 (), tablets=0/0, tabletList= cardinality=0, avgRowSize=8.0, numNodes=1 pushAggOp=NONE @@ -31,7 +31,7 @@ PLAN FRAGMENT 0 0:VOlapScanNode TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: `k1` = 1 OR `k1` = 2 - partitions=0/1, tablets=0/0, tabletList= + partitions=0/1 (), tablets=0/0, tabletList= cardinality=0, avgRowSize=8.0, numNodes=1 pushAggOp=NONE diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy new file mode 100644 index 00000000000..12304b4a71e --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_multi_range_partition") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "set partition_pruning_expand_threshold=10;" + sql "drop table if exists pt" + sql """ + CREATE TABLE `pt` ( + `k1` int(11) NULL COMMENT "", + `k2` int(11) NULL COMMENT "", + `k3` int(11) NULL COMMENT "" + ) + PARTITION BY RANGE(`k1`, `k2`) + (PARTITION p1 VALUES LESS THAN ("3", "1"), + PARTITION p2 VALUES [("3", "1"), ("7", "10")), + PARTITION p3 VALUES [("7", "10"), ("10", "15"))) + DISTRIBUTED BY HASH(`k1`) BUCKETS 10 + PROPERTIES ('replication_num' = '1'); + """ + sql "insert into pt values (7, 0, 0);" + sql "insert into pt(k1) values (7);" + sql "insert into pt(k1) values (0);" + sql "insert into pt values (7, 11, 0);" + + // basic test + explain{ + sql "select * from pt where k1=7;" + contains "partitions=2/3 (p2,p3)" + } + + explain{ + sql "select * from pt where k1=10;" + contains "partitions=1/3 (p3)" + } + + explain{ + sql "select * from pt where k1=11;" + contains "VEMPTYSET" + } + + explain{ + sql "select * from pt where k1=-1;" + contains "partitions=1/3 (p1)" + } + + // ===============function(part_key)=================== + explain{ + sql "select * from pt where 2*k1=20; --漏裁 p1" + contains "partitions=2/3 (p1,p3)" + } + explain{ + sql "select * from pt where 2*(k1+1)=22; --漏裁 p1" + contains "partitions=2/3 (p1,p3)" + } + explain{ + sql "select * from pt where sin(k1)=0" + contains "artitions=3/3 (p1,p2,p3)" + } + + // BUG: p1 missed + explain{ + sql "select * from pt where sin(k1)<>0" + contains "partitions=2/3 (p2,p3)" + } + + // ============= in predicate ====================== + explain{ + sql "select * from pt where k1 in (7, 8);" + contains "partitions=2/3 (p2,p3)" + } + + explain{ + sql "select * from pt where k1 in (15, 18);" + contains "VEMPTYSET" + } + + // =========== is null =================== + explain{ + sql "select * from pt where k1 is null" + contains "partitions=1/3 (p1)" + } + + // BUG: p1 missed + explain{ + sql "select * from pt where k1 is not null" + contains "partitions=2/3 (p2,p3)" + } + + //======== the second partition key ========= + explain{ + sql "select * from pt where k1=7 and (k1>k2);" + contains "partitions=1/3 (p2)" + } + + explain { + sql "select * from pt where k1=7 and not (k1>k2);" + contains "partitions=2/3 (p2,p3)" + } + + // p3 NOT pruned + explain { + sql "select * from pt where k1=7 and (k1<k2);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k1=k2" + contains "partitions=1/3 (p2)" + } + + //BUG: p2 missed + explain { + sql "select * from pt where k1=7 and k1<>k2" + contains "partitions=1/3 (p3)" + } + + //p3 NOT pruned + explain { + sql "select * from pt where k1=7 and (k1 > cast(k2 as bigint));" + contains "partitions=2/3 (p2,p3)" + } + + //BUG: p2 missed + explain { + sql "select * from pt where k1=7 and not (k2 is null);" + contains "VEMPTYSET" + } + + //p3 NOT pruned + explain { + sql "select * from pt where k1=7 and not (k2 is not null);" + contains "partitions=2/3 (p2,p3)" + } + + //BUG: p2 missed + explain { + sql "select * from pt where k1=7 and k2 not in (1, 2);" + contains "partitions=1/3 (p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (1, 12);" + contains "partitions=2/3 (p2,p3)" + } + + //BUG: p2,p3 pruned + explain { + sql "select * from pt where k1=7 and k2 not in (1, 12)" + contains "VEMPTYSET" + } + + explain { + sql" select * from pt where k1=7 and k2 in (0);" + contains "partitions=1/3 (p2)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (11)" + contains "partitions=1/3 (p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (null);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 not in (null);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=10 and k2 in (30);" + contains "VEMPTYSET" + } + + explain { + sql "select * from pt where k1=7 and k1 > k3;" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k1 <> k3;" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k2 in (null);" + contains "partitions=3/3 (p1,p2,p3)" + } + + // p1/p2/p3 NOT pruned + explain { + sql "select * from pt where k2 not in (null)" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 in (0)" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 > 100" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 < -1" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and (k3 is null)" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and not (k3 is null);" + contains "partitions=2/3 (p2,p3)" + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org