This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ae7cf0bccf2c78a2e6a88a2276e9c462c703334e Author: mch_ucchi <[email protected]> AuthorDate: Wed Sep 13 12:04:20 2023 +0800 [fix](Nereids) default partition be prunned by mistake (#24186) ```sql CREATE TABLE IF NOT EXISTS t ( k1 tinyint NOT NULL, k2 smallint NOT NULL, k3 int NOT NULL, k4 bigint NOT NULL, k5 decimal(9, 3) NOT NULL, k8 double max NOT NULL, k9 float sum NOT NULL ) AGGREGATE KEY(k1,k2,k3,k4,k5) PARTITION BY LIST(k1) ( PARTITION p1 VALUES IN ("1","2","3","4"), PARTITION p2 VALUES IN ("5","6","7","8"), PARTITION p3 ) DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1") select * from t where k1=10 ``` The query will return 0 rows because p3 is pruned, we fix it by skip prune default partitions. TODO: prune default partition if filter do not hit it --- .../rules/HiveDefaultPartitionEvaluator.java | 5 +++++ .../rules/OneListPartitionEvaluator.java | 5 +++++ .../expression/rules/OnePartitionEvaluator.java | 10 ++++++++++ .../rules/OneRangePartitionEvaluator.java | 5 +++++ .../rules/expression/rules/PartitionPruner.java | 3 ++- .../rules/UnknownPartitionEvaluator.java | 5 +++++ .../rules/rewrite/PruneOlapScanPartition.java | 1 + .../rules/rewrite/PruneOlapScanPartitionTest.java | 22 ++++++++++++++++++++++ 8 files changed, 55 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java index 1249dd0577..04785e1db1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java @@ -60,4 +60,9 @@ public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator { public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) { return BooleanLiteral.TRUE; } + + @Override + public boolean isDefaultPartition() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java index fcdd9c2a45..dd71ed8e99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java @@ -98,4 +98,9 @@ public class OneListPartitionEvaluator public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) { return expression.accept(this, currentInputs); } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java index a2e5c332a7..c51252b44a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.expression.rules; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import java.util.List; import java.util.Map; @@ -45,4 +46,13 @@ public interface OnePartitionEvaluator { * we will return a context which result expression is BooleanLiteral.FALSE */ Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs); + + default Expression evaluateWithDefaultPartition(Expression expression, Map<Slot, PartitionSlotInput> inputs) { + if (isDefaultPartition()) { + return BooleanLiteral.TRUE; + } + return evaluate(expression, inputs); + } + + boolean isDefaultPartition(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index d03d3276ef..f985764685 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -665,4 +665,9 @@ public class OneRangePartitionEvaluator this.childrenResult = childrenResult; } } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 7e89a97915..8479c27776 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -83,6 +83,7 @@ public class PartitionPruner { .collect(ImmutableList.toImmutableList()); PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate); + //TODO: we keep default partition because it's too hard to prune it, we return false in canPrune(). return partitionPruner.prune(); } @@ -110,7 +111,7 @@ public class PartitionPruner { private boolean canPrune(OnePartitionEvaluator evaluator) { List<Map<Slot, PartitionSlotInput>> onePartitionInputs = evaluator.getOnePartitionInputs(); for (Map<Slot, PartitionSlotInput> currentInputs : onePartitionInputs) { - Expression result = evaluator.evaluate(partitionPredicate, currentInputs); + Expression result = evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs); if (!result.equals(BooleanLiteral.FALSE)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java index 1325635eb7..ae313ca09d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java @@ -52,4 +52,9 @@ public class UnknownPartitionEvaluator implements OnePartitionEvaluator { // do not prune return expression; } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 4c10f339bf..1bdff7f6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -68,6 +68,7 @@ public class PruneOlapScanPartition extends OneRewriteRuleFactory { List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune( partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext, PartitionTableType.OLAP)); + List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions(); if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) { prunedPartitions.retainAll(manuallySpecifiedPartitions); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java index 829feda16c..85fde57b51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java @@ -196,6 +196,28 @@ class PruneOlapScanPartitionTest extends TestWithFeService implements MemoPatter test("testOlapScanPartitionPruneWithMultiColumnCase", "cast(col1 as bigint) + 1 = 5", 1); } + @Test + public void prunePartitionWithDefaultPartition() throws Exception { + createTable("CREATE TABLE IF NOT EXISTS test_default_in_parts (\n" + + " k1 tinyint NOT NULL, \n" + + " k2 smallint NOT NULL, \n" + + " k3 int NOT NULL, \n" + + " k4 bigint NOT NULL, \n" + + " k5 decimal(9, 3) NOT NULL,\n" + + " k8 double max NOT NULL, \n" + + " k9 float sum NOT NULL ) \n" + + " AGGREGATE KEY(k1,k2,k3,k4,k5)\n" + + " PARTITION BY LIST(k1) ( \n" + + " PARTITION p1 VALUES IN (\"1\",\"2\",\"3\",\"4\"), \n" + + " PARTITION p2 VALUES IN (\"5\",\"6\",\"7\",\"8\"), \n" + + " PARTITION p3 ) \n" + + " DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(\"replication_num\" = \"1\")"); + test("test_default_in_parts", "k1 = 10", 1); + test("test_default_in_parts", "k1 > 5", 2); + test("test_default_in_parts", "k1 > 2", 3); + test("test_default_in_parts", "(k1 > 1 and k1 < 8)", 3); + } + @Test public void prunePartitionWithOrPredicate() { test("test_list_parts", "(part = 9 and id <= 500) or (part = 3)", 1); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
