This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 2a264808005 branch-4.0: [feature](partition prune) support remove
predicates that are always true after partition prune in list partition #57169
#57688 #57817 (#58040)
2a264808005 is described below
commit 2a2648080058d6bad5039d6b4ffcd11c307a23c3
Author: feiniaofeiafei <[email protected]>
AuthorDate: Thu Nov 27 16:04:01 2025 +0800
branch-4.0: [feature](partition prune) support remove predicates that are
always true after partition prune in list partition #57169 #57688 #57817
(#58040)
picked from #57169 #57688 #57817
---
.../org/apache/doris/nereids/StatementContext.java | 9 +
.../nereids/load/NereidsLoadPlanInfoCollector.java | 3 +-
.../rules/expression/rules/PartitionPruner.java | 131 ++++-
.../rules/rewrite/PruneFileScanPartition.java | 8 +-
.../rules/rewrite/PruneOlapScanPartition.java | 41 +-
.../trees/plans/commands/DeleteFromCommand.java | 7 +-
.../trees/plans/commands/ExplainCommand.java | 3 +
.../java/org/apache/doris/qe/SessionVariable.java | 11 +
.../nereids/mv/OptimizeGetAvailableMvsTest.java | 4 +-
.../nereids/rules/rewrite/PartitionPrunerTest.java | 570 +++++++++++++++++++++
.../infer_predicate/infer_unequal_predicates.out | 19 +-
.../partition_prune/list_prune_predicate.out | 16 +
.../partition_prune/hive_partition_prune.groovy | 2 +-
.../partition_prune/list_prune_predicate.groovy | 367 +++++++++++++
14 files changed, 1130 insertions(+), 61 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index f3beb227528..8fd9aba8b1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -282,6 +282,7 @@ public class StatementContext implements Closeable {
private final Set<List<String>> materializationRewrittenSuccessSet = new
HashSet<>();
private boolean isInsert = false;
+ private boolean skipPrunePredicate = false;
private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates =
Optional.empty();
@@ -1039,4 +1040,12 @@ public class StatementContext implements Closeable {
this.icebergRewriteFileScanTasks = null;
return tasks;
}
+
+ public boolean isSkipPrunePredicate() {
+ return skipPrunePredicate;
+ }
+
+ public void setSkipPrunePredicate(boolean skipPrunePredicate) {
+ this.skipPrunePredicate = skipPrunePredicate;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
index 210249263a6..6cc6ca60cf0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
@@ -49,6 +49,7 @@ import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
+import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
@@ -499,7 +500,7 @@ public class NereidsLoadPlanInfoCollector extends
DefaultPlanVisitor<Void, PlanT
List<Long> prunedPartitions = PartitionPruner.prune(
partitionSlots, filterPredicate, idToPartitions,
CascadesContext.initContext(new StatementContext(),
logicalPlan, PhysicalProperties.ANY),
- PartitionPruner.PartitionTableType.OLAP,
sortedPartitionRanges);
+ PartitionTableType.OLAP, sortedPartitionRanges).first;
return prunedPartitions;
} else {
return null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index 6e1f04f354d..badc925ad38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.catalog.ListPartitionItem;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
@@ -35,7 +36,11 @@ import
org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.types.DateTimeType;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
@@ -46,6 +51,7 @@ import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -110,17 +116,21 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
}
/** prune */
- public <K extends Comparable<K>> List<K> prune() {
+ public <K extends Comparable<K>> Pair<List<K>, Boolean> prune() {
Builder<K> scanPartitionIdents = ImmutableList.builder();
+ boolean canPredicatePruned = true;
for (OnePartitionEvaluator partition : partitions) {
- if (!canBePrunedOut(partitionPredicate, partition)) {
+ Pair<Boolean, Boolean> res = canBePrunedOut(partitionPredicate,
partition);
+ if (!res.first) {
+ canPredicatePruned = canPredicatePruned && res.second;
scanPartitionIdents.add((K) partition.getPartitionIdent());
}
}
- return scanPartitionIdents.build();
+ return Pair.of(scanPartitionIdents.build(), canPredicatePruned);
}
- public static <K extends Comparable<K>> List<K> prune(List<Slot>
partitionSlots, Expression partitionPredicate,
+ public static <K extends Comparable<K>> Pair<List<K>,
Optional<Expression>> prune(List<Slot> partitionSlots,
+ Expression partitionPredicate,
Map<K, PartitionItem> idToPartitions, CascadesContext
cascadesContext,
PartitionTableType partitionTableType) {
return prune(partitionSlots, partitionPredicate, idToPartitions,
@@ -130,7 +140,8 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
/**
* prune partition with `idToPartitions` as parameter.
*/
- public static <K extends Comparable<K>> List<K> prune(List<Slot>
partitionSlots, Expression partitionPredicate,
+ public static <K extends Comparable<K>> Pair<List<K>,
Optional<Expression>> prune(List<Slot> partitionSlots,
+ Expression partitionPredicate,
Map<K, PartitionItem> idToPartitions, CascadesContext
cascadesContext,
PartitionTableType partitionTableType,
Optional<SortedPartitionRanges<K>> sortedPartitionRanges) {
long startAt = System.currentTimeMillis();
@@ -146,14 +157,15 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
}
}
- private static <K extends Comparable<K>> List<K> pruneInternal(List<Slot>
partitionSlots,
+ private static <K extends Comparable<K>> Pair<List<K>,
Optional<Expression>> pruneInternal(
+ List<Slot> partitionSlots,
Expression partitionPredicate,
Map<K, PartitionItem> idToPartitions, CascadesContext
cascadesContext,
PartitionTableType partitionTableType,
Optional<SortedPartitionRanges<K>> sortedPartitionRanges) {
partitionPredicate = PartitionPruneExpressionExtractor.extract(
partitionPredicate, ImmutableSet.copyOf(partitionSlots),
cascadesContext);
+ Expression originalPartitionPredicate = partitionPredicate;
partitionPredicate =
PredicateRewriteForPartitionPrune.rewrite(partitionPredicate, cascadesContext);
-
int expandThreshold = cascadesContext.getAndCacheSessionVariable(
"partitionPruningExpandThreshold",
10, sessionVariable ->
sessionVariable.partitionPruningExpandThreshold);
@@ -161,25 +173,37 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
partitionPredicate = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(
partitionPredicate, new
ExpressionRewriteContext(cascadesContext));
if (BooleanLiteral.TRUE.equals(partitionPredicate)) {
- return Utils.fastToImmutableList(idToPartitions.keySet());
+ // The partition column predicate is always true and can be
deleted, the partition cannot be pruned
+ return Pair.of(Utils.fastToImmutableList(idToPartitions.keySet()),
Optional.of(originalPartitionPredicate));
} else if (BooleanLiteral.FALSE.equals(partitionPredicate) ||
partitionPredicate.isNullLiteral()) {
- return ImmutableList.of();
+ // The partition column predicate is always false, and all
partitions can be pruned.
+ return Pair.of(ImmutableList.of(), Optional.empty());
}
if (sortedPartitionRanges.isPresent()) {
RangeSet<MultiColumnBound> predicateRanges =
partitionPredicate.accept(
new PartitionPredicateToRange(partitionSlots), null);
if (predicateRanges != null) {
- return binarySearchFiltering(
+ Pair<List<K>, Boolean> res = binarySearchFiltering(
sortedPartitionRanges.get(), partitionSlots,
partitionPredicate, cascadesContext,
expandThreshold, predicateRanges
);
+ if (res.second) {
+ return Pair.of(res.first,
Optional.of(originalPartitionPredicate));
+ } else {
+ return Pair.of(res.first, Optional.empty());
+ }
}
}
- return sequentialFiltering(
+ Pair<List<K>, Boolean> res = sequentialFiltering(
idToPartitions, partitionSlots, partitionPredicate,
cascadesContext, expandThreshold
);
+ if (res.second) {
+ return Pair.of(res.first, Optional.of(originalPartitionPredicate));
+ } else {
+ return Pair.of(res.first, Optional.empty());
+ }
}
/**
@@ -198,7 +222,7 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
}
}
- private static <K extends Comparable<K>> List<K> binarySearchFiltering(
+ private static <K extends Comparable<K>> Pair<List<K>, Boolean>
binarySearchFiltering(
SortedPartitionRanges<K> sortedPartitionRanges, List<Slot>
partitionSlots,
Expression partitionPredicate, CascadesContext cascadesContext,
int expandThreshold,
RangeSet<MultiColumnBound> predicateRanges) {
@@ -206,6 +230,7 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
Set<K> selectedIdSets = Sets.newTreeSet();
int leftIndex = 0;
+ boolean canPredicatePruned = true;
for (Range<MultiColumnBound> predicateRange :
predicateRanges.asRanges()) {
int rightIndex = sortedPartitions.size();
if (leftIndex >= rightIndex) {
@@ -246,8 +271,10 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
OnePartitionEvaluator<K> partitionEvaluator =
toPartitionEvaluator(
partitionId, partition.partitionItem, partitionSlots,
cascadesContext, expandThreshold);
- if (!canBePrunedOut(partitionPredicate, partitionEvaluator)) {
+ Pair<Boolean, Boolean> res =
canBePrunedOut(partitionPredicate, partitionEvaluator);
+ if (!res.first) {
selectedIdSets.add(partitionId);
+ canPredicatePruned = canPredicatePruned && res.second;
}
}
}
@@ -256,15 +283,17 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
K partitionId = defaultPartition.id;
OnePartitionEvaluator<K> partitionEvaluator = toPartitionEvaluator(
partitionId, defaultPartition.partitionItem,
partitionSlots, cascadesContext, expandThreshold);
- if (!canBePrunedOut(partitionPredicate, partitionEvaluator)) {
+ Pair<Boolean, Boolean> res = canBePrunedOut(partitionPredicate,
partitionEvaluator);
+ if (!res.first) {
selectedIdSets.add(partitionId);
+ canPredicatePruned = canPredicatePruned && res.second;
}
}
- return Utils.fastToImmutableList(selectedIdSets);
+ return Pair.of(Utils.fastToImmutableList(selectedIdSets),
canPredicatePruned);
}
- private static <K extends Comparable<K>> List<K> sequentialFiltering(
+ private static <K extends Comparable<K>> Pair<List<K>, Boolean>
sequentialFiltering(
Map<K, PartitionItem> idToPartitions, List<Slot> partitionSlots,
Expression partitionPredicate, CascadesContext cascadesContext,
int expandThreshold) {
List<OnePartitionEvaluator<?>> evaluators =
Lists.newArrayListWithCapacity(idToPartitions.size());
@@ -278,18 +307,70 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
}
/**
- * return true if partition is not qualified. that is, can be pruned out.
+ * return Pair
+ * pair.first is true if partition can be pruned
+ * pair.second is true if partitionPredicate is always true in this
partition
*/
- private static <K> boolean canBePrunedOut(Expression partitionPredicate,
OnePartitionEvaluator<K> evaluator) {
+ private static <K> Pair<Boolean, Boolean> canBePrunedOut(Expression
partitionPredicate,
+ OnePartitionEvaluator<K> evaluator) {
List<Map<Slot, PartitionSlotInput>> onePartitionInputs =
evaluator.getOnePartitionInputs();
- for (Map<Slot, PartitionSlotInput> currentInputs : onePartitionInputs)
{
- // evaluate whether there's possible for this partition to accept
this predicate
- Expression result =
evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
- if (!result.equals(BooleanLiteral.FALSE) && !(result instanceof
NullLiteral)) {
- return false;
+ if (evaluator instanceof OneListPartitionEvaluator) {
+ // if a table has default partition, the predicate should not be
pruned,
+ // because evaluateWithDefaultPartition always return true in
default partition
+ // e.g. PARTITION BY LIST(k1) (
+ // PARTITION p1 VALUES IN ("1","2","3","4"),
+ // PARTITION p2 VALUES IN ("5","6","7","8"),
+ // PARTITION p3 ) p3 is default partition
+ boolean notDefaultPartition = !evaluator.isDefaultPartition();
+ Pair<Boolean, Boolean> res = Pair.of(notDefaultPartition,
notDefaultPartition);
+ for (Map<Slot, PartitionSlotInput> currentInputs :
onePartitionInputs) {
+ // evaluate whether there's possible for this partition to
accept this predicate
+ Expression result =
evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
+ if (result.equals(BooleanLiteral.FALSE) || (result instanceof
NullLiteral)) {
+ // Indicates that there is a partition value that does not
satisfy the predicate
+ res.second = false;
+ } else if (result.equals(BooleanLiteral.TRUE)) {
+ // Indicates that there is a partition value that
satisfies the predicate
+ res.first = false;
+ } else {
+ // Indicates that this partition value may or may not
satisfy the predicate
+ res.second = false;
+ res.first = false;
+ }
+ if (!res.first && !res.second) {
+ break;
+ }
}
+ return res;
+ } else {
+ // only prune partition predicates in list partition, therefore
set pair.second always be false,
+ // meaning not to prune partition predicates in range partition
+ for (Map<Slot, PartitionSlotInput> currentInputs :
onePartitionInputs) {
+ Expression result =
evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
+ if (!result.equals(BooleanLiteral.FALSE) && !(result
instanceof NullLiteral)) {
+ return Pair.of(false, false);
+ }
+ }
+ // only have false result: Can be pruned out. have other exprs:
CanNot be pruned out
+ return Pair.of(true, false);
+ }
+ }
+
+ /** remove predicates that are always true*/
+ public static Plan prunePredicate(boolean skipPrunePredicate,
Optional<Expression> prunedPredicates,
+ LogicalFilter<? extends Plan> filter, LogicalRelation scan) {
+ if (!skipPrunePredicate && prunedPredicates.isPresent()) {
+ Set<Expression> conjuncts = new
LinkedHashSet<>(filter.getConjuncts());
+ Expression deletedPredicate = prunedPredicates.get();
+ Set<Expression> deletedPredicateSet =
ExpressionUtils.extractConjunctionToSet(deletedPredicate);
+ conjuncts.removeAll(deletedPredicateSet);
+ if (conjuncts.isEmpty()) {
+ return scan;
+ } else {
+ return filter.withConjunctsAndChild(conjuncts, scan);
+ }
+ } else {
+ return filter.withChildren(ImmutableList.of(scan));
}
- // only have false result: Can be pruned out. have other exprs: CanNot
be pruned out
- return true;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index 0fb8e68c486..456dde5c5bc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.SupportBinarySearchFilteringPartitions;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.nereids.CascadesContext;
@@ -28,6 +29,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
@@ -100,10 +102,10 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
sortedPartitionRanges = (Optional) partitionsCacheManager.get(
(SupportBinarySearchFilteringPartitions)
externalTable, scan);
}
-
- List<String> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
+ Pair<List<String>, Optional<Expression>> res = PartitionPruner.prune(
partitionSlots, filter.getPredicate(), nameToPartitionItem,
ctx,
- PartitionTableType.EXTERNAL, sortedPartitionRanges));
+ PartitionTableType.EXTERNAL, sortedPartitionRanges);
+ List<String> prunedPartitions = new ArrayList<>(res.first);
for (String name : prunedPartitions) {
selectedPartitionItems.put(name, nameToPartitionItem.get(name));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
index 023a17bec5a..937f7ed5d50 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager;
import org.apache.doris.nereids.pattern.MatchingContext;
import org.apache.doris.nereids.rules.Rule;
@@ -32,6 +33,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -70,7 +72,7 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
// Case1: sql without filter condition, e.g. SELECT *
FROM tbl (${tabletID})
LogicalOlapScan scan = ctx.root;
OlapTable table = scan.getTable();
- return prunePartition(scan, table, null, ctx);
+ return prunePartition(scan, table, null, ctx).first;
}).toRule(RuleType.OLAP_SCAN_PARTITION_PRUNE),
logicalFilter(logicalOlapScan()
.whenNot(LogicalOlapScan::isPartitionPruned))
@@ -79,44 +81,49 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
LogicalFilter<LogicalOlapScan> filter = ctx.root;
LogicalOlapScan scan = filter.child();
OlapTable table = scan.getTable();
- LogicalRelation rewrittenLogicalRelation =
prunePartition(scan, table, filter, ctx);
+ Pair<LogicalRelation, Optional<Expression>> prunedRes
+ = prunePartition(scan, table, filter, ctx);
+ LogicalRelation rewrittenLogicalRelation =
prunedRes.first;
if (rewrittenLogicalRelation == null) {
return null;
}
if (rewrittenLogicalRelation instanceof
LogicalEmptyRelation) {
return rewrittenLogicalRelation;
} else {
- LogicalOlapScan rewrittenScan = (LogicalOlapScan)
rewrittenLogicalRelation;
- return
filter.withChildren(ImmutableList.of(rewrittenScan));
+ return PartitionPruner.prunePredicate(
+
ctx.connectContext.getSessionVariable().skipPrunePredicate
+ ||
ctx.statementContext.isSkipPrunePredicate(),
+ prunedRes.second, filter,
rewrittenLogicalRelation);
}
}).toRule(RuleType.OLAP_SCAN_PARTITION_PRUNE)
);
}
- private LogicalRelation prunePartition(LogicalOlapScan scan,
+ private Pair<LogicalRelation, Optional<Expression>>
prunePartition(LogicalOlapScan scan,
OlapTable table,
LogicalFilter filter,
MatchingContext ctx) {
- List<Long> prunedPartitionsByFilters = prunePartitionByFilters(scan,
table, filter, ctx);
- List<Long> prunedPartitions = prunePartitionByTabletIds(scan, table,
prunedPartitionsByFilters);
+ Pair<List<Long>, Optional<Expression>> prunedPartitionsByFilters =
+ prunePartitionByFilters(scan, table, filter, ctx);
+ List<Long> prunedPartitions = prunePartitionByTabletIds(scan, table,
prunedPartitionsByFilters.first);
if (prunedPartitions == null) {
- return null;
+ return Pair.of(null, Optional.empty());
}
if (prunedPartitions.isEmpty()) {
- return new LogicalEmptyRelation(
+ return Pair.of(new LogicalEmptyRelation(
ConnectContext.get().getStatementContext().getNextRelationId(),
- ctx.root.getOutput());
+ ctx.root.getOutput()), Optional.empty());
}
- return scan.withSelectedPartitionIds(prunedPartitions);
+ return Pair.of(scan.withSelectedPartitionIds(prunedPartitions),
prunedPartitionsByFilters.second);
}
- private List<Long> prunePartitionByFilters(LogicalOlapScan scan,
+ private Pair<List<Long>, Optional<Expression>>
prunePartitionByFilters(LogicalOlapScan scan,
OlapTable table,
LogicalFilter filter,
MatchingContext ctx) {
Set<String> partitionColumnNameSet =
Utils.execWithReturnVal(table::getPartitionColumnNames);
if (partitionColumnNameSet.isEmpty()) {
- return null;
+ return Pair.of(null, Optional.empty());
}
List<Slot> output = scan.getOutput();
PartitionInfo partitionInfo = table.getPartitionInfo();
@@ -132,7 +139,7 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
}
}
if (partitionSlot == null) {
- return null;
+ return Pair.of(null, Optional.empty());
} else {
partitionSlots.add(partitionSlot);
}
@@ -156,14 +163,14 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
.collect(Collectors.toMap(Function.identity(),
allPartitions::get));
}
if (filter != null) {
- List<Long> prunedPartitions = PartitionPruner.prune(
+ Pair<List<Long>, Optional<Expression>> prunedPartitions =
PartitionPruner.prune(
partitionSlots, filter.getPredicate(), idToPartitions,
ctx.cascadesContext,
PartitionTableType.OLAP, sortedPartitionRanges);
return prunedPartitions;
} else if (!manuallySpecifiedPartitions.isEmpty()) {
- return Utils.fastToImmutableList(idToPartitions.keySet());
+ return Pair.of(Utils.fastToImmutableList(idToPartitions.keySet()),
Optional.empty());
} else {
- return null;
+ return Pair.of(null, Optional.empty());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index 944378ed528..bc2bba1b325 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -129,7 +129,10 @@ public class DeleteFromCommand extends Command implements
ForwardWithSync, Expla
public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
LogicalPlanAdapter logicalPlanAdapter = new
LogicalPlanAdapter(logicalQuery, ctx.getStatementContext());
updateSessionVariableForDelete(ctx.getSessionVariable());
- NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
+ StatementContext statementContext = ctx.getStatementContext();
+ // delete not prune predicate after partition prune
+ statementContext.setSkipPrunePredicate(true);
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
boolean originalIsSkipAuth = ctx.isSkipAuth();
// delete not need select priv
ctx.setSkipAuth(true);
@@ -293,7 +296,7 @@ public class DeleteFromCommand extends Command implements
ForwardWithSync, Expla
List<Long> prunedPartitions = PartitionPruner.prune(
partitionSlots, filter.getPredicate(), idToPartitions,
CascadesContext.initContext(new StatementContext(), this,
PhysicalProperties.ANY),
- PartitionTableType.OLAP, sortedPartitionRanges);
+ PartitionTableType.OLAP, sortedPartitionRanges).first;
return
prunedPartitions.stream().map(olapTable::getPartition).collect(Collectors.toList());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index c13b284e2b4..834007ad81d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -87,6 +87,9 @@ public class ExplainCommand extends Command implements
NoForward {
|| explainable instanceof UpdateCommand) {
ctx.getStatementContext().setIsInsert(true);
}
+ if (explainable instanceof DeleteFromCommand) {
+ ctx.getStatementContext().setSkipPrunePredicate(true);
+ }
explainPlan = ((LogicalPlan) explainable.getExplainPlan(ctx));
NereidsPlanner planner = explainable.getExplainPlanner(explainPlan,
ctx.getStatementContext()).orElseGet(() ->
new NereidsPlanner(ctx.getStatementContext())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 2ed41fdde85..13c1543f7c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -185,6 +185,7 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT =
"enable_rewrite_element_at_to_slot";
public static final String ENABLE_ODBC_TRANSCATION =
"enable_odbc_transcation";
public static final String ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS =
"enable_binary_search_filtering_partitions";
+ public static final String SKIP_PRUNE_PREDICATE = "skip_prune_predicate";
public static final String ENABLE_SQL_CACHE = "enable_sql_cache";
public static final String ENABLE_HIVE_SQL_CACHE = "enable_hive_sql_cache";
public static final String ENABLE_QUERY_CACHE = "enable_query_cache";
@@ -1287,6 +1288,16 @@ public class SessionVariable implements Serializable,
Writable {
)
public boolean enableBinarySearchFilteringPartitions = true;
+ @VariableMgr.VarAttr(name = SKIP_PRUNE_PREDICATE, fuzzy = true,
+ description = {
+ "是否跳过“在分区裁剪后删除恒真谓词”的优化。默认为OFF（即执行此优化）。",
+ "Skips the removal of always-true predicates after
partition pruning. "
+ + "Defaults to OFF (optimization is active)."
+ }
+ )
+ public boolean skipPrunePredicate = false;
+
+
@VariableMgr.VarAttr(name = ENABLE_SQL_CACHE, fuzzy = true)
public boolean enableSqlCache = true;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
index 7508890be96..44c9cd6b86a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
@@ -173,10 +173,10 @@ public class OptimizeGetAvailableMvsTest extends
SqlTestBase {
new MockUp<PartitionPruner>() {
@Mock
- public <K extends Comparable<K>> List<Long> prune(List<Slot>
partitionSlots, Expression partitionPredicate,
+ public <K extends Comparable<K>> Pair<List<K>,
Optional<Expression>> prune(List<Slot> partitionSlots, Expression
partitionPredicate,
Map<K, PartitionItem> idToPartitions, CascadesContext
cascadesContext,
PartitionTableType partitionTableType,
Optional<SortedPartitionRanges<K>> sortedPartitionRanges) {
- return Lists.newArrayList(1L);
+ return Pair.of((List) Lists.newArrayList(1L),
Optional.empty());
}
};
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
new file mode 100644
index 00000000000..0378cc86dba
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
@@ -0,0 +1,570 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
+import
org.apache.doris.nereids.rules.expression.rules.OneListPartitionEvaluator;
+import org.apache.doris.nereids.rules.expression.rules.OnePartitionEvaluator;
+import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+public class PartitionPrunerTest extends TestWithFeService {
+ private Method canBePrunedOutMethod;
+ private final Column partitionColumn = new Column("a", PrimitiveType.INT);
+ private CascadesContext cascadesContext;
+ private final SlotReference slotA = new SlotReference("a",
IntegerType.INSTANCE);
+ private final SlotReference slotB = new SlotReference("b",
IntegerType.INSTANCE);
+ private final SlotReference slotC = new SlotReference("c",
IntegerType.INSTANCE);
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ Class<?> clazz = PartitionPruner.class;
+ canBePrunedOutMethod = clazz.getDeclaredMethod("canBePrunedOut",
Expression.class, OnePartitionEvaluator.class);
+ canBePrunedOutMethod.setAccessible(true);
+ cascadesContext = createCascadesContext("select * from t1");
+ }
+
+ // test canBePrunedOut res
+ // list partition p1, partition value is 1
+ // predicate: a = 1
+ @Test
+ public void testEqualPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue = new PartitionValue("1");
+ PartitionKey partitionKey =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+ Expression predicate = new EqualTo(slotA, Literal.of(1));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertTrue(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2
+ // predicate: a = 1
+ @Test
+ public void testEqualPredicate2()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
+ ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
+ ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+ Expression predicate = new EqualTo(slotA, Literal.of(1));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1
+ // predicate: a = 2
+ @Test
+ public void testEqualPredicate3()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue = new PartitionValue("1");
+ PartitionKey partitionKey =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+ Expression predicate = new EqualTo(slotA, Literal.of(2));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertTrue(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1
+ // predicate: a = NULL
+ @Test
+ public void testNullPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue = new PartitionValue("1");
+ PartitionKey partitionKey =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new EqualTo(slotA, NullLiteral.INSTANCE);
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertTrue(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2, 3
+ // predicate: a IN (1, 2)
+ @Test
+ public void testInPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionValue partitionValue3 = new PartitionValue("3");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey3 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue3),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2, partitionKey3);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new InPredicate(slotA,
ImmutableList.of(Literal.of(1), Literal.of(2)));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2
+ // predicate: a IN (1, 2)
+ @Test
+ public void testInPredicateExactMatch()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new InPredicate(slotA,
ImmutableList.of(Literal.of(1), Literal.of(2)));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertTrue(result.second);
+ }
+
+
+ // list partition p1, partition value (1, "a"), (2, "b")
+ // predicate: a = 1 AND b = "a"
+ @Test
+ public void testMultiColumnPartition()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ Column partitionColumn2 = new Column("b", PrimitiveType.VARCHAR);
+ SlotReference slot2 = new SlotReference("b",
VarcharType.createVarcharType(10));
+
+ PartitionValue partitionValue1a = new PartitionValue("1");
+ PartitionValue partitionValue1b = new PartitionValue("a");
+ PartitionValue partitionValue2a = new PartitionValue("2");
+ PartitionValue partitionValue2b = new PartitionValue("b");
+
+ PartitionKey partitionKey1 = PartitionKey.createPartitionKey(
+ ImmutableList.of(partitionValue1a, partitionValue1b),
+ ImmutableList.of(partitionColumn, partitionColumn2));
+ PartitionKey partitionKey2 = PartitionKey.createPartitionKey(
+ ImmutableList.of(partitionValue2a, partitionValue2b),
+ ImmutableList.of(partitionColumn, partitionColumn2));
+
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA, slot2), partitionItem,
cascadesContext);
+
+ Expression predicate = new And(
+ new EqualTo(slotA, Literal.of(1)),
+ new EqualTo(slot2, Literal.of("a"))
+ );
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2
+ // predicate: a = 1 OR a = 3
+ @Test
+ public void testOrPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new Or(
+ new EqualTo(slotA, Literal.of(1)),
+ new EqualTo(slotA, Literal.of(3))
+ );
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1
+ // predicate: NOT (a = 1)
+ @Test
+ public void testNotPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue = new PartitionValue("1");
+ PartitionKey partitionKey =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new Not(new EqualTo(slotA, Literal.of(1)));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertTrue(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2, 3
+ // predicate: a > 2
+ @Test
+ public void testGreaterThanPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionValue partitionValue3 = new PartitionValue("3");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey3 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue3),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2, partitionKey3);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new GreaterThan(slotA, Literal.of(2));
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // list partition p1, partition value is 1, 2, 3
+ // predicate: (a = 1 OR a = 2) AND a > 0
+ @Test
+ public void testComplexNestedPredicate()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ PartitionValue partitionValue1 = new PartitionValue("1");
+ PartitionValue partitionValue2 = new PartitionValue("2");
+ PartitionValue partitionValue3 = new PartitionValue("3");
+ PartitionKey partitionKey1 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue1),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey2 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue2),
ImmutableList.of(partitionColumn));
+ PartitionKey partitionKey3 =
PartitionKey.createPartitionKey(ImmutableList.of(partitionValue3),
ImmutableList.of(partitionColumn));
+ List<PartitionKey> partitionKeys = ImmutableList.of(partitionKey1,
partitionKey2, partitionKey3);
+ ListPartitionItem partitionItem = new ListPartitionItem(partitionKeys);
+ OneListPartitionEvaluator<String> partitionEvaluator = new
OneListPartitionEvaluator<>(
+ "p1", ImmutableList.of(slotA), partitionItem, cascadesContext);
+
+ Expression predicate = new And(
+ new Or(
+ new EqualTo(slotA, Literal.of(1)),
+ new EqualTo(slotA, Literal.of(2))
+ ),
+ new GreaterThan(slotA, Literal.of(0))
+ );
+ Pair<Boolean, Boolean> result = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(null, predicate, partitionEvaluator);
+ Assertions.assertFalse(result.first);
+ Assertions.assertFalse(result.second);
+ }
+
+ // test prune predicate
+ // Test basis: some predicates are pruned
+ @Test
+ public void testPrunePartialPredicates() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ LessThan lt = new LessThan(slotB, Literal.of(20));
+ predicates.add(gt);
+ predicates.add(lt);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
+ Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
+ }
+
+ // all predicates are pruned
+ @Test
+ public void testPruneAllPredicates() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ predicates.add(gt);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
+ }
+
+ // no predicates are pruned
+ @Test
+ public void testPruneNoPredicates() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ LessThan lt = new LessThan(slotB, Literal.of(20));
+ predicates.add(gt);
+ predicates.add(lt);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ EqualTo nonExistentPredicate = new EqualTo(slotC, Literal.of(30));
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(nonExistentPredicate), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
+ }
+
+ @Test
+ public void testPruneCompoundPredicate() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ LessThan lt = new LessThan(slotB, Literal.of(20));
+ EqualTo eq = new EqualTo(slotC, Literal.of(30));
+ predicates.add(gt);
+ predicates.add(lt);
+ predicates.add(eq);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ // (a > 10 AND b < 20)
+ And compoundPredicate = new And(gt, lt);
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq));
+ Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
+ Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
+ }
+
+ @Test
+ public void testSkipPrunePredicate() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ predicates.add(gt);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(true,
Optional.of(gt), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
+ }
+
+ @Test
+ public void testEmptyPrunedPredicates() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ predicates.add(gt);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ // prunedPredicates is empty
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.empty(), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
+ }
+
+ @Test
+ public void testPruneDuplicatePredicates() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt1 = new GreaterThan(slotA, Literal.of(10));
+ GreaterThan gt2 = new GreaterThan(slotA, Literal.of(10)); //
duplicated predicate
+ predicates.add(gt1);
+ predicates.add(gt2);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt1), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
+ }
+
+ @Test
+ public void testPruneWithNullLiteral() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ EqualTo nullEq = new EqualTo(slotB, new NullLiteral());
+ predicates.add(gt);
+ predicates.add(nullEq);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(nullEq));
+ }
+
+ @Test
+ public void testPruneMultiplePredicatesPartially() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ LessThan lt = new LessThan(slotB, Literal.of(20));
+ EqualTo eq1 = new EqualTo(slotC, Literal.of(30));
+ EqualTo eq2 = new EqualTo(slotC, Literal.of(40));
+ predicates.add(gt);
+ predicates.add(lt);
+ predicates.add(eq1);
+ predicates.add(eq2);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ // (a > 10 AND b < 20)
+ And compoundPredicate = new And(gt, lt);
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq1));
+ Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq2));
+ Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
+ Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
+ }
+
+ @Test
+ public void testPruneNestedCompoundPredicate() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ LessThan lt = new LessThan(slotB, Literal.of(20));
+ EqualTo eq = new EqualTo(slotC, Literal.of(30));
+ predicates.add(gt);
+ predicates.add(lt);
+ predicates.add(eq);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ // (a > 10 AND (b < 20 AND c = 30))
+ And innerAnd = new And(lt, eq);
+ And outerAnd = new And(gt, innerAnd);
+
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(outerAnd), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
+ }
+
+ @Test
+ public void testPruneWhenFilterContainsOr() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ Or orPredicate = new Or(
+ new GreaterThan(slotA, Literal.of(10)),
+ new LessThan(slotB, Literal.of(20))
+ );
+ predicates.add(orPredicate);
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
+
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
+ }
+
+ @Test
+ public void testPruneWhenFilterContainsAndOrMix() {
+ Set<Expression> predicates = new LinkedHashSet<>();
+ // filter :a > 10 AND (b < 20 OR c = 30)
+ Or orPredicate = new Or(
+ new LessThan(slotB, Literal.of(20)),
+ new EqualTo(slotC, Literal.of(30))
+ );
+ GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
+
+ predicates.add(gt);
+ predicates.add(orPredicate);
+
+ LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
+ LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
+ // a > 10
+ Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
+ Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
+ LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
+
+ Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
+
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
+ }
+}
+
+
diff --git
a/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out
b/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out
index 30e82ec957c..aa498a3bf47 100644
---
a/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out
+++
b/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out
@@ -32,7 +32,7 @@ PhysicalResultSink
-- !should_infer_because_d_is_partition_column --
PhysicalResultSink
---filter((t1.c < 10) and (t1.d < 10) and (t1.d < t1.c))
+--filter((t1.c < 10) and (t1.d < t1.c))
----PhysicalOlapScan[infer_unequal_predicates_t1]
-- !infer_with_equal --
@@ -53,17 +53,17 @@ PhysicalResultSink
-- !infer_long_chain_same_table_infer_a_and_d --
PhysicalResultSink
---filter((t1.a < 10) and (t1.a < t1.d) and (t1.c < 10) and (t1.d < 10) and
(t1.d < t1.c))
+--filter((t1.a < 10) and (t1.a < t1.d) and (t1.c < 10) and (t1.d < t1.c))
----PhysicalOlapScan[infer_unequal_predicates_t1]
-- !infer_long_chain_same_table_not_infer_c --
PhysicalResultSink
---filter((t1.a < 10) and (t1.a < t1.c) and (t1.c < t1.d) and (t1.d < 10))
+--filter((t1.a < 10) and (t1.a < t1.c) and (t1.c < t1.d))
----PhysicalOlapScan[infer_unequal_predicates_t1]
-- !remove_useless_input_predicate_c_less_than_10 --
PhysicalResultSink
---filter((t1.a < 10) and (t1.a < t1.c) and (t1.c < t1.d) and (t1.d < 10))
+--filter((t1.a < 10) and (t1.a < t1.c) and (t1.c < t1.d))
----PhysicalOlapScan[infer_unequal_predicates_t1]
-- !remove_useless_predicate --
@@ -78,7 +78,7 @@ PhysicalResultSink
--NestedLoopJoin[INNER_JOIN](t1.a < t2.d)
----filter((t1.a < 10))
------PhysicalOlapScan[infer_unequal_predicates_t1]
-----filter((t2.c < 10) and (t2.d < 10) and (t2.d < t2.c))
+----filter((t2.c < 10) and (t2.d < t2.c))
------PhysicalOlapScan[infer_unequal_predicates_t2]
-- !infer_with_constant_and_columns --
@@ -86,7 +86,7 @@ PhysicalResultSink
--hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.c)) otherCondition=()
----filter((t1.a > 1))
------PhysicalOlapScan[infer_unequal_predicates_t1]
-----filter((t2.c < t2.d) and (t2.c > 1) and (t2.d > 1))
+----filter((t2.c < t2.d) and (t2.c > 1))
------PhysicalOlapScan[infer_unequal_predicates_t2]
-- !no_infer --
@@ -107,7 +107,7 @@ PhysicalResultSink
--NestedLoopJoin[INNER_JOIN](t1.a < t2.a)
----filter((t1.a < 10))
------PhysicalOlapScan[infer_unequal_predicates_t1]
-----filter((t2.a < 10) and (t2.a < t2.c) and (t2.c < t2.d) and (t2.d < 10))
+----filter((t2.a < 10) and (t2.a < t2.c) and (t2.c < t2.d))
------PhysicalOlapScan[infer_unequal_predicates_t2]
-- !infer_cast_int --
@@ -126,10 +126,9 @@ PhysicalResultSink
-- !no_redundant_predicates --
PhysicalResultSink
--hashJoin[INNER_JOIN] hashCondition=((t1.d = t2.d)) otherCondition=()
-----filter((t1.c > 1) and (t1.d < 10) and (t1.d = t1.c) and (t1.d > 1))
+----filter((t1.c > 1) and (t1.d = t1.c))
------PhysicalOlapScan[infer_unequal_predicates_t1]
-----filter((t2.d < 10) and (t2.d > 1))
-------PhysicalOlapScan[infer_unequal_predicates_t2]
+----PhysicalOlapScan[infer_unequal_predicates_t2]
-- !expr_unequal_infer_same_table1 --
PhysicalResultSink
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/list_prune_predicate.out
b/regression-test/data/nereids_rules_p0/partition_prune/list_prune_predicate.out
new file mode 100644
index 00000000000..fd6fe7e046c
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/list_prune_predicate.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !default_partition --
+10 1 1 1 24453.325 1.0 1.0
+
+-- !update --
+1 2025-01-01 10
+2 2025-01-02 1
+3 2025-01-03 1
+4 2025-01-04 1
+5 2025-01-05 1
+6 2025-01-06 1
+7 2025-01-07 1
+8 2025-01-08 1
+9 2025-01-09 1
+10 2025-01-10 1
+
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
index ebf4d35d3a1..452e5224d16 100644
---
a/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
@@ -68,7 +68,7 @@ suite("hive_partition_prune") {
}
explain {
sql "SELECT * FROM test_hive_partition WHERE p not in (15,6,1,
'2021-01-02 00:00:00')"
- contains("partition=4/6")
+ contains("partition=0/6")
}
explain {
sql "SELECT * FROM test_hive_partition WHERE p not in (1, 5,6,null)"
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/list_prune_predicate.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/list_prune_predicate.groovy
new file mode 100644
index 00000000000..b44dd8aa873
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/list_prune_predicate.groovy
@@ -0,0 +1,367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("list_prune_predicate") {
+ multi_sql """
+ drop table if exists test_list_date;
+ create table test_list_date (id int, d_date date)
+ PARTITION BY LIST(d_date)
+ (
+ PARTITION `p20250101` VALUES IN ('2025-01-01'),
+ PARTITION `p20250102` VALUES IN ('2025-01-02'),
+ PARTITION `p20250103` VALUES IN ('2025-01-03'),
+ PARTITION `p20250104` VALUES IN ('2025-01-04'),
+ PARTITION `p20250105` VALUES IN ('2025-01-05'),
+ PARTITION `p20250106` VALUES IN ('2025-01-06'),
+ PARTITION `p20250107` VALUES IN ('2025-01-07'),
+ PARTITION `p20250108` VALUES IN ('2025-01-08'),
+ PARTITION `p20250109` VALUES IN ('2025-01-09'),
+ PARTITION `p20250110` VALUES IN ('2025-01-10')
+ )
+ properties('replication_num'='1');
+
+ insert into test_list_date values
+ (1,'2025-01-01') , (2, '2025-01-02') , (3, '2025-01-03'), (4,
'2025-01-04') , (5, '2025-01-05'), (6,'2025-01-06') ,
+ (7, '2025-01-07') , (8, '2025-01-08'), (9, '2025-01-09') , (10,
'2025-01-10'); """
+
+ // =
+ explain {
+ sql "select * from test_list_date where d_date='2025-01-07';"
+ contains("partitions=1/10 (p20250107)")
+ notContains("PREDICATES")
+ }
+ // >
+ explain {
+ sql "select * from test_list_date where d_date>'2025-01-07';"
+ contains("partitions=3/10 (p20250108,p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+
+ // <
+ explain {
+ sql "select * from test_list_date where d_date<'2025-01-07';"
+ contains("partitions=6/10
(p20250101,p20250102,p20250103,p20250104,p20250105,p20250106)")
+ notContains("PREDICATES")
+ }
+ // >=
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-07';"
+ contains("partitions=4/10 (p20250107,p20250108,p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+ // <=
+ explain {
+ sql "select * from test_list_date where d_date<='2025-01-07';"
+ contains("partitions=7/10
(p20250101,p20250102,p20250103,p20250104,p20250105,p20250106,p20250107)")
+ notContains("PREDICATES")
+ }
+ // empty paritition
+ explain {
+ sql "select * from test_list_date where d_date<='2024-01-01'"
+ contains("VEMPTYSET")
+ notContains("PREDICATES")
+ }
+
+ // and
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-06' and
d_date<='2025-01-07';"
+ contains("partitions=2/10 (p20250106,p20250107)")
+ notContains("PREDICATES")
+ }
+ // or
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-08' or
d_date<='2025-01-03';"
+ contains("partitions=6/10
(p20250101,p20250102,p20250103,p20250108,p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+
+ // or
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-06' or
d_date<='2025-01-07';"
+ contains("partitions=10/10
(p20250101,p20250102,p20250103,p20250104,p20250105,p20250106,p20250107,p20250108,p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+
+ // have other predicates
+ // =
+ explain {
+ sql "select * from test_list_date where d_date='2025-01-07' and id >
8;"
+ contains("partitions=1/10 (p20250107)")
+ contains("PREDICATES: (id[#0] > 8)")
+ }
+ // >
+ explain {
+ sql "select * from test_list_date where d_date>'2025-01-07' and id >
8;"
+ contains("partitions=3/10 (p20250108,p20250109,p20250110)")
+ contains("PREDICATES: (id[#0] > 8)")
+ }
+ // and
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-06' and
d_date<='2025-01-07' and id <2;"
+ contains("partitions=2/10 (p20250106,p20250107)")
+ contains("PREDICATES: (id[#0] < 2)")
+ }
+ // or
+ explain {
+ sql "select * from test_list_date where (d_date>='2025-01-08' or
d_date<='2025-01-03') and id < 2;"
+ contains("partitions=6/10
(p20250101,p20250102,p20250103,p20250108,p20250109,p20250110)")
+ contains("PREDICATES: (id[#0] < 2)")
+ }
+
+ // or
+ explain {
+ sql "select * from test_list_date where d_date>='2025-01-08' or
(d_date<='2025-01-03' and id < 2);"
+ contains("partitions=6/10
(p20250101,p20250102,p20250103,p20250108,p20250109,p20250110)")
+ contains("PREDICATES: ((d_date[#1] >= '2025-01-08') OR ((d_date[#1] <=
'2025-01-03') AND (id[#0] < 2)))")
+ }
+ // or
+ explain {
+ sql "select * from test_list_date where (d_date>='2025-01-06' or
d_date<='2025-01-07') and id < 2;"
+ contains("PREDICATES: (id[#0] < 2)")
+ contains("partitions=10/10
(p20250101,p20250102,p20250103,p20250104,p20250105,p20250106,p20250107,p20250108,p20250109,p20250110)")
+ }
+
+ // not in
+ explain {
+ sql "select * from test_list_date where d_date not in ('2025-01-01',
'2025-01-02');"
+ contains("partitions=8/10
(p20250103,p20250104,p20250105,p20250106,p20250107,p20250108,p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+
+ // in
+ explain {
+ sql "select * from test_list_date where d_date in ('2025-01-01',
'2025-01-03', '2025-01-05');"
+ contains("partitions=3/10 (p20250101,p20250103,p20250105)")
+ notContains("PREDICATES")
+ }
+
+ // between
+ explain {
+ sql "select * from test_list_date where d_date between '2025-01-03'
and '2025-01-05';"
+ contains("partitions=3/10 (p20250103,p20250104,p20250105)")
+ notContains("PREDICATES")
+ }
+
+ // and or
+ explain {
+ sql "select * from test_list_date where (d_date >= '2025-01-03' and
d_date <= '2025-01-05') or (d_date >= '2025-01-08' and d_date <= '2025-01-09');"
+ contains("partitions=5/10
(p20250103,p20250104,p20250105,p20250108,p20250109)")
+ notContains("PREDICATES")
+ }
+
+ explain {
+ sql "select * from test_list_date where (d_date = '2025-01-05' or
(d_date = '2025-01-06' and id > 5)) and id < 10;"
+ contains("partitions=2/10 (p20250105,p20250106)")
+ }
+
+ explain {
+ sql "select * from test_list_date where ((d_date >= '2025-01-03' and
d_date <= '2025-01-05') or d_date = '2025-01-08') and (id = 3 or id = 4 or id =
8);"
+ contains("partitions=4/10 (p20250103,p20250104,p20250105,p20250108)")
+ contains("PREDICATES: id[#0] IN (3, 4, 8)")
+ }
+
+ explain {
+ sql "select * from test_list_date where (d_date = '2025-01-03' and id
> 2) or (d_date = '2025-01-07' and id < 8);"
+ contains("partitions=2/10 (p20250103,p20250107)")
+ }
+
+ explain {
+ sql "select * from test_list_date where (d_date in
('2025-01-02','2025-01-04') and id % 2 = 0) or (d_date = '2025-01-06' and id %
2 = 1);"
+ contains("partitions=3/10 (p20250102,p20250104,p20250106)")
+ }
+ // is null
+ explain {
+ sql "select * from test_list_date where d_date is null;"
+ contains("VEMPTYSET")
+ }
+
+ // func
+ explain {
+ sql "select * from test_list_date where year(d_date) = 2025;"
+ contains("partitions=10/10
(p20250101,p20250102,p20250103,p20250104,p20250105,p20250106,p20250107,p20250108,p20250109,p20250110)")
+ }
+
+ // test date(dt) predicate rewrite before partition prune
+ multi_sql """
+ drop table if exists test_list_datetime;
+ create table test_list_datetime (id int, d_date datetime)
+ PARTITION BY LIST(d_date)
+ (
+ PARTITION `p20250101` VALUES IN ('2025-01-01 00:00:00'),
+ PARTITION `p20250102` VALUES IN ('2025-01-02 00:00:00'),
+ PARTITION `p20250103` VALUES IN ('2025-01-03 00:00:00'),
+ PARTITION `p20250104` VALUES IN ('2025-01-04 00:00:00'),
+ PARTITION `p20250105` VALUES IN ('2025-01-05 00:00:00'),
+ PARTITION `p20250106` VALUES IN ('2025-01-06 00:00:00'),
+ PARTITION `p20250107` VALUES IN ('2025-01-07 00:00:00'),
+ PARTITION `p20250108` VALUES IN ('2025-01-08 00:00:00'),
+ PARTITION `p20250109` VALUES IN ('2025-01-09 00:00:00'),
+ PARTITION `p20250110` VALUES IN ('2025-01-10 00:00:00')
+ )
+ properties('replication_num'='1');
+ insert into test_list_datetime values
+ (1,'2025-01-01') , (2, '2025-01-02') , (3, '2025-01-03'), (4,
'2025-01-04') , (5, '2025-01-05'), (6,'2025-01-06') ,
+ (7, '2025-01-07') , (8, '2025-01-08'), (9, '2025-01-09') , (10,
'2025-01-10');"""
+
+ explain {
+ sql "select * from test_list_datetime where date(d_date) in
('2025-01-09','2025-01-10');"
+ contains("partitions=2/10 (p20250109,p20250110)")
+ notContains("PREDICATES")
+ }
+
+ // multi column partition
+ multi_sql """
+ drop table if exists test_list_multi_column;
+ create table test_list_multi_column (id int, d_date date)
+ PARTITION BY LIST(id,d_date)
+ (
+ PARTITION `p20250101` VALUES IN ((1,'2025-01-01')),
+ PARTITION `p20250102` VALUES IN ((2,'2025-01-02')),
+ PARTITION `p20250103` VALUES IN ((3,'2025-01-03')),
+ PARTITION `p20250104` VALUES IN ((4,'2025-01-04')),
+ PARTITION `p20250105` VALUES IN ((5,'2025-01-05')),
+ PARTITION `p20250106` VALUES IN ((6,'2025-01-06')),
+ PARTITION `p20250107` VALUES IN ((7,'2025-01-07')),
+ PARTITION `p20250108` VALUES IN ((8,'2025-01-08')),
+ PARTITION `p20250109` VALUES IN ((9,'2025-01-09')),
+ PARTITION `p20250110` VALUES IN ((10,'2025-01-10'))
+ )
+ properties('replication_num'='1');
+ insert into test_list_multi_column values
+ (1,'2025-01-01') , (2, '2025-01-02') , (3, '2025-01-03'), (4,
'2025-01-04') , (5, '2025-01-05'), (6,'2025-01-06') ,
+ (7, '2025-01-07') , (8, '2025-01-08'), (9, '2025-01-09') , (10,
'2025-01-10');
+ """
+ explain {
+ sql "select * from test_list_multi_column where id=1 and
d_date='2025-01-01';"
+ contains("partitions=1/10 (p20250101)")
+ notContains("PREDICATES")
+ }
+ explain {
+ sql "select * from test_list_multi_column where id=1 and
d_date='2025-01-02';"
+ contains("VEMPTYSET")
+ notContains("PREDICATES")
+ }
+ explain {
+ sql "select * from test_list_multi_column where (id=1 and
d_date='2025-01-01') or id=2;"
+ contains("partitions=2/10 (p20250101,p20250102)")
+ notContains("PREDICATES")
+ }
+
+ explain {
+ sql "select * from test_list_multi_column where (id = 1 and d_date =
'2025-01-01') or (id = 2 and d_date = '2025-01-02');"
+ contains("partitions=2/10 (p20250101,p20250102)")
+ notContains("PREDICATES")
+ }
+
+ explain {
+ sql "select * from test_list_multi_column where id = 1 and (d_date =
'2025-01-01' or d_date = '2025-01-02');"
+ contains("partitions=1/10 (p20250101)")
+ notContains("PREDICATES")
+ }
+
+ explain {
+ sql "select * from test_list_multi_column where (id = 1 or id = 2) and
d_date = '2025-01-01';"
+ contains("partitions=1/10 (p20250101)")
+ notContains("PREDICATES")
+ }
+
+ // test default partition
+ multi_sql """
+ drop table if exists list_par_data_migration;
+ CREATE TABLE IF NOT EXISTS list_par_data_migration (
+ k1 tinyint NOT NULL,
+ k2 smallint NOT NULL,
+ k3 int NOT NULL,
+ k4 bigint NOT NULL,
+ k5 decimal(9, 3) NOT NULL,
+ k8 double max NOT NULL,
+ k9 float sum NOT NULL )
+ AGGREGATE KEY(k1,k2,k3,k4,k5)
+ PARTITION BY LIST(k1) (
+ PARTITION p1 VALUES IN ("1","2","3","4"),
+ PARTITION p2 VALUES IN ("5","6","7","8"),
+ PARTITION p3 )
+ DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1");
+ """
+ sql """insert into list_par_data_migration values
(1,1,1,1,24453.325,1,1)"""
+ sql """insert into list_par_data_migration values
(10,1,1,1,24453.325,1,1)"""
+ sql """insert into list_par_data_migration values
(11,1,1,1,24453.325,1,1)"""
+ qt_default_partition """
+ select * from list_par_data_migration partition p3 where k1=10 order by
k1;"""
+ // test manuallySpecifiedPartitions
+ explain {
+ sql "select * from list_par_data_migration partition p2 where k1=2
order by k1;"
+ contains("VEMPTYSET")
+ }
+ explain {
+ sql "select * from list_par_data_migration partition p1 where k1=2
order by k1;"
+ contains("partitions=1/3 (p1)")
+ // k1=2 can not be deleted because k1=2 is not always true in p1
+ contains("PREDICATES: (k1[#0] = 2)")
+ }
+ explain {
+ sql "select * from list_par_data_migration partition p1 where k1=1 or
k1=2 or k1=3 or k1=4 order by k1;"
+ contains("partitions=1/3 (p1)")
+ notContains("PREDICATES")
+ }
+
+ // test variable skip_prune_predicate
+ sql "set skip_prune_predicate = true"
+ explain {
+ sql "select * from list_par_data_migration partition p1 where k1=1 or
k1=2 or k1=3 or k1=4 order by k1;"
+ contains("partitions=1/3 (p1)")
+ contains("PREDICATES")
+ }
+ sql "set skip_prune_predicate = false"
+
+ sql " drop table if exists test_list_multi_column_unique;"
+ sql """ create table test_list_multi_column_unique (id int, d_date date,
c02 int default 1) unique key (id, d_date)
+ PARTITION BY LIST(id,d_date)
+ (
+ PARTITION `p20250101` VALUES IN ((1,'2025-01-01')),
+ PARTITION `p20250102` VALUES IN ((2,'2025-01-02')),
+ PARTITION `p20250103` VALUES IN ((3,'2025-01-03')),
+ PARTITION `p20250104` VALUES IN ((4,'2025-01-04')),
+ PARTITION `p20250105` VALUES IN ((5,'2025-01-05')),
+ PARTITION `p20250106` VALUES IN ((6,'2025-01-06')),
+ PARTITION `p20250107` VALUES IN ((7,'2025-01-07')),
+ PARTITION `p20250108` VALUES IN ((8,'2025-01-08')),
+ PARTITION `p20250109` VALUES IN ((9,'2025-01-09')),
+ PARTITION `p20250110` VALUES IN ((10,'2025-01-10'))
+ ) DISTRIBUTED BY HASH(id) BUCKETS 10
+ properties('replication_num'='1');"""
+ sql """insert into test_list_multi_column_unique(id, d_date) values
+ (1,'2025-01-01') , (2, '2025-01-02') , (3, '2025-01-03'), (4,
'2025-01-04') , (5, '2025-01-05'), (6,'2025-01-06') ,
+ (7, '2025-01-07') , (8, '2025-01-08'), (9, '2025-01-09') , (10,
'2025-01-10');
+ """
+
+ // add test for delete(not support optimize)
+ explain {
+ sql "delete from test_list_multi_column_unique where id=1 and
d_date='2025-01-01';"
+ contains("partitions=1/10 (p20250101)")
+ contains("(id[#0] = 1) AND (d_date[#1] = '2025-01-01')")
+ }
+
+ // add test for update(support optimize)
+ explain {
+ sql "update test_list_multi_column_unique set c02 = 1 where id=1 and
d_date='2025-01-01';"
+ contains("partitions=1/10 (p20250101)")
+ notContains("(id[#0] = 1) AND (d_date[#1] = '2025-01-01')")
+ }
+ sql """update test_list_multi_column_unique set c02 = 10 where id=1 and
d_date='2025-01-01';"""
+ qt_update "select * from test_list_multi_column_unique order by id;"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
