This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fb27aebed63 [fix](partition_prune) Move the pruning of predicates that
are always true after partition pruning into the PlanPostProcessor (#63111)
fb27aebed63 is described below
commit fb27aebed633df2c340b612d27a2efcb221c8f58
Author: feiniaofeiafei <[email protected]>
AuthorDate: Thu May 14 18:27:08 2026 +0800
[fix](partition_prune) Move the pruning of predicates that are always true
after partition pruning into the PlanPostProcessor (#63111)
### What problem does this PR solve?
Related PR: #57169
Problem Summary:
After partition pruning, predicates that are evaluated to constant true
are removed as an optimization (introduced by #57169). However, this
introduced a bug in materialized view rewriting: when such predicates
are removed, they are not compensated above the rewritten materialized
view. Since the materialized view itself is not partitioned, this leads
to incorrect query results.
This PR fixes the issue by moving the removal of constant-true
predicates to the planPostProcessors phase. This ensures that the
predicate removal does not interfere with materialized view rewriting,
preserving correctness while still retaining the optimization benefit.
---------
Co-authored-by: Copilot <[email protected]>
---
.../nereids/processor/post/PlanPostProcessors.java | 1 +
.../processor/post/PrunePartitionPredicate.java | 146 ++++++++++++
.../exploration/mv/SyncMaterializationContext.java | 11 +-
.../rules/expression/rules/PartitionPruner.java | 23 --
.../LogicalOlapScanToPhysicalOlapScan.java | 3 +-
.../rules/rewrite/PruneOlapScanPartition.java | 30 ++-
.../trees/plans/PartitionPrunablePredicate.java | 95 ++++++++
.../trees/plans/logical/LogicalOlapScan.java | 100 ++++++--
.../trees/plans/physical/PhysicalOlapScan.java | 63 ++++-
.../nereids/rules/rewrite/PartitionPrunerTest.java | 256 ---------------------
.../partition_prune/prune_predicates_mv_test.out | 14 ++
.../prune_predicates_mv_test.groovy | 182 +++++++++++++++
12 files changed, 615 insertions(+), 309 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
index 4f305554685..31f8d716c75 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
@@ -63,6 +63,7 @@ public class PlanPostProcessors {
// add processor if we need
Builder<PlanPostProcessor> builder = ImmutableList.builder();
builder.add(new PushDownFilterThroughProject());
+ builder.add(new PrunePartitionPredicate());
builder.add(new RemoveUselessProjectPostProcessor());
builder.add(new ShuffleKeyPruner());
builder.add(new RecomputeLogicalPropertiesProcessor());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
new file mode 100644
index 00000000000..01ecf88e595
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.processor.post;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Removes partition-prunable conjuncts that were registered by {@link
+ * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in
+ * the logical plan during cascades. Doing the removal here, after
+ * materialized-view rewrite has finished, ensures MV matching observes the
+ * original predicates; otherwise the MV view-predicate may incorrectly cover
+ * the dropped partition predicate and produce extra rows.
+ */
+public class PrunePartitionPredicate extends PlanPostProcessor {
+
+ @Override
+ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter,
CascadesContext context) {
+ filter = (PhysicalFilter<? extends Plan>) super.visit(filter, context);
+ Plan child = filter.child();
+ if (!(child instanceof PhysicalOlapScan)) {
+ return filter;
+ }
+ PhysicalOlapScan scan = (PhysicalOlapScan) child;
+ Optional<PartitionPrunablePredicate> entryOpt =
scan.getPartitionPrunablePredicates();
+ if (!entryOpt.isPresent()) {
+ return filter;
+ }
+ boolean skipPrunePredicate =
context.getConnectContext().getSessionVariable().skipPrunePredicate
+ || context.getStatementContext().isDelete();
+ if (skipPrunePredicate) {
+ return filter;
+ }
+ Set<Long> scanPartitions = new
HashSet<>(scan.getSelectedPartitionIds());
+ Map<String, Slot> nameToOutputSlot = buildNameToSlotMap(scan);
+
+ Set<Expression> remaining = new LinkedHashSet<>(filter.getConjuncts());
+ boolean changed = false;
+ PartitionPrunablePredicate entry = entryOpt.get();
+ if (entry.getSelectedPartitionIds().containsAll(scanPartitions)) {
+ Map<Expression, Expression> slotReplaceMap =
+ buildSlotReplaceMap(entry.getSnapshotPartitionSlots(),
nameToOutputSlot);
+ if (slotReplaceMap != null) {
+ for (Expression conjunct : entry.getPrunableConjuncts()) {
+ Expression rewritten = slotReplaceMap.isEmpty()
+ ? conjunct : ExpressionUtils.replace(conjunct,
slotReplaceMap);
+ if (remaining.remove(rewritten)) {
+ changed = true;
+ }
+ }
+ }
+ }
+ if (!changed) {
+ return filter;
+ }
+ if (remaining.isEmpty()) {
+ return scan;
+ }
+ return filter.withConjunctsAndChild(remaining, scan)
+ .copyStatsAndGroupIdFrom((AbstractPhysicalPlan) filter);
+ }
+
+ private static Map<String, Slot> buildNameToSlotMap(PhysicalOlapScan scan)
{
+ OlapTable table = scan.getTable();
+ List<Slot> slots = scan.getOutput();
+ Map<String, Slot> map = new HashMap<>(slots.size());
+ if (scan.getSelectedIndexId() == table.getBaseIndexId()) {
+ for (Slot slot : slots) {
+ map.put(slot.getName().toLowerCase(), slot);
+ }
+ } else {
+ for (Slot slot : slots) {
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<Column> columnOptional =
slotReference.getOriginalColumn();
+ if (!columnOptional.isPresent()) {
+ continue;
+ }
+ Expr expr = columnOptional.get().getDefineExpr();
+ if (!(expr instanceof SlotRef)) {
+ continue;
+ }
+ map.put(((SlotRef) expr).getColumnName().toLowerCase(), slot);
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Map each recorded snapshot slot to the scan's current output slot of the
+ * same column name. Returns null when any snapshot slot cannot be located,
+ * so the caller can skip the entry.
+ */
+ private static Map<Expression, Expression> buildSlotReplaceMap(
+ List<Slot> snapshotSlots, Map<String, Slot> nameToOutputSlot) {
+ Map<Expression, Expression> replaceMap = new
HashMap<>(snapshotSlots.size());
+ for (Slot snapshot : snapshotSlots) {
+ Slot current =
nameToOutputSlot.get(snapshot.getName().toLowerCase());
+ if (current == null) {
+ return null;
+ }
+ if (!snapshot.equals(current)) {
+ replaceMap.put(snapshot, current);
+ }
+ }
+ return replaceMap;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
index 95c8d372665..0aa5108efdf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
@@ -122,13 +122,18 @@ public class SyncMaterializationContext extends
MaterializationContext {
return scanPlan.accept(new DefaultPlanRewriter<Void>() {
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan,
Void context) {
- if
(!queryStructInfoRelations.get(0).getTable().getFullQualifiers().equals(
+ LogicalOlapScan queryScan = (LogicalOlapScan)
queryStructInfoRelations.get(0);
+ if (!queryScan.getTable().getFullQualifiers().equals(
olapScan.getTable().getFullQualifiers())) {
// Only the same table, we can do partition prue
return olapScan;
}
- return olapScan.withSelectedPartitionIds(
- ((LogicalOlapScan)
queryStructInfoRelations.get(0)).getSelectedPartitionIds());
+ // Carry partition-prunable predicates from the original
query scan onto
+ // the rewritten MV scan so the post-processor can still
drop the
+ // predicates that have already been enforced by partition
pruning.
+ return olapScan
+
.withSelectedPartitionIds(queryScan.getSelectedPartitionIds())
+
.withPartitionPrunablePredicates(queryScan.getPartitionPrunablePredicates());
}
}, null);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index 83e7ad6e9cd..b49c8b74362 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -36,11 +36,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.types.DateTimeType;
-import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
@@ -51,7 +47,6 @@ import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -390,22 +385,4 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
return Pair.of(true, false);
}
}
-
- /** remove predicates that are always true*/
- public static Plan prunePredicate(boolean skipPrunePredicate,
Optional<Expression> prunedPredicates,
- LogicalFilter<? extends Plan> filter, LogicalRelation scan) {
- if (!skipPrunePredicate && prunedPredicates.isPresent()) {
- Set<Expression> conjuncts = new
LinkedHashSet<>(filter.getConjuncts());
- Expression deletedPredicate = prunedPredicates.get();
- Set<Expression> deletedPredicateSet =
ExpressionUtils.extractConjunctionToSet(deletedPredicate);
- conjuncts.removeAll(deletedPredicateSet);
- if (conjuncts.isEmpty()) {
- return scan;
- } else {
- return filter.withConjunctsAndChild(conjuncts, scan);
- }
- } else {
- return filter.withChildren(ImmutableList.of(scan));
- }
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
index 0b1e483fc21..6a8b24c2c11 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
@@ -72,7 +72,8 @@ public class LogicalOlapScanToPhysicalOlapScan extends
OneImplementationRuleFact
olapScan.getScoreRangeInfo(),
olapScan.getAnnOrderKeys(),
olapScan.getAnnLimit(),
- olapScan.getTableAlias())
+ olapScan.getTableAlias(),
+ olapScan.getPartitionPrunablePredicates())
).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
index beaef5f9fd8..aa5f722587f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
@@ -36,10 +36,12 @@ import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.Partition
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
@@ -47,6 +49,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -92,12 +95,29 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
}
if (rewrittenLogicalRelation instanceof
LogicalEmptyRelation) {
return rewrittenLogicalRelation;
- } else {
- return PartitionPruner.prunePredicate(
-
ctx.connectContext.getSessionVariable().skipPrunePredicate
- || ctx.statementContext.isDelete(),
- prunedRes.second, filter,
rewrittenLogicalRelation);
}
+ boolean skipPrunePredicate =
ctx.connectContext.getSessionVariable().skipPrunePredicate
+ || ctx.statementContext.isDelete();
+ if (!skipPrunePredicate &&
prunedRes.second.isPresent()) {
+ // Defer the predicate removal to
PlanPostProcessor so that materialized-view
+ // rewrite still sees the original predicates.
Otherwise, partition predicates
+ // that are equivalent to the surviving partition
list would be silently
+ // dropped, leading to wrong results when an MV
definition predicate matches
+ // the remaining conjuncts.
+ LogicalOlapScan prunedScan = (LogicalOlapScan)
rewrittenLogicalRelation;
+ Set<Expression> prunableConjuncts =
ExpressionUtils.extractConjunctionToSet(
+ prunedRes.second.get());
+ List<Slot> partitionSlots =
getPartitionSlots(prunedScan, prunedScan.getTable());
+ if (partitionSlots != null) {
+ PartitionPrunablePredicate entry = new
PartitionPrunablePredicate(
+ new
HashSet<>(prunedScan.getSelectedPartitionIds()),
+ partitionSlots,
+ prunableConjuncts);
+ rewrittenLogicalRelation =
prunedScan.withPartitionPrunablePredicates(
+ Optional.of(entry));
+ }
+ }
+ return
filter.withChildren(ImmutableList.of(rewrittenLogicalRelation));
}).toRule(RuleType.OLAP_SCAN_PARTITION_PRUNE)
);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
new file mode 100644
index 00000000000..54ba19493b8
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Records that, on the scan whose partition list equals {@link
+ * #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed to
+ * evaluate to TRUE for every surviving row.
+ *
+ * <p>The predicate is registered by {@link
+ * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in
+ * the logical filter during cascades. The actual removal happens later in
+ * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} so
+ * that materialized-view rewrite still sees the original predicates. Keeping
+ * the predicate in the plan avoids the wrong-result problem in which the MV
+ * view-predicate happens to cover the remaining conjuncts after the partition
+ * predicate has been silently dropped.
+ *
+ * <p>The predicate lives on the scan itself (see {@code LogicalOlapScan} and
+ * {@code PhysicalOlapScan}) so we no longer need to match it back to its scan
+ * via a table identifier. Because rewrites between recording and removal may
+ * rebuild the scan with fresh slot ids, {@link #snapshotPartitionSlots}
+ * captures the slots that appear in the recorded conjuncts. The post-processor
+ * maps them onto the actual scan's output slots by column name before
+ * performing the conjunct removal.
+ */
+public class PartitionPrunablePredicate {
+ private final Set<Long> selectedPartitionIds;
+ private final List<Slot> snapshotPartitionSlots;
+ private final Set<Expression> prunableConjuncts;
+
+ public PartitionPrunablePredicate(Set<Long> selectedPartitionIds,
+ List<Slot> snapshotPartitionSlots,
+ Set<Expression> prunableConjuncts) {
+ this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds);
+ this.snapshotPartitionSlots =
ImmutableList.copyOf(snapshotPartitionSlots);
+ this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PartitionPrunablePredicate that = (PartitionPrunablePredicate) o;
+ return selectedPartitionIds.equals(that.selectedPartitionIds)
+ && snapshotPartitionSlots.equals(that.snapshotPartitionSlots)
+ && prunableConjuncts.equals(that.prunableConjuncts);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(selectedPartitionIds, snapshotPartitionSlots,
prunableConjuncts);
+ }
+
+ public Set<Long> getSelectedPartitionIds() {
+ return selectedPartitionIds;
+ }
+
+ public List<Slot> getSnapshotPartitionSlots() {
+ return snapshotPartitionSlots;
+ }
+
+ public Set<Expression> getPrunableConjuncts() {
+ return prunableConjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 8351fbbfbd0..833e198d886 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -161,6 +162,16 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
protected final List<OrderKey> annOrderKeys;
protected final Optional<Long> annLimit;
+ /**
+ * Conjuncts that are guaranteed to be TRUE on the current scan because
they
+ * were already enforced by partition pruning. The deferred removal happens
+ * in the {@link
org.apache.doris.nereids.processor.post.PrunePartitionPredicate}
+ * post-processor so that materialized-view rewrite still observes the
+ * original predicates. The set is preserved through {@code with*} rewrites
+ * and copied onto MV rewrite outputs.
+ */
+ private final Optional<PartitionPrunablePredicate>
partitionPrunablePredicates;
+
public LogicalOlapScan(RelationId id, OlapTable table) {
this(id, table, ImmutableList.of());
}
@@ -256,6 +267,29 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
List<OrderKey> annOrderKeys, Optional<Long> annLimit, String
tableAlias) {
+ this(id, table, qualifier, groupExpression, logicalProperties,
selectedPartitionIds, partitionPruned,
+ hasPartitionPredicate, selectedTabletIds, selectedIndexId,
indexSelected, preAggStatus,
+ specifiedPartitions, hints, cacheSlotWithSlotName,
cachedOutput, tableSample, directMvScan,
+ colToSubPathsMap, specifiedTabletIds, operativeSlots,
virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ Optional.empty());
+ }
+
+ /**
+ * Constructor for LogicalOlapScan.
+ */
+ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
+ List<Long> selectedPartitionIds, boolean partitionPruned, boolean
hasPartitionPredicate,
+ List<Long> selectedTabletIds, long selectedIndexId, boolean
indexSelected,
+ PreAggStatus preAggStatus, List<Long> specifiedPartitions,
+ List<String> hints, Map<Pair<Long, String>, Slot>
cacheSlotWithSlotName,
+ Optional<List<Slot>> cachedOutput, Optional<TableSample>
tableSample, boolean directMvScan,
+ Map<String, Set<List<String>>> colToSubPathsMap, List<Long>
specifiedTabletIds,
+ Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
+ List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
+ List<OrderKey> annOrderKeys, Optional<Long> annLimit, String
tableAlias,
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
operativeSlots, virtualColumns, groupExpression,
logicalProperties, tableAlias);
Preconditions.checkArgument(selectedPartitionIds != null,
@@ -294,6 +328,9 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
this.scoreRangeInfo = scoreRangeInfo;
this.annOrderKeys = Utils.fastToImmutableList(annOrderKeys);
this.annLimit = annLimit;
+ this.partitionPrunablePredicates = partitionPrunablePredicates == null
+ ? Optional.empty()
+ : partitionPrunablePredicates;
}
public List<Long> getSelectedPartitionIds() {
@@ -304,6 +341,29 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
return hasPartitionPredicate;
}
+ public Optional<PartitionPrunablePredicate>
getPartitionPrunablePredicates() {
+ return partitionPrunablePredicates;
+ }
+
+ /**
+ * Returns a new {@code LogicalOlapScan} carrying the supplied
+ * {@link PartitionPrunablePredicate}. It is preserved across all other
+ * {@code with*} builders so partition-derived conjuncts can be removed
+ * safely after MV rewrite has had a chance to match the plan.
+ */
+ public LogicalOlapScan withPartitionPrunablePredicates(
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
+ return AbstractPlan.copyWithSameId(this, () ->
+ new LogicalOlapScan(relationId, (Table) table, qualifier,
+ groupExpression, Optional.of(getLogicalProperties()),
+ selectedPartitionIds, partitionPruned, hasPartitionPredicate,
selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
+ hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
+ colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
+ }
+
@Override
public String getFingerprint() {
String partitions = "";
@@ -371,7 +431,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
&& Objects.equals(scoreLimit, that.scoreLimit)
&& Objects.equals(scoreRangeInfo, that.scoreRangeInfo)
&& Objects.equals(annOrderKeys, that.annOrderKeys)
- && Objects.equals(annLimit, that.annLimit);
+ && Objects.equals(annLimit, that.annLimit)
+ && Objects.equals(partitionPrunablePredicates,
that.partitionPrunablePredicates);
}
@Override
@@ -388,7 +449,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
@Override
@@ -400,7 +462,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
/**
@@ -422,7 +485,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
/**
@@ -438,7 +502,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
indexId, true, PreAggStatus.unset(),
manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
cachedOutput, tableSample, directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds,
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
scoreRangeInfo,
- annOrderKeys, annLimit, tableAlias));
+ annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
/**
@@ -452,7 +516,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns, scoreOrderKeys,
- scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias));
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias, partitionPrunablePredicates));
}
/**
@@ -466,7 +530,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
/**
@@ -480,7 +545,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
/**
@@ -494,7 +560,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
@Override
@@ -507,7 +574,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, Maps.newHashMap(), Optional.empty(), tableSample,
directMvScan,
colToSubPathsMap, selectedTabletIds, operativeSlots,
virtualColumns, scoreOrderKeys,
- scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias));
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias, partitionPrunablePredicates));
}
@Override
@@ -519,7 +586,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias));
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ partitionPrunablePredicates));
}
/**
@@ -540,7 +608,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias));
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
/**
@@ -563,7 +631,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots,
mergedVirtualColumns, scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias);
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates);
}
/**
@@ -592,7 +660,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots,
mergedVirtualColumns, scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias);
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates);
}
@Override
@@ -939,7 +1007,7 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias));
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
@VisibleForTesting
@@ -1021,7 +1089,7 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, Optional.of(outputSlots),
tableSample, directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias));
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
index d2d4bcd8fd3..9205f008cf4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -78,6 +79,15 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
private final List<OrderKey> annOrderKeys;
private final Optional<Long> annLimit;
+ /**
+ * Predicates known to be TRUE on this scan thanks to partition pruning.
+ * Carried alongside the scan so the
+ * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate}
+ * post-processor can strip them from the surrounding filter after MV
+ * rewrite has finished its matching work.
+ */
+ private final Optional<PartitionPrunablePredicate>
partitionPrunablePredicates;
+
/**
* Constructor for PhysicalOlapScan.
*/
@@ -144,6 +154,26 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
List<OrderKey> annOrderKeys, Optional<Long> annLimit, String
tableAlias) {
+ this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds,
selectedPartitionIds,
+ hasPartitionPredicate, distributionSpec, preAggStatus,
baseOutputs, groupExpression,
+ logicalProperties, physicalProperties, statistics,
tableSample, operativeSlots, virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, tableAlias,
+ Optional.empty());
+ }
+
+ /**
+ * Ultimate constructor for PhysicalOlapScan.
+ */
+ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List<String>
qualifier, long selectedIndexId,
+ List<Long> selectedTabletIds, List<Long> selectedPartitionIds,
boolean hasPartitionPredicate,
+ DistributionSpec distributionSpec, PreAggStatus preAggStatus,
List<Slot> baseOutputs,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics,
+ Optional<TableSample> tableSample,
+ Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
+ List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
+ List<OrderKey> annOrderKeys, Optional<Long> annLimit, String
tableAlias,
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier,
groupExpression, logicalProperties, physicalProperties,
statistics, operativeSlots, tableAlias);
this.selectedIndexId = selectedIndexId;
@@ -161,6 +191,9 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
this.scoreRangeInfo = scoreRangeInfo;
this.annOrderKeys = ImmutableList.copyOf(annOrderKeys);
this.annLimit = annLimit;
+ this.partitionPrunablePredicates = partitionPrunablePredicates == null
+ ? Optional.empty()
+ : partitionPrunablePredicates;
}
@Override
@@ -181,6 +214,25 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
return hasPartitionPredicate;
}
+ public Optional<PartitionPrunablePredicate>
getPartitionPrunablePredicates() {
+ return partitionPrunablePredicates;
+ }
+
+ /**
+ * Returns a new {@code PhysicalOlapScan} carrying the supplied
+ * {@link PartitionPrunablePredicate} set. The set is preserved through all
+ * other {@code with*} builders so the post-processor can remove the
+ * derived conjuncts after MV rewrite has had a chance to match them.
+ */
+ public PhysicalOlapScan withPartitionPrunablePredicates(
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
+ return AbstractPlan.copyWithSameId(this, () -> new
PhysicalOlapScan(relationId, getTable(), qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
hasPartitionPredicate,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(),
+ getPhysicalProperties(), statistics, tableSample,
operativeSlots, virtualColumns, scoreOrderKeys,
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias, partitionPrunablePredicates));
+ }
+
@Override
public OlapTable getTable() {
return (OlapTable) table;
@@ -306,7 +358,8 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
&& Objects.equals(scoreLimit, olapScan.scoreLimit)
&& Objects.equals(scoreRangeInfo, olapScan.scoreRangeInfo)
&& Objects.equals(annOrderKeys, olapScan.annOrderKeys)
- && Objects.equals(annLimit, olapScan.annLimit);
+ && Objects.equals(annLimit, olapScan.annLimit)
+ && Objects.equals(partitionPrunablePredicates,
olapScan.partitionPrunablePredicates);
}
@Override
@@ -325,7 +378,7 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
selectedIndexId, selectedTabletIds, selectedPartitionIds,
hasPartitionPredicate,
distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(), null, null,
tableSample, operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit, scoreRangeInfo,
- annOrderKeys, annLimit, tableAlias));
+ annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
@Override
@@ -335,7 +388,7 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
selectedIndexId, selectedTabletIds, selectedPartitionIds,
hasPartitionPredicate,
distributionSpec, preAggStatus, baseOutputs, groupExpression,
logicalProperties.get(), null, null,
tableSample, operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit, scoreRangeInfo,
- annOrderKeys, annLimit, tableAlias));
+ annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
@Override
@@ -345,7 +398,7 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
selectedIndexId, selectedTabletIds, selectedPartitionIds,
hasPartitionPredicate,
distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(),
physicalProperties, statistics, tableSample, operativeSlots,
virtualColumns, scoreOrderKeys,
- scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias));
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
tableAlias, partitionPrunablePredicates));
}
@Override
@@ -373,7 +426,7 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(),
getPhysicalProperties(), statistics, tableSample,
operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit, tableAlias));
+ scoreRangeInfo, annOrderKeys, annLimit, tableAlias,
partitionPrunablePredicates));
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
index 137791ba5a5..f4f78da1e30 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
@@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ListPartitionItem;
-import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionKey;
import org.apache.doris.catalog.PrimitiveType;
@@ -38,16 +37,11 @@ import
org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
-import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.RelationId;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.utframe.TestWithFeService;
@@ -59,11 +53,9 @@ import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
public class PartitionPrunerTest extends TestWithFeService {
private Method canBePrunedOutMethod;
@@ -324,254 +316,6 @@ public class PartitionPrunerTest extends
TestWithFeService {
Assertions.assertFalse(result.second);
}
- // test prune predicate
- // Test basis: some predicates are pruned
- @Test
- public void testPrunePartialPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- predicates.add(gt);
- predicates.add(lt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- }
-
- // all predicates are pruned
- @Test
- public void testPruneAllPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- // no predicates are pruned
- @Test
- public void testPruneNoPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- predicates.add(gt);
- predicates.add(lt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- EqualTo nonExistentPredicate = new EqualTo(slotC, Literal.of(30));
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(nonExistentPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testPruneCompoundPredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq = new EqualTo(slotC, Literal.of(30));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND b < 20)
- And compoundPredicate = new And(gt, lt);
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testSkipPrunePredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(true,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- }
-
- @Test
- public void testEmptyPrunedPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // prunedPredicates is empty
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.empty(), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- }
-
- @Test
- public void testPruneDuplicatePredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt1 = new GreaterThan(slotA, Literal.of(10));
- GreaterThan gt2 = new GreaterThan(slotA, Literal.of(10)); //
duplicated predicate
- predicates.add(gt1);
- predicates.add(gt2);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt1), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- @Test
- public void testPruneWithNullLiteral() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- EqualTo nullEq = new EqualTo(slotB, new NullLiteral());
- predicates.add(gt);
- predicates.add(nullEq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(nullEq));
- }
-
- @Test
- public void testPruneMultiplePredicatesPartially() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq1 = new EqualTo(slotC, Literal.of(30));
- EqualTo eq2 = new EqualTo(slotC, Literal.of(40));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq1);
- predicates.add(eq2);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND b < 20)
- And compoundPredicate = new And(gt, lt);
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq1));
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq2));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testPruneNestedCompoundPredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq = new EqualTo(slotC, Literal.of(30));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND (b < 20 AND c = 30))
- And innerAnd = new And(lt, eq);
- And outerAnd = new And(gt, innerAnd);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(outerAnd), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- @Test
- public void testPruneWhenFilterContainsOr() {
- Set<Expression> predicates = new LinkedHashSet<>();
- Or orPredicate = new Or(
- new GreaterThan(slotA, Literal.of(10)),
- new LessThan(slotB, Literal.of(20))
- );
- predicates.add(orPredicate);
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
-
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
- }
-
- @Test
- public void testPruneWhenFilterContainsAndOrMix() {
- Set<Expression> predicates = new LinkedHashSet<>();
- // filter :a > 10 AND (b < 20 OR c = 30)
- Or orPredicate = new Or(
- new LessThan(slotB, Literal.of(20)),
- new EqualTo(slotC, Literal.of(30))
- );
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
-
- predicates.add(gt);
- predicates.add(orPredicate);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
- // a > 10
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
-
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
-
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
- }
-
@Test
public void testPruneWithResultIgnoresNonPruningPartitionPredicate()
throws AnalysisException {
Map<String, PartitionItem> idToPartitions = ImmutableMap.of(
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
new file mode 100644
index 00000000000..cc704a5c7fe
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !mv_1 --
+1
+2
+
+-- !mv_2 --
+1
+2
+
+-- !query_3 --
+a 1 5
+a 2 7
+a 3 0
+
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
new file mode 100644
index 00000000000..cf33f0a5f1d
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("prune_predicates_mv_test") {
+ String currentDb = context.config.getDbNameByFile(context.file)
+ sql """
+ drop table if exists base_t;
+ CREATE TABLE base_t (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+
+ // case 1:
+ def mv_1 = """
+ SELECT top_asset, tag_key, SUM(frame_count) AS frame_count
+ FROM base_t
+ WHERE frame_count != 0
+ GROUP BY top_asset, tag_key;
+ """
+
+ def query_1 = """
+ SELECT /*+ USE_MV(mv_1) */ tag_key FROM base_t
+ WHERE tag_key IN (1, 2, 3) AND frame_count != 0
+ GROUP BY tag_key
+ ORDER BY tag_key;
+ """
+
+ //Execute (force rewrite):
+ //1. verify result is correct
+ //2. verify shape plan contains filter
+ //3. verify rewrite succeeded: async_mv_rewrite_success
+
+ // The base table is partitioned and the predicate was removed after
partition prune;
+ // the MV is not partitioned, so verify the MV did not drop the predicate.
+ async_mv_rewrite_success(currentDb, mv_1, query_1, "mv_1")
+ order_qt_mv_1 query_1
+ explain {
+ sql "shape plan ${query_1}"
+ contains "filter"
+ }
+
+ // case2: the MV is also partitioned; verify that partition pruning on the
MV
+ // is performed and the always-true predicate is removed after the prune.
+
+ def async_partition_mv_rewrite_success = { db, mv_sql, query_sql, mv_name,
partition, expected_pre_rewrite_strategys = [] ->
+ if (!mvShouldContinueCheck(expected_pre_rewrite_strategys)) {
+ return;
+ }
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ ${partition}
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+ def job_name = getJobName(db, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ // force meta sync to avoid stale meta data on follower fe
+ sql """sync;"""
+ mv_rewrite_success(query_sql, mv_name, true,
expected_pre_rewrite_strategys)
+ }
+
+ sql """
+ drop table if exists base_t2;
+ CREATE TABLE base_t2 (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t2 VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+ def mv_2 = """
+ SELECT top_asset, tag_key, SUM(frame_count) AS frame_count
+ FROM base_t2
+ WHERE frame_count != 0
+ GROUP BY top_asset, tag_key;
+ """
+ def query_2 = """
+ SELECT /*+use_mv(mv_2)*/ tag_key FROM base_t2
+ WHERE tag_key IN (1, 2, 3) AND frame_count != 0
+ GROUP BY tag_key
+ ORDER BY tag_key;
+ """
+
+ async_partition_mv_rewrite_success(currentDb, mv_2, query_2, "mv_2",
"PARTITION BY (tag_key)")
+ order_qt_mv_2 query_2
+ explain {
+ sql "physical plan ${query_2}"
+ contains "partitions(2/6)"
+ notContains "PhysicalFilter"
+ }
+
+ sql """
+ drop table if exists base_t3;
+ CREATE TABLE base_t3 (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ duplicate KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t3 VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+ create_sync_mv(currentDb, "base_t3", "mv_3", """
+ SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum
+ FROM base_t3
+ GROUP BY top_asset, tag_key;
+ """)
+
+ def query_3 = """
+ SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum
+ FROM base_t3
+ where tag_key in (1,2,3)
+ GROUP BY top_asset, tag_key;
+ """
+ mv_rewrite_success(query_3, "mv_3")
+ order_qt_query_3 query_3
+ explain {
+ sql "physical plan ${query_3}"
+ notContains "PhysicalFilter"
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
