This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 3341ad74a32 branch-4.2: [fix](partition) Preserve lexicographic range
partition boundaries #67888 (#70056)
3341ad74a32 is described below
commit 3341ad74a32ef31cc2b2aea95df1d8336429ebf8
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:17:07 2026 +0800
branch-4.2: [fix](partition) Preserve lexicographic range partition
boundaries #67888 (#70056)
picked from master #67888
---
.../rules/OneRangePartitionEvaluator.java | 341 ++++++++++++++++-----
.../nereids/rules/rewrite/PartitionPrunerTest.java | 198 ++++++++++++
.../test_lexicographic_range_partition.out | 21 ++
.../test_lexicographic_range_partition.groovy | 93 ++++++
4 files changed, 584 insertions(+), 69 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 2a58b4d7c73..21d309ab6e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -23,7 +23,6 @@ import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator.EvaluateRangeInput;
import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator.EvaluateRangeResult;
@@ -52,9 +51,9 @@ import
org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Preconditions;
-import com.google.common.collect.BoundType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
@@ -93,8 +92,26 @@ public class OneRangePartitionEvaluator<K>
// whether the Expression in partition range may be null.
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
+ // An impossible expander/evaluator state makes pruning unsafe. Production
keeps the partition;
+ // fe_debug throws at the point where the broken invariant is observed.
+ private boolean disablePruning;
- /** OneRangePartitionEvaluator */
+ /**
+ * Create an evaluator for one RANGE partition.
+ *
+ * <p>The constructor converts the tuple endpoints to Nereids literals,
classifies every
+ * partition coordinate as {@code CONST}, {@code RANGE}, or {@code OTHER},
and expands the
+ * enumerable prefix up to {@code expandThreshold}. It also records
whether each coordinate can
+ * contain NULL so expression evaluation can fold null-sensitive
predicates safely. An unknown
+ * coordinate type violates the expander/evaluator contract: fe_debug
reports it immediately,
+ * while production marks this evaluator as non-prunable.
+ *
+ * @param partitionIdent identifier returned when this partition must be
scanned
+ * @param partitionSlots partition-key slots in lexicographic order
+ * @param partitionItem inclusive-lower/exclusive-upper RANGE partition
definition
+ * @param cascadesContext planner context used by expression rewrite rules
+ * @param expandThreshold maximum number of enumerable values expanded
from the partition range
+ */
public OneRangePartitionEvaluator(K partitionIdent, List<Slot>
partitionSlots,
RangePartitionItem partitionItem, CascadesContext cascadesContext,
int expandThreshold) {
this.partitionIdent = partitionIdent;
@@ -140,7 +157,10 @@ public class OneRangePartitionEvaluator<K>
maybeNull = true;
break;
default:
- throw new AnalysisException("Unknown partition slot
type: " + partitionSlotType);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while deriving
nullability: " + partitionSlotType);
+ maybeNull = true;
+ break;
}
partitionSlotContainsNull.put(slot, maybeNull);
}
@@ -171,12 +191,28 @@ public class OneRangePartitionEvaluator<K>
}
}
+ /**
+ * Evaluate a partition predicate under one projected input row.
+ *
+ * <p>The projected ranges seed expression evaluation and are then
narrowed by the predicate.
+ * If input construction or range refinement discovers an invalid internal
state, returning any
+ * folded result could incorrectly remove data. In production this method
therefore returns the
+ * original predicate, which {@link PartitionPruner} treats as unknown and
keeps the partition.
+ * In fe_debug the invalid state is thrown before this fallback is reached.
+ *
+ * @param expression partition predicate to simplify
+ * @param currentInputs replacement expression and projected range for
every partition slot
+ * @return the simplified predicate, or {@code expression} when pruning
has been disabled
+ */
@Override
public Expression evaluate(Expression expression, Map<Slot,
PartitionSlotInput> currentInputs) {
+ if (disablePruning) {
+ return expression;
+ }
Map<Expression, ColumnRange> defaultColumnRanges =
currentInputs.values().iterator().next().columnRanges;
Map<Expression, ColumnRange> rangeMap = new
HashMap<>(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new
EvaluateRangeInput(currentInputs, rangeMap));
- return result.result;
+ return disablePruning ? expression : result.result;
}
@Override
@@ -408,6 +444,19 @@ public class OneRangePartitionEvaluator<K>
return result;
}
+ /**
+ * Evaluate a conjunction and refine the independent column ranges with
the tuple boundary.
+ *
+ * <p>The normal child evaluation first intersects the ranges contributed
by every conjunct.
+ * A composite RANGE partition needs one additional pass: a suffix column
is constrained by a
+ * lower or upper endpoint only while every preceding coordinate is still
equal to that endpoint.
+ * Applying this pass after merging the children also recovers prefix
coordinates whose equality
+ * expression was folded to a literal and therefore disappeared from the
child range map.
+ *
+ * @param and conjunction being evaluated
+ * @param context replacement values and projected ranges for the current
expanded partition input
+ * @return the folded conjunction together with its lexicographically
refined column ranges
+ */
@Override
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
@@ -425,8 +474,8 @@ public class OneRangePartitionEvaluator<K>
}
// shrink range and prune the other type: if previous column is
literal and equals to the bound
- andResult = determinateRangeOfOtherType(andResult, lowers, true);
- andResult = determinateRangeOfOtherType(andResult, uppers, false);
+ andResult = determinateRangeOfOtherType(andResult, lowers, true,
context.rangeMap);
+ andResult = determinateRangeOfOtherType(andResult, uppers, false,
context.rangeMap);
return andResult;
}
@@ -527,60 +576,83 @@ public class OneRangePartitionEvaluator<K>
}
}
+ /**
+ * Refine the first unresolved suffix column against one lexicographic
partition endpoint.
+ *
+ * <p>Partition slot types have the shape {@code CONST*, RANGE, OTHER*}.
The scan proceeds from
+ * left to right and keeps an endpoint active only while the observed
singleton values equal its
+ * prefix. For the first unresolved {@code OTHER} coordinate, the active
lower endpoint contributes
+ * {@code >= bound}; an active upper endpoint contributes {@code <=
bound}, except that the final
+ * partition column uses {@code < bound} because RANGE partitions are
upper-exclusive. The method
+ * returns immediately after that coordinate because once it may differ
from the endpoint, later
+ * coordinates are lexicographically unconstrained.
+ *
+ * <p>Constant folding can remove an expanded {@code RANGE} slot from
{@code context.columnRanges}.
+ * For that slot only, the method reads the singleton from {@code
defaultColumnRanges} so the
+ * omitted equal-prefix coordinate can still participate in prefix
comparison. A missing
+ * {@code OTHER} slot is different: its default range describes only the
partition boundary, not
+ * the predicate. Adding it to the predicate result would let {@code NOT}
complement a synthetic
+ * range and could incorrectly prune the partition, so a missing {@code
OTHER} stops refinement.
+ *
+ * @param context result of merging all conjunct ranges
+ * @param partitionBound lower or upper endpoint of the composite partition
+ * @param isLowerBound whether {@code partitionBound} is the inclusive
lower endpoint
+ * @param defaultColumnRanges projected ranges for the current expanded
partition input
+ * @return {@code context} refined at the first decisive suffix
coordinate, or an equivalent
+ * false result if the refined range is empty
+ */
private EvaluateRangeResult determinateRangeOfOtherType(
- EvaluateRangeResult context, List<Literal> partitionBound, boolean
isLowerBound) {
+ EvaluateRangeResult context, List<Literal> partitionBound, boolean
isLowerBound,
+ Map<Expression, ColumnRange> defaultColumnRanges) {
if (context.result instanceof Literal) {
return context;
}
- Slot qualifiedSlot = null;
- ColumnRange qualifiedRange = null;
+ LexicographicBoundState boundState = new LexicographicBoundState(
+ partitionBound, isLowerBound, partitionSlots.size());
for (int i = 0; i < partitionSlotTypes.size(); i++) {
PartitionSlotType partitionSlotType = partitionSlotTypes.get(i);
Slot slot = partitionSlots.get(i);
- if (!context.columnRanges.containsKey(slot)) {
- return context;
- }
+ ColumnRange columnRange = context.columnRanges.get(slot);
switch (partitionSlotType) {
case CONST: continue;
case RANGE:
- ColumnRange columnRange = context.columnRanges.get(slot);
- if (!columnRange.isSingleton()
- ||
!columnRange.getLowerBound().getValue().equals(partitionBound.get(i))) {
+ // Expanded RANGE literals can disappear after constant
folding. Recover only this
+ // equal-prefix coordinate from the partition projection.
+ if (columnRange == null) {
+ columnRange = defaultColumnRanges.get(slot);
+ }
+ if (columnRange == null || !columnRange.isSingleton()) {
+ return context;
+ }
+
boundState.observeLiteral(columnRange.getLowerBound().getValue(), i);
+ if (!boundState.hasEqualPrefix()) {
return context;
}
continue;
case OTHER:
- columnRange = context.columnRanges.get(slot);
+ // Do not expose a default-only partition range to the
predicate tree. In
+ // particular, visitNot() must never complement a range
that the predicate did
+ // not contribute.
+ if (columnRange == null) {
+ return context;
+ }
if (columnRange.isSingleton()
- &&
columnRange.getLowerBound().getValue().equals(partitionBound.get(i))) {
+ &&
columnRange.getLowerBound().getValue().equals(partitionBound.get(i))
+ && i + 1 < partitionSlots.size()) {
continue;
}
-
- qualifiedSlot = slot;
- if (isLowerBound) {
- qualifiedRange =
ColumnRange.atLeast(partitionBound.get(i));
- } else {
- qualifiedRange = i + 1 == partitionSlots.size()
- ? ColumnRange.lessThen(partitionBound.get(i))
- : ColumnRange.atMost(partitionBound.get(i));
- }
- break;
+ ColumnRange newRange =
boundState.constrainFirstUnresolvedColumn(columnRange, i);
+ Map<Expression, ColumnRange> newRanges = replaceExprRange(
+ context.columnRanges, slot, newRange);
+ return newRange.isEmptyRange()
+ ? new EvaluateRangeResult(BooleanLiteral.FALSE,
newRanges, context.childrenResult)
+ : new EvaluateRangeResult(context.result,
newRanges, context.childrenResult);
default:
- throw new AnalysisException("Unknown partition slot type:
" + partitionSlotType);
- }
- }
-
- if (qualifiedSlot != null) {
- ColumnRange origin = context.columnRanges.get(qualifiedSlot);
- ColumnRange newRange = origin.intersect(qualifiedRange);
-
- Map<Expression, ColumnRange> newRanges =
replaceExprRange(context.columnRanges, qualifiedSlot, newRange);
-
- if (newRange.isEmptyRange()) {
- return new EvaluateRangeResult(BooleanLiteral.FALSE,
newRanges, context.childrenResult);
- } else {
- return new EvaluateRangeResult(context.result, newRanges,
context.childrenResult);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while refining a
lexicographic bound: "
+ + partitionSlotType);
+ return context;
}
}
return context;
@@ -754,52 +826,69 @@ public class OneRangePartitionEvaluator<K>
return ImmutableList.of(slotToInputs);
}
+ /**
+ * Build evaluator inputs for every expanded representation of this
composite RANGE partition.
+ *
+ * <p>Range expansion replaces enumerable coordinates with literals and
leaves unexpanded
+ * coordinates as slots. Separate lower- and upper-bound states track
whether the literal prefix
+ * of each generated input still equals the corresponding endpoint. Only
the first unresolved
+ * coordinate while a state is active receives that endpoint's constraint;
after it can diverge,
+ * all suffix coordinates remain unbounded. This preserves tuple ordering
instead of incorrectly
+ * treating each partition column as an independent interval.
+ *
+ * <p>The returned {@link PartitionSlotInput}s all contain the complete
projected range map for
+ * their generated input. Expression evaluation can therefore recover
ranges for slots that were
+ * replaced by literals and removed by constant folding.
+ *
+ * <p>A {@code CONST} coordinate must have been expanded to a literal. If
it is not, the input is
+ * structurally invalid: fe_debug throws, while production makes the
coordinate unbounded and
+ * disables pruning for the whole evaluator so planning can continue
without dropping data.
+ *
+ * @return one slot-to-input map for each Cartesian-product row produced
by range expansion
+ */
private List<Map<Slot, PartitionSlotInput>>
commonComputeOnePartitionInputs() {
List<Map<Slot, PartitionSlotInput>> onePartitionInputs =
Lists.newArrayListWithCapacity(inputs.size());
for (List<Expression> input : inputs) {
- boolean previousIsLowerBoundLiteral = true;
- boolean previousIsUpperBoundLiteral = true;
+ LexicographicBoundState lowerState = new LexicographicBoundState(
+ lowers, true, partitionSlots.size());
+ LexicographicBoundState upperState = new LexicographicBoundState(
+ uppers, false, partitionSlots.size());
Builder<Slot, PartitionSlotInput> slotToInputs =
ImmutableMap.builderWithExpectedSize(16);
for (int i = 0; i < partitionSlots.size(); ++i) {
Slot partitionSlot = partitionSlots.get(i);
// partitionSlot will be replaced to this expression
Expression expression = input.get(i);
- ColumnRange slotRange = null;
+ ColumnRange slotRange;
PartitionSlotType partitionSlotType =
partitionSlotTypes.get(i);
if (expression instanceof Literal) {
// const or expanded range
slotRange = ColumnRange.singleton((Literal) expression);
- if (!expression.equals(lowers.get(i))) {
- previousIsLowerBoundLiteral = false;
- }
- if (!expression.equals(uppers.get(i))) {
- previousIsUpperBoundLiteral = false;
- }
+ lowerState.observeLiteral(expression, i);
+ upperState.observeLiteral(expression, i);
} else {
- // un expanded range
+ // The first unresolved column carries every still-active
lexicographic bound.
+ // Once that column can diverge, every suffix column must
remain unbounded.
switch (partitionSlotType) {
case RANGE:
- boolean isLastPartitionColumn = i + 1 ==
partitionSlots.size();
- BoundType rightBoundType = isLastPartitionColumn
- ? BoundType.OPEN : BoundType.CLOSED;
- slotRange = ColumnRange.range(
- lowers.get(i), BoundType.CLOSED,
uppers.get(i), rightBoundType);
- break;
case OTHER:
- if (previousIsLowerBoundLiteral) {
- slotRange = ColumnRange.atLeast(lowers.get(i));
- } else if (previousIsUpperBoundLiteral) {
- slotRange =
ColumnRange.lessThen(uppers.get(i));
- } else {
- // unknown range
- slotRange = ColumnRange.all();
- }
+ slotRange =
lowerState.constrainFirstUnresolvedColumn(ColumnRange.all(), i);
+ slotRange =
upperState.constrainFirstUnresolvedColumn(slotRange, i);
+ break;
+ case CONST:
+ disablePruningOnInvalidState("CONST partition
input must be a literal: slot="
+ + partitionSlot + ", expression=" +
expression);
+ slotRange = ColumnRange.all();
+ lowerState.diverge();
+ upperState.diverge();
break;
default:
- throw new AnalysisException("Unknown partition
slot type: " + partitionSlotType);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while
building evaluator inputs: "
+ + partitionSlotType);
+ slotRange = ColumnRange.all();
+ lowerState.diverge();
+ upperState.diverge();
}
- previousIsLowerBoundLiteral = false;
- previousIsUpperBoundLiteral = false;
}
ImmutableMap<Expression, ColumnRange> slotToRange =
ImmutableMap.of(partitionSlot, slotRange);
slotToInputs.put(partitionSlot, new
PartitionSlotInput(expression, slotToRange));
@@ -811,6 +900,120 @@ public class OneRangePartitionEvaluator<K>
return onePartitionInputs;
}
+ /**
+ * Handle a state that violates the contract between {@link
PartitionRangeExpander} and this evaluator.
+ *
+ * <p>Tests and debugging sessions set {@code fe_debug=true}, so they fail
immediately and expose
+ * the broken invariant. Production planning must remain available: after
logging the problem,
+ * evaluation returns the original predicate for this partition, which
conservatively keeps it.
+ *
+ * @param message description of the invalid state
+ */
+ private void disablePruningOnInvalidState(String message) {
+ SessionVariable.throwAnalysisExceptionWhenFeDebug(message);
+ disablePruning = true;
+ }
+
+ /** Describes whether the coordinates already consumed are still equal to
an endpoint prefix. */
+ private enum BoundPrefixState {
+ /** No consumed coordinate differs from the tracked lower or upper
endpoint. */
+ EQUAL_PREFIX,
+
+ /** An earlier coordinate can differ, so this endpoint cannot
constrain any later coordinate. */
+ DIVERGED
+ }
+
+ /**
+ * Tracks one lower or upper endpoint while projected ranges are built or
refined.
+ *
+ * <p>Lexicographic comparison makes an endpoint relevant only as long as
all preceding
+ * coordinates equal its prefix. For example, with lower endpoint {@code
(1, 10, 100)}, an input
+ * whose first coordinate is {@code 1} must constrain the first unresolved
coordinate to
+ * {@code >= 10}. If the first coordinate is {@code 2}, the lower endpoint
is already satisfied
+ * and neither the second nor third coordinate receives a lower constraint.
+ *
+ * <p>The state is monotonic: it starts at {@link
BoundPrefixState#EQUAL_PREFIX} and can transition
+ * to {@link BoundPrefixState#DIVERGED} only once.
+ */
+ private static class LexicographicBoundState {
+ private final List<Literal> bound;
+ private final boolean lowerBound;
+ private final int columnCount;
+ private BoundPrefixState prefixState = BoundPrefixState.EQUAL_PREFIX;
+
+ /**
+ * Create state for one endpoint of a composite partition.
+ *
+ * @param bound endpoint values in partition-column order
+ * @param lowerBound true for the inclusive lower endpoint, false for
the exclusive upper endpoint
+ * @param columnCount total number of partition columns, used to
identify the final coordinate
+ */
+ private LexicographicBoundState(List<Literal> bound, boolean
lowerBound, int columnCount) {
+ this.bound = bound;
+ this.lowerBound = lowerBound;
+ this.columnCount = columnCount;
+ }
+
+ /**
+ * Return whether every coordinate observed so far is equal to this
endpoint's prefix.
+ *
+ * @return true while this endpoint may still constrain the next
unresolved coordinate
+ */
+ private boolean hasEqualPrefix() {
+ return prefixState == BoundPrefixState.EQUAL_PREFIX;
+ }
+
+ /**
+ * Consume a literal coordinate and deactivate this endpoint if the
literal differs from it.
+ *
+ * <p>Once a coordinate differs, later values cannot make the tuple
equal to the endpoint
+ * again, so an already-diverged state is intentionally left unchanged.
+ *
+ * @param literal literal selected for the current partition coordinate
+ * @param index zero-based partition-column index of {@code literal}
+ */
+ private void observeLiteral(Expression literal, int index) {
+ if (hasEqualPrefix() && !literal.equals(bound.get(index))) {
+ diverge();
+ }
+ }
+
+ /**
+ * Intersect an unresolved coordinate with this endpoint when its
prefix is still equal.
+ *
+ * <p>The lower endpoint is inclusive on every coordinate. The upper
endpoint is inclusive on
+ * non-terminal coordinates because equality there leaves later
coordinates to decide tuple
+ * membership; only the final coordinate is exclusive. Consuming an
unresolved coordinate
+ * always deactivates the endpoint so no suffix coordinate is
independently constrained.
+ *
+ * @param origin range already inferred for the unresolved coordinate
+ * @param index zero-based partition-column index of that coordinate
+ * @return {@code origin} intersected with the active endpoint, or
unchanged if an earlier
+ * coordinate has already diverged
+ */
+ private ColumnRange constrainFirstUnresolvedColumn(ColumnRange origin,
int index) {
+ if (!hasEqualPrefix()) {
+ return origin;
+ }
+ diverge();
+ Literal boundary = bound.get(index);
+ ColumnRange boundaryRange;
+ if (lowerBound) {
+ boundaryRange = ColumnRange.atLeast(boundary);
+ } else if (index + 1 == columnCount) {
+ boundaryRange = ColumnRange.lessThen(boundary);
+ } else {
+ boundaryRange = ColumnRange.atMost(boundary);
+ }
+ return origin.intersect(boundaryRange);
+ }
+
+ /** Mark this endpoint as satisfied or violated by an earlier decisive
coordinate. */
+ private void diverge() {
+ prefixState = BoundPrefixState.DIVERGED;
+ }
+ }
+
public EvaluateRangeResult visitMonotonic(Expression monotonic,
EvaluateRangeInput context) {
EvaluateRangeResult rangeResult = evaluateChildrenThenThis(monotonic,
context);
if (!rangeResult.result.getClass().equals(monotonic.getClass())) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
index 6d38a17fca1..11ea103bd44 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
@@ -24,11 +24,13 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionKey;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.expression.rules.OneListPartitionEvaluator;
import org.apache.doris.nereids.rules.expression.rules.OnePartitionEvaluator;
+import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionPruneResult;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
@@ -36,11 +38,13 @@ import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
@@ -50,15 +54,19 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Range;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -72,6 +80,7 @@ public class PartitionPrunerTest extends TestWithFeService {
private final SlotReference slotA = new SlotReference("a",
IntegerType.INSTANCE);
private final SlotReference slotB = new SlotReference("b",
IntegerType.INSTANCE);
private final SlotReference slotC = new SlotReference("c",
IntegerType.INSTANCE);
+ private final SlotReference slotD = new SlotReference("d",
IntegerType.INSTANCE);
@Override
protected void runBeforeAll() throws Exception {
@@ -600,6 +609,195 @@ public class PartitionPrunerTest extends
TestWithFeService {
Assertions.assertTrue(result.hasPartitionPredicate);
}
+ /**
+ * Verify lower/upper boundary transitions for a three-column partition in
both evaluator paths.
+ *
+ * <p>The low threshold leaves the RANGE coordinate unexpanded, while the
high threshold expands
+ * it into literals. The witness tuples cover divergence in a suffix
coordinate, an exactly equal
+ * lower endpoint, and tuples at or beyond the exclusive upper endpoint.
+ */
+ @Test
+ public void testThreeColumnLexicographicRangeBoundaries()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 100}, new int[] {100, 20, 200});
+
+ for (int expandThreshold : new int[] {1, 200}) {
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 11, 50);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 100, 19, 250);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 100, 20, 199);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 100);
+
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 1, 10, 99);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 100, 20, 200);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 100, 20, 250);
+ }
+ }
+
+ /**
+ * Verify that only the first unresolved coordinate receives a
lexicographic endpoint constraint.
+ *
+ * <p>Four columns exercise divergence at the first, middle, and final
coordinates. Extreme suffix
+ * values demonstrate that once an earlier coordinate differs, later
columns are intentionally
+ * unbounded; exact-prefix cases verify inclusive lower and exclusive
upper semantics at the end.
+ */
+ @Test
+ public void testFourColumnLexicographicRangeBoundaries()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT),
+ new Column("d", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC, slotD);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 100, 1000}, new int[] {4, 20, 200,
2000});
+
+ for (int expandThreshold : new int[] {1, 10}) {
+ // Once an earlier column diverges, all suffix columns are
unbounded.
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 2, -1000, -1000, -1000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 3, 9999, 9999, 9999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 11, -1, -1);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 19, 9999, 9999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 101, -1);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 20, 199, 9999);
+
+ // An equal prefix leaves the final column to decide the inclusive
lower and exclusive upper bounds.
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 100, 1000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 20, 200, 1999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 1, 10, 100, 999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 4, 20, 200, 2000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 4, 20, 200, 2001);
+ }
+ }
+
+ /**
+ * A range copied only from the partition projection must not become part
of a predicate result.
+ *
+ * <p>This is the reduced tree produced by the Date-IN partition-pruning
rewrite for
+ * {@code NOT(date(c) IN (day2, day3))}. For expanded {@code a = 1}, the
predicate constrains only
+ * {@code c}; injecting the default {@code b >= 10} range would let {@code
NOT} complement it and
+ * incorrectly turn the entire partition into false.
+ */
+ @Test
+ public void testNotDoesNotComplementDefaultOnlySuffixRange()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 1}, new int[] {2, 20, 4});
+ Expression predicate = new Not(new Or(
+ new And(new GreaterThanEqual(slotC, Literal.of(2)), new
LessThan(slotC, Literal.of(3))),
+ new And(new GreaterThanEqual(slotC, Literal.of(3)), new
LessThan(slotC, Literal.of(4)))));
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, 200);
+
+ Pair<Boolean, Boolean> result =
+ (Pair<Boolean, Boolean>) canBePrunedOutMethod.invoke(null,
predicate, evaluator);
+
+ Assertions.assertFalse(result.first);
+ }
+
+ /**
+ * Invalid expander/evaluator states fail in fe_debug but conservatively
keep the partition in production.
+ */
+ @Test
+ public void testInvalidConstInputUsesFeDebugPolicy() throws Exception {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10}, new int[] {1, 20});
+ Expression predicate = new EqualTo(slotA, Literal.of(2));
+ boolean oldFeDebug = connectContext.getSessionVariable().feDebug;
+ try {
+ connectContext.getSessionVariable().feDebug = false;
+ OneRangePartitionEvaluator<String> productionEvaluator =
+ createEvaluatorWithNonLiteralConstInput(partitionItem,
slots);
+ Pair<Boolean, Boolean> productionResult = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(
+ null, predicate, productionEvaluator);
+ Assertions.assertFalse(productionResult.first);
+
+ connectContext.getSessionVariable().feDebug = true;
+ OneRangePartitionEvaluator<String> debugEvaluator =
+ createEvaluatorWithNonLiteralConstInput(partitionItem,
slots);
+ InvocationTargetException exception =
Assertions.assertThrows(InvocationTargetException.class,
+ () -> canBePrunedOutMethod.invoke(null, predicate,
debugEvaluator));
+ Assertions.assertInstanceOf(
+
org.apache.doris.nereids.exceptions.AnalysisException.class,
exception.getCause());
+ } finally {
+ connectContext.getSessionVariable().feDebug = oldFeDebug;
+ }
+ }
+
+ private OneRangePartitionEvaluator<String>
createEvaluatorWithNonLiteralConstInput(
+ RangePartitionItem partitionItem, List<Slot> slots) throws
ReflectiveOperationException {
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, 200);
+ Field inputsField =
OneRangePartitionEvaluator.class.getDeclaredField("inputs");
+ inputsField.setAccessible(true);
+ inputsField.set(evaluator, ImmutableList.of(ImmutableList.of(slotA,
slotB)));
+ return evaluator;
+ }
+
+ /**
+ * Evaluate an equality predicate for one tuple and assert whether the
partition is pruned.
+ *
+ * @param partitionItem composite RANGE partition under test
+ * @param slots partition slots in tuple order
+ * @param expandThreshold threshold selecting expanded or unexpanded
evaluator inputs
+ * @param expectedPruned expected result from {@code
PartitionPruner.canBePrunedOut}
+ * @param values tuple values used to build one equality per partition slot
+ */
+ private void assertRangeTuplePruned(RangePartitionItem partitionItem,
List<Slot> slots,
+ int expandThreshold, boolean expectedPruned, int... values)
+ throws InvocationTargetException, IllegalAccessException {
+ ImmutableList.Builder<Expression> equalities =
ImmutableList.builderWithExpectedSize(values.length);
+ for (int i = 0; i < values.length; i++) {
+ equalities.add(new EqualTo(slots.get(i), Literal.of(values[i])));
+ }
+ Expression predicate = ExpressionUtils.and(equalities.build());
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, expandThreshold);
+ Pair<Boolean, Boolean> result =
+ (Pair<Boolean, Boolean>) canBePrunedOutMethod.invoke(null,
predicate, evaluator);
+ Assertions.assertEquals(expectedPruned, result.first,
+ "tuple=" + Arrays.toString(values) + ", expandThreshold=" +
expandThreshold);
+ }
+
+ /**
+ * Construct a closed-open composite RANGE partition from integer tuple
endpoints.
+ *
+ * @param columns partition columns in key order
+ * @param lowerValues inclusive lower endpoint values
+ * @param upperValues exclusive upper endpoint values
+ * @return partition item representing {@code [lowerValues, upperValues)}
+ * @throws AnalysisException if an endpoint cannot be converted to a typed
partition key
+ */
+ private RangePartitionItem createRangePartitionItem(
+ List<Column> columns, int[] lowerValues, int[] upperValues) throws
AnalysisException {
+ ImmutableList.Builder<PartitionValue> lower =
ImmutableList.builderWithExpectedSize(lowerValues.length);
+ ImmutableList.Builder<PartitionValue> upper =
ImmutableList.builderWithExpectedSize(upperValues.length);
+ for (int value : lowerValues) {
+ lower.add(new PartitionValue(Integer.toString(value)));
+ }
+ for (int value : upperValues) {
+ upper.add(new PartitionValue(Integer.toString(value)));
+ }
+ PartitionKey lowerKey = PartitionKey.createPartitionKey(lower.build(),
columns);
+ PartitionKey upperKey = PartitionKey.createPartitionKey(upper.build(),
columns);
+ return new RangePartitionItem(Range.closedOpen(lowerKey, upperKey));
+ }
+
private ListPartitionItem createListPartitionItem(String... values) throws
AnalysisException {
ImmutableList.Builder<PartitionKey> partitionKeys =
ImmutableList.builder();
for (String value : values) {
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
b/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
new file mode 100644
index 00000000000..4cf8fb2485d
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !expanded_lower_suffix --
+1
+
+-- !expanded_upper_suffix --
+2
+
+-- !expanded_upper_equal_prefix --
+3
+
+-- !unexpanded_lower_suffix --
+1
+
+-- !unexpanded_upper_suffix --
+2
+
+-- !unexpanded_upper_equal_prefix --
+3
+
+-- !negated_multi_date_in --
+42
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
new file mode 100644
index 00000000000..91c2a36ad8c
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lexicographic_range_partition") {
+ sql "DROP TABLE IF EXISTS lexicographic_range_partition"
+ sql """
+ CREATE TABLE lexicographic_range_partition (
+ k1 INT NOT NULL,
+ k2 INT NOT NULL,
+ k3 INT NOT NULL,
+ v INT NOT NULL
+ )
+ DUPLICATE KEY(k1, k2, k3)
+ PARTITION BY RANGE(k1, k2, k3) (
+ PARTITION p_target VALUES [("1", "10", "100"), ("100", "20",
"200"))
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO lexicographic_range_partition VALUES
+ (1, 11, 50, 1),
+ (100, 19, 250, 2),
+ (100, 20, 199, 3),
+ (1, 10, 100, 4)
+ """
+
+ // Verify the optimizer retains the only partition before checking the
query result. A wrong
+ // empty result alone is ambiguous here because it could also come from
execution or test data.
+ def assertTargetPartition = { String query ->
+ explain {
+ sql query
+ contains "partitions=1/1 (p_target)"
+ }
+ }
+
+ sql "SET partition_pruning_expand_threshold=200"
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=1 AND k2=11 AND k3=50")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=19 AND k3=250")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=20 AND k3=199")
+ qt_expanded_lower_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=1 AND k2=11 AND k3=50"
+ qt_expanded_upper_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=100 AND k2=19 AND k3=250"
+ qt_expanded_upper_equal_prefix "SELECT v FROM
lexicographic_range_partition WHERE k1=100 AND k2=20 AND k3=199"
+
+ sql "SET partition_pruning_expand_threshold=1"
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=1 AND k2=11 AND k3=50")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=19 AND k3=250")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=20 AND k3=199")
+ qt_unexpanded_lower_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=1 AND k2=11 AND k3=50"
+ qt_unexpanded_upper_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=100 AND k2=19 AND k3=250"
+ qt_unexpanded_upper_equal_prefix "SELECT v FROM
lexicographic_range_partition WHERE k1=100 AND k2=20 AND k3=199"
+
+ sql "DROP TABLE IF EXISTS lexicographic_range_not_date_in"
+ sql """
+ CREATE TABLE lexicographic_range_not_date_in (
+ k1 INT NOT NULL,
+ k2 INT NOT NULL,
+ k3 DATE NOT NULL,
+ v INT NOT NULL
+ )
+ DUPLICATE KEY(k1, k2, k3)
+ PARTITION BY RANGE(k1, k2, k3) (
+ PARTITION p_target VALUES [("1", "10", "2020-01-01"), ("2", "20",
"2020-01-04"))
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql "INSERT INTO lexicographic_range_not_date_in VALUES (1, 11,
'2020-01-01', 42)"
+
+ // The Date-IN rewrite creates NOT(OR(day ranges)). Default-only k2 ranges
must not leak into
+ // that predicate tree, otherwise NOT turns the valid partition into an
empty set.
+ sql "SET partition_pruning_expand_threshold=200"
+ def negatedDateInQuery = """
+ SELECT v FROM lexicographic_range_not_date_in
+ WHERE NOT(DATE(k3) IN ('2020-01-02', '2020-01-03'))
+ """
+ assertTargetPartition(negatedDateInQuery)
+ qt_negated_multi_date_in negatedDateInQuery
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
