This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 00786a3295c [fix](Nereids) could not prune datev1 partition column
(#24959)
00786a3295c is described below
commit 00786a3295c53a75af188f704e954d2a37e028b3
Author: morrySnow <[email protected]>
AuthorDate: Wed Sep 27 18:41:56 2023 +0800
[fix](Nereids) could not prune datev1 partition column (#24959)
because storage engine could not process date comparison predicates.
we convert it to datetime comparison predicates.
however, partition prunner could not process cast(slot) cp literal.
so, we convert back in partition pruner to let it work well.
TODO:
move convert date to datetime in translate stage
and only convert predicates for storage engine.
---
.../rules/HiveDefaultPartitionEvaluator.java | 4 +-
.../rules/OneRangePartitionEvaluator.java | 18 ++++----
.../rules/expression/rules/PartitionPruner.java | 48 ++++++++++++++++++++--
.../trees/expressions/literal/DateTimeLiteral.java | 4 ++
4 files changed, 59 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java
index 04785e1db14..f50cbe3cf33 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java
@@ -32,8 +32,8 @@ import java.util.Map;
* For any partition predicate, the evaluate() will always return true.
*/
public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator {
- private long id;
- private List<Slot> partitionSlots;
+ private final long id;
+ private final List<Slot> partitionSlots;
public HiveDefaultPartitionEvaluator(long id, List<Slot> partitionSlots) {
this.id = id;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index f9857646852..13ecddd150a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -76,15 +76,15 @@ public class OneRangePartitionEvaluator
extends ExpressionVisitor<EvaluateRangeResult, EvaluateRangeInput>
implements OnePartitionEvaluator {
private final long partitionId;
- private List<Slot> partitionSlots;
- private RangePartitionItem partitionItem;
- private ExpressionRewriteContext expressionRewriteContext;
- private List<PartitionSlotType> partitionSlotTypes;
- private List<Literal> lowers;
- private List<Literal> uppers;
- private List<List<Expression>> inputs;
- private Map<Slot, Boolean> partitionSlotContainsNull;
- private Map<Slot, PartitionSlotType> slotToType;
+ private final List<Slot> partitionSlots;
+ private final RangePartitionItem partitionItem;
+ private final ExpressionRewriteContext expressionRewriteContext;
+ private final List<PartitionSlotType> partitionSlotTypes;
+ private final List<Literal> lowers;
+ private final List<Literal> uppers;
+ private final List<List<Expression>> inputs;
+ private final Map<Slot, Boolean> partitionSlotContainsNull;
+ private final Map<Slot, PartitionSlotType> slotToType;
/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot>
partitionSlots,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index 8479c27776a..6c932f558b0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -22,9 +22,16 @@ import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
+import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.types.DateTimeType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -36,9 +43,9 @@ import java.util.Objects;
/**
* PartitionPruner
*/
-public class PartitionPruner {
- private List<OnePartitionEvaluator> partitions;
- private Expression partitionPredicate;
+public class PartitionPruner extends DefaultExpressionRewriter<Void> {
+ private final List<OnePartitionEvaluator> partitions;
+ private final Expression partitionPredicate;
/** Different type of table may have different partition prune behavior. */
public enum PartitionTableType {
@@ -48,7 +55,40 @@ public class PartitionPruner {
private PartitionPruner(List<OnePartitionEvaluator> partitions, Expression
partitionPredicate) {
this.partitions = Objects.requireNonNull(partitions, "partitions
cannot be null");
- this.partitionPredicate = Objects.requireNonNull(partitionPredicate,
"partitionPredicate cannot be null");
+ this.partitionPredicate =
Objects.requireNonNull(partitionPredicate.accept(this, null),
+ "partitionPredicate cannot be null");
+ }
+
+ @Override
+ public Expression visitComparisonPredicate(ComparisonPredicate cp, Void
context) {
+ // Date cp Date is not supported in BE storage engine. So cast to
DateTime in SimplifyComparisonPredicate
+ // for easy process partition prune, we convert back to date compare
date here
+ // see more info in SimplifyComparisonPredicate
+ Expression left = cp.left();
+ Expression right = cp.right();
+ if (left.getDataType() != DateTimeType.INSTANCE || right.getDataType()
!= DateTimeType.INSTANCE) {
+ return cp;
+ }
+ if (!(left instanceof DateTimeLiteral) && !(right instanceof
DateTimeLiteral)) {
+ return cp;
+ }
+ if (left instanceof DateTimeLiteral && ((DateTimeLiteral)
left).isMidnight()
+ && right instanceof Cast
+ && ((Cast) right).child() instanceof SlotReference
+ && ((Cast) right).child().getDataType().isDateType()) {
+ DateTimeLiteral dt = (DateTimeLiteral) left;
+ Cast cast = (Cast) right;
+ return cp.withChildren(new DateLiteral(dt.getYear(),
dt.getMonth(), dt.getDay()), cast.child());
+ } else if (right instanceof DateTimeLiteral && ((DateTimeLiteral)
right).isMidnight()
+ && left instanceof Cast
+ && ((Cast) left).child() instanceof SlotReference
+ && ((Cast) left).child().getDataType().isDateType()) {
+ DateTimeLiteral dt = (DateTimeLiteral) right;
+ Cast cast = (Cast) left;
+ return cp.withChildren(cast.child(), new DateLiteral(dt.getYear(),
dt.getMonth(), dt.getDay()));
+ } else {
+ return cp;
+ }
}
public List<Long> prune() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
index 3318951c959..aabe6f1e193 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
@@ -93,6 +93,10 @@ public class DateTimeLiteral extends DateLiteral {
this.day = day;
}
+ public boolean isMidnight() {
+ return hour == 0 && minute == 0 && second == 0 && microSecond == 0;
+ }
+
/**
* determine scale by datetime string
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
