This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 8a0d05d9b01 [opt](mtmv) Materialized view partition track supports
date_trunc and optimize the fail reason (#35562) (#36947)
8a0d05d9b01 is described below
commit 8a0d05d9b01760fab1e2e7d6af5f479272289275
Author: seawinde <[email protected]>
AuthorDate: Fri Jul 5 15:12:43 2024 +0800
[opt](mtmv) Materialized view partition track supports date_trunc and
optimize the fail reason (#35562) (#36947)
cherry pick from master #35562
commitId: 43d0f191
---
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 6 -
.../mv/AsyncMaterializationContext.java | 6 +-
.../exploration/mv/MaterializedViewUtils.java | 221 ++++++++++++--
.../mv/rollup/AggFunctionRollUpHandler.java | 5 +-
.../rules/expression/ExpressionNormalization.java | 2 +
.../rules/expression/rules/MergeDateTrunc.java | 78 +++++
.../org/apache/doris/nereids/trees/TreeNode.java | 4 +-
.../trees/expressions/literal/Interval.java | 37 ++-
.../commands/info/MTMVPartitionDefinition.java | 107 ++++---
.../exploration/mv/MaterializedViewUtilsTest.java | 172 ++++++++---
.../expression/ExpressionRewriteTestHelper.java | 6 +
.../rules/expression/MergeDateTruncTest.java | 57 ++++
.../data/mtmv_p0/test_rollup_partition_mtmv.out | 22 ++
.../mtmv_p0/test_rollup_partition_mtmv.groovy | 325 +++++++++++++++++++++
14 files changed, 900 insertions(+), 148 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index 7ca1b7e3e63..ff4060f334a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -21,8 +21,6 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.catalog.Column;
import org.apache.doris.common.AnalysisException;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.gson.annotations.SerializedName;
import java.util.List;
@@ -38,10 +36,6 @@ public class MTMVPartitionInfo {
SELF_MANAGE
}
- public static final ImmutableSet<String> MTMV_PARTITION_FUNCTIONS = new
ImmutableSortedSet.Builder<String>(
- String.CASE_INSENSITIVE_ORDER).add("date_trunc")
- .build();
-
@SerializedName("pt")
private MTMVPartitionType partitionType;
@SerializedName("rt")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
index 9776673de79..d830e9d41cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
@@ -104,9 +104,9 @@ public class AsyncMaterializationContext extends
MaterializationContext {
return Optional.empty();
}
RelationId relationId = null;
- List<LogicalOlapScan> logicalOlapScan =
this.getScanPlan().collectFirst(LogicalOlapScan.class::isInstance);
- if (!logicalOlapScan.isEmpty()) {
- relationId = logicalOlapScan.get(0).getRelationId();
+ Optional<LogicalOlapScan> logicalOlapScan =
this.getScanPlan().collectFirst(LogicalOlapScan.class::isInstance);
+ if (logicalOlapScan.isPresent()) {
+ relationId = logicalOlapScan.get().getRelationId();
}
return Optional.of(Pair.of(relationId,
normalizeStatisticsColumnExpression(mtmvCache.getStatistics())));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 53f775e2177..5c6a666fdb4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -27,12 +27,18 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.StructInfoMap;
+import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -48,10 +54,12 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
@@ -77,9 +85,11 @@ public class MaterializedViewUtils {
* @param materializedViewPlan this should be rewritten or analyzed plan,
should not be physical plan.
* @param column ref column name.
*/
- public static Optional<RelatedTableInfo> getRelatedTableInfo(String
column, Plan materializedViewPlan) {
+ public static RelatedTableInfo getRelatedTableInfo(String column, String
timeUnit,
+ Plan materializedViewPlan, CascadesContext cascadesContext) {
+
List<Slot> outputExpressions = materializedViewPlan.getOutput();
- Slot columnExpr = null;
+ NamedExpression columnExpr = null;
// get column slot
for (Slot outputSlot : outputExpressions) {
if (outputSlot.getName().equalsIgnoreCase(column)) {
@@ -88,14 +98,12 @@ public class MaterializedViewUtils {
}
}
if (columnExpr == null) {
- return Optional.empty();
- }
- if (!(columnExpr instanceof SlotReference)) {
- return Optional.empty();
+ return RelatedTableInfo.failWith("partition column can not find
from sql select column");
}
- SlotReference columnSlot = (SlotReference) columnExpr;
- if (!columnSlot.isColumnFromTable()) {
- return Optional.empty();
+ if (timeUnit != null) {
+ Expression dateTrunc = new DateTrunc(columnExpr, new
VarcharLiteral(timeUnit));
+ columnExpr = new Alias(dateTrunc);
+ materializedViewPlan = new
LogicalProject<>(ImmutableList.of(columnExpr), materializedViewPlan);
}
// Collect table relation map which is used to identify self join
List<CatalogRelation> catalogRelationObjs =
@@ -107,19 +115,20 @@ public class MaterializedViewUtils {
}
// Check sql pattern
IncrementCheckerContext checkContext =
- new IncrementCheckerContext(columnSlot,
tableCatalogRelationMultimapBuilder.build());
+ new IncrementCheckerContext(columnExpr,
tableCatalogRelationMultimapBuilder.build(), cascadesContext);
materializedViewPlan.accept(MaterializedViewIncrementChecker.INSTANCE,
checkContext);
Multimap<TableIf, Column> partitionRelatedTableAndColumnMap =
checkContext.getPartitionRelatedTableAndColumnMap();
if (partitionRelatedTableAndColumnMap.isEmpty()) {
- return Optional.empty();
+ return RelatedTableInfo.failWith(String.format("can't not find
valid partition track column, because %s",
+ String.join(",", checkContext.getFailReasons())));
}
// TODO support to return only one related table info, support multi
later
for (Map.Entry<TableIf, Column> entry :
partitionRelatedTableAndColumnMap.entries()) {
- return Optional.of(new RelatedTableInfo(new
BaseTableInfo(entry.getKey()), true,
- entry.getValue().getName()));
+ return RelatedTableInfo.successWith(new
BaseTableInfo(entry.getKey()), true,
+ entry.getValue().getName(),
checkContext.getPartitionExpression().orElseGet(() -> null));
}
- return Optional.empty();
+ return RelatedTableInfo.failWith("can't not find valid partition track
column finally");
}
/**
@@ -290,10 +299,63 @@ public class MaterializedViewUtils {
DefaultPlanVisitor<Void, IncrementCheckerContext> {
public static final MaterializedViewIncrementChecker INSTANCE = new
MaterializedViewIncrementChecker();
+ public static final Set<Class<? extends Expression>>
SUPPORT_EXPRESSION_TYPES =
+ ImmutableSet.of(DateTrunc.class, SlotReference.class,
Literal.class);
@Override
public Void visitLogicalProject(LogicalProject<? extends Plan>
project, IncrementCheckerContext context) {
- return visit(project, context);
+ NamedExpression mvPartitionColumn = context.getMvPartitionColumn();
+ List<Slot> output = project.getOutput();
+ if (context.getMvPartitionColumn().isColumnFromTable()) {
+ return visit(project, context);
+ }
+ for (Slot projectSlot : output) {
+ if (!projectSlot.equals(mvPartitionColumn.toSlot())) {
+ continue;
+ }
+ if (projectSlot.isColumnFromTable()) {
+ context.setMvPartitionColumn(projectSlot);
+ } else {
+ // should be only use date_trunc
+ Expression shuttledExpression =
+
ExpressionUtils.shuttleExpressionWithLineage(projectSlot, project, new
BitSet());
+ // merge date_trunc
+ shuttledExpression = new
ExpressionNormalization().rewrite(shuttledExpression,
+ new
ExpressionRewriteContext(context.getCascadesContext()));
+
+ List<Expression> expressions =
shuttledExpression.collectToList(Expression.class::isInstance);
+ for (Expression expression : expressions) {
+ if (SUPPORT_EXPRESSION_TYPES.stream().noneMatch(
+ supportExpression ->
supportExpression.isAssignableFrom(expression.getClass()))) {
+ context.addFailReason(
+ String.format("partition column use
invalid implicit expression, invalid "
+ + "expression is %s",
expression));
+ return null;
+ }
+ }
+ List<DateTrunc> dataTruncExpressions =
+
shuttledExpression.collectToList(DateTrunc.class::isInstance);
+ if (dataTruncExpressions.size() != 1) {
+ // mv time unit level is little then query
+ context.addFailReason("partition column time unit
level should be "
+ + "greater than sql select column");
+ return null;
+ }
+ Optional<Slot> columnExpr =
+
shuttledExpression.getArgument(0).collectFirst(Slot.class::isInstance);
+ if (!columnExpr.isPresent() ||
!columnExpr.get().isColumnFromTable()) {
+ context.addFailReason(String.format("partition
reference column should be direct column "
+ + "rather then expression except date_trunc,
columnExpr is %s", columnExpr));
+ return null;
+ }
+ context.setPartitionExpression(shuttledExpression);
+ context.setMvPartitionColumn(columnExpr.get());
+ }
+ return visit(project, context);
+ }
+ context.addFailReason(String.format("partition reference column
should be direct column "
+ + "rather then expression except date_trunc, current
project is %s", project));
+ return null;
}
@Override
@@ -305,6 +367,7 @@ public class MaterializedViewUtils {
public Void visitLogicalJoin(LogicalJoin<? extends Plan, ? extends
Plan> join,
IncrementCheckerContext context) {
if (join.isMarkJoin()) {
+ context.addFailReason("partition track doesn't support mark
join");
return null;
}
Plan left = join.child(0);
@@ -313,7 +376,11 @@ public class MaterializedViewUtils {
&& slot.isColumnFromTable())
.map(slot -> ((SlotReference) slot).getColumn().get())
.collect(Collectors.toSet());
- boolean useLeft =
leftColumnSet.contains(context.getMvPartitionColumn().getColumn().get());
+ SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
+ if (contextPartitionColumn == null) {
+ return null;
+ }
+ boolean useLeft =
leftColumnSet.contains(contextPartitionColumn.getColumn().get());
JoinType joinType = join.getJoinType();
if (joinType.isInnerJoin() || joinType.isCrossJoin()) {
return visit(join, context);
@@ -326,12 +393,16 @@ public class MaterializedViewUtils {
|| joinType.isRightSemiJoin()) && !useLeft) {
return visit(join.right(), context);
}
+ context.addFailReason(String.format("partition column is in un
supported join null generate side, "
+ + "current join type is %s", joinType));
return null;
}
@Override
public Void visitLogicalRelation(LogicalRelation relation,
IncrementCheckerContext context) {
if (!(relation instanceof LogicalCatalogRelation)) {
+ context.addFailReason(String.format("relation should be
LogicalCatalogRelation, "
+ + "but now is %s",
relation.getClass().getSimpleName()));
return null;
}
LogicalCatalogRelation logicalCatalogRelation =
(LogicalCatalogRelation) relation;
@@ -339,25 +410,41 @@ public class MaterializedViewUtils {
// if self join, self join can not partition track now, remove the
partition column correspondingly
if (context.getRelationByTable(table).size() > 1) {
context.getPartitionRelatedTableAndColumnMap().removeAll(table);
+ context.addFailReason(String.format("self join doesn't support
partition update, "
+ + "self join table name is %s", table.getName()));
return null;
}
// TODO: 2024/1/31 support only one partition referenced column,
support multi later
if (!context.getPartitionRelatedTableAndColumnMap().isEmpty()) {
+ context.addFailReason(String.format("partition track already
has an related base table column,"
+ + "track info is %s",
context.getPartitionRelatedTableAndColumnMap()));
return null;
}
if (!(table instanceof MTMVRelatedTableIf)) {
+ context.addFailReason(String.format("relation base table is
not MTMVRelatedTableIf, the table is %s",
+ table.getName()));
return null;
}
MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table;
PartitionType type = relatedTable.getPartitionType();
if (PartitionType.UNPARTITIONED.equals(type)) {
+ context.addFailReason(String.format("related base table is not
partition table, the table is %s",
+ table.getName()));
return null;
}
Set<Column> partitionColumnSet = new
HashSet<>(relatedTable.getPartitionColumns());
- Column mvReferenceColumn =
context.getMvPartitionColumn().getColumn().get();
+ SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
+ if (contextPartitionColumn == null) {
+ return null;
+ }
+ Column mvReferenceColumn =
contextPartitionColumn.getColumn().get();
if (partitionColumnSet.contains(mvReferenceColumn)
&& (!mvReferenceColumn.isAllowNull() ||
relatedTable.isPartitionColumnAllowNull())) {
context.addTableColumn(table, mvReferenceColumn);
+ } else {
+ context.addFailReason(String.format("related base table
partition column doesn't contain the mv"
+ + " partition or partition nullable check
fail, the mvReferenceColumn is %s",
+ mvReferenceColumn));
}
return visit(relation, context);
}
@@ -367,6 +454,7 @@ public class MaterializedViewUtils {
IncrementCheckerContext context) {
Set<Expression> groupByExprSet = new
HashSet<>(aggregate.getGroupByExpressions());
if (groupByExprSet.isEmpty()) {
+ context.addFailReason("group by sets is empty, doesn't contain
the target partition");
return null;
}
Set<Column> originalGroupbyExprSet = new HashSet<>();
@@ -375,7 +463,12 @@ public class MaterializedViewUtils {
originalGroupbyExprSet.add(((SlotReference)
groupExpr).getColumn().get());
}
});
- if
(!originalGroupbyExprSet.contains(context.getMvPartitionColumn().getColumn().get()))
{
+ SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
+ if (contextPartitionColumn == null) {
+ return null;
+ }
+ if
(!originalGroupbyExprSet.contains(contextPartitionColumn.getColumn().get())) {
+ context.addFailReason("group by sets doesn't contain the
target partition");
return null;
}
return visit(aggregate, context);
@@ -389,6 +482,7 @@ public class MaterializedViewUtils {
}
for (NamedExpression namedExpression : windowExpressions) {
if (!checkWindowPartition(namedExpression, context)) {
+ context.addFailReason("window partition sets doesn't
contain the target partition");
return null;
}
}
@@ -421,29 +515,52 @@ public class MaterializedViewUtils {
originalPartitionbyExprSet.add(((SlotReference)
groupExpr).getColumn().get());
}
});
- if
(!originalPartitionbyExprSet.contains(context.getMvPartitionColumn().getColumn().get()))
{
+ SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
+ if (contextPartitionColumn == null) {
+ return false;
+ }
+ if
(!originalPartitionbyExprSet.contains(contextPartitionColumn.getColumn().get()))
{
return false;
}
}
return true;
}
+
+ private SlotReference
getContextPartitionColumn(IncrementCheckerContext context) {
+ if (!context.getMvPartitionColumn().isColumnFromTable()) {
+ context.addFailReason(String.format("context partition column
should be slot from column, "
+ + "context column is %s",
+ context.getMvPartitionColumn()));
+ return null;
+ }
+ return (SlotReference) context.getMvPartitionColumn();
+ }
}
private static final class IncrementCheckerContext {
- private final SlotReference mvPartitionColumn;
+ private NamedExpression mvPartitionColumn;
+ private Optional<Expression> partitionExpression = Optional.empty();
private final Multimap<TableIdentifier, CatalogRelation>
tableAndCatalogRelationMap;
private final Multimap<TableIf, Column>
partitionRelatedTableAndColumnMap = HashMultimap.create();
+ private final Set<String> failReasons = new HashSet<>();
+ private final CascadesContext cascadesContext;
- public IncrementCheckerContext(SlotReference mvPartitionColumn,
- Multimap<TableIdentifier, CatalogRelation>
tableAndCatalogRelationMap) {
+ public IncrementCheckerContext(NamedExpression mvPartitionColumn,
+ Multimap<TableIdentifier, CatalogRelation>
tableAndCatalogRelationMap,
+ CascadesContext cascadesContext) {
this.mvPartitionColumn = mvPartitionColumn;
this.tableAndCatalogRelationMap = tableAndCatalogRelationMap;
+ this.cascadesContext = cascadesContext;
}
- public SlotReference getMvPartitionColumn() {
+ public NamedExpression getMvPartitionColumn() {
return mvPartitionColumn;
}
+ public void setMvPartitionColumn(NamedExpression mvPartitionColumn) {
+ this.mvPartitionColumn = mvPartitionColumn;
+ }
+
public void addTableColumn(TableIf relatedTable, Column
partitionColumn) {
partitionRelatedTableAndColumnMap.put(relatedTable,
partitionColumn);
}
@@ -459,20 +576,56 @@ public class MaterializedViewUtils {
public void addTableAndRelation(TableIf tableIf, CatalogRelation
relation) {
tableAndCatalogRelationMap.put(new TableIdentifier(tableIf),
relation);
}
+
+ public Set<String> getFailReasons() {
+ return failReasons;
+ }
+
+ public void addFailReason(String failReason) {
+ this.failReasons.add(failReason);
+ }
+
+ public CascadesContext getCascadesContext() {
+ return cascadesContext;
+ }
+
+ public Optional<Expression> getPartitionExpression() {
+ return partitionExpression;
+ }
+
+ public void setPartitionExpression(Expression partitionExpression) {
+ this.partitionExpression =
Optional.ofNullable(partitionExpression);
+ }
}
/**
* The related table info that mv relate
*/
public static final class RelatedTableInfo {
- private BaseTableInfo tableInfo;
- private boolean pctPossible;
- private String column;
-
- public RelatedTableInfo(BaseTableInfo tableInfo, boolean pctPossible,
String column) {
+ private final BaseTableInfo tableInfo;
+ private final boolean pctPossible;
+ private final String column;
+ private final Set<String> failReasons = new HashSet<>();
+ // This records the partition expression if exist
+ private Optional<Expression> partitionExpression;
+
+ public RelatedTableInfo(BaseTableInfo tableInfo, boolean pctPossible,
String column, String failReason,
+ Expression partitionExpression) {
this.tableInfo = tableInfo;
this.pctPossible = pctPossible;
this.column = column;
+ this.failReasons.add(failReason);
+ this.partitionExpression =
Optional.ofNullable(partitionExpression);
+ }
+
+ public static RelatedTableInfo failWith(String failReason) {
+ return new RelatedTableInfo(null, false, null, failReason,
+ null);
+ }
+
+ public static RelatedTableInfo successWith(BaseTableInfo tableInfo,
boolean pctPossible, String column,
+ Expression partitionExpression) {
+ return new RelatedTableInfo(tableInfo, pctPossible, column, "",
partitionExpression);
}
public BaseTableInfo getTableInfo() {
@@ -486,5 +639,17 @@ public class MaterializedViewUtils {
public String getColumn() {
return column;
}
+
+ public void addFailReason(String failReason) {
+ this.failReasons.add(failReason);
+ }
+
+ public String getFailReason() {
+ return String.join(",", failReasons);
+ }
+
+ public Optional<Expression> getPartitionExpression() {
+ return partitionExpression;
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java
index 97229af62c3..250d8a83c26 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java
@@ -24,6 +24,8 @@ import
org.apache.doris.nereids.trees.expressions.functions.Function;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.RollUpTrait;
+import com.google.common.collect.ImmutableList;
+
import java.util.List;
import java.util.Set;
@@ -63,7 +65,8 @@ public abstract class AggFunctionRollUpHandler {
protected static List<Expression> extractArguments(Expression
functionWithAny, Function actualFunction) {
Set<Object> exprSetToRemove = functionWithAny.collectToSet(expr ->
!(expr instanceof Any));
return actualFunction.collectFirst(expr ->
- exprSetToRemove.stream().noneMatch(exprToRemove ->
exprToRemove.equals(expr)));
+ exprSetToRemove.stream().noneMatch(exprToRemove ->
exprToRemove.equals(expr)))
+ .map(expr -> ImmutableList.of((Expression)
expr)).orElse(ImmutableList.of());
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index f63ab2eee5f..0e52f2aaaf6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -23,6 +23,7 @@ import
org.apache.doris.nereids.rules.expression.rules.DigitalMaskingConvert;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup;
import
org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule;
+import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
import
org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
import
org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule;
@@ -53,6 +54,7 @@ public class ExpressionNormalization extends
ExpressionRewrite {
DigitalMaskingConvert.INSTANCE,
SimplifyArithmeticComparisonRule.INSTANCE,
ConvertAggStateCast.INSTANCE,
+ MergeDateTrunc.INSTANCE,
CheckCast.INSTANCE
)
);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java
new file mode 100644
index 00000000000..892724cf893
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MergeDateTrunc.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.literal.Interval.TimeUnit;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Rewrite rule to convert
+ * For example:
+ * date_trunc(date_trunc(data_slot, 'hour'), 'day') -> date_trunc(data_slot,
'day')
+ */
+public class MergeDateTrunc implements ExpressionPatternRuleFactory {
+
+ public static MergeDateTrunc INSTANCE = new MergeDateTrunc();
+ public static ImmutableSet<TimeUnit> UN_SUPPORT_TIME_UNIT =
ImmutableSet.of(TimeUnit.WEEK, TimeUnit.QUARTER);
+
+ @Override
+ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+ return ImmutableList.of(
+ matchesTopType(DateTrunc.class)
+ .when(dateTrunc -> dateTrunc.getArgument(0) instanceof
DateTrunc)
+ .then(MergeDateTrunc::rewrite)
+ );
+ }
+
+ private static Expression rewrite(DateTrunc dateTrunc) {
+
+ Expression parentTimeUnitArgument = dateTrunc.getArgument(1);
+ if (!(parentTimeUnitArgument instanceof Literal)) {
+ return dateTrunc;
+ }
+ Optional<TimeUnit> parentTimeUnit = TimeUnit.of(((Literal)
parentTimeUnitArgument).getStringValue());
+ DateTrunc childDateTrunc = (DateTrunc) dateTrunc.getArgument(0);
+ Expression childTimeUnitArgument = childDateTrunc.getArgument(1);
+ if (!(childTimeUnitArgument instanceof Literal)) {
+ return dateTrunc;
+ }
+ Optional<TimeUnit> childTimeUnit = TimeUnit.of(((Literal)
childTimeUnitArgument).getStringValue());
+ if (!parentTimeUnit.isPresent() || !childTimeUnit.isPresent()) {
+ return dateTrunc;
+ }
+ if (UN_SUPPORT_TIME_UNIT.contains(parentTimeUnit.get())
+ || UN_SUPPORT_TIME_UNIT.contains(childTimeUnit.get())) {
+ return dateTrunc;
+ }
+ if (parentTimeUnit.get().getLevel() < childTimeUnit.get().getLevel()) {
+ return dateTrunc;
+ }
+ return new DateTrunc(childDateTrunc.getArgument(0), new
VarcharLiteral(parentTimeUnit.get().toString()));
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
index a4bfab08890..5900584b861 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -271,7 +271,7 @@ public interface TreeNode<NODE_TYPE extends
TreeNode<NODE_TYPE>> {
/**
* Collect the nodes that satisfied the predicate firstly.
*/
- default <T> List<T> collectFirst(Predicate<TreeNode<NODE_TYPE>> predicate)
{
+ default <T> Optional<T> collectFirst(Predicate<TreeNode<NODE_TYPE>>
predicate) {
List<TreeNode<NODE_TYPE>> result = new ArrayList<>();
foreach(node -> {
if (result.isEmpty() && predicate.test(node)) {
@@ -279,7 +279,7 @@ public interface TreeNode<NODE_TYPE extends
TreeNode<NODE_TYPE>> {
}
return !result.isEmpty();
});
- return (List<T>) ImmutableList.copyOf(result);
+ return result.isEmpty() ? Optional.empty() : Optional.of((T)
result.get(0));
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java
index 5cffc0c7fff..454001fb3f1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java
@@ -25,6 +25,9 @@ import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateType;
import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.EnumUtils;
+
+import java.util.Optional;
/**
* Interval for timestamp calculation.
@@ -61,30 +64,46 @@ public class Interval extends Expression implements
LeafExpression, AlwaysNotNul
* Supported time unit.
*/
public enum TimeUnit {
- YEAR("YEAR", false),
- MONTH("MONTH", false),
- WEEK("WEEK", false),
- DAY("DAY", false),
- HOUR("HOUR", true),
- MINUTE("MINUTE", true),
- SECOND("SECOND", true);
+ YEAR("YEAR", false, 800),
+ MONTH("MONTH", false, 700),
+ QUARTER("QUARTER", false, 600),
+ WEEK("WEEK", false, 500),
+ DAY("DAY", false, 400),
+ HOUR("HOUR", true, 300),
+ MINUTE("MINUTE", true, 200),
+ SECOND("SECOND", true, 100);
private final String description;
-
private final boolean isDateTimeUnit;
+ /**
+ * Time unit level, second level is low, year level is high
+ */
+ private final int level;
- TimeUnit(String description, boolean isDateTimeUnit) {
+ TimeUnit(String description, boolean isDateTimeUnit, int level) {
this.description = description;
this.isDateTimeUnit = isDateTimeUnit;
+ this.level = level;
}
public boolean isDateTimeUnit() {
return isDateTimeUnit;
}
+ public int getLevel() {
+ return level;
+ }
+
@Override
public String toString() {
return description;
}
+
+ /**
+ * Construct time unit by name
+ */
+ public static Optional<TimeUnit> of(String name) {
+ return
Optional.ofNullable(EnumUtils.getEnumIgnoreCase(TimeUnit.class, name));
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
index 09b30063b9d..2c5cbb4649e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
@@ -40,7 +40,10 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
import
org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
@@ -51,7 +54,6 @@ import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.Sets;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -59,6 +61,7 @@ import java.util.stream.Collectors;
* MTMVPartitionDefinition
*/
public class MTMVPartitionDefinition {
+ public static final String PARTITION_BY_FUNCTION_NAME = "date_trunc";
private MTMVPartitionType partitionType;
private String partitionCol;
private Expression functionCallExpression;
@@ -78,25 +81,39 @@ public class MTMVPartitionDefinition {
return mtmvPartitionInfo;
}
String partitionColName;
+ String timeUnit;
if (this.partitionType == MTMVPartitionType.EXPR) {
- Expr expr;
- if (functionCallExpression instanceof UnboundFunction) {
- UnboundFunction function = (UnboundFunction)
functionCallExpression;
- expr = new FunctionCallExpr(function.getName(),
- new
FunctionParams(convertToLegacyArguments(function.children())));
+ String functionName = ((UnboundFunction)
functionCallExpression).getName();
+ if (functionCallExpression instanceof UnboundFunction
+ &&
functionName.equalsIgnoreCase(PARTITION_BY_FUNCTION_NAME)) {
+ partitionColName = functionCallExpression.getArgument(0)
instanceof UnboundSlot
+ ? ((UnboundSlot)
functionCallExpression.getArgument(0)).getName() : null;
+ timeUnit =
functionCallExpression.getArguments().get(1).isLiteral()
+ ? ((Literal)
functionCallExpression.getArgument(1)).getStringValue() : null;
} else {
throw new AnalysisException(
"unsupported auto partition expr " +
functionCallExpression.toString());
}
- partitionColName = getColNameFromExpr(expr);
- mtmvPartitionInfo.setExpr(expr);
} else {
partitionColName = this.partitionCol;
+ timeUnit = null;
}
mtmvPartitionInfo.setPartitionCol(partitionColName);
- RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx,
logicalQuery, partitionColName);
+ RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx,
logicalQuery, partitionColName, timeUnit);
mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn());
mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo());
+ if (relatedTableInfo.getPartitionExpression().isPresent()) {
+ // Set mv partition expr by relatedTableInfo, this is used for
partition rollup and so on
+ if
(relatedTableInfo.getPartitionExpression().get().getExpressionName()
+ .equalsIgnoreCase(PARTITION_BY_FUNCTION_NAME)) {
+ DateTrunc dateTrunc = (DateTrunc)
relatedTableInfo.getPartitionExpression().get();
+ // todo use new expression?
+ mtmvPartitionInfo.setExpr(new
FunctionCallExpr(dateTrunc.getName(),
+ new
FunctionParams(convertToLegacyArguments(dateTrunc.children()))));
+ mtmvPartitionInfo.setPartitionType(MTMVPartitionType.EXPR);
+ this.partitionType = MTMVPartitionType.EXPR;
+ }
+ }
if (this.partitionType == MTMVPartitionType.EXPR) {
try {
MTMVPartitionExprFactory.getExprService(mtmvPartitionInfo.getExpr()).analyze(mtmvPartitionInfo);
@@ -107,38 +124,10 @@ public class MTMVPartitionDefinition {
return mtmvPartitionInfo;
}
- /**
- * getColNameFromExpr
- *
- * @param expr expr
- * @return String
- */
- public static String getColNameFromExpr(Expr expr) {
- if (!(expr instanceof FunctionCallExpr)) {
- throw new AnalysisException(
- "auto create partition only support function call expr is:
"
- + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS);
- }
- FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
- List<Expr> paramsExpr = functionCallExpr.getParams().exprs();
- String name = functionCallExpr.getFnName().getFunction();
- if (MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS.contains(name)) {
- for (Expr param : paramsExpr) {
- if (param instanceof SlotRef) {
- return ((SlotRef) param).getColumnName();
- }
- }
- throw new AnalysisException("can not find colName");
- } else {
- throw new AnalysisException(
- "auto create partition only support function call expr is:
"
- + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS);
- }
- }
-
private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner,
ConnectContext ctx, LogicalPlan
logicalQuery,
- String partitionColName) {
+ String partitionColName,
+ String timeUnit) {
CascadesContext cascadesContext = planner.getCascadesContext();
SessionVariable sessionVariable =
cascadesContext.getConnectContext().getSessionVariable();
Set<String> tempDisableRules =
sessionVariable.getDisableNereidsRuleNames();
@@ -149,12 +138,13 @@ public class MTMVPartitionDefinition {
try {
Plan mvRewrittenPlan =
planner.plan(logicalQuery, PhysicalProperties.ANY,
ExplainLevel.REWRITTEN_PLAN);
- Optional<RelatedTableInfo> relatedTableInfo = MaterializedViewUtils
- .getRelatedTableInfo(partitionColName, mvRewrittenPlan);
- if (!relatedTableInfo.isPresent() ||
!relatedTableInfo.get().isPctPossible()) {
- throw new AnalysisException("Unable to find a suitable base
table for partitioning");
+ RelatedTableInfo relatedTableInfo = MaterializedViewUtils
+ .getRelatedTableInfo(partitionColName, timeUnit,
mvRewrittenPlan, cascadesContext);
+ if (!relatedTableInfo.isPctPossible()) {
+ throw new AnalysisException(String.format("Unable to find a
suitable base table for partitioning,"
+ + " the fail reason is %s",
relatedTableInfo.getFailReason()));
}
- MTMVRelatedTableIf mtmvBaseRealtedTable =
MTMVUtil.getRelatedTable(relatedTableInfo.get().getTableInfo());
+ MTMVRelatedTableIf mtmvBaseRealtedTable =
MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo());
Set<String> partitionColumnNames =
Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
try {
partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames());
@@ -162,14 +152,14 @@ public class MTMVPartitionDefinition {
throw new AnalysisException(e.getMessage(), e);
}
- if
(!partitionColumnNames.contains(relatedTableInfo.get().getColumn())) {
- throw new AnalysisException("error related column: " +
relatedTableInfo.get().getColumn());
+ if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) {
+ throw new AnalysisException("error related column: " +
relatedTableInfo.getColumn());
}
if (!(mtmvBaseRealtedTable instanceof HMSExternalTable)
&& partitionColumnNames.size() != 1) {
throw new AnalysisException("only hms table support multi
column partition.");
}
- return relatedTableInfo.get();
+ return relatedTableInfo;
} finally {
// after operate, roll back the disable rules
sessionVariable.setDisableNereidsRules(String.join(",",
tempDisableRules));
@@ -178,15 +168,20 @@ public class MTMVPartitionDefinition {
}
private static List<Expr> convertToLegacyArguments(List<Expression>
children) {
- return children.stream().map(child -> {
- if (child instanceof UnboundSlot) {
- return new SlotRef(null, ((UnboundSlot) child).getName());
- } else if (child instanceof Literal) {
- return new StringLiteral(((Literal) child).getStringValue());
- } else {
- throw new AnalysisException("unsupported argument " +
child.toString());
- }
- }).collect(Collectors.toList());
+ return
children.stream().map(MTMVPartitionDefinition::convertToLegacyRecursion).collect(Collectors.toList());
+ }
+
+ private static Expr convertToLegacyRecursion(Expression expression) {
+ if (expression instanceof Slot) {
+ return new SlotRef(null, ((Slot) expression).getName());
+ } else if (expression instanceof Literal) {
+ return new StringLiteral(((Literal) expression).getStringValue());
+ } else if (expression instanceof Cast) {
+ // mv partition roll up only need the slot in cast
+ return convertToLegacyRecursion(((Cast) expression).child());
+ } else {
+ throw new AnalysisException("unsupported argument " +
expression.toString());
+ }
}
public MTMVPartitionType getPartitionType() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index 1ed6b92129a..3878ab742ca 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -28,8 +28,6 @@ import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.Optional;
-
/**
* Test for materialized view util
*/
@@ -251,8 +249,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "on t1.l_orderkey = o_orderkey;",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem_no_data",
"L_SHIPDATE",
@@ -277,8 +276,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "ON l.L_PARTKEY = ps.PS_PARTKEY and
l.L_SUPPKEY = ps.PS_SUPPKEY",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -303,9 +303,10 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "ON l.L_PARTKEY = ps.PS_PARTKEY and
l.L_SUPPKEY = ps.PS_SUPPKEY",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
-
Assertions.assertTrue(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.isPctPossible());
});
}
@@ -326,8 +327,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "GROUP BY l.L_SHIPDATE, o.O_ORDERDATE ",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("ship_data_alias", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("ship_data_alias", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"l_shipdate",
@@ -356,8 +358,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "t2.O_ORDERSTATUS;",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -375,8 +378,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ " group by l_shipdate, l_orderkey",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem_list_partition",
"l_orderkey",
@@ -394,9 +398,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ " group by t1.l_shipdate, t1.l_orderkey,
t1.l_partkey, t1.l_suppkey",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "self join doesn't support partition
update"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
PlanChecker.from(connectContext)
@@ -407,9 +414,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ " group by t1.l_shipdate, t1.l_orderkey,
t1.l_partkey, t1.l_suppkey",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "self join doesn't support partition
update"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
PlanChecker.from(connectContext)
@@ -420,9 +430,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ " group by t1.l_shipdate, t1.l_orderkey,
t1.l_partkey, t1.l_suppkey",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_orderkey", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "partition column is in un supported join
null generate side"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
}
@@ -447,8 +460,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "t2.O_ORDERSTATUS;",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("o_orderdate", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("o_orderdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"orders",
"O_ORDERDATE",
@@ -477,9 +491,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "t2.O_ORDERSTATUS;",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("o_orderdate", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("o_orderdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "partition column is in un supported join
null generate side"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
}
@@ -494,9 +511,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "ON ps_1.PS_PARTKEY = ps_2.PS_SUPPKEY ",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("PS_SUPPLYCOST", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("PS_SUPPLYCOST", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "self join doesn't support partition
update"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
}
@@ -517,8 +537,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "ON l.L_PARTKEY = ps.PS_PARTKEY and
l.L_SUPPKEY = ps.PS_SUPPKEY",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -543,9 +564,73 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ "ON l.L_PARTKEY = ps.PS_PARTKEY and
l.L_SUPPKEY = ps.PS_SUPPKEY",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", rewrittenPlan);
-
Assertions.assertFalse(relatedTableInfo.isPresent());
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "window partition sets doesn't contain the
target partition"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
+ });
+ }
+
+ @Test
+ public void testPartitionDateTrunc() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT date_trunc(t1.L_SHIPDATE, 'hour') as
date_alias, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS, "
+ + "count(distinct case when t1.L_SUPPKEY > 0
then t2.O_ORDERSTATUS else null end) as cnt_1 "
+ + "from "
+ + " (select * from "
+ + " lineitem "
+ + " where L_SHIPDATE in ('2017-01-30')) t1 "
+ + "left join "
+ + " (select * from "
+ + " orders "
+ + " where O_ORDERDATE in ('2017-01-30')) t2 "
+ + "on t1.L_ORDERKEY = t2.O_ORDERKEY "
+ + "group by "
+ + "t1.L_SHIPDATE, "
+ + "t2.O_ORDERDATE, "
+ + "t1.L_QUANTITY, "
+ + "t2.O_ORDERSTATUS;",
+ nereidsPlanner -> {
+ Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("date_alias", "day",
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+ checkRelatedTableInfo(relatedTableInfo,
+ "lineitem",
+ "L_SHIPDATE",
+ true);
+ });
+ }
+
+ @Test
+ public void testPartitionDateTruncShouldNotTrack() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT date_trunc(t1.L_SHIPDATE, 'day') as
date_alias, t2.O_ORDERDATE, t1.L_QUANTITY, t2.O_ORDERSTATUS, "
+ + "count(distinct case when t1.L_SUPPKEY > 0
then t2.O_ORDERSTATUS else null end) as cnt_1 "
+ + "from "
+ + " (select * from "
+ + " lineitem "
+ + " where L_SHIPDATE in ('2017-01-30')) t1 "
+ + "left join "
+ + " (select * from "
+ + " orders "
+ + " where O_ORDERDATE in ('2017-01-30')) t2 "
+ + "on t1.L_ORDERKEY = t2.O_ORDERKEY "
+ + "group by "
+ + "t1.L_SHIPDATE, "
+ + "t2.O_ORDERDATE, "
+ + "t1.L_QUANTITY, "
+ + "t2.O_ORDERSTATUS;",
+ nereidsPlanner -> {
+ Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("date_alias", "hour",
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+
Assertions.assertTrue(relatedTableInfo.getFailReason().contains(
+ "partition column time unit level should
be greater than sql select column"));
+
Assertions.assertFalse(relatedTableInfo.isPctPossible());
});
}
@@ -581,8 +666,9 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
+ ")t2 on t1.vin_type1 = t2.vin_type2;",
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
- Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("upgrade_day", rewrittenPlan);
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("upgrade_day", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"test1",
"upgrade_day",
@@ -630,12 +716,12 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
});
}
- private void checkRelatedTableInfo(Optional<RelatedTableInfo>
relatedTableInfo,
+ private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo,
String expectTableName,
String expectColumnName,
boolean pctPossible) {
- Assertions.assertTrue(relatedTableInfo.isPresent());
- BaseTableInfo relatedBaseTableInfo =
relatedTableInfo.get().getTableInfo();
+ Assertions.assertNotNull(relatedTableInfo);
+ BaseTableInfo relatedBaseTableInfo = relatedTableInfo.getTableInfo();
try {
TableIf tableIf = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrAnalysisException(relatedBaseTableInfo.getCtlId())
@@ -645,7 +731,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
} catch (Exception exception) {
Assertions.fail();
}
-
Assertions.assertEquals(relatedTableInfo.get().getColumn().toLowerCase(),
expectColumnName.toLowerCase());
+ Assertions.assertEquals(relatedTableInfo.getColumn().toLowerCase(),
expectColumnName.toLowerCase());
Assertions.assertTrue(pctPossible);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
index e245ee2f7e1..ad9db4a8dda 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
@@ -78,6 +78,12 @@ public abstract class ExpressionRewriteTestHelper extends
ExpressionRewrite {
Assertions.assertEquals(expectedExpression, rewrittenExpression);
}
+ protected void assertNotRewrite(Expression expression, Expression
expectedExpression) {
+ expression = typeCoercion(expression);
+ Expression rewrittenExpression = executor.rewrite(expression, context);
+ Assertions.assertNotEquals(expectedExpression, rewrittenExpression);
+ }
+
protected void assertRewriteAfterTypeCoercion(String expression, String
expected) {
Map<String, Slot> mem = Maps.newHashMap();
Expression needRewriteExpression = PARSER.parseExpression(expression);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java
new file mode 100644
index 00000000000..945c647fcb5
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/MergeDateTruncTest.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression;
+
+import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+import org.apache.doris.nereids.types.DateV2Type;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+/**
+ * merge date_trunc test
+ */
+public class MergeDateTruncTest extends ExpressionRewriteTestHelper {
+
+ @Test
+ public void testMergeDateTrunc() {
+ executor = new ExpressionRuleExecutor(ImmutableList.of(
+ bottomUp(MergeDateTrunc.INSTANCE)
+ ));
+ Slot dataSlot = new SlotReference("data_slot", DateV2Type.INSTANCE);
+ assertRewrite(new DateTrunc(new DateTrunc(dataSlot, new
VarcharLiteral("HOUR")),
+ new VarcharLiteral("DAY")),
+ new DateTrunc(dataSlot, new VarcharLiteral("DAY")));
+
+ assertRewrite(new DateTrunc(new DateTrunc(dataSlot, new
VarcharLiteral("DAY")),
+ new VarcharLiteral("YEAR")),
+ new DateTrunc(dataSlot, new VarcharLiteral("YEAR")));
+
+ assertNotRewrite(new DateTrunc(new DateTrunc(dataSlot, new
VarcharLiteral("HOUR")),
+ new VarcharLiteral("WEEK")),
+ new DateTrunc(dataSlot, new VarcharLiteral("WEEK")));
+
+ assertNotRewrite(new DateTrunc(new DateTrunc(dataSlot, new
VarcharLiteral("DAY")),
+ new VarcharLiteral("QUARTER")),
+ new DateTrunc(dataSlot, new VarcharLiteral("QUARTER")));
+ }
+}
diff --git a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
index 5552dac72c3..38e59530d8c 100644
--- a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
@@ -4,6 +4,18 @@
2 2020-01-02
3 2020-02-01
+-- !date_list_month_partition_by_column --
+2020-01-01 1 2020-01-01
+2020-01-01 2 2020-01-02
+2020-02-01 3 2020-02-01
+
+-- !date_list_month_level --
+2020-01-01 1 2020-01-01
+2020-01-02 2 2020-01-02
+2020-02-01 3 2020-02-01
+
+-- !date_list_year_partition_by_column --
+
-- !string_list_month --
1 2020==01==01
2 2020==01==02
@@ -14,3 +26,13 @@
2 2020-01-02
3 2020-02-01
+-- !date_range_month_partition_by_column --
+2020-01-01 1 2020-01-01
+2020-01-01 2 2020-01-02
+2020-02-01 3 2020-02-01
+
+-- !date_range_month_level --
+2020-01-01
+2020-01-02
+2020-02-01
+
diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
index 3a1bfe9f4fd..d0a30e840c9 100644
--- a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
@@ -69,6 +69,82 @@ suite("test_rollup_partition_mtmv") {
waitingMTMVTaskFinished(jobName)
order_qt_date_list_month "SELECT * FROM ${mvName} order by k1,k2"
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (month_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ def date_list_month_partitions = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " +
date_list_month_partitions.toString())
+ assertEquals(2, date_list_month_partitions.size())
+ waitingMTMVTaskFinished(getJobName(dbName, mvName))
+ order_qt_date_list_month_partition_by_column "SELECT * FROM ${mvName}"
+
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(month_alias, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName};
+ """
+ def date_list_month_partitions_level = sql """show partitions from
${mvName}"""
+ logger.info("showPartitionsResult: " +
date_list_month_partitions_level.toString())
+ assertEquals(2, date_list_month_partitions_level.size())
+ waitingMTMVTaskFinished(getJobName(dbName, mvName))
+ order_qt_date_list_month_level "SELECT * FROM ${mvName}"
+
+ // mv partition level should be higher or equal then query, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(month_alias, 'day'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column time unit level
should be greater than sql select column"))
+ }
+
+ // mv partition use a column not in mv sql select, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`k2`, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column can not find from
sql select column"))
+ }
+
sql """drop materialized view if exists ${mvName};"""
// list date year
sql """
@@ -86,6 +162,23 @@ suite("test_rollup_partition_mtmv") {
logger.info("showPartitionsResult: " + showPartitionsResult.toString())
assertEquals(1, showPartitionsResult.size())
+ sql """drop materialized view if exists ${mvName};"""
+ // list date year
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (year_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'year') as year_alias, * FROM ${tableName};
+ """
+ def date_list_year_partitions = sql """show partitions from ${mvName}"""
+ assertEquals(1, date_list_year_partitions.size())
+ order_qt_date_list_year_partition_by_column "SELECT * FROM ${mvName}"
+
// list string month
sql """drop table if exists `${tableName}`"""
sql """drop materialized view if exists ${mvName};"""
@@ -134,6 +227,86 @@ suite("test_rollup_partition_mtmv") {
order_qt_string_list_month "SELECT * FROM ${mvName} order by k1,k2"
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(month_alias, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column use invalid
implicit expression"))
+ }
+
+ // mv partition level should be higher or equal then query, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(month_alias, 'day'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column use invalid
implicit expression"))
+ }
+
+ // mv partition use a column not in mv sql select, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`k2`, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column can not find from
sql select column"))
+ }
+
+ // mv partition column type is date, base table is string, partition
mapping fail
+ // support later
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (month_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_date_format'='%Y==%m==%d'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ }
+
// range date month
sql """drop table if exists `${tableName}`"""
sql """drop materialized view if exists ${mvName};"""
@@ -180,6 +353,86 @@ suite("test_rollup_partition_mtmv") {
waitingMTMVTaskFinished(jobName)
order_qt_date_range_month "SELECT * FROM ${mvName} order by k1,k2"
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (month_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ def date_range_month_partitions = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " +
date_range_month_partitions.toString())
+ assertEquals(2, date_range_month_partitions.size())
+
+ jobName = getJobName(dbName, mvName);
+ log.info(jobName)
+ waitingMTMVTaskFinished(jobName)
+ order_qt_date_range_month_partition_by_column "SELECT * FROM ${mvName}"
+
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(day_alias, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as day_alias FROM ${tableName};
+ """
+ def date_range_month_partitions_level = sql """show partitions from
${mvName}"""
+ logger.info("showPartitionsResult: " +
date_range_month_partitions_level.toString())
+ assertEquals(2, date_range_month_partitions_level.size())
+ waitingMTMVTaskFinished(getJobName(dbName, mvName))
+ order_qt_date_range_month_level "SELECT * FROM ${mvName}"
+
+ // mv partition level should be higher or equal then query, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(month_alias, 'day'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column time unit level
should be greater than sql select column"))
+ }
+
+ // mv partition use a column not in mv sql select, should fail
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`k2`, 'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'day') as day_alias FROM ${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("partition column can not find from
sql select column"))
+ }
+
+
// not support MAXVALUE
sql """drop table if exists `${tableName}`"""
sql """drop materialized view if exists ${mvName};"""
@@ -217,6 +470,24 @@ suite("test_rollup_partition_mtmv") {
log.info(e.getMessage())
}
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (month_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ }
+
// range not support other data type
sql """drop table if exists `${tableName}`"""
@@ -253,6 +524,24 @@ suite("test_rollup_partition_mtmv") {
log.info(e.getMessage())
}
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (month_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ }
+
// not support trunc hour
sql """drop table if exists `${tableName}`"""
sql """drop materialized view if exists ${mvName};"""
@@ -289,4 +578,40 @@ suite("test_rollup_partition_mtmv") {
} catch (Exception e) {
log.info(e.getMessage())
}
+
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (hour_alias)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'hour') as hour_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ }
+
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ partition by (date_trunc(miniute_alias, 'hour'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT date_trunc(`k2`,'miniute') as miniute_alias, * FROM
${tableName};
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
