morrySnow commented on code in PR #35976:
URL: https://github.com/apache/doris/pull/35976#discussion_r1630855506
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java:
##########
@@ -195,58 +178,116 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
finalGroupExpressions.add(groupByExpression);
}
}
- // add project to guarantee group by column ref is slot reference,
- // this is necessary because physical createHash will need
slotReference later
+ List<Expression> queryGroupByExpressions =
queryAggregate.getGroupByExpressions();
+ // handle the scene that query top plan not use the group by in query
bottom aggregate
if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) {
for (Expression expression : queryGroupByExpressions) {
if (queryTopPlanGroupBySet.contains(expression)) {
continue;
}
- Expression queryGroupShuttledExpr =
ExpressionUtils.shuttleExpressionWithLineage(
- expression, queryTopPlan,
queryStructInfo.getTableBitSet());
- AggregateExpressionRewriteContext context = new
AggregateExpressionRewriteContext(true,
- mvExprToMvScanExprQueryBased, queryTopPlan,
queryStructInfo.getTableBitSet());
- // group by expression maybe group by a + b, so we need
expression rewriter
- Expression rewrittenGroupByExpression =
queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER,
- context);
- if (!context.isValid()) {
- // group expr can not rewrite by view
- materializationContext.recordFailReason(queryStructInfo,
- "View dimensions doesn't not cover the query
dimensions in bottom agg ",
- () -> String.format("mvExprToMvScanExprQueryBased
is %s,\n queryGroupShuttledExpr is %s",
- mvExprToMvScanExprQueryBased,
queryGroupShuttledExpr));
+ Expression rewrittenGroupByExpression =
tryRewriteExpression(queryStructInfo, expression,
+ mvExprToMvScanExprQueryBased, true,
materializationContext,
+ "View dimensions doesn't not cover the query
dimensions in bottom agg ",
+ () -> String.format("mvExprToMvScanExprQueryBased is
%s,\n expression is %s",
+ mvExprToMvScanExprQueryBased, expression));
+ if (rewrittenGroupByExpression == null) {
return null;
}
NamedExpression groupByExpression = rewrittenGroupByExpression
instanceof NamedExpression
? (NamedExpression) rewrittenGroupByExpression : new
Alias(rewrittenGroupByExpression);
finalGroupExpressions.add(groupByExpression);
}
}
- List<Expression> copiedFinalGroupExpressions = new
ArrayList<>(finalGroupExpressions);
- List<NamedExpression> projectsUnderAggregate =
copiedFinalGroupExpressions.stream()
- .map(NamedExpression.class::cast)
- .collect(Collectors.toList());
- projectsUnderAggregate.addAll(tempRewritedPlan.getOutput());
- LogicalProject<Plan> mvProject = new
LogicalProject<>(projectsUnderAggregate, tempRewritedPlan);
- // add agg rewrite
- Map<ExprId, Slot> projectOutPutExprIdMap =
mvProject.getOutput().stream()
- .distinct()
- .collect(Collectors.toMap(NamedExpression::getExprId, slot ->
slot));
- // make the expressions to re reference project output
- finalGroupExpressions = finalGroupExpressions.stream()
- .map(expr -> {
- ExprId exprId = ((NamedExpression) expr).getExprId();
- if (projectOutPutExprIdMap.containsKey(exprId)) {
- return projectOutPutExprIdMap.get(exprId);
+ if (queryContainsGroupSets) {
+ // construct group sets for repeat
+ List<List<Expression>> rewrittenGroupSetsExpressions = new
ArrayList<>();
+ List<List<Expression>> groupingSets =
queryAggregate.getSourceRepeat().get().getGroupingSets();
+ for (List<Expression> groupingSet : groupingSets) {
+ if (groupingSet.isEmpty()) {
+ rewrittenGroupSetsExpressions.add(ImmutableList.of());
+ } else {
+ List<Expression> rewrittenGroupSetExpressions = new
ArrayList<>();
+ for (Expression expression : groupingSet) {
+ Expression rewrittenGroupByExpression =
tryRewriteExpression(queryStructInfo, expression,
+ mvExprToMvScanExprQueryBased, true,
materializationContext,
+ "View dimensions doesn't not cover the query
group set dimensions",
+ () ->
String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s",
+ mvExprToMvScanExprQueryBased,
expression));
+ if (rewrittenGroupByExpression == null) {
+ return null;
+ }
+
rewrittenGroupSetExpressions.add(rewrittenGroupByExpression);
}
- return (NamedExpression) expr;
- })
- .collect(Collectors.toList());
- finalOutputExpressions = finalOutputExpressions.stream()
- .map(expr ->
projectOutPutExprIdMap.containsKey(expr.getExprId())
- ? projectOutPutExprIdMap.get(expr.getExprId()) : expr)
- .collect(Collectors.toList());
- return new LogicalAggregate(finalGroupExpressions,
finalOutputExpressions, mvProject);
+
rewrittenGroupSetsExpressions.add(rewrittenGroupSetExpressions);
+ }
+ }
+ LogicalRepeat<Plan> repeat = new
LogicalRepeat<>(rewrittenGroupSetsExpressions,
+ finalOutputExpressions, tempRewritedPlan);
+ return NormalizeRepeat.doNormalize(repeat);
+ }
+ return new LogicalAggregate<>(finalGroupExpressions,
finalOutputExpressions, tempRewritedPlan);
+ }
+
+ /**
+ * Try to rewrite query expression by view, contains both group by
dimension and aggregate function
+ */
+ protected Expression tryRewriteExpression(StructInfo queryStructInfo,
Expression queryExpression,
+ Map<Expression, Expression> mvShuttledExprToMvScanExprQueryBased,
boolean isGroupBy,
+ MaterializationContext materializationContext, String
summaryIfFail, Supplier<String> detailIfFail) {
+ Expression queryFunctionShuttled =
ExpressionUtils.shuttleExpressionWithLineage(
+ queryExpression,
+ queryStructInfo.getTopPlan(),
+ queryStructInfo.getTableBitSet());
+ AggregateExpressionRewriteContext expressionRewriteContext = new
AggregateExpressionRewriteContext(
+ isGroupBy, mvShuttledExprToMvScanExprQueryBased,
queryStructInfo.getTopPlan(),
+ queryStructInfo.getTableBitSet());
+ Expression rewrittenExpression =
queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
+ expressionRewriteContext);
+ if (!expressionRewriteContext.isValid()) {
+ materializationContext.recordFailReason(queryStructInfo,
summaryIfFail, detailIfFail);
+ return null;
+ }
+ return rewrittenExpression;
+ }
+
+ /**
+ * Check query and view aggregate compatibility
+ */
+ private static boolean checkCompatibility(
+ StructInfo queryStructInfo,
+ LogicalAggregate<Plan> queryAggregate, LogicalAggregate<Plan>
viewAggregate,
+ MaterializationContext materializationContext) {
+ // if view is scalar aggregate but query is not. Or if query is scalar
aggregate but view is not
+ // Should not rewrite
+ List<Expression> queryGroupByExpressions =
queryAggregate.getGroupByExpressions();
+ List<Expression> viewGroupByExpressions =
viewAggregate.getGroupByExpressions();
+ if (!queryGroupByExpressions.isEmpty() &&
viewGroupByExpressions.isEmpty()) {
+ materializationContext.recordFailReason(queryStructInfo,
+ "only one the of query or view is scalar aggregate and "
+ + "can not rewrite expression meanwhile",
+ () -> String.format("query aggregate = %s,\n view
aggregate = %s,\n",
+ queryAggregate.treeString(),
+ viewAggregate.treeString()));
+ return false;
+ }
+ boolean queryHasGroupSets = queryAggregate.getSourceRepeat()
+ .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0;
+ boolean viewHasGroupSets = viewAggregate.getSourceRepeat()
+ .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0;
+ // if both query and view has no group sets, maybe rewrite
+ if (!queryHasGroupSets && !viewHasGroupSets) {
+ return true;
+ }
Review Comment:
remove this if
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java:
##########
@@ -195,58 +178,116 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
finalGroupExpressions.add(groupByExpression);
}
}
- // add project to guarantee group by column ref is slot reference,
- // this is necessary because physical createHash will need
slotReference later
+ List<Expression> queryGroupByExpressions =
queryAggregate.getGroupByExpressions();
+ // handle the scene that query top plan not use the group by in query
bottom aggregate
if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) {
for (Expression expression : queryGroupByExpressions) {
if (queryTopPlanGroupBySet.contains(expression)) {
continue;
}
- Expression queryGroupShuttledExpr =
ExpressionUtils.shuttleExpressionWithLineage(
- expression, queryTopPlan,
queryStructInfo.getTableBitSet());
- AggregateExpressionRewriteContext context = new
AggregateExpressionRewriteContext(true,
- mvExprToMvScanExprQueryBased, queryTopPlan,
queryStructInfo.getTableBitSet());
- // group by expression maybe group by a + b, so we need
expression rewriter
- Expression rewrittenGroupByExpression =
queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER,
- context);
- if (!context.isValid()) {
- // group expr can not rewrite by view
- materializationContext.recordFailReason(queryStructInfo,
- "View dimensions doesn't not cover the query
dimensions in bottom agg ",
- () -> String.format("mvExprToMvScanExprQueryBased
is %s,\n queryGroupShuttledExpr is %s",
- mvExprToMvScanExprQueryBased,
queryGroupShuttledExpr));
+ Expression rewrittenGroupByExpression =
tryRewriteExpression(queryStructInfo, expression,
+ mvExprToMvScanExprQueryBased, true,
materializationContext,
+ "View dimensions doesn't not cover the query
dimensions in bottom agg ",
+ () -> String.format("mvExprToMvScanExprQueryBased is
%s,\n expression is %s",
+ mvExprToMvScanExprQueryBased, expression));
+ if (rewrittenGroupByExpression == null) {
return null;
}
NamedExpression groupByExpression = rewrittenGroupByExpression
instanceof NamedExpression
? (NamedExpression) rewrittenGroupByExpression : new
Alias(rewrittenGroupByExpression);
finalGroupExpressions.add(groupByExpression);
}
}
- List<Expression> copiedFinalGroupExpressions = new
ArrayList<>(finalGroupExpressions);
- List<NamedExpression> projectsUnderAggregate =
copiedFinalGroupExpressions.stream()
- .map(NamedExpression.class::cast)
- .collect(Collectors.toList());
- projectsUnderAggregate.addAll(tempRewritedPlan.getOutput());
- LogicalProject<Plan> mvProject = new
LogicalProject<>(projectsUnderAggregate, tempRewritedPlan);
- // add agg rewrite
- Map<ExprId, Slot> projectOutPutExprIdMap =
mvProject.getOutput().stream()
- .distinct()
- .collect(Collectors.toMap(NamedExpression::getExprId, slot ->
slot));
- // make the expressions to re reference project output
- finalGroupExpressions = finalGroupExpressions.stream()
- .map(expr -> {
- ExprId exprId = ((NamedExpression) expr).getExprId();
- if (projectOutPutExprIdMap.containsKey(exprId)) {
- return projectOutPutExprIdMap.get(exprId);
+ if (queryContainsGroupSets) {
+ // construct group sets for repeat
+ List<List<Expression>> rewrittenGroupSetsExpressions = new
ArrayList<>();
+ List<List<Expression>> groupingSets =
queryAggregate.getSourceRepeat().get().getGroupingSets();
+ for (List<Expression> groupingSet : groupingSets) {
+ if (groupingSet.isEmpty()) {
+ rewrittenGroupSetsExpressions.add(ImmutableList.of());
+ } else {
+ List<Expression> rewrittenGroupSetExpressions = new
ArrayList<>();
+ for (Expression expression : groupingSet) {
+ Expression rewrittenGroupByExpression =
tryRewriteExpression(queryStructInfo, expression,
+ mvExprToMvScanExprQueryBased, true,
materializationContext,
+ "View dimensions doesn't not cover the query
group set dimensions",
+ () ->
String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s",
+ mvExprToMvScanExprQueryBased,
expression));
+ if (rewrittenGroupByExpression == null) {
+ return null;
+ }
+
rewrittenGroupSetExpressions.add(rewrittenGroupByExpression);
}
- return (NamedExpression) expr;
- })
- .collect(Collectors.toList());
- finalOutputExpressions = finalOutputExpressions.stream()
- .map(expr ->
projectOutPutExprIdMap.containsKey(expr.getExprId())
- ? projectOutPutExprIdMap.get(expr.getExprId()) : expr)
- .collect(Collectors.toList());
- return new LogicalAggregate(finalGroupExpressions,
finalOutputExpressions, mvProject);
+
rewrittenGroupSetsExpressions.add(rewrittenGroupSetExpressions);
+ }
+ }
+ LogicalRepeat<Plan> repeat = new
LogicalRepeat<>(rewrittenGroupSetsExpressions,
+ finalOutputExpressions, tempRewritedPlan);
+ return NormalizeRepeat.doNormalize(repeat);
+ }
+ return new LogicalAggregate<>(finalGroupExpressions,
finalOutputExpressions, tempRewritedPlan);
+ }
+
+ /**
+ * Try to rewrite query expression by view, contains both group by
dimension and aggregate function
+ */
+ protected Expression tryRewriteExpression(StructInfo queryStructInfo,
Expression queryExpression,
+ Map<Expression, Expression> mvShuttledExprToMvScanExprQueryBased,
boolean isGroupBy,
+ MaterializationContext materializationContext, String
summaryIfFail, Supplier<String> detailIfFail) {
+ Expression queryFunctionShuttled =
ExpressionUtils.shuttleExpressionWithLineage(
+ queryExpression,
+ queryStructInfo.getTopPlan(),
+ queryStructInfo.getTableBitSet());
+ AggregateExpressionRewriteContext expressionRewriteContext = new
AggregateExpressionRewriteContext(
+ isGroupBy, mvShuttledExprToMvScanExprQueryBased,
queryStructInfo.getTopPlan(),
+ queryStructInfo.getTableBitSet());
+ Expression rewrittenExpression =
queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
+ expressionRewriteContext);
+ if (!expressionRewriteContext.isValid()) {
+ materializationContext.recordFailReason(queryStructInfo,
summaryIfFail, detailIfFail);
+ return null;
+ }
+ return rewrittenExpression;
+ }
+
+ /**
+ * Check query and view aggregate compatibility
+ */
+ private static boolean checkCompatibility(
+ StructInfo queryStructInfo,
+ LogicalAggregate<Plan> queryAggregate, LogicalAggregate<Plan>
viewAggregate,
+ MaterializationContext materializationContext) {
+ // if view is scalar aggregate but query is not. Or if query is scalar
aggregate but view is not
+ // Should not rewrite
+ List<Expression> queryGroupByExpressions =
queryAggregate.getGroupByExpressions();
+ List<Expression> viewGroupByExpressions =
viewAggregate.getGroupByExpressions();
+ if (!queryGroupByExpressions.isEmpty() &&
viewGroupByExpressions.isEmpty()) {
+ materializationContext.recordFailReason(queryStructInfo,
+ "only one the of query or view is scalar aggregate and "
+ + "can not rewrite expression meanwhile",
+ () -> String.format("query aggregate = %s,\n view
aggregate = %s,\n",
+ queryAggregate.treeString(),
+ viewAggregate.treeString()));
+ return false;
+ }
+ boolean queryHasGroupSets = queryAggregate.getSourceRepeat()
+ .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0;
+ boolean viewHasGroupSets = viewAggregate.getSourceRepeat()
+ .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0;
+ // if both query and view has no group sets, maybe rewrite
+ if (!queryHasGroupSets && !viewHasGroupSets) {
+ return true;
+ }
+ // if both query and view has group sets, or query doesn't hava, mv
have, not supported
+ if ((queryHasGroupSets && viewHasGroupSets) || (!queryHasGroupSets &&
viewHasGroupSets)) {
Review Comment:
```suggestion
if (viewHasGroupSets) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
