morrySnow commented on code in PR #33362:
URL: https://github.com/apache/doris/pull/33362#discussion_r1573326057
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java:
##########
@@ -43,12 +45,12 @@
* Get from rewrite plan and can also get from plan struct info, if from plan
struct info it depends on
* the nodes from graph.
*/
-public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression,
ExpressionReplaceContext> {
+public class ExpressionLineageReplacer extends DefaultPlanVisitor<Void,
ExpressionReplaceContext> {
Review Comment:
replacer return Void is weird
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java:
##########
@@ -132,18 +139,46 @@ public static boolean containTableQueryOperator(Plan
analyzedPlan) {
/**
* Extract struct info from plan, support to get struct info from logical
plan or plan in group.
*/
- public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext) {
- if (plan.getGroupExpression().isPresent() &&
!plan.getGroupExpression().get().getOwnerGroup().getStructInfos()
- .isEmpty()) {
- return
plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
- } else {
- // build struct info and add them to current group
- List<StructInfo> structInfos = StructInfo.of(plan);
- if (plan.getGroupExpression().isPresent()) {
-
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
+ public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext,
+ BitSet materializedViewTableSet) {
+ // If plan belong to some group, construct it with group struct info
+ if (plan.getGroupExpression().isPresent()) {
+ Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
+ StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
+ structInfoMap.refresh(ownerGroup);
+ Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
+ if (!queryTableSets.isEmpty()) {
+ return queryTableSets.stream()
+ // Just construct the struct info which mv table set
contains all the query table set
+ .filter(queryTableSet ->
materializedViewTableSet.isEmpty()
+ ||
StructInfo.containsAll(materializedViewTableSet, queryTableSet))
+ .map(tableMap -> structInfoMap.getStructInfo(tableMap,
tableMap, ownerGroup, plan))
+ .collect(Collectors.toList());
}
- return structInfos;
}
+ // if plan doesn't belong to any group, construct it directly
+ return ImmutableList.of(StructInfo.of(plan));
+ }
+
+ /**
+ * Generate scan plan for materialized view
+ * if MaterializationContext is already rewritten by materialized view,
then should generate in real time
+ * when query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
+ * should be different
+ */
+ public static Plan generateMvScanPlan(MTMV materializedView,
CascadesContext cascadesContext) {
+ LogicalOlapScan mvScan = new LogicalOlapScan(
+ cascadesContext.getStatementContext().getNextRelationId(),
+ materializedView,
+ ImmutableList.of(materializedView.getQualifiedDbName()),
+ // this must be empty, or it will be used to sample
+ Lists.newArrayList(),
+ Lists.newArrayList(),
Review Comment:
use ImmutalbeList.of()
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java:
##########
@@ -132,18 +139,46 @@ public static boolean containTableQueryOperator(Plan
analyzedPlan) {
/**
* Extract struct info from plan, support to get struct info from logical
plan or plan in group.
*/
- public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext) {
- if (plan.getGroupExpression().isPresent() &&
!plan.getGroupExpression().get().getOwnerGroup().getStructInfos()
- .isEmpty()) {
- return
plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
- } else {
- // build struct info and add them to current group
- List<StructInfo> structInfos = StructInfo.of(plan);
- if (plan.getGroupExpression().isPresent()) {
-
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
+ public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext,
+ BitSet materializedViewTableSet) {
+ // If plan belong to some group, construct it with group struct info
+ if (plan.getGroupExpression().isPresent()) {
+ Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
+ StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
+ structInfoMap.refresh(ownerGroup);
+ Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
+ if (!queryTableSets.isEmpty()) {
+ return queryTableSets.stream()
+ // Just construct the struct info which mv table set
contains all the query table set
+ .filter(queryTableSet ->
materializedViewTableSet.isEmpty()
+ ||
StructInfo.containsAll(materializedViewTableSet, queryTableSet))
+ .map(tableMap -> structInfoMap.getStructInfo(tableMap,
tableMap, ownerGroup, plan))
+ .collect(Collectors.toList());
Review Comment:
use for loop
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java:
##########
@@ -85,52 +87,62 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
*/
public List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext)
{
List<Plan> rewrittenPlans = new ArrayList<>();
- // already rewrite or query is invalid, bail out
- List<StructInfo> queryStructInfos = checkQuery(queryPlan,
cascadesContext);
- if (queryStructInfos.isEmpty()) {
+ // if available materialization list is empty, bail out
+ if (cascadesContext.getMaterializationContexts().isEmpty()) {
return rewrittenPlans;
}
for (MaterializationContext context :
cascadesContext.getMaterializationContexts()) {
if (checkIfRewritten(queryPlan, context)) {
continue;
}
- // TODO Just support only one query struct info, support multi
later.
- StructInfo queryStructInfo = queryStructInfos.get(0);
- try {
- if (rewrittenPlans.size() < cascadesContext.getConnectContext()
-
.getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) {
- rewrittenPlans.addAll(doRewrite(queryStructInfo,
cascadesContext, context));
+ context.tryReGenerateMvScanPlan(cascadesContext);
+ // check mv plan is valid or not
+ if (!checkPattern(context.getStructInfo())) {
+ context.recordFailReason(context.getStructInfo(),
+ "View struct info is invalid", () -> String.format(",
view plan is %s",
+
context.getStructInfo().getOriginalPlan().treeString()));
+ continue;
+ }
+ // get query struct infos according to the view strut info, if
valid query struct infos is empty, bail out
+ List<StructInfo> queryStructInfos =
getValidQueryStructInfos(queryPlan, cascadesContext,
+ context.getStructInfo().getTableBitSet());
+ if (queryStructInfos.isEmpty()) {
+ continue;
+ }
+ for (StructInfo queryStructInfo : queryStructInfos) {
+ try {
+ if (rewrittenPlans.size() <
cascadesContext.getConnectContext()
+
.getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) {
+ rewrittenPlans.addAll(doRewrite(queryStructInfo,
cascadesContext, context));
+ }
+ } catch (Exception exception) {
+ context.recordFailReason(queryStructInfo,
+ "Materialized view rule exec fail",
exception::toString);
}
- } catch (Exception exception) {
- context.recordFailReason(queryStructInfo,
- "Materialized view rule exec fail",
exception::toString);
}
}
return rewrittenPlans;
}
/**
- * Check query is valid or not, if valid return the query struct infos, if
invalid return empty list.
+ * Get valid query struct infos, if invalid record the invalid reason
*/
- protected List<StructInfo> checkQuery(Plan queryPlan, CascadesContext
cascadesContext) {
- List<StructInfo> validQueryStructInfos = new ArrayList<>();
- List<MaterializationContext> materializationContexts =
cascadesContext.getMaterializationContexts();
- if (materializationContexts.isEmpty()) {
- return validQueryStructInfos;
- }
- List<StructInfo> queryStructInfos =
MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext);
- // TODO Just Check query queryPlan firstly, support multi later.
- StructInfo queryStructInfo = queryStructInfos.get(0);
- if (!checkPattern(queryStructInfo)) {
- for (MaterializationContext ctx :
cascadesContext.getMaterializationContexts()) {
- ctx.recordFailReason(queryStructInfo, "Query struct info is
invalid",
- () -> String.format("queryPlan is %s",
queryPlan.treeString())
- );
- }
- return validQueryStructInfos;
- }
- validQueryStructInfos.add(queryStructInfo);
- return validQueryStructInfos;
+ protected List<StructInfo> getValidQueryStructInfos(Plan queryPlan,
CascadesContext cascadesContext,
+ BitSet materializedViewTableSet) {
+ return MaterializedViewUtils.extractStructInfo(queryPlan,
cascadesContext, materializedViewTableSet)
+ .stream()
Review Comment:
use for loop
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
