morrySnow commented on code in PR #27059:
URL: https://github.com/apache/doris/pull/27059#discussion_r1401475045
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java:
##########
@@ -112,6 +113,8 @@ public class CascadesContext implements ScheduleContext {
private final Optional<CTEId> currentTree;
private final Optional<CascadesContext> parent;
+ private List<MaterializationContext> materializationContexts;
Review Comment:
final?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java:
##########
@@ -225,6 +225,24 @@ public enum RuleType {
MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
+
+ MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.REWRITE),
+
+ MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.REWRITE),
+
+ MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.REWRITE),
+ MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
Review Comment:
project always on the top of filter, so we do not need filter project xxx
rules?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java:
##########
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.Group;
+import
org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The abstract class for all materialized view rules
+ */
+public abstract class AbstractMaterializedViewRule {
+
+ protected List<Plan> rewrite(Plan queryPlan, CascadesContext
cascadesContext) {
Review Comment:
all method in this class should have comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java:
##########
@@ -196,6 +199,26 @@ public static Expression combine(Class<? extends
Expression> type, Collection<Ex
.orElse(BooleanLiteral.of(type == And.class));
}
+ // Replace the slot in expression with the lineage identifier from
specified
+ // baseTable sets or target table types
+ // example as following:
+ // select a + 10 as a1, d from (
+ // select b - 5 as a, d from table
+ // );
+ // after shuttle a1 and d is [b - 5 + 10, d]
+ public static List<? extends Expression>
shuttleExpressionWithLineage(List<? extends Expression> expression,
Review Comment:
use javadoc style comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java:
##########
@@ -196,6 +199,26 @@ public static Expression combine(Class<? extends
Expression> type, Collection<Ex
.orElse(BooleanLiteral.of(type == And.class));
}
+ // Replace the slot in expression with the lineage identifier from
specified
+ // baseTable sets or target table types
+ // example as following:
+ // select a + 10 as a1, d from (
+ // select b - 5 as a, d from table
+ // );
+ // after shuttle a1 and d is [b - 5 + 10, d]
+ public static List<? extends Expression>
shuttleExpressionWithLineage(List<? extends Expression> expression,
+ Plan plan,
+ Set<TableType> targetTypes,
+ Set<String> tableIdentifiers) {
+ return ImmutableList.of();
+ }
+
+ // Replace the slot in expressions according to the slotMapping
+ // if any slot cannot be mapped then return null
+ public static List<? extends Expression> permute(List<? extends
Expression> expressions, SlotMapping slotMapping) {
Review Comment:
use javadoc style comment
##########
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java:
##########
@@ -1359,6 +1365,17 @@ public void setEnableLeftZigZag(boolean
enableLeftZigZag) {
+ "considered outdated."})
public int tableStatsHealthThreshold = 60;
+ @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward
= true,
+ description = {"是否开启基于结构信息的透明改写",
Review Comment:
说明里面最好提及一下物化视图
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java:
##########
@@ -259,6 +259,10 @@ private void initCascadesContext(LogicalPlan plan,
PhysicalProperties requirePro
if (statementContext.getConnectContext().getTables() != null) {
cascadesContext.setTables(statementContext.getConnectContext().getTables());
}
+ if
(statementContext.getConnectContext().getSessionVariable().enableMaterializedViewRewrite)
{
Review Comment:
```suggestion
if (statementContext.getConnectContext().getSessionVariable().
isEnableMaterializedViewRewrite()) {
```
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java:
##########
@@ -54,4 +61,46 @@ public Boolean visitAggregateFunction(AggregateFunction
aggregateFunction, Void
return true;
}
}
+
+ /**
+ * Split the expression to equal, range and residual predicate.
+ * Should instance when used.
+ */
+ public static class PredicatesSpliter extends
DefaultExpressionVisitor<Void, Void> {
+
+ private List<Expression> equalPredicates = new ArrayList<>();
+ private List<Expression> rangePredicates = new ArrayList<>();
+ private List<Expression> residualPredicates = new ArrayList<>();
+ private final Expression target;
+
+ public PredicatesSpliter(Expression target) {
+ this.target = target;
+ }
+
+ @Override
+ public Void visitComparisonPredicate(ComparisonPredicate
comparisonPredicate, Void context) {
+ // TODO Smallest implement, complete later
+ if (comparisonPredicate instanceof EqualTo) {
+ Expression argument0 = comparisonPredicate.getArgument(0);
+ Expression argument1 = comparisonPredicate.getArgument(1);
Review Comment:
use left right instead of arg0, arg1
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java:
##########
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.Group;
+import
org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The abstract class for all materialized view rules
+ */
+public abstract class AbstractMaterializedViewRule {
+
+ protected List<Plan> rewrite(Plan queryPlan, CascadesContext
cascadesContext) {
+ List<MaterializationContext> materializationContexts =
cascadesContext.getMaterializationContexts();
+ List<Plan> rewriteResults = new ArrayList<>();
+ if (materializationContexts.isEmpty()) {
+ return rewriteResults;
+ }
+ StructInfo queryStructInfo = extractStructInfo(queryPlan,
cascadesContext);
+ // Check query queryPlan
+ if (!isPatternSupport(queryStructInfo)) {
+ return rewriteResults;
+ }
+
+ for (MaterializationContext materializationContext :
materializationContexts) {
+ Plan mvPlan = materializationContext.getMvPlan();
+ StructInfo viewStructInfo = extractStructInfo(mvPlan,
cascadesContext);
+ if (!isPatternSupport(viewStructInfo)) {
+ continue;
+ }
+ if
(!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(),
viewStructInfo.getHyperGraph())) {
+ continue;
+ }
+ MatchMode matchMode =
decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
+ if (MatchMode.NOT_MATCH == matchMode) {
+ continue;
+ }
+ List<RelationMapping> queryToViewTableMappings =
+ RelationMapping.generate(queryStructInfo.getRelations(),
viewStructInfo.getRelations());
+ for (RelationMapping queryToViewTableMapping :
queryToViewTableMappings) {
+ SplitPredicate compensatePredicates =
predicatesCompensate(queryStructInfo, viewStructInfo,
+ queryToViewTableMapping);
+ // can not compensate, bail out
+ if (compensatePredicates == null ||
compensatePredicates.isEmpty()) {
+ continue;
+ }
+ Plan rewritedPlan;
+ Plan mvScan = materializationContext.getScanPlan();
+ if (compensatePredicates.isAlwaysTrue()) {
+ rewritedPlan = mvScan;
+ } else {
+ // try to rewrite compensate predicates by using mv scan
+ List<NamedExpression> rewriteCompensatePredicates =
rewriteExpression(
+ compensatePredicates.toList(),
+ queryStructInfo,
+ viewStructInfo,
+ queryToViewTableMapping,
+ mvScan);
+ if (rewriteCompensatePredicates.isEmpty()) {
+ continue;
+ }
+ rewritedPlan = new
LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
+ }
+ // rewrite query by view
+ rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo,
viewStructInfo,
+ queryToViewTableMapping, rewritedPlan);
+ if (rewritedPlan == null) {
+ continue;
+ }
+ rewriteResults.add(rewritedPlan);
+ }
+ }
+ return rewriteResults;
+ }
+
+ // Rewrite query by view, for aggregate or join rewriting should be
different inherit class implementation
Review Comment:
use javadoc style comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java:
##########
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * EquivalenceClass, this is used for equality propagation when predicate
compensation
+ */
+public class EquivalenceClass {
+
+ private final Map<SlotReference, Set<SlotReference>> equivalenceSlotMap =
new LinkedHashMap<>();
+
+ public EquivalenceClass() {
+ }
+
+ /**
+ * EquivalenceClass
+ */
+ public void addEquivalenceClass(SlotReference slot0, SlotReference slot1) {
+
+ Set<SlotReference> slot0Sets = equivalenceSlotMap.get(slot0);
+ Set<SlotReference> slot1Sets = equivalenceSlotMap.get(slot1);
Review Comment:
0 1 are not good name
##########
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java:
##########
@@ -1359,6 +1365,17 @@ public void setEnableLeftZigZag(boolean
enableLeftZigZag) {
+ "considered outdated."})
public int tableStatsHealthThreshold = 60;
+ @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward
= true,
+ description = {"是否开启基于结构信息的透明改写",
Review Comment:
说明里面最好提及一下物化视图
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
