This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 62998719dfd [opt](mtmv) Add threshold for relation mapping num when
query rewrite (#34694) (#35378)
62998719dfd is described below
commit 62998719dfdd1842f876b9cad240fbae3a186ea8
Author: seawinde <[email protected]>
AuthorDate: Fri May 24 20:36:29 2024 +0800
[opt](mtmv) Add threshold for relation mapping num when query rewrite
(#34694) (#35378)
if query and mv def is as following:
def mv1_1 = """
select t1.L_LINENUMBER,t2.l_extendedprice, t2.L_ORDERKEY
from lineitem t1
inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
"""
def query1_1 = """
select t1.L_LINENUMBER, t2.L_ORDERKEY
from lineitem t1
inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
"""
this will generate relation mapping by Cartesian, if the num of self join
is too much, this will cause the performance problem
so we add `materialized_view_relation_mapping_max_count` session varaible,
default 8. if actual num is greater than the value, the excess relation mapping
is discarded.
---
.../mv/AbstractMaterializedViewAggregateRule.java | 3 +-
.../mv/AbstractMaterializedViewJoinRule.java | 3 +-
.../mv/AbstractMaterializedViewRule.java | 17 ++++++++--
.../exploration/mv/MaterializedViewScanRule.java | 3 +-
.../java/org/apache/doris/qe/SessionVariable.java | 13 ++++++++
.../mv/availability/materialized_view_switch.out | 29 +++++++++++++++++
.../availability/materialized_view_switch.groovy | 38 ++++++++++++++++++++++
7 files changed, 100 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
index 5a47bb07fda..b37b04d8022 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
@@ -370,7 +371,7 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
* slot reference equals currently.
*/
@Override
- protected boolean checkPattern(StructInfo structInfo) {
+ protected boolean checkPattern(StructInfo structInfo, CascadesContext
cascadesContext) {
PlanCheckContext checkContext =
PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
// if query or mv contains more then one top aggregate, should fail
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER,
checkContext)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
index 2a05fecd33f..3b20cefbba8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
+import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -74,7 +75,7 @@ public abstract class AbstractMaterializedViewJoinRule
extends AbstractMateriali
* Join condition should be slot reference equals currently.
*/
@Override
- protected boolean checkPattern(StructInfo structInfo) {
+ protected boolean checkPattern(StructInfo structInfo, CascadesContext
cascadesContext) {
PlanCheckContext checkContext =
PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER,
checkContext)
&& !checkContext.isContainsTopAggregate();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 2274ad441f8..8442d2485c7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -66,6 +66,8 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.BitSet;
@@ -82,6 +84,8 @@ import java.util.stream.Collectors;
* The abstract class for all materialized view rules
*/
public abstract class AbstractMaterializedViewRule implements
ExplorationRuleFactory {
+
+ public static final Logger LOG =
LogManager.getLogger(AbstractMaterializedViewRule.class);
public static final Set<JoinType> SUPPORTED_JOIN_TYPE_SET =
ImmutableSet.of(
JoinType.INNER_JOIN,
JoinType.LEFT_OUTER_JOIN,
@@ -142,7 +146,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
List<StructInfo> uncheckedStructInfos =
MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext,
materializedViewTableSet);
uncheckedStructInfos.forEach(queryStructInfo -> {
- boolean valid = checkPattern(queryStructInfo) &&
queryStructInfo.isValid();
+ boolean valid = checkPattern(queryStructInfo, cascadesContext) &&
queryStructInfo.isValid();
if (!valid) {
cascadesContext.getMaterializationContexts().forEach(ctx ->
ctx.recordFailReason(queryStructInfo, "Query struct
info is invalid",
@@ -178,6 +182,13 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
"Query to view table mapping is null", () -> "");
return rewriteResults;
}
+ int materializedViewRelationMappingMaxCount =
cascadesContext.getConnectContext().getSessionVariable()
+ .getMaterializedViewRelationMappingMaxCount();
+ if (queryToViewTableMappings.size() >
materializedViewRelationMappingMaxCount) {
+ LOG.warn("queryToViewTableMappings is over limit and be
intercepted");
+ queryToViewTableMappings = queryToViewTableMappings.subList(0,
materializedViewRelationMappingMaxCount);
+ }
+
for (RelationMapping queryToViewTableMapping :
queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping =
materializationContext.getSlotMappingFromCache(queryToViewTableMapping);
@@ -650,7 +661,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
/**
* Check the pattern of query or materializedView is supported or not.
*/
- protected boolean checkPattern(StructInfo structInfo) {
+ protected boolean checkPattern(StructInfo structInfo, CascadesContext
cascadesContext) {
if (structInfo.getRelations().isEmpty()) {
return false;
}
@@ -676,7 +687,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
materializationId);
if (cachedCheckResult == null) {
// need check in real time
- boolean checkResult = checkPattern(context.getStructInfo());
+ boolean checkResult = checkPattern(context.getStructInfo(),
cascadesContext);
if (!checkResult) {
context.recordFailReason(context.getStructInfo(),
"View struct info is invalid", () ->
String.format("view plan is %s",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
index b3e742c64cb..d6d7817d35f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
+import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -75,7 +76,7 @@ public abstract class MaterializedViewScanRule extends
AbstractMaterializedViewR
* Join condition should be slot reference equals currently.
*/
@Override
- protected boolean checkPattern(StructInfo structInfo) {
+ protected boolean checkPattern(StructInfo structInfo, CascadesContext
cascadesContext) {
PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of());
return
structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER,
checkContext)
&& !checkContext.isContainsTopAggregate();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 602c10249d3..b17d6b3a417 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -529,6 +529,9 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE
= "enable_materialized_view_nest_rewrite";
+ public static final String MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT
+ = "materialized_view_relation_mapping_max_count";
+
public static final String CREATE_TABLE_PARTITION_MAX_NUM
= "create_table_partition_max_num";
@@ -1660,6 +1663,12 @@ public class SessionVariable implements Serializable,
Writable {
"The max candidate num which participate in CBO when using
asynchronous materialized views"})
public int materializedViewRewriteSuccessCandidateNum = 3;
+ @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT,
needForward = true,
+ description = {"透明改写过程中，relation mapping最大允许数量，如果超过，进行截取",
+ "During transparent rewriting, relation mapping specifies
the maximum allowed number. "
+ + "If the number exceeds the allowed number, the
number is intercepted"})
+ public int materializedViewRelationMappingMaxCount = 8;
+
@VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_UNION_REWRITE,
needForward = true,
description = {"当物化视图不足以提供查询的全部数据时，是否允许基表和物化视图 union 来响应查询",
"When the materialized view is not enough to provide all
the data for the query, "
@@ -3730,6 +3739,10 @@ public class SessionVariable implements Serializable,
Writable {
return enableMaterializedViewNestRewrite;
}
+ public int getMaterializedViewRelationMappingMaxCount() {
+ return materializedViewRelationMappingMaxCount;
+ }
+
public int getCreateTablePartitionMaxNum() {
return createTablePartitionMaxNum;
}
diff --git
a/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out
b/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out
new file mode 100644
index 00000000000..3e413a21d84
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !query1_0_before --
+4 1
+4 2
+4 3
+4 4
+6 5
+
+-- !query1_0_after --
+4 1
+4 2
+4 3
+4 4
+6 5
+
+-- !query1_1_before --
+4 1
+4 2
+4 3
+4 4
+6 5
+
+-- !query1_1_after --
+4 1
+4 2
+4 3
+4 4
+6 5
+
diff --git
a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
index 29bd112e76f..b4ae8b7bfc5 100644
---
a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
@@ -152,4 +152,42 @@ suite("materialized_view_switch") {
sql "SET enable_materialized_view_rewrite=true"
check_mv_rewrite_success(db, mv_name, query, "mv_name")
sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name"""
+
+ // test when materialized_view_relation_mapping_max_count is 8
+ def mv1_0 = """
+ select t1.L_LINENUMBER, t2.l_extendedprice, t2.L_ORDERKEY
+ from lineitem t1
+ inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
+ """
+ def query1_0 = """
+ select t1.L_LINENUMBER, t2.L_ORDERKEY
+ from lineitem t1
+ inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
+ """
+ order_qt_query1_0_before "${query1_0}"
+ check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0")
+ order_qt_query1_0_after "${query1_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
+
+
+ // test when materialized_view_relation_mapping_max_count is 0
+ sql "SET materialized_view_relation_mapping_max_count = 0"
+
+ def mv1_1 = """
+ select t1.L_LINENUMBER,t2.l_extendedprice, t2.L_ORDERKEY
+ from lineitem t1
+ inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
+ """
+ def query1_1 = """
+ select t1.L_LINENUMBER, t2.L_ORDERKEY
+ from lineitem t1
+ inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY;
+ """
+ order_qt_query1_1_before "${query1_1}"
+ check_mv_rewrite_fail(db, mv1_1, query1_1, "mv1_1")
+ order_qt_query1_1_after "${query1_1}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
+
+ sql "SET materialized_view_relation_mapping_max_count = 8"
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
