This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cd2062ae8df [improvement](mtmv) Support to add use_for_rewrite
property when create materialized view (#40332)
cd2062ae8df is described below
commit cd2062ae8dfca5895d41f651b250afa2969c2215
Author: seawinde <[email protected]>
AuthorDate: Wed Sep 11 19:03:56 2024 +0800
[improvement](mtmv) Support to add use_for_rewrite property when create
materialized view (#40332)
Add `is_used_in_rewrite` property when create mv. Default true;
If `is_used_in_rewrite` is false which means the mv would not partion in
query rewrite.
Such as mv def is as following:
CREATE MATERIALIZED VIEW mv1
BUILD IMMEDIATE REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1',
'use_for_rewrite' = 'false'
)
AS
SELECT k1, k2, count(*) from t1 group by k1, k2;
if we run query as following, this mv would not partition in the query
rewrite:
SELECT k2, count(*) from t1 group by k2;
---
.../main/java/org/apache/doris/catalog/MTMV.java | 13 +++
.../apache/doris/common/util/PropertyAnalyzer.java | 3 +
.../org/apache/doris/mtmv/MTMVPropertyUtil.java | 15 ++-
.../mv/InitMaterializationContextHook.java | 11 ++-
.../suites/mtmv_p0/test_use_for_rewrite.groovy | 103 +++++++++++++++++++++
5 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 3e60a489c93..f93ecc9475c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -245,6 +245,19 @@ public class MTMV extends OlapTable {
}
}
+ public boolean isUseForRewrite() {
+ readMvLock();
+ try {
+ if
(!StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE)))
{
+ return
Boolean.valueOf(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE));
+ }
+ // default is true
+ return true;
+ } finally {
+ readMvUnlock();
+ }
+ }
+
public int getRefreshPartitionNum() {
readMvLock();
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 8441c9307ca..3d95e69cb67 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -186,6 +186,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION =
"enable_nondeterministic_function";
+
+ public static final String PROPERTIES_USE_FOR_REWRITE =
+ "use_for_rewrite";
public static final String PROPERTIES_EXCLUDED_TRIGGER_TABLES =
"excluded_trigger_tables";
public static final String PROPERTIES_REFRESH_PARTITION_NUM =
"refresh_partition_num";
public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
index 12287183886..dbaccaf9247 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
@@ -38,7 +38,8 @@ public class MTMVPropertyUtil {
PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT,
PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT,
PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT,
- PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION
+ PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION,
+ PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE
);
public static void analyzeProperty(String key, String value) {
@@ -65,6 +66,10 @@ public class MTMVPropertyUtil {
analyzePartitionSyncLimit(value);
break;
case PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION:
+ analyzeBooleanProperty(value,
PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION);
+ break;
+ case PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE:
+ analyzeBooleanProperty(value,
PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE);
break;
default:
throw new AnalysisException("illegal key:" + key);
@@ -138,4 +143,12 @@ public class MTMVPropertyUtil {
}
}
+ private static void analyzeBooleanProperty(String propertyValue, String
propertyName) {
+ if (StringUtils.isEmpty(propertyValue)) {
+ return;
+ }
+ if (!"true".equalsIgnoreCase(propertyValue) &&
!"false".equalsIgnoreCase(propertyValue)) {
+ throw new AnalysisException(String.format("valid property %s
fail", propertyName));
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index e1f98afae3f..0ad6f8f74a4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -129,8 +129,8 @@ public class InitMaterializationContextHook implements
PlannerHook {
Set<TableIf> usedTables) {
Set<MTMV> availableMTMVs = getAvailableMTMVs(usedTables,
cascadesContext);
if (availableMTMVs.isEmpty()) {
- LOG.debug(String.format("Enable materialized view rewrite but
availableMTMVs is empty, current queryId "
- + "is %s",
cascadesContext.getConnectContext().getQueryIdentifier()));
+ LOG.debug("Enable materialized view rewrite but availableMTMVs is
empty, current queryId "
+ + "is {}",
cascadesContext.getConnectContext().getQueryIdentifier());
return ImmutableList.of();
}
List<MaterializationContext> asyncMaterializationContext = new
ArrayList<>();
@@ -138,6 +138,13 @@ public class InitMaterializationContextHook implements
PlannerHook {
MTMVCache mtmvCache = null;
try {
mtmvCache =
materializedView.getOrGenerateCache(cascadesContext.getConnectContext());
+ // If mv property use_for_rewrite is set false, should not
partition in
+ // query rewrite by materialized view
+ if (!materializedView.isUseForRewrite()) {
+ LOG.debug("mv doesn't part in query rewrite process
because "
+ + "use_for_rewrite is false, mv is {}",
materializedView.getName());
+ continue;
+ }
if (mtmvCache == null) {
continue;
}
diff --git a/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
b/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
new file mode 100644
index 00000000000..d67f6960dfa
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_use_for_rewrite","mtmv") {
+ String suiteName = "test_use_for_rewrite"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String db = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k1 TINYINT,
+ k2 INT not null,
+ k3 DATE NOT NULL
+ )
+ COMMENT "my first table"
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ insert into ${tableName} values(1,1, '2024-05-01'),(2,2,
'2024-05-02'),(3,3, '2024-05-03'),
+ (1,1, '2024-05-01'),(2,2, '2024-05-02'), (3,3, '2024-05-03');
+ """
+
+ // when not set use_for_rewrite, should rewrite successfully
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_rewrite_success_without_check_chosen("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+ sql """drop materialized view if exists ${mvName};"""
+
+
+ // when set use_for_rewrite = true, should rewrite successfully
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'use_for_rewrite' = 'true'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_rewrite_success_without_check_chosen("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+
+
+ // when set use_for_rewrite = false, should not partition in rewritten
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'use_for_rewrite' = 'false'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_not_part_in("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+
+ sql """drop materialized view if exists ${mvName};"""
+ sql """drop table if exists `${tableName}`"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
