This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 37471390f0f [fix](mtmv) Fix rewrite fail when query direct external
table without group by (#39041)
37471390f0f is described below
commit 37471390f0fa8f3e65b6d232f38cc85b3baa5204
Author: seawinde <[email protected]>
AuthorDate: Tue Aug 20 20:13:42 2024 +0800
[fix](mtmv) Fix rewrite fail when query direct external table without group
by (#39041)
this is brought by #34185
if query external table without group by, rewrite by materialized view
will fail
such as mv def is
select o_custkey, o_orderdate
from ${hive_catalog_name}.${hive_database}.${hive_table};
query is query external table directly as following, this would fail
when try to rewrte by materialized view
select o_custkey
from ${hive_catalog_name}.${hive_database}.${hive_table};
this pr fix the problem.
---
.../mv/MaterializedViewFilterProjectScanRule.java | 13 ++-
.../mv/MaterializedViewFilterScanRule.java | 7 +-
.../mv/MaterializedViewOnlyScanRule.java | 3 +-
.../mv/MaterializedViewProjectFilterScanRule.java | 13 ++-
.../mv/MaterializedViewProjectScanRule.java | 7 +-
.../mv/external_table/single_external_table.out | 17 +++
.../mv/external_table/single_external_table.groovy | 116 +++++++++++++++++++++
7 files changed, 159 insertions(+), 17 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java
index 613b356cdc5..c828be694a9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java
@@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -37,9 +38,11 @@ public class MaterializedViewFilterProjectScanRule extends
AbstractMaterializedV
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
-
logicalFilter(logicalProject(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
- LogicalFilter<LogicalProject<LogicalOlapScan>> root =
ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
+
logicalFilter(logicalProject(any().when(LogicalCatalogRelation.class::isInstance)))
+ .thenApplyMultiNoThrow(
+ ctx -> {
+ LogicalFilter<LogicalProject<Plan>> root =
ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java
index 18f366354ce..e7bbd28b51b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java
@@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import com.google.common.collect.ImmutableList;
@@ -36,8 +37,8 @@ public class MaterializedViewFilterScanRule extends
AbstractMaterializedViewScan
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalFilter(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> {
- LogicalFilter<LogicalOlapScan> root = ctx.root;
+
logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx
-> {
+ LogicalFilter<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_SCAN));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java
index 97fe2153885..24211ac8b08 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import com.google.common.collect.ImmutableList;
@@ -34,7 +35,7 @@ public class MaterializedViewOnlyScanRule extends
AbstractMaterializedViewScanRu
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalOlapScan().thenApplyMultiNoThrow(ctx -> {
+
any().when(LogicalCatalogRelation.class::isInstance).thenApplyMultiNoThrow(ctx
-> {
return rewrite(ctx.root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_SCAN));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java
index d8a92ef59f2..08abf6d6a59 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java
@@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -37,9 +38,11 @@ public class MaterializedViewProjectFilterScanRule extends
AbstractMaterializedV
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
-
logicalProject(logicalFilter(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
- LogicalProject<LogicalFilter<LogicalOlapScan>> root =
ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
+
logicalProject(logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)))
+ .thenApplyMultiNoThrow(
+ ctx -> {
+ LogicalProject<LogicalFilter<Plan>> root =
ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java
index 72a656532cd..1a51eb615b8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java
@@ -19,7 +19,8 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -36,8 +37,8 @@ public class MaterializedViewProjectScanRule extends
AbstractMaterializedViewSca
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalProject(logicalOlapScan()).thenApplyMultiNoThrow(ctx ->
{
- LogicalProject<LogicalOlapScan> root = ctx.root;
+
logicalProject(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx
-> {
+ LogicalProject<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_SCAN));
}
diff --git
a/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
new file mode 100644
index 00000000000..5305ddb7e5c
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !query1_0_before --
+1
+2
+3
+
+-- !query1_0_after --
+1
+2
+3
+
+-- !query1_1_before --
+3
+
+-- !query1_1_after --
+3
+
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
new file mode 100644
index 00000000000..30f3fe64b3f
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
@@ -0,0 +1,116 @@
+package mv.external_table
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("single_external_table", "p0,external,hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test. then doesn't test mv rewrite")
+ return;
+ }
+ // prepare catalog
+ def suite_name = "single_external_table";
+ def externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ def hms_port = context.config.otherConfigs.get("hive2HmsPort")
+ def hive_catalog_name = "${suite_name}_catalog"
+ def hive_database = "${suite_name}_db"
+ def hive_table = "${suite_name}_orders"
+
+ sql """drop catalog if exists ${hive_catalog_name}"""
+ sql """
+ create catalog if not exists ${hive_catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ sql """switch ${hive_catalog_name};"""
+ sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
+ sql """ drop database if exists ${hive_database}"""
+ sql """ create database ${hive_database}"""
+ sql """use ${hive_database}"""
+ sql """
+ CREATE TABLE IF NOT EXISTS ${hive_table} (
+ o_orderkey integer,
+ o_custkey integer,
+ o_orderstatus char(1),
+ o_totalprice decimalv3(15,2),
+ o_orderpriority char(15),
+ o_clerk char(15),
+ o_shippriority integer,
+ o_comment varchar(79),
+ o_orderdate date
+ ) ENGINE=hive
+ PARTITION BY list(o_orderdate)()
+ PROPERTIES (
+ "replication_num" = "1",
+ "file_format"="orc",
+ "compression"="zlib"
+ );
+ """
+
+ sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table}
values(1, 1, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-17');"""
+ sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table}
values(2, 2, 'ok', 109.2, 'c','d',2, 'mm', '2023-10-18');"""
+ sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table}
values(3, 3, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-19');"""
+
+ // prepare table and data in olap
+ def internal_catalog = "internal"
+ def olap_db = context.config.getDbNameByFile(context.file)
+
+ sql """switch ${internal_catalog};"""
+ sql "use ${olap_db};"
+ sql "SET enable_nereids_planner=true;"
+ sql "set runtime_filter_mode=OFF";
+ sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject';"
+ sql "SET materialized_view_rewrite_enable_contain_external_table=true"
+
+
+ // single table without aggregate
+ def mv1_0 = """
+ select o_custkey, o_orderdate
+ from ${hive_catalog_name}.${hive_database}.${hive_table};
+ """
+ def query1_0 = """
+ select o_custkey
+ from ${hive_catalog_name}.${hive_database}.${hive_table};
+ """
+ order_qt_query1_0_before "${query1_0}"
+ check_mv_rewrite_success(olap_db, mv1_0, query1_0, "mv1_0")
+ order_qt_query1_0_after "${query1_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
+
+
+ // single table filter without aggregate
+ def mv1_1 = """
+ select o_custkey, o_orderdate
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 1;
+ """
+ def query1_1 = """
+ select o_custkey
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 2;
+ """
+ order_qt_query1_1_before "${query1_1}"
+ check_mv_rewrite_success(olap_db, mv1_1, query1_1, "mv1_1")
+ order_qt_query1_1_after "${query1_1}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
+
+
+ sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
+ sql """drop database if exists ${hive_catalog_name}.${hive_database}"""
+ sql """drop catalog if exists ${hive_catalog_name}"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
