This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 979777969b9 branch-3.1: [opt](mtmv) Chose mv more easier when mv on
external table #46368 (#51968)
979777969b9 is described below
commit 979777969b96c6d881b5aef6cc1f5a8ed1512580
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 20 11:08:30 2025 +0800
branch-3.1: [opt](mtmv) Chose mv more easier when mv on external table
#46368 (#51968)
Cherry-picked from #46368
Co-authored-by: seawinde <[email protected]>
---
.../org/apache/doris/nereids/cost/CostModelV1.java | 11 +--
.../mv/external_table/single_external_table.out | Bin 205 -> 487 bytes
.../mv/external_table/single_external_table.groovy | 80 +++++++++++++++++++++
3 files changed, 87 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index f6cf30d855f..434a29a5191 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -69,6 +69,9 @@ import java.util.Set;
class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
static final double RANDOM_SHUFFLE_TO_HASH_SHUFFLE_FACTOR = 0.1;
+ // The cost of using external tables should be somewhat higher than using
internal tables,
+ // so when encountering a scan of an external table, a coefficient should
be applied.
+ static final double EXTERNAL_TABLE_SCAN_FACTOR = 5;
private final int beNumber;
private final int parallelInstance;
@@ -188,7 +191,7 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
@Override
public Cost visitPhysicalFileScan(PhysicalFileScan physicalFileScan,
PlanContext context) {
Statistics statistics = context.getStatisticsWithCheck();
- return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
+ return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount() * EXTERNAL_TABLE_SCAN_FACTOR);
}
@Override
@@ -209,19 +212,19 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
@Override
public Cost visitPhysicalJdbcScan(PhysicalJdbcScan physicalJdbcScan,
PlanContext context) {
Statistics statistics = context.getStatisticsWithCheck();
- return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
+ return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount() * EXTERNAL_TABLE_SCAN_FACTOR);
}
@Override
public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan,
PlanContext context) {
Statistics statistics = context.getStatisticsWithCheck();
- return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
+ return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount() * EXTERNAL_TABLE_SCAN_FACTOR);
}
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext
context) {
Statistics statistics = context.getStatisticsWithCheck();
- return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
+ return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount() * EXTERNAL_TABLE_SCAN_FACTOR);
}
@Override
diff --git
a/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
index 5305ddb7e5c..d838545f2ea 100644
Binary files
a/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
and
b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out
differ
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
index d78899a12ec..5bd7e5cf338 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
@@ -111,6 +111,86 @@ suite("single_external_table", "p0,external,hive") {
order_qt_query1_1_after "${query1_1}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
+ // single table and only query with filter
+ def mv1_2 = """
+ select o_custkey, o_orderdate
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ """
+ def query1_2 = """
+ select o_custkey
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 2;
+ """
+ order_qt_query1_2_before "${query1_2}"
+ async_mv_rewrite_success(olap_db, mv1_2, query1_2, "mv1_2")
+ order_qt_query1_2_after "${query1_2}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2"""
+
+
+
+ // single table with aggregate and filter
+ def mv1_3 = """
+ select o_custkey, o_orderdate,
+ count(*)
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ group by
+ o_custkey, o_orderdate;
+ """
+ def query1_3 = """
+ select o_custkey, o_orderdate,
+ count(*)
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 2
+ group by
+ o_custkey, o_orderdate;
+ """
+ order_qt_query1_3_before "${query1_3}"
+ async_mv_rewrite_success(olap_db, mv1_3, query1_3, "mv1_3")
+ order_qt_query1_3_after "${query1_3}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3"""
+
+
+ // single table with aggregate roll up and filter
+ def mv1_4 = """
+ select o_custkey, o_orderdate,
+ count(*)
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ group by
+ o_custkey, o_orderdate;
+ """
+ def query1_4 = """
+ select o_custkey, o_orderdate,
+ count(*)
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 2
+ group by
+ o_custkey, o_orderdate;
+ """
+ order_qt_query1_4_before "${query1_4}"
+ async_mv_rewrite_success(olap_db, mv1_4, query1_4, "mv1_4")
+ order_qt_query1_4_after "${query1_4}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4"""
+
+
+
+ // single table with aggregate roll up and filter
+ def mv1_5 = """
+ select o_custkey, o_orderdate, o_totalprice
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ """
+ def query1_5 = """
+ select o_custkey, o_orderdate,
+ count(*)
+ from ${hive_catalog_name}.${hive_database}.${hive_table}
+ where o_custkey > 2
+ group by
+ o_custkey, o_orderdate;
+ """
+ order_qt_query1_5_before "${query1_5}"
+ async_mv_rewrite_success(olap_db, mv1_5, query1_5, "mv1_5")
+ order_qt_query1_5_after "${query1_5}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5"""
+
sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
sql """drop database if exists ${hive_catalog_name}.${hive_database}"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
