This is an automated email from the ASF dual-hosted git repository.
bobhan1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 396ad795b05 [fix](test) Isolate TPC-H MOR-as-DUP tables from
compaction (#67566)
396ad795b05 is described below
commit 396ad795b059a44fef51942f8950588f83e7be0a
Author: bobhan1 <[email protected]>
AuthorDate: Tue Sep 8 16:43:35 2026 +0800
[fix](test) Isolate TPC-H MOR-as-DUP tables from compaction (#67566)
### What problem does this PR solve?
Issue Number: None
Related PR: #61762
Problem Summary:
The TPC-H SF100 MOR suite loads the same data twice into MOR and DUP
tables, then compares their results with `read_mor_as_dup_tables`
enabled. Automatic compaction can merge the MOR versions before the
comparison, leaving fewer rows than the DUP tables.
Add eight dedicated `*_mor_as_dup` tables with automatic compaction
disabled. Prepare them in the existing two-round `load.groovy` flow by
reusing the existing S3 import SQL with a distinct table name and load
label. Route the MOR-as-DUP queries to these tables, preserving the
shared MOR table definitions used by the normal TPC-H and
predicate-pushdown cases.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test
- [x] No need to test or manual test. Explain why:
- [x] Other reason: Test fixture change. Both modified Groovy scripts
passed compilation with the existing regression framework's Groovy
compiler; all eight dedicated DDL schemas and import substitution
anchors were checked; shared MOR DDLs are unchanged; `git diff --check`
passed. The full SF100 suite was not run locally.
- Behavior changed:
- [x] No. Test setup only.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../ddl/customer_mor_as_dup.sql | 18 +++++++++++++++
.../ddl/lineitem_mor_as_dup.sql | 26 ++++++++++++++++++++++
.../ddl/nation_mor_as_dup.sql | 14 ++++++++++++
.../ddl/orders_mor_as_dup.sql | 19 ++++++++++++++++
.../ddl/part_mor_as_dup.sql | 19 ++++++++++++++++
.../ddl/partsupp_mor_as_dup.sql | 15 +++++++++++++
.../ddl/region_mor_as_dup.sql | 13 +++++++++++
.../ddl/supplier_mor_as_dup.sql | 17 ++++++++++++++
.../suites/tpch_sf100_unique_mor_p2/load.groovy | 12 +++++++++-
.../test_read_mor_as_dup.groovy | 12 ++++++----
10 files changed, 160 insertions(+), 5 deletions(-)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/customer_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/customer_mor_as_dup.sql
new file mode 100644
index 00000000000..c025c137cbb
--- /dev/null
+++
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/customer_mor_as_dup.sql
@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS customer_mor_as_dup (
+ c_custkey int NOT NULL,
+ c_name VARCHAR(25) NOT NULL,
+ c_address VARCHAR(40) NOT NULL,
+ c_nationkey int NOT NULL,
+ c_phone VARCHAR(15) NOT NULL,
+ c_acctbal decimal(15, 2) NOT NULL,
+ c_mktsegment VARCHAR(10) NOT NULL,
+ c_comment VARCHAR(117) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`c_custkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 24
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/lineitem_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/lineitem_mor_as_dup.sql
new file mode 100644
index 00000000000..72ed310afbf
--- /dev/null
+++
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/lineitem_mor_as_dup.sql
@@ -0,0 +1,26 @@
+CREATE TABLE IF NOT EXISTS lineitem_mor_as_dup (
+ l_shipdate DATE NOT NULL,
+ l_orderkey bigint NOT NULL,
+ l_linenumber int not null,
+ l_partkey int NOT NULL,
+ l_suppkey int not null,
+ l_quantity decimal(15, 2) NOT NULL,
+ l_extendedprice decimal(15, 2) NOT NULL,
+ l_discount decimal(15, 2) NOT NULL,
+ l_tax decimal(15, 2) NOT NULL,
+ l_returnflag VARCHAR(1) NOT NULL,
+ l_linestatus VARCHAR(1) NOT NULL,
+ l_commitdate DATE NOT NULL,
+ l_receiptdate DATE NOT NULL,
+ l_shipinstruct VARCHAR(25) NOT NULL,
+ l_shipmode VARCHAR(10) NOT NULL,
+ l_comment VARCHAR(44) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`l_shipdate`, `l_orderkey`,`l_linenumber`,`l_partkey`,`l_suppkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/nation_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/nation_mor_as_dup.sql
new file mode 100644
index 00000000000..cd6f7e29c38
--- /dev/null
+++ b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/nation_mor_as_dup.sql
@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS nation_mor_as_dup (
+ `n_nationkey` int(11) NOT NULL,
+ `n_name` varchar(25) NOT NULL,
+ `n_regionkey` int(11) NOT NULL,
+ `n_comment` varchar(152) NULL
+) ENGINE=OLAP
+UNIQUE KEY(`N_NATIONKEY`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`N_NATIONKEY`) BUCKETS 1
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+);
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/orders_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/orders_mor_as_dup.sql
new file mode 100644
index 00000000000..dcba391f18e
--- /dev/null
+++ b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/orders_mor_as_dup.sql
@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS orders_mor_as_dup (
+ o_orderkey bigint NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_custkey int NOT NULL,
+ o_orderstatus VARCHAR(1) NOT NULL,
+ o_totalprice decimal(15, 2) NOT NULL,
+ o_orderpriority VARCHAR(15) NOT NULL,
+ o_clerk VARCHAR(15) NOT NULL,
+ o_shippriority int NOT NULL,
+ o_comment VARCHAR(79) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`o_orderkey`, `o_orderdate`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/part_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/part_mor_as_dup.sql
new file mode 100644
index 00000000000..a931808aa37
--- /dev/null
+++ b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/part_mor_as_dup.sql
@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS part_mor_as_dup (
+ p_partkey int NOT NULL,
+ p_name VARCHAR(55) NOT NULL,
+ p_mfgr VARCHAR(25) NOT NULL,
+ p_brand VARCHAR(10) NOT NULL,
+ p_type VARCHAR(25) NOT NULL,
+ p_size int NOT NULL,
+ p_container VARCHAR(10) NOT NULL,
+ p_retailprice decimal(15, 2) NOT NULL,
+ p_comment VARCHAR(23) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`p_partkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/partsupp_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/partsupp_mor_as_dup.sql
new file mode 100644
index 00000000000..352e5fddade
--- /dev/null
+++
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/partsupp_mor_as_dup.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS partsupp_mor_as_dup (
+ ps_partkey int NOT NULL,
+ ps_suppkey int NOT NULL,
+ ps_availqty int NOT NULL,
+ ps_supplycost decimal(15, 2) NOT NULL,
+ ps_comment VARCHAR(199) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`ps_partkey`,`ps_suppkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`ps_partkey`) BUCKETS 24
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/region_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/region_mor_as_dup.sql
new file mode 100644
index 00000000000..cd4448291cb
--- /dev/null
+++ b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/region_mor_as_dup.sql
@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS region_mor_as_dup (
+ r_regionkey int NOT NULL,
+ r_name VARCHAR(25) NOT NULL,
+ r_comment VARCHAR(152)
+)ENGINE=OLAP
+UNIQUE KEY(`r_regionkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`r_regionkey`) BUCKETS 1
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/supplier_mor_as_dup.sql
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/supplier_mor_as_dup.sql
new file mode 100644
index 00000000000..09f86ed07d8
--- /dev/null
+++
b/regression-test/suites/tpch_sf100_unique_mor_p2/ddl/supplier_mor_as_dup.sql
@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS supplier_mor_as_dup (
+ s_suppkey int NOT NULL,
+ s_name VARCHAR(25) NOT NULL,
+ s_address VARCHAR(40) NOT NULL,
+ s_nationkey int NOT NULL,
+ s_phone VARCHAR(15) NOT NULL,
+ s_acctbal decimal(15, 2) NOT NULL,
+ s_comment VARCHAR(101) NOT NULL
+)ENGINE=OLAP
+UNIQUE KEY(`s_suppkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 12
+PROPERTIES (
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "3"
+)
diff --git a/regression-test/suites/tpch_sf100_unique_mor_p2/load.groovy
b/regression-test/suites/tpch_sf100_unique_mor_p2/load.groovy
index 36c55cbf392..0ab5f2ac2a6 100644
--- a/regression-test/suites/tpch_sf100_unique_mor_p2/load.groovy
+++ b/regression-test/suites/tpch_sf100_unique_mor_p2/load.groovy
@@ -55,13 +55,16 @@ suite("load") {
// Load data twice to create overlapping rowsets with duplicate keys,
// which triggers the MOR (Merge-On-Read) unique key merge semantics.
- // Also load DUP tables twice for read_mor_as_dup result comparison.
+ // Also load independent MOR-as-DUP and DUP tables twice for comparison.
for (int loadRound = 1; loadRound <= 2; loadRound++) {
tables.each { table, rows ->
if (loadRound == 1) {
// drop and recreate MOR table on first round
sql "DROP TABLE IF EXISTS ${table}"
sql new
File("""${context.file.parent}/ddl/${table}.sql""").text
+ // keep the MOR-as-DUP fixture independent of the shared MOR
tables
+ sql "DROP TABLE IF EXISTS ${table}_mor_as_dup"
+ sql new
File("${context.file.parent}/ddl/${table}_mor_as_dup.sql").text
// drop and recreate DUP table on first round
sql "DROP TABLE IF EXISTS ${table}_dup"
sql new
File("""${context.file.parent}/ddl/${table}_dup.sql""").text
@@ -72,6 +75,12 @@ suite("load") {
morSql = morSql.replaceAll("\\\$\\{loadLabel\\}", morLabel) +
s3WithProperties
loadAndWait(morLabel, morSql)
+ // reuse the same import SQL for the MOR-as-DUP table
+ def morAsDupLabel = table + "_mor_as_dup_round${loadRound}_" +
uniqueID
+ def morAsDupSql = morSql.replace("LOAD LABEL ${morLabel}", "LOAD
LABEL ${morAsDupLabel}")
+ .replace("INTO TABLE ${table}", "INTO TABLE
${table}_mor_as_dup")
+ loadAndWait(morAsDupLabel, morAsDupSql)
+
// load DUP table
def dupLabel = table + "_dup_round${loadRound}_" + uniqueID
def dupSql = new
File("""${context.file.parent}/ddl/${table}_dup_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}",
s3BucketName)
@@ -93,5 +102,6 @@ suite("load") {
logger.info("DUP table ${table}_dup row count after double load:
${dupRowCount[0][0]}, expected: ${rows * 2}")
assertTrue(dupRowCount[0][0] == rows * 2, "DUP table ${table}_dup row
count ${dupRowCount[0][0]} != expected ${rows * 2}")
sql """ ANALYZE TABLE ${table}_dup WITH SYNC """
+ sql "ANALYZE TABLE ${table}_mor_as_dup WITH SYNC"
}
}
diff --git
a/regression-test/suites/tpch_sf100_unique_mor_p2/test_read_mor_as_dup.groovy
b/regression-test/suites/tpch_sf100_unique_mor_p2/test_read_mor_as_dup.groovy
index 78277b66c53..11f6c17ec1d 100644
---
a/regression-test/suites/tpch_sf100_unique_mor_p2/test_read_mor_as_dup.groovy
+++
b/regression-test/suites/tpch_sf100_unique_mor_p2/test_read_mor_as_dup.groovy
@@ -18,8 +18,8 @@
// This test runs all 22 TPC-H queries on MOR (Merge-On-Read) unique key tables
// with read_mor_as_dup_tables enabled, and compares the results against actual
// DUPLICATE KEY tables loaded with the same data (loaded twice).
-// MOR tables with read_mor_as_dup should produce identical results to DUP
tables
-// since both skip merge and expose all row versions.
+// MOR tables with read_mor_as_dup should produce identical results to DUP
tables.
+// The dedicated MOR tables disable compaction to preserve both loaded
versions.
suite("test_read_mor_as_dup") {
sql "SET query_timeout = 1800"
@@ -34,9 +34,13 @@ suite("test_read_mor_as_dup") {
def queryName = queryFile.name.replace('.sql', '')
def querySql = queryFile.text
- // Run on MOR tables with read_mor_as_dup_tables enabled
+ // Run on the dedicated MOR tables with read_mor_as_dup_tables enabled
+ def morSql = querySql
+ for (def tbl : tableNames) {
+ morSql = morSql.replaceAll("\\b${tbl}\\b", "${tbl}_mor_as_dup")
+ }
sql "SET read_mor_as_dup_tables = '*'"
- def morResult = sql querySql
+ def morResult = sql morSql
// Run the same query on actual DUP tables (replace table names with
_dup suffix)
sql "SET read_mor_as_dup_tables = ''"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
