This is an automated email from the ASF dual-hosted git repository.
suxiaogang223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b69168cbe5a [test](paimon) Verify Paimon null partition refresh and
rewrite (#70201)
b69168cbe5a is described below
commit b69168cbe5a39ee77a88e5b126eacb4e0861c297
Author: Socrates <[email protected]>
AuthorDate: Thu Nov 5 16:51:56 2026 +0800
[test](paimon) Verify Paimon null partition refresh and rewrite (#70201)
### What problem does this PR solve?
Related PRs: #65904, #64304
The existing Paimon MTMV regression covers the initial refresh of a NULL
partition. This test covers an out-of-band Spark append to that
partition after the initial refresh. It verifies that the stale
partition is ineligible for rewrite, automatic refresh touches only the
NULL partition, and the refreshed MV is chosen again. The string
partition 'null' is a control.
The suite compares live SQL results and checks the rewrite plan without
a golden output file. This PR adds regression coverage only.
### Release note
None
### Check List (For Author)
- Test: `git diff --check` passed. Doris compilation and regression
execution were not run.
- Behavior changed: No
- Does this need documentation: No
---
.../test_paimon_null_partition_mtmv_refresh.groovy | 121 +++++++++++++++++++++
1 file changed, 121 insertions(+)
diff --git
a/regression-test/suites/mtmv_p0/test_paimon_null_partition_mtmv_refresh.groovy
b/regression-test/suites/mtmv_p0/test_paimon_null_partition_mtmv_refresh.groovy
new file mode 100644
index 00000000000..2f7b9a596cc
--- /dev/null
+++
b/regression-test/suites/mtmv_p0/test_paimon_null_partition_mtmv_refresh.groovy
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_null_partition_mtmv_refresh",
"p0,external,mtmv,external_docker,external_docker_doris") {
+ if
(!"true".equalsIgnoreCase(context.config.otherConfigs.get("enablePaimonTest")))
{
+ return
+ }
+
+ String dbName = context.config.getDbNameByFile(context.file)
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ sql "DROP MATERIALIZED VIEW IF EXISTS paimon_null_partition_mtmv"
+ sql "DROP CATALOG IF EXISTS paimon_null_partition_mtmv_catalog"
+
+ spark_paimon_multi """
+ CREATE DATABASE IF NOT EXISTS paimon.paimon_null_partition_mtmv_db;
+ DROP TABLE IF EXISTS paimon.paimon_null_partition_mtmv_db.events;
+ CREATE TABLE paimon.paimon_null_partition_mtmv_db.events (id INT,
region STRING)
+ USING paimon PARTITIONED BY (region);
+ INSERT INTO paimon.paimon_null_partition_mtmv_db.events VALUES (1,
NULL), (2, 'null');
+ """
+
+ // Read the latest snapshot after the Spark commit without an explicit
metadata refresh.
+ sql """
+ CREATE CATALOG paimon_null_partition_mtmv_catalog PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = 's3://warehouse/wh/',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.region' = 'us-east-1',
+ 'meta.cache.paimon.table.ttl-second' = '0'
+ )
+ """
+ sql "SET materialized_view_rewrite_enable_contain_external_table = true"
+ sql "ANALYZE TABLE
paimon_null_partition_mtmv_catalog.paimon_null_partition_mtmv_db.events WITH
SYNC"
+ // As in the other Paimon rewrite suite, make the MV cheaper than scanning
the base fixture.
+ sql """
+ ALTER TABLE
paimon_null_partition_mtmv_catalog.paimon_null_partition_mtmv_db.events
+ MODIFY COLUMN region SET STATS ('row_count' = '20')
+ """
+ sql """
+ CREATE MATERIALIZED VIEW paimon_null_partition_mtmv
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ PARTITION BY (region)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS SELECT region, COUNT(*) AS row_count
+ FROM
paimon_null_partition_mtmv_catalog.paimon_null_partition_mtmv_db.events
+ GROUP BY region
+ """
+
+ String nullRegionQuery = """
+ SELECT region, COUNT(*) AS row_count
+ FROM
paimon_null_partition_mtmv_catalog.paimon_null_partition_mtmv_db.events
+ WHERE region IS NULL
+ GROUP BY region
+ ORDER BY region
+ """
+ String baseNullRegionQuery = nullRegionQuery.replace("SELECT region",
+ "SELECT /*+ SET_VAR(enable_materialized_view_rewrite=false) */
region")
+ String mvNullRegionQuery = """
+ SELECT region, row_count FROM paimon_null_partition_mtmv WHERE region
IS NULL ORDER BY region
+ """
+
+ sql "REFRESH MATERIALIZED VIEW paimon_null_partition_mtmv AUTO"
+ waitingMTMVTaskFinishedByMvName("paimon_null_partition_mtmv")
+ quickTest("initial_null_partition", mvNullRegionQuery, baseNullRegionQuery)
+ mv_rewrite_success(nullRegionQuery, "paimon_null_partition_mtmv")
+
+ spark_paimon "INSERT INTO paimon.paimon_null_partition_mtmv_db.events
VALUES (3, NULL)"
+
+ // A query restricted to the now-stale NULL partition must read current
base data.
+ explain {
+ sql("memo plan ${nullRegionQuery}")
+ notContains(".paimon_null_partition_mtmv chose")
+ notContains(".paimon_null_partition_mtmv not chose")
+ }
+ quickTest("null_partition_while_stale", nullRegionQuery,
baseNullRegionQuery)
+
+ sql "REFRESH MATERIALIZED VIEW paimon_null_partition_mtmv AUTO"
+ waitingMTMVTaskFinishedByMvName("paimon_null_partition_mtmv")
+
+ // The automatic refresh must touch only the SQL NULL partition and leave
literal 'null' alone.
+ quickTest("refreshed_null_partition", """
+ SELECT json_extract_string(NeedRefreshPartitions, '\$[0]'),
+ json_extract_string(CompletedPartitions, '\$[0]')
+ FROM (
+ SELECT RefreshMode, NeedRefreshPartitions, CompletedPartitions
+ FROM tasks('type' = 'mv')
+ WHERE MvDatabaseName = '${dbName}' AND MvName =
'paimon_null_partition_mtmv'
+ ORDER BY CreateTime DESC, TaskId DESC
+ LIMIT 1
+ ) latest
+ WHERE RefreshMode = 'PARTIAL'
+ AND JSON_LENGTH(NeedRefreshPartitions) = 1
+ AND JSON_LENGTH(CompletedPartitions) = 1
+ """, """
+ SELECT MAX(PARTITION_NAME), MAX(PARTITION_NAME) FROM
information_schema.partitions
+ WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME =
'paimon_null_partition_mtmv'
+ AND PARTITION_NAME = 'pn_NULL'
+ """)
+
+ quickTest("incremental_null_partition", mvNullRegionQuery,
baseNullRegionQuery)
+ mv_rewrite_success(nullRegionQuery, "paimon_null_partition_mtmv")
+ quickTest("null_partition_after_rewrite", nullRegionQuery,
baseNullRegionQuery)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
