This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20676-a05e56c90bc9cf05c445769881ecc9272e2e6130
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 06dc47c2d4666d36c64fa39de4980a90bb64cde9
Author: Danny Chan <[email protected]>
AuthorDate: Thu Nov 19 03:40:42 2026 +0000

    test(spark): cover MOR log merging in keyed SDP flows (#20676)
    
    * test(spark): cover MOR log merging in keyed SDP flows
    
    * test(spark): tighten keyed MOR acceptance assertions
---
 hudi-integ-test/spark-pipelines/KEYED_FLOWS.md | 13 +++++--
 hudi-integ-test/spark-pipelines/keyed_flows.py | 50 ++++++++++++++++++++------
 2 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md 
b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
index 888f642483fb..cec90f50410f 100644
--- a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
+++ b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
@@ -27,7 +27,10 @@ python hudi-integ-test/spark-pipelines/keyed_flows.py \
   --bundle /path/to/candidate-hudi-spark4.1-bundle.jar
 ```
 
-The generated SQL defines partitioned COW tables with `primaryKey=id`,
+Pass `--table-type mor` to run the same matrix with MOR; the default is COW.
+Run each table type separately with isolated targets and checkpoints.
+
+The generated SQL defines partitioned tables with `primaryKey=id`,
 `preCombineField=ts`, and `hoodie.record.merge.mode=EVENT_TIME_ORDERING`.
 A boolean `_hoodie_is_deleted` carries deletes. The scenario checks a newly
 created target and a pre-existing catalog table, then restarts each through the
@@ -46,7 +49,13 @@ Metadata indexing is disabled. This is delete input carried 
by a flow, not CDC
 propagation of upstream deletes. It does not cover full refresh, table 
chaining,
 fan-in or failure between commit and checkpoint (separate recovery coverage).
 
-MOR log-merge coverage remains deferred under #20408; this matrix covers COW 
only.
+For MOR, inline and asynchronous compaction are disabled. After the newer 
update,
+the test requires log files and verifies that read-optimized base-file output 
matches
+the exact stage-1 rows while the snapshot matches the updated rows. This 
proves the
+snapshot must merge logs. Subsequent late updates, deletes and reinsertion 
must also
+produce exact snapshots. Both table
+types verify that an unchanged restart adds no completed commit, reporting the
+before/after commit filenames for each table if the check fails.
 The flows rely on the default streaming upsert operation, without a session 
override.
 The catalog regression uses `PARTITIONED BY (PART)` against schema field `part`
 to verify case-insensitive normalization. The SDP scenario uses exact spelling
diff --git a/hudi-integ-test/spark-pipelines/keyed_flows.py 
b/hudi-integ-test/spark-pipelines/keyed_flows.py
index 4d1d2ed6b570..445ff0b9624e 100644
--- a/hudi-integ-test/spark-pipelines/keyed_flows.py
+++ b/hudi-integ-test/spark-pipelines/keyed_flows.py
@@ -24,7 +24,7 @@ import sys
 import tempfile
 
 SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import 
SparkSession\nspark = 
SparkSession.active()\[email protected]_view(name="input_events")\ndef 
input_events():\n    return spark.readStream.schema("id INT, ts LONG, part 
STRING, value STRING, _hoodie_is_deleted 
BOOLEAN").json(spark.conf.get("hudi.test.input"))\n'
-PROPERTIES = "'type'='cow', 'primaryKey'='id', 'preCombineField'='ts', 
'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'"
+PROPERTIES = "'primaryKey'='id', 'preCombineField'='ts', 
'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'"
 TABLES = ("keyed_new", "keyed_existing")
 
 
@@ -42,26 +42,40 @@ def client(mode, remote, root):
     spark = SparkSession.builder.remote(remote).getOrCreate()
     try:
         require(spark.conf.get("spark.hudi.test.run.id") == root.name, 
"Connected to the wrong test server")
+        table_type = (root / "table-type").read_text()
+        properties = f"'type'='{table_type}', {PROPERTIES}"
         if mode == "prepare":
             spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts 
BIGINT, part STRING, value STRING, "
-                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({PROPERTIES})").collect()
+                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({properties})").collect()
         else:
             expected = json.loads((root / "expected.json").read_text())
             for table in TABLES:
                 actual = [list(row) for row in spark.table(f"sdp_test.{table}")
                           .select("id", "ts", "part", 
"value").orderBy("id").collect()]
                 require(actual == expected, (table, actual, expected))
-                print(f"PASS {mode} {table}: {actual}")
+                print(f"PASS {mode} {table_type} {table}: {actual}")
+                if table_type == "mor" and mode == "verify-2":
+                    location = root / "warehouse" / "sdp_test.db" / table
+                    require(any(location.rglob("*.log.*")), f"No MOR log files 
in {location}")
+                    base_rows = [list(row) for row in spark.read.format("hudi")
+                                 .option("hoodie.datasource.query.type", 
"read_optimized")
+                                 .load(location.as_uri()).select("id", "ts", 
"part", "value")
+                                 .orderBy("id").collect()]
+                    expected_base = json.loads((root / 
"expected-base.json").read_text())
+                    require(base_rows == expected_base, (table, "MOR base 
rows", base_rows, expected_base))
+                    print(f"PASS MOR log merge {table}: base rows {base_rows}, 
snapshot {actual}")
     finally:
         spark.stop()
 
 
-def scenario(h):
+def scenario(h, table_type):
+    (h.root / "table-type").write_text(table_type)
+    properties = f"'type'='{table_type}', {PROPERTIES}"
     source = h.root / "input"
     source.mkdir()
     (h.root / "source.py").write_text(SOURCE)
     flows = "\n".join(
-        f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) 
TBLPROPERTIES ({PROPERTIES}) "
+        f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) 
TBLPROPERTIES ({properties}) "
         "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM 
input_events;" for table in TABLES)
     (h.root / "flows.sql").write_text(flows)
     spec = {"name": "keyed_flows", "storage": (h.root / 
"checkpoints").as_uri(),
@@ -69,6 +83,8 @@ def scenario(h):
             "libraries": [{"glob": {"include": "source.py"}}, {"glob": 
{"include": "flows.sql"}}],
             "configuration": {"hudi.test.input": source.as_uri(), 
"spark.sql.shuffle.partitions": "2",
                               "spark.hoodie.metadata.enable": "false",
+                              "spark.hoodie.compact.inline": "false",
+                              
"spark.hoodie.datasource.write.streaming.disable.compaction": "true",
                               "spark.sql.pipelines.maxFlowRetryAttempts": "0"}}
     path = h.root / "spark-pipeline.yml"
     path.write_text(json.dumps(spec, indent=2))
@@ -83,20 +99,31 @@ def scenario(h):
         ([(1, 2, "a", "late-delete", True)], [[1, 3, "a", "updated"], [2, 1, 
"b", "keep"]]),
         ([(1, 4, "a", "deleted", True)], [[2, 1, "b", "keep"]]),
         ([(1, 5, "a", "reinserted", False)], [[1, 5, "a", "reinserted"], [2, 
1, "b", "keep"]]))
+    (h.root / "expected-base.json").write_text(json.dumps(stages[0][1]))
     for i, (rows, expected) in enumerate(stages, 1):
         (source / f"{i}.json").write_text("".join(json.dumps(dict(zip(
             ("id", "ts", "part", "value", "_hoodie_is_deleted"), row))) + "\n" 
for row in rows))
         (h.root / "expected.json").write_text(json.dumps(expected))
         h.run_pipeline(path, f"stage-{i}")
         h.client(Path(__file__).resolve(), f"verify-{i}")
+    def completed_commits():
+        return {table: sorted(p.name for p in
+                (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / 
"timeline").glob("*")
+                if p.suffix in (".commit", ".deltacommit", ".replacecommit")) 
for table in TABLES}
+
+    before_restart = completed_commits()
+    require(all(before_restart.values()), "Expected completed commits before 
restart")
     h.run_pipeline(path, "restart")
+    after_restart = completed_commits()
+    require(after_restart == before_restart,
+            ("Unchanged restart wrote an extra commit", before_restart, 
after_restart))
     h.client(Path(__file__).resolve(), "verify-restart")
     invalid = (
-        ("missing_key", PROPERTIES.replace("'primaryKey'='id'", 
"'primaryKey'='missing_key'"), "part", SOURCE, "Can't find primaryKey"),
-        ("missing_order", PROPERTIES.replace("'preCombineField'='ts'", 
"'preCombineField'='missing_order'"), "part", SOURCE, "Can't find ordering 
fields"),
-        ("missing_partition", PROPERTIES, "missing_partition", SOURCE, 
"Partition column 'missing_partition'"),
-        ("invalid_marker", PROPERTIES, "part", 
SOURCE.replace("_hoodie_is_deleted BOOLEAN", "_hoodie_is_deleted STRING"), "has 
to be BOOLEAN"),
-        ("changed_identity", PROPERTIES.replace("'primaryKey'='id'", 
"'primaryKey'='value'"), "part", SOURCE, "Can't set primaryKey to 'value' on 
Hudi table"))
+        ("missing_key", properties.replace("'primaryKey'='id'", 
"'primaryKey'='missing_key'"), "part", SOURCE, "Can't find primaryKey"),
+        ("missing_order", properties.replace("'preCombineField'='ts'", 
"'preCombineField'='missing_order'"), "part", SOURCE, "Can't find ordering 
fields"),
+        ("missing_partition", properties, "missing_partition", SOURCE, 
"Partition column 'missing_partition'"),
+        ("invalid_marker", properties, "part", 
SOURCE.replace("_hoodie_is_deleted BOOLEAN", "_hoodie_is_deleted STRING"), "has 
to be BOOLEAN"),
+        ("changed_identity", properties.replace("'primaryKey'='id'", 
"'primaryKey'='value'"), "part", SOURCE, "Can't set primaryKey to 'value' on 
Hudi table"))
     for name, props, partition, definition, error_text in invalid:
         target = "keyed_existing" if name == "changed_identity" else 
f"bad_{name}"
         (h.root / "source.py").write_text(definition)
@@ -128,6 +155,7 @@ def main():
         return
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--harness-dir", required=True, type=Path)
+    parser.add_argument("--table-type", choices=("cow", "mor"), default="cow")
     parser.add_argument("--bundle", required=True, type=Path)
     parser.add_argument("--logs", type=Path, 
default=Path("target/keyed-acceptance-logs"))
     args = parser.parse_args()
@@ -138,7 +166,7 @@ def main():
     print(f"Keyed logs: {logs}", flush=True)
     signal.signal(signal.SIGTERM, interrupted)
     with ConnectHarness(args.bundle, logs) as harness:
-        scenario(harness)
+        scenario(harness, args.table_type)
 
 
 if __name__ == "__main__":

Reply via email to