This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20675-f757583aa3e07d2a6e3394e9765d7398a31b8fa2
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit a05e56c90bc9cf05c445769881ecc9272e2e6130
Author: Danny Chan <[email protected]>
AuthorDate: Thu Nov 19 03:32:07 2026 +0000

    test(spark): cover keyless aggregate SDP materialized views (#20675)
---
 .../spark-pipelines/MATERIALIZED_VIEWS.md          | 30 ++++++++++++------
 .../spark-pipelines/materialized_views.py          | 36 +++++++++++++++++++---
 2 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md 
b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md
index 2fd3f2d0ee8b..08cd4c35de66 100644
--- a/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md
+++ b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md
@@ -27,16 +27,26 @@ python 
hudi-integ-test/spark-pipelines/materialized_views.py \
   --bundle /path/to/candidate-hudi-spark4.1-bundle.jar
 ```
 
-Two Hudi materialized views read a JSON source and a Hudi source respectively.
-Four real CLI runs replace the source contents with `(1, one), (2, two)`,
-then `(2, updated), (3, three)`, then `(4, four)`, then zero rows.
-The JSON-backed target is partitioned by `value`. After every refresh, each
-view must exactly match its defining query's expected rows. Removing old rows
-ensures that accidental append behavior fails. Repeated catalog reconciliation
-exercises metadata-column deletes and unchanged properties.
-
-Keyless aggregate materialized views remain deferred under #20405.
-Land this scenario after the #20453 harness dependency. Checks remain active 
under
+Three Hudi materialized views cover keyed JSON and Hudi sources and a keyless
+JSON aggregate. The aggregate declares only `type=cow`, with no primary key,
+and groups by `value`, computing `sum(id)` and `count(*)`.
+
+Four real CLI runs use the following exact inputs and aggregate expectations:
+
+| Run | Input `(id, value)` | Aggregate `(value, total, rows)` |
+| --- | --- | --- |
+| 1 | `(1, one), (2, one), (3, two)` | `(one, 3, 2), (two, 3, 1)` |
+| 2 | `(2, one), (4, one), (3, three)` | `(one, 6, 2), (three, 3, 1)` |
+| 3 | `(4, four)` | `(four, 4, 1)` |
+| 4 | empty | empty |
+
+After every refresh, all three views must match exact expected business rows.
+The repeated `one` group changes its aggregate, old groups disappear, and the
+last refresh removes every group. This detects stale rows and duplicate 
aggregate
+rows even when Hudi generates record keys. The keyed JSON target remains
+partitioned by `value`; the aggregate is unpartitioned.
+
+Checks remain active under
 `python -O`, and SIGTERM unwinds the harness for server and 
temporary-directory cleanup.
 
 This is Phase 2 coverage on today's purge-style truncate. See the
diff --git a/hudi-integ-test/spark-pipelines/materialized_views.py 
b/hudi-integ-test/spark-pipelines/materialized_views.py
index 2eb650c0773b..6533bdaa8c02 100644
--- a/hudi-integ-test/spark-pipelines/materialized_views.py
+++ b/hudi-integ-test/spark-pipelines/materialized_views.py
@@ -23,7 +23,25 @@ from pathlib import Path
 import sys
 import tempfile
 
-SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import 
SparkSession\nspark = 
SparkSession.active()\n\[email protected]_view(name="json_mv", format="hudi", 
partition_cols=["value"], table_properties={"type": "cow", "primaryKey": 
"id"})\ndef json_mv():\n    return spark.read.schema("id INT, value 
STRING").json(spark.conf.get("hudi.test.input"))\n\[email protected]_view(name="hudi_mv",
 format="hudi", table_properties={"type": "cow", "primaryKey": "id"})\ndef 
hudi_mv():\n     [...]
+SOURCE = """from pyspark import pipelines as dp
+from pyspark.sql import SparkSession, functions as F
+spark = SparkSession.active()
+
[email protected]_view(name="json_mv", format="hudi", partition_cols=["value"],
+                      table_properties={"type": "cow", "primaryKey": "id"})
+def json_mv():
+    return spark.read.schema("id INT, value 
STRING").json(spark.conf.get("hudi.test.input"))
+
[email protected]_view(name="hudi_mv", format="hudi", table_properties={"type": 
"cow", "primaryKey": "id"})
+def hudi_mv():
+    return 
spark.read.format("hudi").load(spark.conf.get("hudi.test.source")).select("id", 
"value")
+
[email protected]_view(name="aggregate_mv", format="hudi", 
table_properties={"type": "cow"})
+def aggregate_mv():
+    return (spark.read.schema("id INT, value 
STRING").json(spark.conf.get("hudi.test.input"))
+            .groupBy("value").agg(F.sum("id").alias("total"), 
F.count("*").alias("rows")))
+"""
+
 
 def require(condition, message):
     if not condition:
@@ -54,6 +72,11 @@ def client(mode, remote, root):
                           .select("id", "value").orderBy("id").collect()]
                 require(actual == expected, (name, actual, expected))
                 print(f"PASS {mode} {name}: {actual}")
+            expected_aggregate = json.loads((root / 
"expected-aggregate.json").read_text())
+            actual = [list(row) for row in spark.table("sdp_test.aggregate_mv")
+                      .select("value", "total", 
"rows").orderBy("value").collect()]
+            require(actual == expected_aggregate, ("aggregate_mv", actual, 
expected_aggregate))
+            print(f"PASS {mode} aggregate_mv: {actual}")
     finally:
         spark.stop()
 
@@ -75,15 +98,20 @@ def scenario(h):
     (h.logs / "mv.py").write_text(SOURCE)
     (h.logs / path.name).write_text(path.read_text())
     # Remove and change rows as well as adding them: stale append output must 
fail.
-    for i, expected in enumerate(([[1, "one"], [2, "two"]],
-                                   [[2, "updated"], [3, "three"]], [[4, 
"four"]], []), 1):
+    stages = (
+        ([[1, "one"], [2, "one"], [3, "two"]], [["one", 3, 2], ["two", 3, 1]]),
+        ([[2, "one"], [3, "three"], [4, "one"]], [["one", 6, 2], ["three", 3, 
1]]),
+        ([[4, "four"]], [["four", 4, 1]]),
+        ([], []))
+    for i, (expected, aggregate) in enumerate(stages, 1):
+        (h.root / "expected-aggregate.json").write_text(json.dumps(aggregate))
         (h.root / "expected.json").write_text(json.dumps(expected))
         (source / "rows.json").write_text("".join(json.dumps(dict(zip(("id", 
"value"), row))) + "\n"
                                                    for row in expected))
         h.client(Path(__file__).resolve(), f"prepare-{i}")
         h.run_pipeline(path, f"refresh-{i}")
         h.client(Path(__file__).resolve(), f"verify-{i}")
-    print("PASS: four exact MV refreshes over JSON and Hudi sources")
+    print("PASS: four exact keyed and keyless aggregate MV refreshes")
 
 
 def main():

Reply via email to