wombatu-kun commented on code in PR #20676:
URL: https://github.com/apache/hudi/pull/20676#discussion_r4612031324


##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -42,33 +42,48 @@ def client(mode, remote, root):
     spark = SparkSession.builder.remote(remote).getOrCreate()
     try:
         require(spark.conf.get("spark.hudi.test.run.id") == root.name, 
"Connected to the wrong test server")
+        table_type = (root / "table-type").read_text()
+        properties = f"'type'='{table_type}', {PROPERTIES}"
         if mode == "prepare":
             spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts 
BIGINT, part STRING, value STRING, "
-                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({PROPERTIES})").collect()
+                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({properties})").collect()
         else:
             expected = json.loads((root / "expected.json").read_text())
             for table in TABLES:
                 actual = [list(row) for row in spark.table(f"sdp_test.{table}")
                           .select("id", "ts", "part", 
"value").orderBy("id").collect()]
                 require(actual == expected, (table, actual, expected))
-                print(f"PASS {mode} {table}: {actual}")
+                print(f"PASS {mode} {table_type} {table}: {actual}")
+                if table_type == "mor" and mode == "verify-2":
+                    location = root / "warehouse" / "sdp_test.db" / table
+                    require(any(location.rglob("*.log.*")), f"No MOR log files 
in {location}")
+                    base_rows = [list(row) for row in spark.read.format("hudi")
+                                 .option("hoodie.datasource.query.type", 
"read_optimized")
+                                 .load(location.as_uri()).select("id", "ts", 
"part", "value")
+                                 .orderBy("id").collect()]
+                    require(base_rows != expected, "Update must require log 
merging, not only base-file reads")

Review Comment:
   `base_rows != expected` also passes for empty or otherwise incorrect 
base-file output, so it does not establish the pre-update state that the 
snapshot must merge with the log. Compare `base_rows` with the exact stage-1 
rows instead.



##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -89,14 +104,22 @@ def scenario(h):
         (h.root / "expected.json").write_text(json.dumps(expected))
         h.run_pipeline(path, f"stage-{i}")
         h.client(Path(__file__).resolve(), f"verify-{i}")
+    def completed_commits():
+        return {table: sorted(p.name for p in
+                (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / 
"timeline").glob("*")
+                if p.suffix in (".commit", ".deltacommit", ".replacecommit")) 
for table in TABLES}
+
+    before_restart = completed_commits()
+    require(all(before_restart.values()), "Expected completed commits before 
restart")
     h.run_pipeline(path, "restart")
+    require(completed_commits() == before_restart, "Unchanged restart wrote an 
extra commit")

Review Comment:
   The restart assertion discards both commit lists, so a failure does not 
identify the changed table or instant. Capture `after_restart = 
completed_commits()` and include `(before_restart, after_restart)` in the 
`require` message.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to