yihua commented on code in PR #19348:
URL: https://github.com/apache/hudi/pull/19348#discussion_r3633176308
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala:
##########
@@ -503,6 +503,83 @@ class TestPayloadDeprecationFlow extends
SparkClientFunctionalTestHarness {
.build()
}
+ /**
+ * End-to-end reproduction for a Copy-on-Write, pre-v9 Postgres-Debezium
table whose payload class
+ * is supplied ONLY via the write config
(hoodie.datasource.write.payload.class) and is NOT
+ * persisted in the table properties. A Debezium delete
(_change_operation_type='d') for a key that
+ * is absent from the base file it gets merged into goes down the
FileGroupReader write path's
+ * previousRecord==null branch. Before the fix the reader cannot derive the
Debezium delete markers
+ * (getTableMergeProperties resolves the payload class from the table
config, which does not have
+ * it), so the delete is classified isDelete=false, routed into
+ * PayloadUpdateProcessor.handleNonDeletes, and .get()s the empty Option ->
NoSuchElementException.
+ *
+ * The parameter is the "similar test case": stripPayloadClass=false keeps
the payload class in the
+ * table properties (delete detected, always works) and acts as the control;
stripPayloadClass=true
+ * reproduces the incident (broken before the fix, correct after).
+ */
+ @ParameterizedTest
+ @ValueSource(strings = Array("true", "false"))
+ def
testDebeziumDeleteForAbsentKeyWithPayloadClassNotInTableConfig(stripPayloadClassStr:
String): Unit = {
+ val stripPayloadClass = stripPayloadClassStr.equals("true")
+ val payloadClazz = classOf[PostgresDebeziumAvroPayload].getName
+ // Payload class is only a write option, never a table property (mirrors
gw-agent provisioning).
+ val opts: Map[String, String] = Map(
+ HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key() -> payloadClazz,
+ HoodieMetadataConfig.ENABLE.key() -> "false")
+ // Single-bucket index so every record (including the new delete key)
hashes to the one existing
+ // file group and is merged against its base file -- deterministically
forcing the FileGroupReader
+ // write path's previousRecord==null branch rather than a fresh insert
file group.
Review Comment:
Use `indexOpts`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
