rahil-c commented on code in PR #14001:
URL: https://github.com/apache/hudi/pull/14001#discussion_r2394957022
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala:
##########
@@ -108,46 +122,57 @@ class TestPayloadDeprecationFlow extends
SparkClientFunctionalTestHarness {
(11, 1L, "rider-X", "driver-X", 19.10, "i", "11.1", 11, 1, "i"),
(12, 1L, "rider-X", "driver-X", 20.10, "D", "12.1", 12, 1, "d"),
(11, 2L, "rider-Y", "driver-Y", 27.70, "u", "11.1", 11, 1, "u"))
- val firstUpdate = spark.createDataFrame(firstUpdateData).toDF(columns: _*)
- firstUpdate.write.format("hudi").
- option(OPERATION.key(), "upsert").
- option(HoodieCompactionConfig.INLINE_COMPACT.key(), "false").
- option(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), "8").
- option(HoodieTableConfig.ORDERING_FIELDS.key(), originalOrderingFields).
- options(opts).
- mode(SaveMode.Append).
- save(basePath)
+ performUpsert(firstUpdateData, columns, serviceOpts, opts, basePath,
+ tableVersion = Some("8"), orderingFields = Some(originalOrderingFields))
// Validate table version.
- metaClient = HoodieTableMetaClient.reload(metaClient)
+ metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(storageConf())
+ .build()
assertEquals(8, metaClient.getTableConfig.getTableVersion.versionCode())
val firstUpdateInstantTime =
metaClient.getActiveTimeline.getInstants.get(1).requestedTime()
- // 3. Add an update. This is expected to trigger the upgrade
+ // 3. Add mixed ordering test data to validate proper ordering handling
+ // This tests that updates/deletes with lower ordering values are ignored
+ // while higher ordering values are applied
+ val mixedOrderingData = Seq(
+ // Update rider-C with HIGHER ordering - should be APPLIED
+ (11, 3L, "rider-CC", "driver-CC", 35.00, "u", "15.1", 15, 1, "u"),
+ // Update rider-C with LOWER ordering - should be IGNORED (rider-C has
ts=10 originally)
Review Comment:
discussed offline, for now any payload using commit time we dont set the
ordering field value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
