rahil-c commented on code in PR #14001:
URL: https://github.com/apache/hudi/pull/14001#discussion_r2386362827
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala:
##########
@@ -206,14 +319,49 @@ class TestPayloadDeprecationFlow extends
SparkClientFunctionalTestHarness {
expectedTimeTravelDf.except(timeTravelDf).isEmpty
&& timeTravelDf.except(expectedTimeTravelDf).isEmpty)
}
+
+ // 8. Downgrade from v9 to v8
+ val writeConfig = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withSchema(spark.read.format("hudi").load(basePath).schema.json)
+ .build()
+
+ new UpgradeDowngrade(metaClient, writeConfig, context,
SparkUpgradeDowngradeHelper.getInstance)
+ .run(HoodieTableVersion.EIGHT, null)
+
+ // Reload metaClient to get updated table config
+ metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(storageConf())
+ .build()
+
+ // Validate table version is 8
+ assertEquals(8, metaClient.getTableConfig.getTableVersion.versionCode())
+
+ // Validate downgrade configs
+ val downgradedTableConfig = metaClient.getTableConfig
+ expectedDowngradeConfigs.foreach { case (key, expectedValue) =>
+ if (expectedValue != null) {
+ assertEquals(expectedValue, downgradedTableConfig.getString(key),
s"Config $key should be $expectedValue after downgrade")
+ } else {
+ assertFalse(downgradedTableConfig.contains(key), s"Config $key should
not be present after downgrade")
+ }
+ }
+
+ // Validate data consistency after downgrade
+ val downgradeDf = spark.read.format("hudi").load(basePath)
+ val downgradeFinalDf = downgradeDf.select("ts", "_event_lsn", "rider",
"driver", "fare", "Op", "_event_seq",
DebeziumConstants.FLATTENED_FILE_COL_NAME,
DebeziumConstants.FLATTENED_POS_COL_NAME,
DebeziumConstants.FLATTENED_OP_COL_NAME).sort("_event_lsn")
+ assertTrue(expectedDf.except(downgradeFinalDf).isEmpty &&
downgradeFinalDf.except(expectedDf).isEmpty,
+ "Data should remain consistent after downgrade")
Review Comment:
Let me try doing this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
