nsivabalan commented on code in PR #6753:
URL: https://github.com/apache/hudi/pull/6753#discussion_r980554115
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java:
##########
@@ -664,6 +655,43 @@ public void testBulkInsertsAndUpsertsWithBootstrap()
throws Exception {
assertTrue(fieldNames.containsAll(expectedFieldNames));
}
+ @Test
+ public void testModifiedTableConfigs() throws Exception {
+ String tableBasePath = dfsBasePath + "/test_table";
+
+ // Initial bulk insert
+ HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath,
WriteOperationType.BULK_INSERT);
+ testRecordCount(cfg,1000,tableBasePath,"00000",1);
+
+ // No new data => no commits.
+ cfg.sourceLimit = 0;
+ testRecordCount(cfg,1000,tableBasePath,"00000",1);
+
+ // add disallowed config update to recordkey field. An exception should be
thrown
+ cfg.sourceLimit = 2000;
+ cfg.operation = WriteOperationType.UPSERT;
+ cfg.configs.add(HoodieTableConfig.RECORDKEY_FIELDS.key() +
"=differentval");
+ assertThrows(HoodieException.class, () ->
testRecordCount(cfg,1000,tableBasePath,"00000",1));
+ List<Row> counts = TestHelpers.countsPerCommit(tableBasePath, sqlContext);
+ assertEquals(1000, counts.stream().mapToLong(entry ->
entry.getLong(1)).sum());
+
+
+ //perform the upsert and now with the original config, the commit should
go through
+ HoodieDeltaStreamer.Config newCfg = TestHelpers.makeConfig(tableBasePath,
WriteOperationType.BULK_INSERT);
+ newCfg.sourceLimit = 2000;
+ newCfg.operation = WriteOperationType.UPSERT;
+ testRecordCount(newCfg,1950,tableBasePath,"00001",2);
+ List<Row> counts2 = TestHelpers.countsPerCommit(tableBasePath, sqlContext);
+ assertEquals(1950, counts2.stream().mapToLong(entry ->
entry.getLong(1)).sum());
+ }
+
+ private void testRecordCount(HoodieDeltaStreamer.Config cfg, Integer
expected, String tableBasePath, String metadata, Integer totalCommits) throws
Exception {
Review Comment:
a good name would be "syncAndAssertRecordCount"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
