smtwilio commented on code in PR #19811:
URL: https://github.com/apache/hudi/pull/19811#discussion_r3908499781
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieMultiTableDeltaStreamer.java:
##########
@@ -243,6 +246,89 @@ public void testMultiTableExecutionWithParquetSource()
throws IOException {
}
}
+ @Test
+ public void testFailFastOnContinuousDefaultsToFalse() {
+ HoodieMultiTableDeltaStreamer.Config cfg = new
HoodieMultiTableDeltaStreamer.Config();
+ assertFalse(cfg.failFastOnContinuousMode);
+ }
+
+ @Test
+ public void testMultiTableContinuousModeSyncsAllTablesInParallel() throws
IOException {
+ HoodieMultiTableDeltaStreamer streamer =
setupContinuousStreamer("parquetContinuous", false);
+ List<TableExecutionContext> contexts =
streamer.getTableExecutionContexts();
+ // Let each table stop on its own once it has ingested its data, so the
test does not run forever.
+ setTerminationStrategy(contexts);
+
+ streamer.sync();
+
+ assertEquals(2, streamer.getSuccessTables().size());
+ assertTrue(streamer.getFailedTables().isEmpty());
+ assertRecordCount(10, contexts.get(0).getConfig().targetBasePath,
sqlContext);
+ assertRecordCount(5, contexts.get(1).getConfig().targetBasePath,
sqlContext);
+ }
+
+ @Test
+ public void testFailFastOnContinuousThrowsWhenATableFails() throws
IOException, InterruptedException {
+ HoodieMultiTableDeltaStreamer streamer =
setupContinuousStreamer("parquetFailFast", true);
+ List<TableExecutionContext> contexts =
streamer.getTableExecutionContexts();
+ // Table 1 blocks after the barrier, so only fail fast interrupting it can
end its sync. Table 2 fails after the
+ // barrier. This proves fail fast tears down a sibling that is still
actively running, not one that stopped itself.
+
contexts.get(0).getProperties().setProperty(ContinuousTestSource.BLOCK_UNTIL_INTERRUPTED,
"true");
+
contexts.get(1).getProperties().setProperty(ContinuousTestSource.FAIL_AFTER_BARRIER,
"true");
+
+ assertThrows(HoodieException.class, streamer::sync);
+ assertFalse(streamer.getFailedTables().isEmpty());
+ // sync() returns only after the blocked sibling was interrupted, so the
latch must already be counted down.
Review Comment:
Right, renaming it and dropping the timeout param.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
