This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7567eaef2c0 [HUDI-4123] Enchancing deltastreamer sql source tests
(#6781)
7567eaef2c0 is described below
commit 7567eaef2c0de77696eb08b52d17ea057fefcb7a
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Sun Jun 23 16:06:26 2024 -0700
[HUDI-4123] Enchancing deltastreamer sql source tests (#6781)
---
.../hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index f067eb3f9b0..ffc34e51c8c 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -2415,11 +2415,18 @@ public class TestHoodieDeltaStreamer extends
HoodieDeltaStreamerTestBase {
String tableBasePath = basePath + "/test_sql_source_table" + testNum++;
HoodieDeltaStreamer deltaStreamer =
new HoodieDeltaStreamer(TestHelpers.makeConfig(
- tableBasePath, WriteOperationType.INSERT,
SqlSource.class.getName(),
+ tableBasePath, WriteOperationType.UPSERT,
SqlSource.class.getName(),
Collections.emptyList(), PROPS_FILENAME_TEST_SQL_SOURCE, false,
- false, 1000, false, null, null, "timestamp", null, true), jsc);
+ false, 2000, false, null, null, "timestamp", null, true), jsc);
deltaStreamer.sync();
assertRecordCount(SQL_SOURCE_NUM_RECORDS, tableBasePath, sqlContext);
+ // Data generation
+ String sourceRoot = basePath + "sqlSourceFiles";
+ HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator();
+ generateSqlSourceTestTable(sourceRoot, "2", "1000",
SQL_SOURCE_NUM_RECORDS, dataGenerator);
+
+ deltaStreamer.sync();
+ assertRecordCount(SQL_SOURCE_NUM_RECORDS * 2, tableBasePath, sqlContext);
}
@Test
