pratyakshsharma commented on a change in pull request #2768:
URL: https://github.com/apache/hudi/pull/2768#discussion_r774117926
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java
##########
@@ -41,4 +67,84 @@ public void testInitHiveIncrementalPuller() {
}
+ private HiveIncrementalPuller.Config getHivePullerConfig(String
incrementalSql) throws IOException {
+ config.hiveJDBCUrl = hiveSyncConfig.jdbcUrl;
+ config.hiveUsername = hiveSyncConfig.hiveUser;
+ config.hivePassword = hiveSyncConfig.hivePass;
+ config.hoodieTmpDir =
Files.createTempDirectory("hivePullerTest").toUri().toString();
+ config.sourceDb = hiveSyncConfig.databaseName;
+ config.sourceTable = hiveSyncConfig.tableName;
+ config.targetDb = "tgtdb";
+ config.targetTable = "test2";
+ config.tmpDb = "tmp_db";
+ config.fromCommitTime = "100";
+ createIncrementalSqlFile(incrementalSql, config);
+ return config;
+ }
+
+ private void createIncrementalSqlFile(String text,
HiveIncrementalPuller.Config cfg) throws IOException {
+ java.nio.file.Path path = Paths.get(cfg.hoodieTmpDir +
"/incremental_pull.txt");
+ Files.createDirectories(path.getParent());
+ Files.createFile(path);
+ try (FileWriter fr = new FileWriter(new File(path.toUri()))) {
+ fr.write(text);
+ } catch (Exception e) {
+ // no-op
+ }
+ cfg.incrementalSQLFile = path.toString();
+ }
+
+ private void createSourceTable() throws IOException, URISyntaxException {
+ String instantTime = "101";
+ HiveTestUtil.createCOWTable(instantTime, 5, true);
+ hiveSyncConfig.syncMode = "jdbc";
+ HiveTestUtil.hiveSyncConfig.batchSyncNum = 3;
+ HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig,
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private void createTargetTable() throws IOException, URISyntaxException {
+ String instantTime = "100";
+ targetBasePath = Files.createTempDirectory("hivesynctest1" +
Instant.now().toEpochMilli()).toUri().toString();
+ HiveTestUtil.createCOWTable(instantTime, 5, true,
+ targetBasePath, "tgtdb", "test2");
+ HiveSyncTool tool = new
HiveSyncTool(getTargetHiveSyncConfig(targetBasePath),
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private HiveSyncConfig getTargetHiveSyncConfig(String basePath) {
+ HiveSyncConfig config = HiveSyncConfig.copy(hiveSyncConfig);
+ config.databaseName = "tgtdb";
+ config.tableName = "test2";
+ config.basePath = basePath;
+ config.batchSyncNum = 3;
+ config.syncMode = "jdbc";
+ return config;
+ }
+
+ private void createTables() throws IOException, URISyntaxException {
+ createSourceTable();
+ createTargetTable();
+ }
+
+ @Test
+ public void testPullerWithoutIncrementalClause() throws IOException,
URISyntaxException {
Review comment:
I was expecting this comment :)
I am actually facing some error with the happy flow test case. Once I am
able to fix it, will add it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
