lw309637554 commented on a change in pull request #2379:
URL: https://github.com/apache/hudi/pull/2379#discussion_r554430490
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
##########
@@ -672,14 +681,72 @@ public void testInlineClustering() throws Exception {
cfg.configs.add(String.format("%s=false",
HoodieCompactionConfig.AUTO_CLEAN_PROP));
cfg.configs.add(String.format("%s=%s",
HoodieClusteringConfig.INLINE_CLUSTERING_PROP, "true"));
cfg.configs.add(String.format("%s=%s",
HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT_PROP, "2"));
-
- deltaStreamerTestRunner(cfg, (r) -> {
+ HoodieDeltaStreamer ds = new HoodieDeltaStreamer(cfg, jsc);
+ deltaStreamerTestRunner(ds, cfg, (r) -> {
HoodieTableMetaClient metaClient = new
HoodieTableMetaClient(this.dfs.getConf(), tableBasePath, true);
int pendingReplaceSize =
metaClient.getActiveTimeline().filterPendingReplaceTimeline().getInstants().toArray().length;
int completeReplaceSize =
metaClient.getActiveTimeline().getCompletedReplaceTimeline().getInstants().toArray().length;
LOG.info("PendingReplaceSize=" + pendingReplaceSize +
",completeReplaceSize = " + completeReplaceSize);
return completeReplaceSize > 0;
});
+ HoodieTableMetaClient metaClient = new
HoodieTableMetaClient(this.dfs.getConf(), tableBasePath, true);
+ assertEquals(1,
metaClient.getActiveTimeline().getCompletedReplaceTimeline().getInstants().toArray().length);
+ }
+
+ private HoodieClusteringJob.Config buildHoodieClusteringUtilConfig(String
basePath,
+ String
clusteringInstantTime, boolean runSchedule) {
+ HoodieClusteringJob.Config config = new HoodieClusteringJob.Config();
+ config.basePath = basePath;
+ config.clusteringInstantTime = clusteringInstantTime;
+ config.runSchedule = runSchedule;
+ config.propsFilePath = dfsBasePath + "/clusteringjob.properties";
+ return config;
+ }
+
+ @Test
+ public void testHoodieAsyncClusteringJob() throws Exception {
Review comment:
Yes, it just test if clustering works with deltastreamer ,and find two
issues. After natively clustering support in
https://issues.apache.org/jira/browse/HUDI-1482 and
https://issues.apache.org/jira/browse/HUDI-1483. I will refactored it in
https://issues.apache.org/jira/browse/HUDI-1516
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
