[hudi] branch master updated: [HUDI-5038] Increase default num_instants to fetch for incremental source (#6955)

[sivabalan]

This is an automated email from the ASF dual-hosted git repository.

sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 47499225e8 [HUDI-5038] Increase default num_instants to fetch for 
incremental source (#6955)
47499225e8 is described below

commit 47499225e89c5a234459aff758deb3caf8be9c22
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Mon Oct 24 20:09:01 2022 -0700

    [HUDI-5038] Increase default num_instants to fetch for incremental source 
(#6955)
---
 .../main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java  | 2 +-
 .../org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java  | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
index a6b979fcbf..3c162a0522 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java
@@ -52,7 +52,7 @@ public class HoodieIncrSource extends RowSource {
      * {@value #NUM_INSTANTS_PER_FETCH} allows the max number of instants 
whose changes can be incrementally fetched.
      */
     static final String NUM_INSTANTS_PER_FETCH = 
"hoodie.deltastreamer.source.hoodieincr.num_instants";
-    static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 1;
+    static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 5;
 
     /**
      * {@value #HOODIE_SRC_PARTITION_FIELDS} specifies partition fields that 
needs to be added to source table after
diff --git 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
index ec163aec33..8218bcd00e 100644
--- 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
+++ 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
@@ -2101,6 +2101,7 @@ public class TestHoodieDeltaStreamer extends 
HoodieDeltaStreamerTestBase {
     HoodieDeltaStreamer.Config downstreamCfg =
         TestHelpers.makeConfigForHudiIncrSrc(tableBasePath, 
downstreamTableBasePath,
             WriteOperationType.BULK_INSERT, true, null);
+    
downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=1");
     new HoodieDeltaStreamer(downstreamCfg, jsc).sync();
 
     insertInTable(tableBasePath, 9, WriteOperationType.UPSERT);
@@ -2112,6 +2113,8 @@ public class TestHoodieDeltaStreamer extends 
HoodieDeltaStreamerTestBase {
       downstreamCfg.configs = new ArrayList<>();
     }
 
+    // Remove source.hoodieincr.num_instants config
+    downstreamCfg.configs.remove(downstreamCfg.configs.size() - 1);
     
downstreamCfg.configs.add(DataSourceReadOptions.INCREMENTAL_FALLBACK_TO_FULL_TABLE_SCAN_FOR_NON_EXISTING_FILES().key()
 + "=true");
     //Adding this conf to make testing easier :)
     
downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=10");