[hudi] branch master updated: HUDI-942 Increase default value number of delta commits for inline compaction (#1664)

[vbalaji]

This is an automated email from the ASF dual-hosted git repository.

vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new df2e0c7  HUDI-942 Increase default value number of delta commits for 
inline compaction (#1664)
df2e0c7 is described below

commit df2e0c760e7df0bd1b200867b3f0d2ca3a3f1fce
Author: sathyaprakashg <sathyapraka...@gmail.com>
AuthorDate: Wed Jun 10 16:16:44 2020 -0700

    HUDI-942 Increase default value number of delta commits for inline 
compaction (#1664)
    
    Co-authored-by: Sathyaprakash Govindasamy <sathyapraka...@zillowgroup.com>
---
 docker/demo/compaction.commands                                        | 2 +-
 .../src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java   | 2 +-
 .../src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java | 2 ++
 hudi-spark/src/test/java/HoodieJavaApp.java                            | 3 +++
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docker/demo/compaction.commands b/docker/demo/compaction.commands
index 9853a35..9bb8eb8 100644
--- a/docker/demo/compaction.commands
+++ b/docker/demo/compaction.commands
@@ -17,6 +17,6 @@
 
 connect --path /user/hive/warehouse/stock_ticks_mor
 compactions show all
-compaction schedule
+compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1
 compaction run --parallelism 2 --sparkMemory 1G  --schemaFilePath 
/var/demo/config/schema.avsc --retry 1 
 
diff --git 
a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java 
b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
index f89fc06..4ec0485 100644
--- 
a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
+++ 
b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
@@ -103,7 +103,7 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
   private static final String DEFAULT_AUTO_CLEAN = "true";
   private static final String DEFAULT_INLINE_COMPACT = "false";
   private static final String DEFAULT_INCREMENTAL_CLEANER = "true";
-  private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "1";
+  private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "5";
   private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = "3";
   private static final String DEFAULT_CLEANER_COMMITS_RETAINED = "10";
   private static final String DEFAULT_MAX_COMMITS_TO_KEEP = "30";
diff --git 
a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
 
b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
index 25e97c9..8e56b31 100644
--- 
a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
+++ 
b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.TableSchemaResolver;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieCompactionConfig;
 import org.apache.hudi.config.HoodieIndexConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieInsertException;
@@ -467,6 +468,7 @@ public class TestTableSchemaEvolution extends 
HoodieClientTestBase {
   private HoodieWriteConfig getWriteConfig(String schema) {
     return getConfigBuilder(schema)
         
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.INMEMORY).build())
+        
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(1).build())
         .withAvroSchemaValidate(true)
         .build();
   }
diff --git a/hudi-spark/src/test/java/HoodieJavaApp.java 
b/hudi-spark/src/test/java/HoodieJavaApp.java
index a179c42..93493a6 100644
--- a/hudi-spark/src/test/java/HoodieJavaApp.java
+++ b/hudi-spark/src/test/java/HoodieJavaApp.java
@@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions;
 import org.apache.hudi.HoodieDataSourceHelpers;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.config.HoodieCompactionConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.hive.MultiPartKeysValueExtractor;
 import org.apache.hudi.hive.NonPartitionedExtractor;
@@ -173,6 +174,7 @@ public class HoodieJavaApp {
         .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(),
             nonPartitionedTable ? 
NonpartitionedKeyGenerator.class.getCanonicalName()
                 : SimpleKeyGenerator.class.getCanonicalName()) // Add Key 
Extractor
+        .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP, 
"1")
         .option(HoodieWriteConfig.TABLE_NAME, tableName).mode(SaveMode.Append);
 
     updateHiveSyncConfig(writer);
@@ -197,6 +199,7 @@ public class HoodieJavaApp {
         .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(),
             nonPartitionedTable ? 
NonpartitionedKeyGenerator.class.getCanonicalName()
                 : SimpleKeyGenerator.class.getCanonicalName()) // Add Key 
Extractor
+        .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP, 
"1")
         .option(HoodieWriteConfig.TABLE_NAME, tableName).mode(SaveMode.Append);
 
     updateHiveSyncConfig(writer);