This is an automated email from the ASF dual-hosted git repository.

vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 372fbc4  Fixes HUDI-9 : Check precondition minInstantsToKeep > 
cleanerCommitsRetained
372fbc4 is described below

commit 372fbc4733824d6f61db7f4b7d5e289dc902e634
Author: Vinoth Chandar <[email protected]>
AuthorDate: Tue Mar 26 18:52:33 2019 -0700

    Fixes HUDI-9 : Check precondition minInstantsToKeep > cleanerCommitsRetained
    
     - Added a precondition check, otherwise incr pull could miss commits
     - Lowered default cleaner retention to 10, to enable simpler understanding 
for newbies
     - Bumped down min/max instants to retain as well
---
 .../uber/hoodie/config/HoodieCompactionConfig.java | 33 ++++++++++++++--------
 .../com/uber/hoodie/config/HoodieWriteConfig.java  |  4 +--
 .../uber/hoodie/config/HoodieWriteConfigTest.java  |  5 ++--
 .../uber/hoodie/io/TestHoodieCommitArchiveLog.java | 10 +++----
 4 files changed, 31 insertions(+), 21 deletions(-)

diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
index b170995..2cde401 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
@@ -43,8 +43,8 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
   public static final String CLEANER_FILE_VERSIONS_RETAINED_PROP =
       "hoodie.cleaner.fileversions" + ".retained";
   public static final String CLEANER_COMMITS_RETAINED_PROP = 
"hoodie.cleaner.commits.retained";
-  public static final String MAX_COMMITS_TO_KEEP = "hoodie.keep.max.commits";
-  public static final String MIN_COMMITS_TO_KEEP = "hoodie.keep.min.commits";
+  public static final String MAX_COMMITS_TO_KEEP_PROP = 
"hoodie.keep.max.commits";
+  public static final String MIN_COMMITS_TO_KEEP_PROP = 
"hoodie.keep.min.commits";
   // Upsert uses this file size to compact new data onto existing files..
   public static final String PARQUET_SMALL_FILE_LIMIT_BYTES = 
"hoodie.parquet.small.file.limit";
   // Turned off by default
@@ -101,9 +101,9 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
   private static final String DEFAULT_INLINE_COMPACT = "false";
   private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "1";
   private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = "3";
-  private static final String DEFAULT_CLEANER_COMMITS_RETAINED = "24";
-  private static final String DEFAULT_MAX_COMMITS_TO_KEEP = 
String.valueOf(128);
-  private static final String DEFAULT_MIN_COMMITS_TO_KEEP = String.valueOf(96);
+  private static final String DEFAULT_CLEANER_COMMITS_RETAINED = "10";
+  private static final String DEFAULT_MAX_COMMITS_TO_KEEP = "30";
+  private static final String DEFAULT_MIN_COMMITS_TO_KEEP = "20";
   public static final String TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP = 
"hoodie.compaction.daybased.target"
       + ".partitions";
   // 500GB of target IO per compaction (both read and write)
@@ -168,8 +168,8 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
     }
 
     public Builder archiveCommitsWith(int minToKeep, int maxToKeep) {
-      props.setProperty(MIN_COMMITS_TO_KEEP, String.valueOf(minToKeep));
-      props.setProperty(MAX_COMMITS_TO_KEEP, String.valueOf(maxToKeep));
+      props.setProperty(MIN_COMMITS_TO_KEEP_PROP, String.valueOf(minToKeep));
+      props.setProperty(MAX_COMMITS_TO_KEEP_PROP, String.valueOf(maxToKeep));
       return this;
     }
 
@@ -254,9 +254,9 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
           CLEANER_FILE_VERSIONS_RETAINED_PROP, 
DEFAULT_CLEANER_FILE_VERSIONS_RETAINED);
       setDefaultOnCondition(props, 
!props.containsKey(CLEANER_COMMITS_RETAINED_PROP),
           CLEANER_COMMITS_RETAINED_PROP, DEFAULT_CLEANER_COMMITS_RETAINED);
-      setDefaultOnCondition(props, !props.containsKey(MAX_COMMITS_TO_KEEP), 
MAX_COMMITS_TO_KEEP,
+      setDefaultOnCondition(props, 
!props.containsKey(MAX_COMMITS_TO_KEEP_PROP), MAX_COMMITS_TO_KEEP_PROP,
           DEFAULT_MAX_COMMITS_TO_KEEP);
-      setDefaultOnCondition(props, !props.containsKey(MIN_COMMITS_TO_KEEP), 
MIN_COMMITS_TO_KEEP,
+      setDefaultOnCondition(props, 
!props.containsKey(MIN_COMMITS_TO_KEEP_PROP), MIN_COMMITS_TO_KEEP_PROP,
           DEFAULT_MIN_COMMITS_TO_KEEP);
       setDefaultOnCondition(props, 
!props.containsKey(PARQUET_SMALL_FILE_LIMIT_BYTES),
           PARQUET_SMALL_FILE_LIMIT_BYTES, 
DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES);
@@ -283,10 +283,19 @@ public class HoodieCompactionConfig extends 
DefaultHoodieConfig {
           TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP, 
DEFAULT_TARGET_PARTITIONS_PER_DAYBASED_COMPACTION);
 
       HoodieCleaningPolicy.valueOf(props.getProperty(CLEANER_POLICY_PROP));
-      
Preconditions.checkArgument(Integer.parseInt(props.getProperty(MAX_COMMITS_TO_KEEP))
 > Integer
-          .parseInt(props.getProperty(MIN_COMMITS_TO_KEEP)));
+
+      // Ensure minInstantsToKeep > cleanerCommitsRetained, otherwise we will 
archive some
+      // commit instant on timeline, that still has not been cleaned. Could 
miss some data via incr pull
+      int minInstantsToKeep = 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP));
+      int maxInstantsToKeep = 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP));
+      int cleanerCommitsRetained = Integer
+          
.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP));
+      Preconditions.checkArgument(maxInstantsToKeep > minInstantsToKeep);
+      Preconditions.checkArgument(minInstantsToKeep > cleanerCommitsRetained,
+          String.format("Increase %s=%d to be greater than %s=%d. Otherwise, 
there is risk of incremental pull "
+                  + "missing data from few instants.", 
HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP,
+              minInstantsToKeep, 
HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, cleanerCommitsRetained));
       return config;
     }
-
   }
 }
diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java 
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
index d1a5e46..67b38d5 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
@@ -176,11 +176,11 @@ public class HoodieWriteConfig extends 
DefaultHoodieConfig {
   }
 
   public int getMaxCommitsToKeep() {
-    return 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP));
+    return 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP));
   }
 
   public int getMinCommitsToKeep() {
-    return 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP));
+    return 
Integer.parseInt(props.getProperty(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP));
   }
 
   public int getParquetSmallFileLimit() {
diff --git 
a/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java 
b/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java
index 3e6fc8e..0835e67 100644
--- 
a/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java
+++ 
b/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java
@@ -34,8 +34,9 @@ public class HoodieWriteConfigTest {
   public void testPropertyLoading() throws IOException {
     Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp");
     Map<String, String> params = Maps.newHashMap();
-    params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP, "5");
-    params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP, "2");
+    params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, "1");
+    params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP, "5");
+    params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP, "2");
     ByteArrayOutputStream outStream = saveParamsIntoOutputStream(params);
     ByteArrayInputStream inputStream = new 
ByteArrayInputStream(outStream.toByteArray());
     try {
diff --git 
a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java
 
b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java
index 76c3641..2d6f957 100644
--- 
a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java
+++ 
b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java
@@ -134,7 +134,7 @@ public class TestHoodieCommitArchiveLog {
     HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(basePath)
         
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
         .withCompactionConfig(
-            HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 
4).build())
+            
HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 
4).build())
         .forTable("test-trip-table").build();
     HoodieTestUtils.init(hadoopConf, basePath);
     // Requested Compaction
@@ -279,7 +279,7 @@ public class TestHoodieCommitArchiveLog {
     HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(basePath)
         
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
         .forTable("test-trip-table").withCompactionConfig(
-            HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 
5).build()).build();
+            
HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 
5).build()).build();
     HoodieTableMetaClient metaClient = new 
HoodieTableMetaClient(dfs.getConf(), basePath);
     HoodieCommitArchiveLog archiveLog = new HoodieCommitArchiveLog(cfg, 
metaClient);
     // Requested Compaction
@@ -346,7 +346,7 @@ public class TestHoodieCommitArchiveLog {
     HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(basePath)
         
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
         .forTable("test-trip-table").withCompactionConfig(
-            HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 
5).build()).build();
+            
HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 
5).build()).build();
     HoodieTableMetaClient metaClient = new 
HoodieTableMetaClient(dfs.getConf(), basePath);
     HoodieCommitArchiveLog archiveLog = new HoodieCommitArchiveLog(cfg, 
metaClient);
     HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf());
@@ -372,7 +372,7 @@ public class TestHoodieCommitArchiveLog {
     HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(basePath)
         
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
         .forTable("test-trip-table").withCompactionConfig(
-            HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 
5).build()).build();
+            
HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 
5).build()).build();
     HoodieTableMetaClient metaClient = new 
HoodieTableMetaClient(dfs.getConf(), basePath);
     HoodieCommitArchiveLog archiveLog = new HoodieCommitArchiveLog(cfg, 
metaClient);
     HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf());
@@ -404,7 +404,7 @@ public class TestHoodieCommitArchiveLog {
     HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(basePath)
         
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
         .forTable("test-trip-table").withCompactionConfig(
-            HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 
5).build()).build();
+            
HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 
5).build()).build();
     HoodieTableMetaClient metaClient = new 
HoodieTableMetaClient(dfs.getConf(), basePath);
     HoodieCommitArchiveLog archiveLog = new HoodieCommitArchiveLog(cfg, 
metaClient);
     HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf());

Reply via email to