[incubator-hudi] branch master updated: HUDI-147 Compaction Inflight Rollback not deleting Marker directory

[vinoth]

This is an automated email from the ASF dual-hosted git repository.

vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 0651732  HUDI-147 Compaction Inflight Rollback not deleting Marker 
directory
0651732 is described below

commit 065173211e2d884e22d48173f75763c8f6b6c7e3
Author: Balaji Varadarajan <varad...@uber.com>
AuthorDate: Sat Jun 8 01:45:19 2019 -0700

    HUDI-147 Compaction Inflight Rollback not deleting Marker directory
---
 .../uber/hoodie/table/HoodieCopyOnWriteTable.java  | 18 +++++-----------
 .../java/com/uber/hoodie/table/HoodieTable.java    | 24 +++++++++++++++++-----
 2 files changed, 24 insertions(+), 18 deletions(-)

diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java 
b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
index 0735b87..ac81a43 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
@@ -389,21 +389,13 @@ public class HoodieCopyOnWriteTable<T extends 
HoodieRecordPayload> extends Hoodi
    */
   protected void deleteInflightInstant(boolean deleteInstant, 
HoodieActiveTimeline activeTimeline,
       HoodieInstant instantToBeDeleted) {
+    // Remove marker files always on rollback
+    deleteMarkerDir(instantToBeDeleted.getTimestamp());
+
     // Remove the rolled back inflight commits
     if (deleteInstant) {
-      try {
-        //TODO: Cleanup Hoodie 1.0 rollback to simply call 
super.cleanFailedWrites with consistency check disabled
-        // and empty WriteStat list.
-        Path markerDir = new 
Path(metaClient.getMarkerFolderPath(instantToBeDeleted.getTimestamp()));
-        logger.info("Removing marker directory=" + markerDir);
-        if (metaClient.getFs().exists(markerDir)) {
-          metaClient.getFs().delete(markerDir, true);
-        }
-        activeTimeline.deleteInflight(instantToBeDeleted);
-        logger.info("Deleted inflight commit " + instantToBeDeleted);
-      } catch (IOException e) {
-        throw new HoodieIOException(e.getMessage(), e);
-      }
+      activeTimeline.deleteInflight(instantToBeDeleted);
+      logger.info("Deleted inflight commit " + instantToBeDeleted);
     } else {
       logger.warn("Rollback finished without deleting inflight instant file. 
Instant=" + instantToBeDeleted);
     }
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java 
b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
index d818248..8e8ded9 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
@@ -295,6 +295,24 @@ public abstract class HoodieTable<T extends 
HoodieRecordPayload> implements Seri
   }
 
   /**
+   * Delete Marker directory corresponding to an instant
+   * @param instantTs Instant Time
+   */
+  protected void deleteMarkerDir(String instantTs) {
+    try {
+      FileSystem fs = getMetaClient().getFs();
+      Path markerDir = new Path(metaClient.getMarkerFolderPath(instantTs));
+      if (fs.exists(markerDir)) {
+        // For append only case, we do not write to marker dir. Hence, the 
above check
+        logger.info("Removing marker directory=" + markerDir);
+        fs.delete(markerDir, true);
+      }
+    } catch (IOException ioe) {
+      throw new HoodieIOException(ioe.getMessage(), ioe);
+    }
+  }
+
+  /**
    * Reconciles WriteStats and marker files to detect and safely delete 
duplicate data files created because of Spark
    * retries.
    *
@@ -364,11 +382,7 @@ public abstract class HoodieTable<T extends 
HoodieRecordPayload> implements Seri
         }
       }
       // Now delete the marker directory
-      if (fs.exists(markerDir)) {
-        // For append only case, we do not write to marker dir. Hence, the 
above check
-        logger.info("Removing marker directory=" + markerDir);
-        fs.delete(markerDir, true);
-      }
+      deleteMarkerDir(instantTs);
     } catch (IOException ioe) {
       throw new HoodieIOException(ioe.getMessage(), ioe);
     }