[incubator-hudi] branch master updated: HUDI-168 Ensure getFileStatus calls for files getting written is done after close() is called (#788)

[vinoth]

This is an automated email from the ASF dual-hosted git repository.

vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 3d408ee  HUDI-168 Ensure getFileStatus calls for files getting written 
is done after close() is called (#788)
3d408ee is described below

commit 3d408ee96b7cc8eb15756276dbab031eb549e6e2
Author: Balaji Varadarajan <varad...@uber.com>
AuthorDate: Tue Jul 16 17:33:34 2019 -0700

    HUDI-168 Ensure getFileStatus calls for files getting written is done after 
close() is called (#788)
---
 .../java/com/uber/hoodie/table/HoodieMergeOnReadTable.java  | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java 
b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java
index af45052..8542999 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java
@@ -505,20 +505,18 @@ public class HoodieMergeOnReadTable<T extends 
HoodieRecordPayload> extends
         }).forEach(wStat -> {
           HoodieLogFormat.Writer writer = null;
           String baseCommitTime = 
fileIdToBaseCommitTimeForLogMap.get(wStat.getFileId());
+          boolean success = false;
           try {
             writer = HoodieLogFormat.newWriterBuilder().onParentPath(
                 FSUtils.getPartitionPath(this.getMetaClient().getBasePath(), 
partitionPath))
                 .withFileId(wStat.getFileId()).overBaseCommit(baseCommitTime)
                 .withFs(this.metaClient.getFs())
                 .withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
-            Long numRollbackBlocks = 0L;
             // generate metadata
             Map<HeaderMetadataType, String> header = generateHeader(commit);
             // if update belongs to an existing log file
             writer = writer.appendBlock(new HoodieCommandBlock(header));
-            numRollbackBlocks++;
-            filesToNumBlocksRollback.put(this.getMetaClient().getFs()
-                .getFileStatus(writer.getLogFile().getPath()), 
numRollbackBlocks);
+            success = true;
           } catch (IOException | InterruptedException io) {
             throw new HoodieRollbackException(
                 "Failed to rollback for commit " + commit, io);
@@ -527,6 +525,13 @@ public class HoodieMergeOnReadTable<T extends 
HoodieRecordPayload> extends
               if (writer != null) {
                 writer.close();
               }
+              if (success) {
+                // This step is intentionally done after writer is closed. 
Guarantees that
+                // getFileStatus would reflect correct stats and 
FileNotFoundException is not thrown in
+                // cloud-storage : HUDI-168
+                filesToNumBlocksRollback.put(this.getMetaClient().getFs()
+                    .getFileStatus(writer.getLogFile().getPath()), 1L);
+              }
             } catch (IOException io) {
               throw new UncheckedIOException(io);
             }