[carbondata] branch master updated: [CARBONDATA-3488] Check the file size after move local file to carbon path

[jackylk]

This is an automated email from the ASF dual-hosted git repository.

jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new d5db3f4  [CARBONDATA-3488] Check the file size after move local file 
to carbon path
d5db3f4 is described below

commit d5db3f48708d766452f59bf7bc8509bbcfdd3d1e
Author: Zhang Zhichao <441586...@qq.com>
AuthorDate: Thu Aug 8 14:02:41 2019 +0800

    [CARBONDATA-3488] Check the file size after move local file to carbon path
    
    Problem:
    the row num saved in carbonindex file is non zero but the file size of 
relevant carbondata file is 0.
    
    Solution:
    In CarbonUtil.copyCarbonDataFileToCarbonStorePath, check the file size of 
carbon file whether is the same as the size fo local file after move local file 
to carbon path.
    
    This closes #3350
---
 .../apache/carbondata/core/util/CarbonUtil.java    | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 7eeff90..a86690c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2739,14 +2739,30 @@ public final class CarbonUtil {
     LOGGER.info(String.format("Copying %s to %s, operation id %d", 
localFilePath,
         carbonDataDirectoryPath, copyStartTime));
     try {
-      CarbonFile localCarbonFile =
-          FileFactory.getCarbonFile(localFilePath, 
FileFactory.getFileType(localFilePath));
+      CarbonFile localCarbonFile = FileFactory.getCarbonFile(localFilePath);
+      // the size of local carbon file must be greater than 0
+      if (localCarbonFile.getSize() == 0L) {
+        LOGGER.error("The size of local carbon file: " + localFilePath + " is 
0.");
+        throw new CarbonDataWriterException("The size of local carbon file is 
0.");
+      }
       String carbonFilePath = carbonDataDirectoryPath + localFilePath
           .substring(localFilePath.lastIndexOf(File.separator));
       copyLocalFileToCarbonStore(carbonFilePath, localFilePath,
           CarbonCommonConstants.BYTEBUFFER_SIZE,
           getMaxOfBlockAndFileSize(fileSizeInBytes, 
localCarbonFile.getSize()));
-    } catch (IOException e) {
+      CarbonFile targetCarbonFile = FileFactory.getCarbonFile(carbonFilePath);
+      // the size of carbon file must be greater than 0
+      // and the same as the size of local carbon file
+      if (targetCarbonFile.getSize() == 0L ||
+          (targetCarbonFile.getSize() != localCarbonFile.getSize())) {
+        LOGGER.error("The size of carbon file: " + carbonFilePath + " is 0 "
+            + "or is not the same as the size of local carbon file: ("
+            + "carbon file size=" + targetCarbonFile.getSize()
+            + ", local carbon file size=" + localCarbonFile.getSize() + ")");
+        throw new CarbonDataWriterException("The size of carbon file is 0 "
+            + "or is not the same as the size of local carbon file.");
+      }
+    } catch (Exception e) {
       throw new CarbonDataWriterException(
           "Problem while copying file from local store to carbon store", e);
     }