Repository: carbondata
Updated Branches:
  refs/heads/master 067b833f1 -> ceaddeacb


[CARBONDATA-2183] Fix compaction when segment is delete during compaction and 
remove unnecessary parameters in functions

Problem:
when compaction is started and job is running, and parallelly the segment 
involved in the compaction is deleted using DeleteSegmentByID, then
compaction is success.

Solution:
when compaction is started and job is running, and parallelly the segment 
involved in the compaction is deleted using DeleteSegmentByID, then
compaction should be aborted and failed. and proper error message should thrown 
to user. THis PR also removes the unnecessary parameters in functions.

This closes #1979


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ceaddeac
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ceaddeac
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ceaddeac

Branch: refs/heads/master
Commit: ceaddeacb6264a0ed92bc3c0c6cd874333d1fc51
Parents: 067b833
Author: akashrn5 <akashnilu...@gmail.com>
Authored: Wed Feb 14 15:45:04 2018 +0530
Committer: manishgupta88 <tomanishgupt...@gmail.com>
Committed: Fri Feb 23 12:29:35 2018 +0530

----------------------------------------------------------------------
 .../core/datastore/filesystem/AbstractDFSCarbonFile.java       | 5 -----
 .../carbondata/core/datastore/filesystem/CarbonFile.java       | 3 ---
 .../carbondata/core/datastore/filesystem/LocalCarbonFile.java  | 5 -----
 .../org/apache/carbondata/core/datastore/impl/FileFactory.java | 5 -----
 .../org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala | 6 +++++-
 5 files changed, 5 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceaddeac/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 7b634d2..927cef5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -461,11 +461,6 @@ public abstract  class AbstractDFSCarbonFile implements 
CarbonFile {
   }
 
   @Override
-  public void setPermission(String directoryPath, FsPermission permission, 
String username,
-      String group) throws IOException {
-  }
-
-  @Override
   public CarbonFile[] listFiles() {
     FileStatus[] listStatus = null;
     try {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceaddeac/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
index 5f172fc..80c0510 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
@@ -135,9 +135,6 @@ public interface CarbonFile {
 
   boolean createNewLockFile(String filePath, FileFactory.FileType fileType) 
throws IOException;
 
-  void setPermission(String directoryPath, FsPermission permission, String 
username, String group)
-      throws IOException;
-
   /**
    * Returns locations of the file
    * @return

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceaddeac/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 5df5a81..24022ad 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -419,11 +419,6 @@ public class LocalCarbonFile implements CarbonFile {
     return file.createNewFile();
   }
 
-  @Override
-  public void setPermission(String directoryPath, FsPermission permission, 
String username,
-      String group) throws IOException {
-  }
-
   @Override public CarbonFile[] locationAwareListFiles() throws IOException {
     return listFiles();
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceaddeac/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java 
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index e6fbd04..38ed2b7 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -448,9 +448,4 @@ public final class FileFactory {
     }
   }
 
-  public static void setPermission(String directoryPath, FsPermission 
permission, String username,
-      String group) throws IOException {
-    getCarbonFile(directoryPath).setPermission(directoryPath, permission, 
username, group);
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceaddeac/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
 
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index bfe4e41..e7bdff8 100644
--- 
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -271,7 +271,11 @@ class CarbonTableCompactor(carbonLoadModel: 
CarbonLoadModel,
           false
       }
       operationContext.setProperty("commitComplete", commitComplete)
-      if (!statusFileUpdation && !commitComplete) {
+      // here either of the conditions can be true, when delete segment is 
fired after compaction
+      // has started, statusFileUpdation will be false , but at the same time 
commitComplete can be
+      // true because compaction for all datamaps will be finished at a time 
to the maximum level
+      // possible (level 1, 2 etc). so we need to check for either condition
+      if (!statusFileUpdation || !commitComplete) {
         LOGGER.audit(s"Compaction request failed for table ${ 
carbonLoadModel.getDatabaseName }." +
                      s"${ carbonLoadModel.getTableName }")
         LOGGER.error(s"Compaction request failed for table ${ 
carbonLoadModel.getDatabaseName }." +

Reply via email to