Repository: carbondata Updated Branches: refs/heads/branch-1.3 aff3b39ef -> 7beef112b
[CARBONDATA-2183] Fix compaction when segment is delete during compaction and remove unnecessary parameters in functions Problem: when compaction is started and job is running, and parallelly the segment involved in the compaction is deleted using DeleteSegmentByID, then compaction is success. Solution: when compaction is started and job is running, and parallelly the segment involved in the compaction is deleted using DeleteSegmentByID, then compaction should be aborted and failed. and proper error message should thrown to user. THis PR also removes the unnecessary parameters in functions. This closes #1979 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7beef112 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7beef112 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7beef112 Branch: refs/heads/branch-1.3 Commit: 7beef112b59c9ccfe14baca87ae841cfe77e4dce Parents: aff3b39 Author: akashrn5 <akashnilu...@gmail.com> Authored: Wed Feb 14 15:45:04 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Fri Feb 23 12:32:07 2018 +0530 ---------------------------------------------------------------------- .../core/datastore/filesystem/AbstractDFSCarbonFile.java | 5 ----- .../carbondata/core/datastore/filesystem/CarbonFile.java | 3 --- .../carbondata/core/datastore/filesystem/LocalCarbonFile.java | 5 ----- .../org/apache/carbondata/core/datastore/impl/FileFactory.java | 5 ----- .../org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala | 6 +++++- 5 files changed, 5 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/7beef112/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java index 7b634d2..927cef5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java @@ -461,11 +461,6 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile { } @Override - public void setPermission(String directoryPath, FsPermission permission, String username, - String group) throws IOException { - } - - @Override public CarbonFile[] listFiles() { FileStatus[] listStatus = null; try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/7beef112/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java index 5f172fc..80c0510 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java @@ -135,9 +135,6 @@ public interface CarbonFile { boolean createNewLockFile(String filePath, FileFactory.FileType fileType) throws IOException; - void setPermission(String directoryPath, FsPermission permission, String username, String group) - throws IOException; - /** * Returns locations of the file * @return http://git-wip-us.apache.org/repos/asf/carbondata/blob/7beef112/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java index 5df5a81..24022ad 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java @@ -419,11 +419,6 @@ public class LocalCarbonFile implements CarbonFile { return file.createNewFile(); } - @Override - public void setPermission(String directoryPath, FsPermission permission, String username, - String group) throws IOException { - } - @Override public CarbonFile[] locationAwareListFiles() throws IOException { return listFiles(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7beef112/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java index e6fbd04..38ed2b7 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java @@ -448,9 +448,4 @@ public final class FileFactory { } } - public static void setPermission(String directoryPath, FsPermission permission, String username, - String group) throws IOException { - getCarbonFile(directoryPath).setPermission(directoryPath, permission, username, group); - } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7beef112/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala index bfe4e41..e7bdff8 100644 --- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala +++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala @@ -271,7 +271,11 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel, false } operationContext.setProperty("commitComplete", commitComplete) - if (!statusFileUpdation && !commitComplete) { + // here either of the conditions can be true, when delete segment is fired after compaction + // has started, statusFileUpdation will be false , but at the same time commitComplete can be + // true because compaction for all datamaps will be finished at a time to the maximum level + // possible (level 1, 2 etc). so we need to check for either condition + if (!statusFileUpdation || !commitComplete) { LOGGER.audit(s"Compaction request failed for table ${ carbonLoadModel.getDatabaseName }." + s"${ carbonLoadModel.getTableName }") LOGGER.error(s"Compaction request failed for table ${ carbonLoadModel.getDatabaseName }." +
