[CARBONDATA-2137] Delete query performance improved

Following is the configuration used :

SPARK_EXECUTOR_MEMORY : 200G
SPARK_DRIVER_MEMORY : 20G
SPARK_EXECUTOR_CORES : 32
SPARK_EXECUTOR_INSTANCEs : 3

Earlier it was taking 20 minute now it is taking approx 5 minute

This closes #1937


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7c05f5f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7c05f5f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7c05f5f1

Branch: refs/heads/carbonstore-rebase
Commit: 7c05f5f18a815fdf23de4a703888f9962071fad7
Parents: 1be27b0
Author: rahulforallp <rahul.ku...@knoldus.in>
Authored: Tue Feb 6 18:41:35 2018 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Fri Feb 9 16:47:37 2018 +0530

----------------------------------------------------------------------
 .../carbondata/core/locks/HdfsFileLock.java     |  2 +-
 .../carbondata/core/locks/LocalFileLock.java    |  2 +-
 .../core/mutate/DeleteDeltaBlockDetails.java    | 27 +++++++++++++++-----
 3 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index 1a46770..cc98b03 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -85,7 +85,7 @@ public class HdfsFileLock extends AbstractCarbonLock {
       return true;
 
     } catch (IOException e) {
-      LOGGER.error(e, e.getMessage());
+      LOGGER.info(e.getMessage());
       return false;
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index c3dfb57..4fee4c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -118,7 +118,7 @@ public class LocalFileLock extends AbstractCarbonLock {
         return false;
       }
     } catch (IOException e) {
-      LOGGER.error(e, e.getMessage());
+      LOGGER.info(e.getMessage());
       return false;
     }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
 
b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index 4275cca..7affe12 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -19,7 +19,10 @@ package org.apache.carbondata.core.mutate;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -31,7 +34,7 @@ public class DeleteDeltaBlockDetails implements Serializable {
 
   private static final long serialVersionUID = 1206104914918495724L;
 
-  private List<DeleteDeltaBlockletDetails> blockletDetails;
+  private Map<String, DeleteDeltaBlockletDetails> blockletDetailsMap;
   private String blockName;
 
   /**
@@ -42,7 +45,7 @@ public class DeleteDeltaBlockDetails implements Serializable {
 
   public DeleteDeltaBlockDetails(String blockName) {
     this.blockName = blockName;
-    blockletDetails = new ArrayList<DeleteDeltaBlockletDetails>();
+    blockletDetailsMap = new TreeMap<>();
   }
 
   public String getBlockName() {
@@ -68,15 +71,25 @@ public class DeleteDeltaBlockDetails implements 
Serializable {
   }
 
   public List<DeleteDeltaBlockletDetails> getBlockletDetails() {
-    return blockletDetails;
+
+    List<DeleteDeltaBlockletDetails> deleteDeltaBlockletDetailsList = new 
ArrayList<>();
+    Iterator<Map.Entry<String, DeleteDeltaBlockletDetails>> iterator =
+        blockletDetailsMap.entrySet().iterator();
+    while (iterator.hasNext()) {
+      deleteDeltaBlockletDetailsList.add(iterator.next().getValue());
+    }
+    return deleteDeltaBlockletDetailsList;
   }
 
   public boolean addBlockletDetails(DeleteDeltaBlockletDetails blocklet) {
-    int index = blockletDetails.indexOf(blocklet);
-    if (blockletDetails.isEmpty() || index == -1) {
-      return blockletDetails.add(blocklet);
+    DeleteDeltaBlockletDetails deleteDeltaBlockletDetails =
+        blockletDetailsMap.get(blocklet.getBlockletKey());
+    if (null == deleteDeltaBlockletDetails) {
+      blockletDetailsMap.put(blocklet.getBlockletKey(), blocklet);
+      return true;
     } else {
-      return 
blockletDetails.get(index).addDeletedRows(blocklet.getDeletedRows());
+      deleteDeltaBlockletDetails.addDeletedRows(blocklet.getDeletedRows());
+      return true;
     }
   }
 

Reply via email to