[carbondata] branch master updated: [HOTFIX] Remove Useless filepath truncation During prunning

[jackylk]

This is an automated email from the ASF dual-hosted git repository.

jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 6ebf10b  [HOTFIX] Remove Useless filepath truncation During prunning
6ebf10b is described below

commit 6ebf10b3a2ee996a31d7a830391b83f9bd5c9a7a
Author: haomarch <marchp...@126.com>
AuthorDate: Fri Mar 6 21:08:42 2020 +0800

    [HOTFIX] Remove Useless filepath truncation During prunning
    
    Why is this PR needed?
    During prunning, it will truncate the filepath with the keyword "/Part0". 
But it is useless.
    Specially, for partition table, the filepath is like 
'/tablename/partitionkey=value/segment0/.carbondata', it even have no keywork 
'/Part0'.
    Then for non-partition table, it is also meaningless to do this work.
    If there are a huge amount of blocklets, such as 400 million blocklets are 
going to do prunned, the truncation hampers the query performance heavily.
    
    What changes were proposed in this PR?
    Remove the filepath truncation.
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No
    
    This closes #3660
---
 .../core/indexstore/blockletindex/BlockDataMap.java          | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
index 0ffb7c0..5437093 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
@@ -103,6 +103,10 @@ public class BlockDataMap extends CoarseGrainDataMap
    * partition table and non transactional table
    */
   protected boolean isFilePathStored;
+  /**
+   * flag to be used for partition table
+   */
+  protected boolean isPartitionTable;
 
   @Override
   public void init(DataMapModel dataMapModel) throws IOException {
@@ -125,8 +129,8 @@ public class BlockDataMap extends CoarseGrainDataMap
     // store file path only in case of partition table, non transactional 
table and flat folder
     // structure
     byte[] filePath;
-    boolean isPartitionTable = 
blockletDataMapInfo.getCarbonTable().isHivePartitionTable();
-    if (isPartitionTable || 
!blockletDataMapInfo.getCarbonTable().isTransactionalTable() ||
+    this.isPartitionTable = 
blockletDataMapInfo.getCarbonTable().isHivePartitionTable();
+    if (this.isPartitionTable || 
!blockletDataMapInfo.getCarbonTable().isTransactionalTable() ||
         blockletDataMapInfo.getCarbonTable().isSupportFlatFolder() ||
         // if the segment data is written in tablepath then no need to store 
whole path of file.
         !blockletDataMapInfo.getFilePath().startsWith(
@@ -785,7 +789,9 @@ public class BlockDataMap extends CoarseGrainDataMap
       byte[][] minValue, boolean[] minMaxFlag, String filePath, int 
blockletId) {
     BitSet bitSet = null;
     if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
-      String uniqueBlockPath = 
filePath.substring(filePath.lastIndexOf("/Part") + 1);
+      String uniqueBlockPath = !isPartitionTable ?
+                filePath.substring(filePath.lastIndexOf("/Part") + 1) :
+                filePath;
       // this case will come in case of old store where index file does not 
contain the
       // blocklet information
       if (blockletId != -1) {