This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 6ebf10b [HOTFIX] Remove Useless filepath truncation During prunning 6ebf10b is described below commit 6ebf10b3a2ee996a31d7a830391b83f9bd5c9a7a Author: haomarch <marchp...@126.com> AuthorDate: Fri Mar 6 21:08:42 2020 +0800 [HOTFIX] Remove Useless filepath truncation During prunning Why is this PR needed? During prunning, it will truncate the filepath with the keyword "/Part0". But it is useless. Specially, for partition table, the filepath is like '/tablename/partitionkey=value/segment0/.carbondata', it even have no keywork '/Part0'. Then for non-partition table, it is also meaningless to do this work. If there are a huge amount of blocklets, such as 400 million blocklets are going to do prunned, the truncation hampers the query performance heavily. What changes were proposed in this PR? Remove the filepath truncation. Does this PR introduce any user interface change? No Is any new testcase added? No This closes #3660 --- .../core/indexstore/blockletindex/BlockDataMap.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java index 0ffb7c0..5437093 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java @@ -103,6 +103,10 @@ public class BlockDataMap extends CoarseGrainDataMap * partition table and non transactional table */ protected boolean isFilePathStored; + /** + * flag to be used for partition table + */ + protected boolean isPartitionTable; @Override public void init(DataMapModel dataMapModel) throws IOException { @@ -125,8 +129,8 @@ public class BlockDataMap extends CoarseGrainDataMap // store file path only in case of partition table, non transactional table and flat folder // structure byte[] filePath; - boolean isPartitionTable = blockletDataMapInfo.getCarbonTable().isHivePartitionTable(); - if (isPartitionTable || !blockletDataMapInfo.getCarbonTable().isTransactionalTable() || + this.isPartitionTable = blockletDataMapInfo.getCarbonTable().isHivePartitionTable(); + if (this.isPartitionTable || !blockletDataMapInfo.getCarbonTable().isTransactionalTable() || blockletDataMapInfo.getCarbonTable().isSupportFlatFolder() || // if the segment data is written in tablepath then no need to store whole path of file. !blockletDataMapInfo.getFilePath().startsWith( @@ -785,7 +789,9 @@ public class BlockDataMap extends CoarseGrainDataMap byte[][] minValue, boolean[] minMaxFlag, String filePath, int blockletId) { BitSet bitSet = null; if (filterExecuter instanceof ImplicitColumnFilterExecutor) { - String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1); + String uniqueBlockPath = !isPartitionTable ? + filePath.substring(filePath.lastIndexOf("/Part") + 1) : + filePath; // this case will come in case of old store where index file does not contain the // blocklet information if (blockletId != -1) {