Repository: carbondata Updated Branches: refs/heads/master 9aa3a8c22 -> 16ed99a11
[HOTFIX] Implementing getMemorySize in BlockletDataMapIndexWrapper This closes #2330 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/16ed99a1 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/16ed99a1 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/16ed99a1 Branch: refs/heads/master Commit: 16ed99a119f95b70a01c95120b20e632405d0a54 Parents: 9aa3a8c Author: dhatchayani <[email protected]> Authored: Tue May 22 12:33:24 2018 +0530 Committer: ravipesala <[email protected]> Committed: Tue May 22 22:03:31 2018 +0530 ---------------------------------------------------------------------- .../core/indexstore/BlockletDataMapIndexStore.java | 9 ++------- .../core/indexstore/BlockletDataMapIndexWrapper.java | 10 +++++++++- 2 files changed, 11 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/16ed99a1/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java index ba4193e..db49976 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java @@ -78,7 +78,6 @@ public class BlockletDataMapIndexStore try { SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore(); Set<String> filesRead = new HashSet<>(); - long memorySize = 0L; String segmentFilePath = identifier.getIndexFilePath(); Map<String, BlockMetaInfo> carbonDataFileBlockMetaInfoMapping = BlockletDataMapUtil .createCarbonDataFileBlockMetaInfoMapping(segmentFilePath); @@ -89,7 +88,6 @@ public class BlockletDataMapIndexStore carbonDataFileBlockMetaInfoMapping); BlockletDataMap blockletDataMap = loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap); - memorySize += blockletDataMap.getMemorySize(); dataMaps.add(blockletDataMap); blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(dataMaps); } else { @@ -103,13 +101,12 @@ public class BlockletDataMapIndexStore carbonDataFileBlockMetaInfoMapping); BlockletDataMap blockletDataMap = loadAndGetDataMap(blockIndexUniqueIdentifier, indexFileStore, blockMetaInfoMap); - memorySize += blockletDataMap.getMemorySize(); dataMaps.add(blockletDataMap); } blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(dataMaps); } lruCache.put(identifier.getUniqueTableSegmentIdentifier(), blockletDataMapIndexWrapper, - memorySize); + blockletDataMapIndexWrapper.getMemorySize()); } catch (Throwable e) { // clear all the memory used by datamaps loaded for (DataMap dataMap : dataMaps) { @@ -189,7 +186,6 @@ public class BlockletDataMapIndexStore if (lock == null) { lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier); } - long memorySize = 0L; // As dataMap will use unsafe memory, it is not recommended to overwrite an existing entry // as in that case clearing unsafe memory need to be taken card. If at all datamap entry // in the cache need to be overwritten then use the invalidate interface @@ -201,10 +197,9 @@ public class BlockletDataMapIndexStore try { for (BlockletDataMap blockletDataMap: dataMaps) { blockletDataMap.convertToUnsafeDMStore(); - memorySize += blockletDataMap.getMemorySize(); } lruCache.put(tableBlockIndexUniqueIdentifier.getUniqueTableSegmentIdentifier(), wrapper, - memorySize); + wrapper.getMemorySize()); } catch (Throwable e) { // clear all the memory acquired by data map in case of any failure for (DataMap blockletDataMap : dataMaps) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/16ed99a1/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java index d674cb4..ea85930 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java @@ -30,8 +30,16 @@ public class BlockletDataMapIndexWrapper implements Cacheable, Serializable { private List<BlockletDataMap> dataMaps; + // size of the wrapper. basically the total size of the datamaps this wrapper is holding + private long wrapperSize; + public BlockletDataMapIndexWrapper(List<BlockletDataMap> dataMaps) { this.dataMaps = dataMaps; + this.wrapperSize = 0L; + // add the size of each and every datamap in this wrapper + for (BlockletDataMap dataMap : dataMaps) { + this.wrapperSize += dataMap.getMemorySize(); + } } @Override public long getFileTimeStamp() { @@ -43,7 +51,7 @@ public class BlockletDataMapIndexWrapper implements Cacheable, Serializable { } @Override public long getMemorySize() { - return 0; + return wrapperSize; } public List<BlockletDataMap> getDataMaps() {
