Repository: carbondata Updated Branches: refs/heads/master 23bd6e9af -> 0ad90820a
[Compatibility] fix legacy store compatibility issue Problem: Migrate a table from 1.1 version with one segment to the latest version and do one load. Now the table will have 1 legacy segment and 1 non-legacy segment. During query when the blocklet count is calculated for both the blocks. Legacy segment should give 0 to avoid blocklet pruning as min max is not there and nonLegacy segment should give 1 so that all the blocklets are scanned for query. Datamap schema is cached to avoid calculating it again. For segment2 the datamap schema of segment1 is being used due to which isLegacy flag is being misused. Solution: Pass 0 as blocklet count for legacy store and calculate blocklet count only for new segments. Be sure to do all of the following checklist to help us incorporate your contribution quickly and easily: This closes #2746 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0ad90820 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0ad90820 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0ad90820 Branch: refs/heads/master Commit: 0ad90820acd78b8158edb4025e559df363af6800 Parents: 23bd6e9 Author: kunal642 <[email protected]> Authored: Fri Sep 21 18:43:22 2018 +0530 Committer: ravipesala <[email protected]> Committed: Mon Sep 24 16:53:35 2018 +0530 ---------------------------------------------------------------------- .../indexstore/blockletindex/BlockDataMap.java | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ad90820/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java index 8a1538e..0cf9914 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java @@ -238,6 +238,10 @@ public class BlockDataMap extends CoarseGrainDataMap blockMetaInfo, updatedMinValues, updatedMaxValues, minMaxFlag); } } + List<Short> blockletCountList = new ArrayList<>(); + blockletCountList.add((short) 0); + byte[] blockletCount = convertRowCountFromShortToByteArray(blockletCountList); + summaryRow.setByteArray(blockletCount, taskSummarySchema.length - 1); setMinMaxFlagForTaskSummary(summaryRow, taskSummarySchema, segmentProperties, minMaxFlag); return summaryRow; } @@ -627,17 +631,12 @@ public class BlockDataMap extends CoarseGrainDataMap // get total blocklet number in this datamap protected int getTotalBlocklets() { - if (isLegacyStore) { - // dummy value - return 0; - } else { - ByteBuffer byteBuffer = ByteBuffer.wrap(getBlockletRowCountForEachBlock()); - int sum = 0; - while (byteBuffer.hasRemaining()) { - sum += byteBuffer.getShort(); - } - return sum; + ByteBuffer byteBuffer = ByteBuffer.wrap(getBlockletRowCountForEachBlock()); + int sum = 0; + while (byteBuffer.hasRemaining()) { + sum += byteBuffer.getShort(); } + return sum; } private List<Blocklet> prune(FilterResolverIntf filterExp) { @@ -1000,7 +999,7 @@ public class BlockDataMap extends CoarseGrainDataMap SegmentPropertiesAndSchemaHolder.getInstance() .getSegmentPropertiesWrapper(segmentPropertiesIndex); try { - return segmentPropertiesWrapper.getTaskSummarySchema(!isLegacyStore, isFilePathStored); + return segmentPropertiesWrapper.getTaskSummarySchema(true, isFilePathStored); } catch (MemoryException e) { throw new RuntimeException(e); }
