[CARBONDATA-2753][Compatibility] Row count of page is calculated wrong for old store(V2 store)
Row count of page is calculated wrong for V2 store. Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d7203720 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d7203720 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d7203720 Branch: refs/heads/branch-1.4 Commit: d72037205e5f9f5f3be12e65827cb195f31828d3 Parents: 92eed69 Author: dhatchayani <[email protected]> Authored: Wed Jul 25 14:41:58 2018 +0530 Committer: ravipesala <[email protected]> Committed: Tue Jul 31 00:11:26 2018 +0530 ---------------------------------------------------------------------- .../blockletindex/BlockletDataRefNode.java | 18 +++++++++++++----- .../scan/scanner/impl/BlockletFullScanner.java | 9 +-------- 2 files changed, 14 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7203720/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java index a11ae8d..5681528 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java @@ -61,18 +61,26 @@ public class BlockletDataRefNode implements DataRefNode { int numberOfPagesCompletelyFilled = detailInfo.getRowCount(); // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the number // of pages filled - if (blockInfo.getVersion() == ColumnarFormatVersion.V2) { + int lastPageRowCount; + int fullyFilledRowsCount; + if (blockInfo.getVersion() == ColumnarFormatVersion.V2 + || blockInfo.getVersion() == ColumnarFormatVersion.V1) { numberOfPagesCompletelyFilled /= CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2; + lastPageRowCount = detailInfo.getRowCount() + % CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2; + fullyFilledRowsCount = + CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2; } else { numberOfPagesCompletelyFilled /= CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; + lastPageRowCount = detailInfo.getRowCount() + % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; + fullyFilledRowsCount = + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; } - int lastPageRowCount = detailInfo.getRowCount() - % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; for (int i = 0; i < numberOfPagesCompletelyFilled; i++) { - pageRowCount[i] = - CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; + pageRowCount[i] = fullyFilledRowsCount; } if (lastPageRowCount > 0) { pageRowCount[pageRowCount.length - 1] = lastPageRowCount; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7203720/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java index c3d4df8..f61a8b1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java @@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.scanner.impl; import java.io.IOException; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; import org.apache.carbondata.core.datastore.DataRefNode; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; @@ -123,13 +122,7 @@ public class BlockletFullScanner implements BlockletScanner { if (numberOfRows == null) { numberOfRows = new int[rawBlockletColumnChunks.getDataBlock().numberOfPages()]; for (int i = 0; i < numberOfRows.length; i++) { - numberOfRows[i] = - CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; - } - int lastPageSize = rawBlockletColumnChunks.getDataBlock().numRows() - % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; - if (lastPageSize > 0) { - numberOfRows[numberOfRows.length - 1] = lastPageSize; + numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i); } } scannedResult.setPageFilteredRowCount(numberOfRows);
