[CARBONDATA-2113]compatibility fix for v2 Fixes related to backword compatibility:
Count() issue: when count() is run on old store where file format version is V2, it was unable to get the files, because when forming the filepath while creating the table block info object from index file info, the path was formed with double slash(//), beacuse in v2, local path was stored. so when trying to get this path with key, it was failing. So form the correct file path. Select * issue: when select * is run, then only the datachunck was considered as whole data and while uncompressing the measure data, it was failing. So, read proper data and datachunck before uncompressing. Read Metadata file: when readMetadataFile is called , the while reading the schema, it was explicitly returning null, so columns will be null and hence some nullpointerxception was coming. so do not return null, return proper schema, by reading the footer. Vesion compatibility: calculation the number of pages to be filled based on row count was not handled for V2 version, handled same, as no. of rows per page is different in V2 and V3 This closes #1901 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1202e209 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1202e209 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1202e209 Branch: refs/heads/branch-1.3 Commit: 1202e209eccda172f9671fa0cc2deeebeb4af456 Parents: 1248bd4 Author: akashrn5 <[email protected]> Authored: Thu Feb 1 12:42:49 2018 +0530 Committer: ravipesala <[email protected]> Committed: Thu Feb 1 22:16:39 2018 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonVersionConstants.java | 5 +++++ .../v2/CompressedMeasureChunkFileBasedReaderV2.java | 15 ++++++++++----- .../blockletindex/BlockletDataRefNodeWrapper.java | 13 +++++++++++-- .../carbondata/core/mutate/CarbonUpdateUtil.java | 10 ++++------ .../core/util/AbstractDataFileFooterConverter.java | 3 ++- .../core/util/DataFileFooterConverter2.java | 2 +- 6 files changed, 33 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java index 2d58b0b..22fbaf2 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java @@ -49,6 +49,11 @@ public final class CarbonVersionConstants { */ public static final String CARBONDATA_BUILD_DATE; + /** + * number of rows per blocklet column page default value for V2 version + */ + public static final int NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2 = 120000; + static { // create input stream for CARBONDATA_VERSION_INFO_FILE InputStream resourceStream = Thread.currentThread().getContextClassLoader() http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java index 2ddc202..d61f98a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java @@ -52,7 +52,14 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun throws IOException { int dataLength = 0; if (measureColumnChunkOffsets.size() - 1 == columnIndex) { - dataLength = measureColumnChunkLength.get(columnIndex); + DataChunk2 metadataChunk = null; + synchronized (fileReader) { + metadataChunk = CarbonUtil.readDataChunk(ByteBuffer.wrap(fileReader + .readByteArray(filePath, measureColumnChunkOffsets.get(columnIndex), + measureColumnChunkLength.get(columnIndex))), 0, + measureColumnChunkLength.get(columnIndex)); + } + dataLength = measureColumnChunkLength.get(columnIndex) + metadataChunk.data_page_length; } else { long currentMeasureOffset = measureColumnChunkOffsets.get(columnIndex); dataLength = (int) (measureColumnChunkOffsets.get(columnIndex + 1) - currentMeasureOffset); @@ -115,9 +122,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun ByteBuffer rawData = measureRawColumnChunk.getRawData(); DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex)); - if (measureColumnChunkOffsets.size() - 1 != blockIndex) { - copyPoint += measureColumnChunkLength.get(blockIndex); - } + copyPoint += measureColumnChunkLength.get(blockIndex); ColumnPage page = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint); page.setNullBits(getNullBitSet(measureColumnChunk.presence)); @@ -130,7 +135,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta(); byte[] encodedMeta = encoder_meta.get(0).array(); - ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta); + ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta); ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta); byte[] rawData = measureRawColumnChunk.getRawData().array(); return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length); http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java index b672c58..4e49ede 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache; import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; +import org.apache.carbondata.core.constants.CarbonVersionConstants; import org.apache.carbondata.core.datastore.DataRefNode; import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.block.TableBlockInfo; @@ -56,8 +57,16 @@ public class BlockletDataRefNodeWrapper implements DataRefNode { detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount()); detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId()); int[] pageRowCount = new int[detailInfo.getPagesCount()]; - int numberOfPagesCompletelyFilled = detailInfo.getRowCount() - / CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; + int numberOfPagesCompletelyFilled = detailInfo.getRowCount(); + // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the number + // of pages filled + if (blockInfo.getVersion() == ColumnarFormatVersion.V2) { + numberOfPagesCompletelyFilled /= + CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2; + } else { + numberOfPagesCompletelyFilled /= + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; + } int lastPageRowCount = detailInfo.getRowCount() % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; for (int i = 0; i < numberOfPagesCompletelyFilled; i++) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index 0e4eec7..c5f61c2 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java @@ -587,26 +587,24 @@ public class CarbonUpdateUtil { */ private static void deleteStaleCarbonDataFiles(LoadMetadataDetails segment, CarbonFile[] allSegmentFiles, SegmentUpdateStatusManager updateStatusManager) { - boolean doForceDelete = true; - boolean isAbortedFile = true; CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles, - isAbortedFile); + true); // now for each invalid delta file need to check the query execution time out // and then delete. for (CarbonFile invalidFile : invalidUpdateDeltaFiles) { - compareTimestampsAndDelete(invalidFile, doForceDelete, false); + compareTimestampsAndDelete(invalidFile, true, false); } // do the same for the index files. CarbonFile[] invalidIndexFiles = updateStatusManager .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles, - isAbortedFile); + true); // now for each invalid index file need to check the query execution time out // and then delete. for (CarbonFile invalidFile : invalidIndexFiles) { - compareTimestampsAndDelete(invalidFile, doForceDelete, false); + compareTimestampsAndDelete(invalidFile, true, false); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index c7bc6aa..94a041a 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -208,7 +208,8 @@ public abstract class AbstractDataFileFooterConverter { if (fileName.lastIndexOf("/") > 0) { fileName = fileName.substring(fileName.lastIndexOf("/")); } - tableBlockInfo.setFilePath(parentPath + "/" + fileName); + fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/"); + tableBlockInfo.setFilePath(parentPath + fileName); return tableBlockInfo; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java index 8cd437f..863e1df 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java @@ -141,6 +141,6 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter { } @Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException { - return null; + return new DataFileFooterConverter().getSchema(tableBlockInfo); } }
