This is an automated email from the ASF dual-hosted git repository.
ravipesala pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 5bd345b [CARBONDATA-3650] Remove file format V1 and V2 reader
5bd345b is described below
commit 5bd345ba0aa1fe822831e7dabfcbfa88ec635614
Author: Jacky Li <[email protected]>
AuthorDate: Sun Dec 29 21:55:05 2019 +0800
[CARBONDATA-3650] Remove file format V1 and V2 reader
V1 and V2 file format is deprecated in CarbonData 2.0
This closes #3543
---
.../core/constants/CarbonVersionConstants.java | 5 -
.../impl/VariableLengthDimensionColumnPage.java | 15 --
.../chunk/reader/CarbonDataReaderFactory.java | 30 +--
.../reader/dimension/AbstractChunkReader.java | 96 ----------
...rmat.java => AbstractDimensionChunkReader.java} | 48 ++++-
.../CompressedDimensionChunkFileBasedReaderV1.java | 181 ------------------
.../CompressedDimensionChunkFileBasedReaderV2.java | 203 ---------------------
...aderV3.java => DimensionChunkPageReaderV3.java} | 6 +-
...edReaderV3.java => DimensionChunkReaderV3.java} | 6 +-
.../reader/measure/AbstractMeasureChunkReader.java | 86 +++++++--
.../AbstractMeasureChunkReaderV2V3Format.java | 111 -----------
.../CompressedMeasureChunkFileBasedReaderV1.java | 112 ------------
.../CompressedMeasureChunkFileBasedReaderV2.java | 152 ---------------
...ReaderV3.java => MeasureChunkPageReaderV3.java} | 6 +-
...asedReaderV3.java => MeasureChunkReaderV3.java} | 6 +-
.../datastore/page/encoding/EncodingFactory.java | 8 -
.../statistics/PrimitivePageStatsCollector.java | 6 +-
.../blockletindex/BlockletDataRefNode.java | 27 +--
.../core/keygenerator/mdkey/NumberCompressor.java | 181 ------------------
.../core/metadata/ColumnarFormatVersion.java | 4 +-
.../core/metadata/blocklet/BlockletInfo.java | 69 -------
.../core/metadata/datatype/DataType.java | 2 -
.../core/metadata/datatype/DataTypes.java | 5 -
.../core/metadata/datatype/LegacyLongType.java | 33 ----
.../apache/carbondata/core/util/CarbonUtil.java | 15 --
.../core/util/DataFileFooterConverterFactory.java | 7 +-
.../apache/carbondata/core/util/DataTypeUtil.java | 4 -
.../carbondata/core/util/path/CarbonTablePath.java | 8 +-
.../mdkey/NumberCompressorUnitTest.java | 116 ------------
.../carbondata/core/util/CarbonTestUtil.java | 3 -
.../carbondata/core/util/CarbonUtilTest.java | 10 +-
.../CarbonV1toV3CompatabilityTestCase.scala | 98 ----------
.../LoadTableWithLocalDictionaryTestCase.scala | 4 +-
.../TestNonTransactionalCarbonTable.scala | 4 +-
.../LocalDictionarySupportLoadTableTest.scala | 4 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../processing/store/CarbonDataWriterFactory.java | 5 +-
37 files changed, 164 insertions(+), 1514 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
index 2382bd8..50c8ffd 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
@@ -50,11 +50,6 @@ public final class CarbonVersionConstants {
*/
public static final String CARBONDATA_BUILD_DATE;
- /**
- * number of rows per blocklet column page default value for V2 version
- */
- public static final int NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2 =
120000;
-
static {
// create input stream for CARBONDATA_VERSION_INFO_FILE
InputStream resourceStream = Thread.currentThread().getContextClassLoader()
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
index 2e941b2..2a71934 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
@@ -36,21 +36,6 @@ public class VariableLengthDimensionColumnPage extends
AbstractDimensionColumnPa
* @param invertedIndexReverse reverse inverted index
* @param numberOfRows number of rows
* @param dictionary carbon local dictionary for string column.
- */
- public VariableLengthDimensionColumnPage(byte[] dataChunks, int[]
invertedIndex,
- int[] invertedIndexReverse, int numberOfRows, DimensionStoreType
dimStoreType,
- CarbonDictionary dictionary, int dataLength) {
- this(dataChunks, invertedIndex, invertedIndexReverse, numberOfRows,
dimStoreType, dictionary,
- null, dataLength);
- }
-
- /**
- * Constructor for this class
- * @param dataChunks data chunk
- * @param invertedIndex inverted index
- * @param invertedIndexReverse reverse inverted index
- * @param numberOfRows number of rows
- * @param dictionary carbon local dictionary for string column.
* @param vectorInfo vector to be filled with decoded column page.
*/
public VariableLengthDimensionColumnPage(byte[] dataChunks, int[]
invertedIndex,
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index 2ea0ea8..5549574 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -17,14 +17,10 @@
package org.apache.carbondata.core.datastore.chunk.reader;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimChunkFileBasedPageLevelReaderV3;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMeasureChunkFileBasedReaderV3;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMsrChunkFileBasedPageLevelReaderV3;
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.DimensionChunkPageReaderV3;
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.DimensionChunkReaderV3;
+import
org.apache.carbondata.core.datastore.chunk.reader.measure.v3.MeasureChunkPageReaderV3;
+import
org.apache.carbondata.core.datastore.chunk.reader.measure.v3.MeasureChunkReaderV3;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -68,18 +64,12 @@ public class CarbonDataReaderFactory {
BlockletInfo blockletInfo, int[] eachColumnValueSize, String filePath,
boolean readPagebyPage) {
switch (version) {
- case V1:
- return new CompressedDimensionChunkFileBasedReaderV1(blockletInfo,
eachColumnValueSize,
- filePath);
- case V2:
- return new CompressedDimensionChunkFileBasedReaderV2(blockletInfo,
eachColumnValueSize,
- filePath);
case V3:
if (readPagebyPage) {
- return new
CompressedDimChunkFileBasedPageLevelReaderV3(blockletInfo, eachColumnValueSize,
+ return new DimensionChunkPageReaderV3(blockletInfo,
eachColumnValueSize,
filePath);
} else {
- return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo,
eachColumnValueSize,
+ return new DimensionChunkReaderV3(blockletInfo, eachColumnValueSize,
filePath);
}
default:
@@ -98,15 +88,11 @@ public class CarbonDataReaderFactory {
public MeasureColumnChunkReader
getMeasureColumnChunkReader(ColumnarFormatVersion version,
BlockletInfo blockletInfo, String filePath, boolean readPagebyPage) {
switch (version) {
- case V1:
- return new CompressedMeasureChunkFileBasedReaderV1(blockletInfo,
filePath);
- case V2:
- return new CompressedMeasureChunkFileBasedReaderV2(blockletInfo,
filePath);
case V3:
if (readPagebyPage) {
- return new
CompressedMsrChunkFileBasedPageLevelReaderV3(blockletInfo, filePath);
+ return new MeasureChunkPageReaderV3(blockletInfo, filePath);
} else {
- return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo,
filePath);
+ return new MeasureChunkReaderV3(blockletInfo, filePath);
}
default:
throw new UnsupportedOperationException("Unsupported columnar format
version: " + version);
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
deleted file mode 100644
index 6a567a1..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.dimension;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * Class which will have all the common properties and behavior among all type
- * of reader
- */
-public abstract class AbstractChunkReader implements
DimensionColumnChunkReader {
-
- /**
- * compressor will be used to uncompress the data
- */
- protected Compressor compressor;
-
- /**
- * size of the each column value
- * for no dictionary column it will be -1
- */
- protected int[] eachColumnValueSize;
-
- /**
- * full qualified path of the data file from
- * which data will be read
- */
- protected String filePath;
-
- /**
- * this will be used to uncompress the
- * row id and rle chunk
- */
- protected NumberCompressor numberCompressor;
-
- /**
- * number of element in each chunk
- */
- protected int numberOfRows;
-
- /**
- * Constructor to get minimum parameter to create
- * instance of this class
- *
- * @param eachColumnValueSize size of the each column value
- * @param filePath file from which data will be read
- */
- public AbstractChunkReader(final int[] eachColumnValueSize, final String
filePath,
- int numberOfRows) {
- this.eachColumnValueSize = eachColumnValueSize;
- this.filePath = filePath;
- int numberOfElement = 0;
- try {
- numberOfElement = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
- CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
- } catch (NumberFormatException exception) {
- numberOfElement =
Integer.parseInt(CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- }
- this.numberCompressor = new NumberCompressor(numberOfElement);
- this.numberOfRows = numberOfRows;
- }
-
- @Override
- public void decodeColumnPageAndFillVector(DimensionRawColumnChunk
dimensionRawColumnChunk,
- int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer
reusableDataBuffer)
- throws IOException, MemoryException {
- throw new UnsupportedOperationException(
- "This operation is not supported in this reader " +
this.getClass().getName());
- }
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractDimensionChunkReader.java
similarity index 73%
rename from
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
rename to
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractDimensionChunkReader.java
index 4f2361d..8e09d04 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractDimensionChunkReader.java
@@ -21,31 +21,68 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import
org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
- * Abstract class for V2, V3 format dimension column reader
+ * Class which will have all the common properties and behavior among all type
+ * of reader
*/
-public abstract class AbstractChunkReaderV2V3Format extends
AbstractChunkReader {
+public abstract class AbstractDimensionChunkReader implements
DimensionColumnChunkReader {
+
+ /**
+ * compressor will be used to uncompress the data
+ */
+ protected Compressor compressor;
+
+ /**
+ * size of the each column value
+ * for no dictionary column it will be -1
+ */
+ protected int[] eachColumnValueSize;
+
+ /**
+ * full qualified path of the data file from
+ * which data will be read
+ */
+ protected String filePath;
/**
* dimension chunks offset
*/
protected List<Long> dimensionChunksOffset;
-
/**
* dimension chunks length
*/
protected List<Integer> dimensionChunksLength;
- public AbstractChunkReaderV2V3Format(final BlockletInfo blockletInfo,
+ /**
+ * Constructor to get minimum parameter to create
+ * instance of this class
+ * @param eachColumnValueSize size of the each column value
+ * @param filePath file from which data will be read
+ */
+ public AbstractDimensionChunkReader(final BlockletInfo blockletInfo,
final int[] eachColumnValueSize, final String filePath) {
- super(eachColumnValueSize, filePath, blockletInfo.getNumberOfRows());
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.filePath = filePath;
dimensionChunksOffset = blockletInfo.getDimensionChunkOffsets();
dimensionChunksLength = blockletInfo.getDimensionChunksLength();
}
+ @Override
+ public void decodeColumnPageAndFillVector(DimensionRawColumnChunk
dimensionRawColumnChunk,
+ int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer
reusableDataBuffer)
+ throws IOException, MemoryException {
+ throw new UnsupportedOperationException(
+ "This operation is not supported in this reader " +
this.getClass().getName());
+ }
+
/**
* Below method will be used to read the chunk based on block indexes
* Reading logic of below method is:
@@ -112,5 +149,4 @@ public abstract class AbstractChunkReaderV2V3Format extends
AbstractChunkReader
*/
protected abstract DimensionRawColumnChunk[]
readRawDimensionChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws
IOException;
-
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
deleted file mode 100644
index 4b7b5f4..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.FileReader;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
-import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
-import
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-/**
- * Compressed dimension chunk reader class
- */
-public class CompressedDimensionChunkFileBasedReaderV1 extends
AbstractChunkReader {
-
- /**
- * data chunk list which holds the information
- * about the data block metadata
- */
- private final List<DataChunk> dimensionColumnChunk;
-
- /**
- * Constructor to get minimum parameter to create instance of this class
- *
- * @param blockletInfo blocklet info
- * @param eachColumnValueSize size of the each column value
- * @param filePath file from which data will be read
- */
- public CompressedDimensionChunkFileBasedReaderV1(final BlockletInfo
blockletInfo,
- final int[] eachColumnValueSize, final String filePath) {
- super(eachColumnValueSize, filePath, blockletInfo.getNumberOfRows());
- this.dimensionColumnChunk = blockletInfo.getDimensionColumnChunk();
- // for v1 store, the compressor is snappy
- this.compressor =
CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor();
- }
-
- /**
- * Below method will be used to read the raw chunk based on block indexes
- *
- * @param fileReader file reader to read the blocks from file
- * @param columnIndexRange blocks to be read
- * @return dimension column chunks
- */
- @Override
- public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader
fileReader,
- int[][] columnIndexRange) throws IOException {
- DimensionRawColumnChunk[] dataChunks = new
DimensionRawColumnChunk[dimensionColumnChunk.size()];
- for (int i = 0; i < columnIndexRange.length; i++) {
- for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
- dataChunks[j] = readRawDimensionChunk(fileReader, j);
- }
- }
- return dataChunks;
- }
-
- /**
- * Below method will be used to read the raw chunk based on block index
- *
- * @param fileReader file reader to read the blocks from file
- * @param columnIndex column to be read
- * @return dimension column chunk
- */
- @Override
- public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
- int columnIndex) throws IOException {
- DataChunk dataChunk = dimensionColumnChunk.get(columnIndex);
- ByteBuffer buffer = null;
- synchronized (fileReader) {
- buffer = fileReader
- .readByteBuffer(filePath, dataChunk.getDataPageOffset(),
dataChunk.getDataPageLength());
- }
- DimensionRawColumnChunk rawColumnChunk = new
DimensionRawColumnChunk(columnIndex, buffer, 0,
- dataChunk.getDataPageLength(), this);
- rawColumnChunk.setFileReader(fileReader);
- rawColumnChunk.setPagesCount(1);
- rawColumnChunk.setRowCount(new int[] { numberOfRows });
- return rawColumnChunk;
- }
-
- @Override
- public DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk
dimensionRawColumnChunk,
- int pageNumber, ReusableDataBuffer reusableDataBuffer) throws
IOException {
- int blockIndex = dimensionRawColumnChunk.getColumnIndex();
- byte[] dataPage = null;
- int[] invertedIndexes = new int[0];
- int[] invertedIndexesReverse = new int[0];
- int[] rlePage = null;
- FileReader fileReader = dimensionRawColumnChunk.getFileReader();
- ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
- int uncompressedSize = compressor
- .unCompressedLength(rawData.array(), (int)
dimensionRawColumnChunk.getOffSet(),
- dimensionRawColumnChunk.getLength());
- if (null != reusableDataBuffer) {
- dataPage = reusableDataBuffer.getDataBuffer(uncompressedSize);
- } else {
- dataPage = new byte[uncompressedSize];
- }
- compressor.rawUncompress(rawData.array(), (int)
dimensionRawColumnChunk.getOffSet(),
- dimensionRawColumnChunk.getLength(), dataPage);
- // if row id block is present then read the row id chunk and uncompress it
- DataChunk dataChunk = dimensionColumnChunk.get(blockIndex);
- if (CarbonUtil.hasEncoding(dataChunk.getEncodingList(),
- Encoding.INVERTED_INDEX)) {
- byte[] columnIndexData;
- synchronized (fileReader) {
- columnIndexData = fileReader
- .readByteArray(filePath, dataChunk.getRowIdPageOffset(),
- dataChunk.getRowIdPageLength());
- }
- invertedIndexes = CarbonUtil
- .getUnCompressColumnIndex(dataChunk.getRowIdPageLength(),
- columnIndexData, numberCompressor, 0);
- // get the reverse index
- invertedIndexesReverse =
CarbonUtil.getInvertedReverseIndex(invertedIndexes);
- }
- // if rle is applied then read the rle block chunk and then uncompress
- //then actual data based on rle block
- if (CarbonUtil
- .hasEncoding(dataChunk.getEncodingList(), Encoding.RLE)) {
- // read and uncompress the rle block
- byte[] key;
- synchronized (fileReader) {
- key = fileReader
- .readByteArray(filePath, dataChunk.getRlePageOffset(),
- dataChunk.getRlePageLength());
- }
- rlePage = numberCompressor
- .unCompress(key, 0, dataChunk.getRlePageLength());
- // uncompress the data with rle indexes
- dataPage = UnBlockIndexer
- .uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex],
uncompressedSize);
- uncompressedSize = dataPage.length;
- rlePage = null;
- }
- // fill chunk attributes
- DimensionColumnPage columnDataChunk = null;
- // if no dictionary column then first create a no dictionary column chunk
- // and set to data chunk instance
- if (!CarbonUtil
- .hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) {
- columnDataChunk =
- new VariableLengthDimensionColumnPage(dataPage, invertedIndexes,
invertedIndexesReverse,
- numberOfRows,
DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_SHORT_LENGTH,
- null, uncompressedSize);
- } else {
- // to store fixed length column chunk values
- columnDataChunk =
- new FixedLengthDimensionColumnPage(dataPage, invertedIndexes,
invertedIndexesReverse,
- numberOfRows, eachColumnValueSize[blockIndex], uncompressedSize);
- }
- return columnDataChunk;
- }
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
deleted file mode 100644
index bdc4ae6..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.datastore.FileReader;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
-import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
-import
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.format.DataChunk2;
-import org.apache.carbondata.format.Encoding;
-
-/**
- * Compressed dimension chunk reader class for version 2
- */
-public class CompressedDimensionChunkFileBasedReaderV2 extends
AbstractChunkReaderV2V3Format {
-
- /**
- * Constructor to get minimum parameter to create instance of this class
- *
- * @param blockletInfo
- * @param eachColumnValueSize
- * @param filePath
- */
- public CompressedDimensionChunkFileBasedReaderV2(final BlockletInfo
blockletInfo,
- final int[] eachColumnValueSize, final String filePath) {
- super(blockletInfo, eachColumnValueSize, filePath);
- // for v2 store, the compressor is snappy
- this.compressor =
CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor();
- }
-
- /**
- * Below method will be used to read the chunk based on block index
- *
- * @param fileReader file reader to read the blocks from file
- * @param columnIndex column to be read
- * @return dimension column chunk
- */
- public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
int columnIndex)
- throws IOException {
- int length = 0;
- if (dimensionChunksOffset.size() - 1 == columnIndex) {
- // Incase of last block read only for datachunk and read remaining while
converting it.
- length = dimensionChunksLength.get(columnIndex);
- } else {
- long currentDimensionOffset = dimensionChunksOffset.get(columnIndex);
- length = (int) (dimensionChunksOffset.get(columnIndex + 1) -
currentDimensionOffset);
- }
- ByteBuffer buffer = null;
- synchronized (fileReader) {
- buffer =
- fileReader.readByteBuffer(filePath,
dimensionChunksOffset.get(columnIndex), length);
- }
- DimensionRawColumnChunk rawColumnChunk =
- new DimensionRawColumnChunk(columnIndex, buffer, 0, length, this);
- rawColumnChunk.setFileReader(fileReader);
- rawColumnChunk.setPagesCount(1);
- rawColumnChunk.setRowCount(new int[] { numberOfRows });
- return rawColumnChunk;
- }
-
- /**
- * Below method will be used to read measure chunk data in group.
- * This method will be useful to avoid multiple IO while reading the
- * data from
- *
- * @param fileReader file reader to read the data
- * @param startColumnBlockletIndex first column blocklet index to be read
- * @param endColumnBlockletIndex end column blocklet index to be read
- * @return measure raw chunkArray
- * @throws IOException
- */
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader
fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws
IOException {
- long currentDimensionOffset =
dimensionChunksOffset.get(startColumnBlockletIndex);
- ByteBuffer buffer = null;
- synchronized (fileReader) {
- buffer = fileReader.readByteBuffer(filePath, currentDimensionOffset,
- (int) (dimensionChunksOffset.get(endColumnBlockletIndex + 1) -
currentDimensionOffset));
- }
- DimensionRawColumnChunk[] dataChunks =
- new DimensionRawColumnChunk[endColumnBlockletIndex -
startColumnBlockletIndex + 1];
- int index = 0;
- int runningLength = 0;
- for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
- int currentLength = (int) (dimensionChunksOffset.get(i + 1) -
dimensionChunksOffset.get(i));
- dataChunks[index] =
- new DimensionRawColumnChunk(i, buffer, runningLength, currentLength,
this);
- dataChunks[index].setFileReader(fileReader);
- dataChunks[index].setPagesCount(1);
- dataChunks[index].setRowCount(new int[] { numberOfRows });
- runningLength += currentLength;
- index++;
- }
- return dataChunks;
- }
-
- public DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk
dimensionRawColumnChunk,
- int pageNumber, ReusableDataBuffer reusableDataBuffer) throws
IOException {
- byte[] dataPage = null;
- int[] invertedIndexes = new int[0];
- int[] invertedIndexesReverse = new int[0];
- int[] rlePage = null;
- DataChunk2 dimensionColumnChunk = null;
- int copySourcePoint = (int) dimensionRawColumnChunk.getOffSet();
- int blockIndex = dimensionRawColumnChunk.getColumnIndex();
- ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
- if (dimensionChunksOffset.size() - 1 == blockIndex) {
- dimensionColumnChunk =
- CarbonUtil.readDataChunk(rawData, copySourcePoint,
dimensionRawColumnChunk.getLength());
- int totalDimensionDataLength =
- dimensionColumnChunk.data_page_length +
dimensionColumnChunk.rle_page_length
- + dimensionColumnChunk.rowid_page_length;
- synchronized (dimensionRawColumnChunk.getFileReader()) {
- rawData =
dimensionRawColumnChunk.getFileReader().readByteBuffer(filePath,
- dimensionChunksOffset.get(blockIndex) +
dimensionChunksLength.get(blockIndex),
- totalDimensionDataLength);
- }
- } else {
- dimensionColumnChunk =
- CarbonUtil.readDataChunk(rawData, copySourcePoint,
dimensionChunksLength.get(blockIndex));
- copySourcePoint += dimensionChunksLength.get(blockIndex);
- }
- // first read the data and uncompressed it
- int uncompressedSize = compressor.unCompressedLength(rawData.array(),
copySourcePoint,
- dimensionColumnChunk.data_page_length);
- if (null != reusableDataBuffer) {
- dataPage = reusableDataBuffer.getDataBuffer(uncompressedSize);
- } else {
- dataPage = new byte[uncompressedSize];
- }
- compressor
- .rawUncompress(rawData.array(), copySourcePoint,
dimensionColumnChunk.data_page_length,
- dataPage);
- copySourcePoint += dimensionColumnChunk.data_page_length;
- // if row id block is present then read the row id chunk and uncompress it
- if (CarbonUtil.hasEncoding(dimensionColumnChunk.encoders,
Encoding.INVERTED_INDEX)) {
- byte[] dataInv = new byte[dimensionColumnChunk.rowid_page_length];
- rawData.position(copySourcePoint);
- rawData.get(dataInv);
- invertedIndexes = CarbonUtil
- .getUnCompressColumnIndex(dimensionColumnChunk.rowid_page_length,
dataInv,
- numberCompressor, 0);
- copySourcePoint += dimensionColumnChunk.rowid_page_length;
- // get the reverse index
- invertedIndexesReverse =
CarbonUtil.getInvertedReverseIndex(invertedIndexes);
- }
- // if rle is applied then read the rle block chunk and then uncompress
- //then actual data based on rle block
- if (CarbonUtil.hasEncoding(dimensionColumnChunk.encoders, Encoding.RLE)) {
- byte[] dataRle = new byte[dimensionColumnChunk.rle_page_length];
- rawData.position(copySourcePoint);
- rawData.get(dataRle);
- rlePage = numberCompressor.unCompress(dataRle, 0,
dimensionColumnChunk.rle_page_length);
- // uncompress the data with rle indexes
- dataPage = UnBlockIndexer
- .uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex],
uncompressedSize);
- uncompressedSize = dataPage.length;
- }
- // fill chunk attributes
- DimensionColumnPage columnDataChunk = null;
-
- // if no dictionary column then first create a no dictionary column chunk
- // and set to data chunk instance
- if (!CarbonUtil.hasEncoding(dimensionColumnChunk.encoders,
Encoding.DICTIONARY)) {
- columnDataChunk =
- new VariableLengthDimensionColumnPage(dataPage, invertedIndexes,
invertedIndexesReverse,
- numberOfRows,
DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_SHORT_LENGTH,
- null, uncompressedSize);
- } else {
- // to store fixed length column chunk values
- columnDataChunk =
- new FixedLengthDimensionColumnPage(dataPage, invertedIndexes,
invertedIndexesReverse,
- numberOfRows, eachColumnValueSize[blockIndex], uncompressedSize);
- }
- return columnDataChunk;
- }
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkPageReaderV3.java
similarity index 97%
rename from
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
rename to
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkPageReaderV3.java
index 0f9735f..f88624f 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkPageReaderV3.java
@@ -47,15 +47,15 @@ import org.apache.carbondata.format.Encoding;
* <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
* <File Footer>
*/
-public class CompressedDimChunkFileBasedPageLevelReaderV3
- extends CompressedDimensionChunkFileBasedReaderV3 {
+public class DimensionChunkPageReaderV3
+ extends DimensionChunkReaderV3 {
/**
* end position of last dimension in carbon data file
*/
private long lastDimensionOffsets;
- public CompressedDimChunkFileBasedPageLevelReaderV3(BlockletInfo
blockletInfo,
+ public DimensionChunkPageReaderV3(BlockletInfo blockletInfo,
int[] eachColumnValueSize, String filePath) {
super(blockletInfo, eachColumnValueSize, filePath);
lastDimensionOffsets = blockletInfo.getDimensionOffset();
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkReaderV3.java
similarity index 98%
rename from
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
rename to
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkReaderV3.java
index 7c50bb7..56f8a74 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkReaderV3.java
@@ -30,7 +30,7 @@ import
org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractDimensionChunkReader;
import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
import
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
@@ -63,7 +63,7 @@ import org.apache.commons.lang.ArrayUtils;
* <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
* <File Footer>
*/
-public class CompressedDimensionChunkFileBasedReaderV3 extends
AbstractChunkReaderV2V3Format {
+public class DimensionChunkReaderV3 extends AbstractDimensionChunkReader {
private EncodingFactory encodingFactory =
DefaultEncodingFactory.getInstance();
@@ -72,7 +72,7 @@ public class CompressedDimensionChunkFileBasedReaderV3
extends AbstractChunkRead
*/
private long lastDimensionOffsets;
- public CompressedDimensionChunkFileBasedReaderV3(BlockletInfo blockletInfo,
+ public DimensionChunkReaderV3(BlockletInfo blockletInfo,
int[] eachColumnValueSize, String filePath) {
super(blockletInfo, eachColumnValueSize, filePath);
lastDimensionOffsets = blockletInfo.getDimensionOffset();
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
index bb60a81..364257a 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
@@ -18,15 +18,15 @@
package org.apache.carbondata.core.datastore.chunk.reader.measure;
import java.io.IOException;
+import java.util.List;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import
org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.datastore.compression.Compressor;
import
org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory;
import org.apache.carbondata.core.datastore.page.encoding.EncodingFactory;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
/**
* Measure block reader abstract class
@@ -40,27 +40,85 @@ public abstract class AbstractMeasureChunkReader implements
MeasureColumnChunkRe
* file path from which blocks will be read
*/
protected String filePath;
-
/**
- * number of rows for blocklet
+ * measure column chunks offset
+ */
+ protected List<Long> measureColumnChunkOffsets;
+ /**
+ * measure column chunks length
*/
- protected int numberOfRows;
+ protected List<Integer> measureColumnChunkLength;
/**
* Constructor to get minimum parameter to create instance of this class
*
* @param filePath file from which data will be read
*/
- public AbstractMeasureChunkReader(String filePath, int numberOfRows) {
+ public AbstractMeasureChunkReader(final BlockletInfo blockletInfo, String
filePath) {
this.filePath = filePath;
- this.numberOfRows = numberOfRows;
+ this.measureColumnChunkOffsets = blockletInfo.getMeasureChunkOffsets();
+ this.measureColumnChunkLength = blockletInfo.getMeasureChunksLength();
}
- @Override
- public void decodeColumnPageAndFillVector(MeasureRawColumnChunk
measureRawColumnChunk,
- int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer
reusableDataBuffer)
- throws IOException, MemoryException {
- throw new UnsupportedOperationException(
- "This operation is not supported in this class " +
getClass().getName());
+ /**
+ * Below method will be used to read the chunk based on block indexes
+ * Reading logic of below method is: Except last column all the column chunk
+ * can be read in group if not last column then read data of all the column
+ * present in block index together then process it. For last column read is
+ * separately and process
+ *
+ * @param fileReader file reader to read the blocks from file
+ * @param columnIndexRange blocks range to be read, columnIndexGroup[i] is
one group, inside the
+ * group, columnIndexGroup[i][0] is start column
index,
+ * and columnIndexGroup[i][1] is end column index
+ * @return measure column chunks
+ * @throws IOException
+ */
+ public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
+ // read the column chunk based on block index and add
+ MeasureRawColumnChunk[] dataChunks =
+ new MeasureRawColumnChunk[measureColumnChunkOffsets.size()];
+ if (columnIndexRange.length == 0) {
+ return dataChunks;
+ }
+ MeasureRawColumnChunk[] groupChunk = null;
+ int index = 0;
+ for (int i = 0; i < columnIndexRange.length - 1; i++) {
+ index = 0;
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
+ dataChunks[j] = groupChunk[index++];
+ }
+ }
+ if (columnIndexRange[columnIndexRange.length - 1][0] ==
measureColumnChunkOffsets.size() - 1) {
+ dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
+ readRawMeasureChunk(fileReader,
columnIndexRange[columnIndexRange.length - 1][0]);
+ } else {
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[columnIndexRange.length - 1][0],
+ columnIndexRange[columnIndexRange.length - 1][1]);
+ index = 0;
+ for (int j = columnIndexRange[columnIndexRange.length - 1][0];
+ j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
+ dataChunks[j] = groupChunk[index++];
+ }
+ }
+ return dataChunks;
}
+
+ /**
+ * Below method will be used to read measure chunk data in group.
+ * This method will be useful to avoid multiple IO while reading the
+ * data from
+ *
+ * @param fileReader file reader to read the data
+ * @param startColumnIndex first column index to be read
+ * @param endColumnIndex end column index to be read
+ * @return measure raw chunkArray
+ * @throws IOException
+ */
+ protected abstract MeasureRawColumnChunk[]
readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException;
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
deleted file mode 100644
index 4a610ec..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.measure;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.FileReader;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-
-/**
- * Abstract class for V2, V3 format measure column reader
- */
-public abstract class AbstractMeasureChunkReaderV2V3Format extends
AbstractMeasureChunkReader {
-
- /**
- * measure column chunks offset
- */
- protected List<Long> measureColumnChunkOffsets;
-
- /**
- * measure column chunks length
- */
- protected List<Integer> measureColumnChunkLength;
-
- public AbstractMeasureChunkReaderV2V3Format(final BlockletInfo blockletInfo,
- final String filePath) {
- super(filePath, blockletInfo.getNumberOfRows());
- this.measureColumnChunkOffsets = blockletInfo.getMeasureChunkOffsets();
- this.measureColumnChunkLength = blockletInfo.getMeasureChunksLength();
- }
-
- /**
- * Below method will be used to read the chunk based on block indexes
- * Reading logic of below method is: Except last column all the column chunk
- * can be read in group if not last column then read data of all the column
- * present in block index together then process it. For last column read is
- * separately and process
- *
- * @param fileReader file reader to read the blocks from file
- * @param columnIndexRange blocks range to be read, columnIndexGroup[i] is
one group, inside the
- * group, columnIndexGroup[i][0] is start column
index,
- * and columnIndexGroup[i][1] is end column index
- * @return measure column chunks
- * @throws IOException
- */
- public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
- int[][] columnIndexRange) throws IOException {
- // read the column chunk based on block index and add
- MeasureRawColumnChunk[] dataChunks =
- new MeasureRawColumnChunk[measureColumnChunkOffsets.size()];
- if (columnIndexRange.length == 0) {
- return dataChunks;
- }
- MeasureRawColumnChunk[] groupChunk = null;
- int index = 0;
- for (int i = 0; i < columnIndexRange.length - 1; i++) {
- index = 0;
- groupChunk = readRawMeasureChunksInGroup(
- fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
- for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
- dataChunks[j] = groupChunk[index++];
- }
- }
- if (columnIndexRange[columnIndexRange.length - 1][0] ==
measureColumnChunkOffsets.size() - 1) {
- dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
- readRawMeasureChunk(fileReader,
columnIndexRange[columnIndexRange.length - 1][0]);
- } else {
- groupChunk = readRawMeasureChunksInGroup(
- fileReader, columnIndexRange[columnIndexRange.length - 1][0],
- columnIndexRange[columnIndexRange.length - 1][1]);
- index = 0;
- for (int j = columnIndexRange[columnIndexRange.length - 1][0];
- j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
- dataChunks[j] = groupChunk[index++];
- }
- }
- return dataChunks;
- }
-
- /**
- * Below method will be used to read measure chunk data in group.
- * This method will be useful to avoid multiple IO while reading the
- * data from
- *
- * @param fileReader file reader to read the data
- * @param startColumnIndex first column index to be read
- * @param endColumnIndex end column index to be read
- * @return measure raw chunkArray
- * @throws IOException
- */
- protected abstract MeasureRawColumnChunk[]
readRawMeasureChunksInGroup(FileReader fileReader,
- int startColumnIndex, int endColumnIndex) throws IOException;
-
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
deleted file mode 100644
index 45b47c8..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.measure.v1;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.FileReader;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-
-/**
- * Compressed measure chunk reader
- */
-public class CompressedMeasureChunkFileBasedReaderV1 extends
AbstractMeasureChunkReader {
-
- /**
- * measure chunk have the information about the metadata present in the file
- */
- private final List<DataChunk> measureColumnChunks;
-
- /**
- * Constructor to get minimum parameter to create instance of this class
- *
- * @param blockletInfo BlockletInfo
- * @param filePath file from which data will be read
- */
- public CompressedMeasureChunkFileBasedReaderV1(final BlockletInfo
blockletInfo,
- final String filePath) {
- super(filePath, blockletInfo.getNumberOfRows());
- this.measureColumnChunks = blockletInfo.getMeasureColumnChunk();
- }
-
- /**
- * Method to read the blocks data based on block indexes
- *
- * @param fileReader file reader to read the blocks
- * @param columnIndexRange blocks to be read
- * @return measure data chunks
- */
- @Override
- public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
- int[][] columnIndexRange) throws IOException {
- MeasureRawColumnChunk[] datChunk = new
MeasureRawColumnChunk[measureColumnChunks.size()];
- for (int i = 0; i < columnIndexRange.length; i++) {
- for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
- datChunk[j] = readRawMeasureChunk(fileReader, j);
- }
- }
- return datChunk;
- }
-
- /**
- * Method to read the blocks data based on block index
- *
- * @param fileReader file reader to read the blocks
- * @param columnIndex column to be read
- * @return measure data chunk
- */
- @Override
- public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int
columnIndex)
- throws IOException {
- DataChunk dataChunk = measureColumnChunks.get(columnIndex);
- ByteBuffer buffer = fileReader
- .readByteBuffer(filePath, dataChunk.getDataPageOffset(),
dataChunk.getDataPageLength());
- MeasureRawColumnChunk rawColumnChunk = new
MeasureRawColumnChunk(columnIndex, buffer, 0,
- dataChunk.getDataPageLength(), this);
- rawColumnChunk.setFileReader(fileReader);
- rawColumnChunk.setPagesCount(1);
- rawColumnChunk.setRowCount(new int[] { numberOfRows });
- return rawColumnChunk;
- }
-
- @Override
- public ColumnPage decodeColumnPage(MeasureRawColumnChunk
measureRawColumnChunk, int pageNumber,
- ReusableDataBuffer reusableDataBuffer) throws IOException,
MemoryException {
- int blockIndex = measureRawColumnChunk.getColumnIndex();
- DataChunk dataChunk = measureColumnChunks.get(blockIndex);
- ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
- ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta,
- CompressorFactory.NativeSupportedCompressor.SNAPPY.getName());
- ColumnPage decodedPage =
codec.decode(measureRawColumnChunk.getRawData().array(),
- (int) measureRawColumnChunk.getOffSet(),
dataChunk.getDataPageLength());
- decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn());
-
- return decodedPage;
- }
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
deleted file mode 100644
index 834fd7f..0000000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.reader.measure.v2;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.FileReader;
-import org.apache.carbondata.core.datastore.ReusableDataBuffer;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.format.DataChunk2;
-
-/**
- * Class to read the measure column data for version 2
- */
-public class CompressedMeasureChunkFileBasedReaderV2 extends
AbstractMeasureChunkReaderV2V3Format {
-
- /**
- * Constructor to get minimum parameter to create instance of this class
- *
- * @param blockletInfo BlockletInfo
- * @param filePath file from which data will be read
- */
- public CompressedMeasureChunkFileBasedReaderV2(final BlockletInfo
blockletInfo,
- final String filePath) {
- super(blockletInfo, filePath);
- this.compressor =
CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor();
- }
-
- @Override
- public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int
columnIndex)
- throws IOException {
- int dataLength = 0;
- if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
- DataChunk2 metadataChunk = null;
- synchronized (fileReader) {
- metadataChunk = CarbonUtil.readDataChunk(ByteBuffer.wrap(fileReader
- .readByteArray(filePath,
measureColumnChunkOffsets.get(columnIndex),
- measureColumnChunkLength.get(columnIndex))), 0,
- measureColumnChunkLength.get(columnIndex));
- }
- dataLength = measureColumnChunkLength.get(columnIndex) +
metadataChunk.data_page_length;
- } else {
- long currentMeasureOffset = measureColumnChunkOffsets.get(columnIndex);
- dataLength = (int) (measureColumnChunkOffsets.get(columnIndex + 1) -
currentMeasureOffset);
- }
- ByteBuffer buffer = null;
- synchronized (fileReader) {
- buffer = fileReader
- .readByteBuffer(filePath,
measureColumnChunkOffsets.get(columnIndex), dataLength);
- }
- MeasureRawColumnChunk rawColumnChunk =
- new MeasureRawColumnChunk(columnIndex, buffer, 0, dataLength, this);
- rawColumnChunk.setFileReader(fileReader);
- rawColumnChunk.setPagesCount(1);
- rawColumnChunk.setRowCount(new int[] { numberOfRows });
- return rawColumnChunk;
- }
-
- /**
- * Below method will be used to read measure chunk data in group.
- * This method will be useful to avoid multiple IO while reading the
- * data from
- *
- * @param fileReader file reader to read the data
- * @param startColumnIndex first column blocklet index to be read
- * @param endColumnIndex end column blocklet index to be read
- * @return measure raw chunkArray
- * @throws IOException
- */
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader
fileReader,
- int startColumnIndex, int endColumnIndex) throws IOException {
- long currentMeasureOffset =
measureColumnChunkOffsets.get(startColumnIndex);
- ByteBuffer buffer = null;
- synchronized (fileReader) {
- buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset,
- (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) -
currentMeasureOffset));
- }
- MeasureRawColumnChunk[] dataChunks =
- new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
- int runningLength = 0;
- int index = 0;
- for (int i = startColumnIndex; i <= endColumnIndex; i++) {
- int currentLength =
- (int) (measureColumnChunkOffsets.get(i + 1) -
measureColumnChunkOffsets.get(i));
- MeasureRawColumnChunk measureRawColumnChunk =
- new MeasureRawColumnChunk(i, buffer, runningLength, currentLength,
this);
- measureRawColumnChunk.setRowCount(new int[] { numberOfRows });
- measureRawColumnChunk.setFileReader(fileReader);
- measureRawColumnChunk.setPagesCount(1);
- dataChunks[index] = measureRawColumnChunk;
- runningLength += currentLength;
- index++;
- }
- return dataChunks;
- }
-
- public ColumnPage decodeColumnPage(MeasureRawColumnChunk
measureRawColumnChunk,
- int pageNumber, ReusableDataBuffer reusableDataBuffer) throws
IOException, MemoryException {
- int copyPoint = (int) measureRawColumnChunk.getOffSet();
- int blockIndex = measureRawColumnChunk.getColumnIndex();
- ByteBuffer rawData = measureRawColumnChunk.getRawData();
- DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData,
copyPoint,
- measureColumnChunkLength.get(blockIndex));
- copyPoint += measureColumnChunkLength.get(blockIndex);
-
- ColumnPage page =
- decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint,
reusableDataBuffer);
- page.setNullBits(QueryUtil.getNullBitSet(measureColumnChunk.presence,
this.compressor));
- return page;
- }
-
- protected ColumnPage decodeMeasure(MeasureRawColumnChunk
measureRawColumnChunk,
- DataChunk2 measureColumnChunk, int copyPoint, ReusableDataBuffer
reusableDataBuffer)
- throws MemoryException, IOException {
- assert (measureColumnChunk.getEncoder_meta().size() > 0);
- List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
- byte[] encodedMeta = encoder_meta.get(0).array();
-
- ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta);
- ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta,
- CompressorFactory.NativeSupportedCompressor.SNAPPY.getName());
- byte[] rawData = measureRawColumnChunk.getRawData().array();
- return codec
- .decode(rawData, copyPoint, measureColumnChunk.data_page_length);
- }
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkPageReaderV3.java
similarity index 97%
rename from
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
rename to
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkPageReaderV3.java
index 5e5cc34..239f771 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkPageReaderV3.java
@@ -47,15 +47,15 @@ import org.apache.carbondata.format.DataChunk3;
* <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
* <File Footer>
*/
-public class CompressedMsrChunkFileBasedPageLevelReaderV3
- extends CompressedMeasureChunkFileBasedReaderV3 {
+public class MeasureChunkPageReaderV3
+ extends MeasureChunkReaderV3 {
/**
* end position of last measure in carbon data file
*/
private long measureOffsets;
- public CompressedMsrChunkFileBasedPageLevelReaderV3(BlockletInfo
blockletInfo, String filePath) {
+ public MeasureChunkPageReaderV3(BlockletInfo blockletInfo, String filePath) {
super(blockletInfo, filePath);
measureOffsets = blockletInfo.getMeasureOffsets();
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkReaderV3.java
similarity index 98%
rename from
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
rename to
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkReaderV3.java
index 6eb63f3..f2ca4fa 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkReaderV3.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import
org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
+import
org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
@@ -53,14 +53,14 @@ import org.apache.commons.lang.ArrayUtils;
* <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
* <File Footer>
*/
-public class CompressedMeasureChunkFileBasedReaderV3 extends
AbstractMeasureChunkReaderV2V3Format {
+public class MeasureChunkReaderV3 extends AbstractMeasureChunkReader {
/**
* end position of last measure in carbon data file
*/
private long measureOffsets;
- public CompressedMeasureChunkFileBasedReaderV3(BlockletInfo blockletInfo,
String filePath) {
+ public MeasureChunkReaderV3(BlockletInfo blockletInfo, String filePath) {
super(blockletInfo, filePath);
measureOffsets = blockletInfo.getMeasureOffsets();
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index d3070b4..cb39bb9 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -216,14 +216,6 @@ public abstract class EncodingFactory {
new ColumnPageEncoderMeta(spec, stats.getDataType(), stats,
compressor);
meta.setFillCompleteVector(fullVectorFill);
return new DirectCompressCodec(stats.getDataType()).createDecoder(meta);
- } else if (dataType == DataTypes.LEGACY_LONG) {
- // In case of older versions like in V1 format it has special datatype
to handle
- AdaptiveIntegralCodec adaptiveCodec =
- new AdaptiveIntegralCodec(DataTypes.LONG, DataTypes.LONG, stats,
false);
- ColumnPageEncoderMeta meta =
- new ColumnPageEncoderMeta(spec, adaptiveCodec.getTargetDataType(),
stats, compressor);
- meta.setFillCompleteVector(fullVectorFill);
- return adaptiveCodec.createDecoder(meta);
} else {
throw new RuntimeException("unsupported data type: " +
stats.getDataType());
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index e8ca5a0..ab885b3 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -105,8 +105,7 @@ public class PrimitivePageStatsCollector implements
ColumnPageStatsCollector, Si
} else if (dataType == DataTypes.INT) {
instance.minInt = (int) meta.getMinValue();
instance.maxInt = (int) meta.getMaxValue();
- } else if (dataType == DataTypes.LEGACY_LONG || dataType == DataTypes.LONG
- || dataType == DataTypes.TIMESTAMP) {
+ } else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
instance.minLong = (long) meta.getMinValue();
instance.maxLong = (long) meta.getMaxValue();
} else if (dataType == DataTypes.DOUBLE) {
@@ -142,8 +141,7 @@ public class PrimitivePageStatsCollector implements
ColumnPageStatsCollector, Si
} else if (dataType == DataTypes.INT) {
minInt = Integer.MAX_VALUE;
maxInt = Integer.MIN_VALUE;
- } else if (dataType == DataTypes.LEGACY_LONG || dataType == DataTypes.LONG
- || dataType == DataTypes.TIMESTAMP) {
+ } else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
minLong = Long.MAX_VALUE;
maxLong = Long.MIN_VALUE;
} else if (dataType == DataTypes.DOUBLE) {
diff --git
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
index 41cbfff..42dae6d 100644
---
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
+++
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.constants.CarbonVersionConstants;
import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
import org.apache.carbondata.core.datastore.DataRefNode;
@@ -59,27 +58,15 @@ public class BlockletDataRefNode implements DataRefNode {
detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
int[] pageRowCount = new int[detailInfo.getPagesCount()];
- int numberOfPagesCompletelyFilled = detailInfo.getRowCount();
+ int numberOfPagesCompletelyFilled = detailInfo.getRowCount() /
+
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+
// no. of rows to a page is 120000 in V2 and 32000 in V3, same is
handled to get the number
// of pages filled
- int lastPageRowCount;
- int fullyFilledRowsCount;
- if (blockInfo.getVersion() == ColumnarFormatVersion.V2
- || blockInfo.getVersion() == ColumnarFormatVersion.V1) {
- numberOfPagesCompletelyFilled /=
-
CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
- lastPageRowCount = detailInfo.getRowCount()
- %
CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
- fullyFilledRowsCount =
-
CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
- } else {
- numberOfPagesCompletelyFilled /=
-
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- lastPageRowCount = detailInfo.getRowCount()
- %
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- fullyFilledRowsCount =
-
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
+ int lastPageRowCount = detailInfo.getRowCount()
+ %
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ int fullyFilledRowsCount =
+
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
pageRowCount[i] = fullyFilledRowsCount;
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
deleted file mode 100644
index b79fa85..0000000
---
a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.keygenerator.mdkey;
-
-/**
- * It compresses the data as per max cardinality. It takes only the required
bits for each key.
- */
-public class NumberCompressor {
-
- /**
- * Bits MAX_LENGTH
- */
- private static final int MAX_LENGTH = 63;
-
- private static final int LONG_LENGTH = 64;
-
- private static final int BYTE_LENGTH = 8;
-
- /**
- * LONG_MAX.
- */
- private static final long LONG_MAX = 0x7fffffffffffffffL;
-
- private byte bitsLength;
-
- public NumberCompressor(int cardinality) {
- bitsLength = (byte) Long.toBinaryString(cardinality).length();
- }
-
- public byte[] compress(int[] keys) {
- int[] sizes = getWordsAndByteSize(keys.length);
- long[] words = get(keys, sizes[0]);
-
- return getByteValues(sizes, words);
- }
-
- private byte[] getByteValues(int[] sizes, long[] words) {
- byte[] bytes = new byte[sizes[1]];
-
- int l = sizes[1] - 1;
- for (int i = 0; i < words.length; i++) {
- long val = words[i];
-
- for (int j = BYTE_LENGTH - 1; j > 0 && l > 0; j--) {
- bytes[l] = (byte) val;
- val >>>= 8;
- l--;
- }
- bytes[l] = (byte) val;
- l--;
- }
- return bytes;
- }
-
- protected long[] get(int[] keys, int wsize) {
- long[] words = new long[wsize];
- int ll = 0;
- int index = 0;
- int pos = 0;
- int nextIndex = 0;
- for (int i = keys.length - 1; i >= 0; i--) {
-
- long val = keys[i];
-
- index = ll >> 6; // divide by 64 to get the new word index
- pos = ll & 0x3f; // to ignore sign bit and consider the remaining
- // val = val & controlBits;
- long mask = (val << pos);
- long word = words[index];
- words[index] = (word | mask);
- ll += bitsLength;
-
- nextIndex = ll >> 6; // This is divide by 64
-
- if (nextIndex != index) {
- int consideredBits = bitsLength - ll & 0x3f;
- // Check for spill over only if all the bits are not considered
- if (consideredBits < bitsLength) {
- // Check for spill over
- mask = (val >> (bitsLength - ll & 0x3f));
- words[nextIndex] |= mask;
- }
- }
-
- }
- return words;
- }
-
- public int[] unCompress(byte[] key, int offset, int length) {
- int arrayLength = (length * BYTE_LENGTH) / bitsLength;
- long[] words = new long[getWordsSizeFromBytesSize(length)];
- unCompressVal(key, length, words, offset);
- return getArray(words, arrayLength);
- }
-
- private void unCompressVal(byte[] key, int ls, long[] words, int offset) {
- for (int i = 0; i < words.length; i++) {
- long l = 0;
- ls -= BYTE_LENGTH;
- int m = 0;
- if (ls < 0) {
- m = ls + BYTE_LENGTH;
- ls = 0;
- } else {
- m = ls + BYTE_LENGTH;
- }
- for (int j = ls; j < m; j++) {
- l <<= BYTE_LENGTH;
- l ^= key[offset + j] & 0xFF;
- }
- words[i] = l;
- }
- }
-
- private int[] getArray(long[] words, int arrayLength) {
- int[] vals = new int[arrayLength];
- int ll = 0;
- long globalMask = LONG_MAX >>> (MAX_LENGTH - bitsLength);
- for (int i = arrayLength - 1; i >= 0; i--) {
-
- int index = ll >> 6;
- int pos = ll & 0x3f;
- long val = words[index];
- long mask = globalMask << pos;
- long value = (val & mask) >>> pos;
- ll += bitsLength;
-
- int nextIndex = ll >> 6;
- if (nextIndex != index) {
- pos = ll & 0x3f;
- // Number of bits pending for current key is zero, no spill over
- if (pos != 0) {
- mask = (LONG_MAX >>> (MAX_LENGTH - pos));
- val = words[nextIndex];
- value = value | ((val & mask) << (bitsLength - pos));
- }
- }
- vals[i] = (int) value;
- }
- return vals;
- }
-
- private int[] getWordsAndByteSize(int arrayLength) {
- int length = arrayLength * bitsLength;
- int wsize = length / LONG_LENGTH;
- int byteSize = length / BYTE_LENGTH;
-
- if (length % LONG_LENGTH != 0) {
- wsize++;
- }
-
- if (length % BYTE_LENGTH != 0) {
- byteSize++;
- }
- return new int[] { wsize, byteSize };
- }
-
- private int getWordsSizeFromBytesSize(int byteSize) {
- int wsize = byteSize / BYTE_LENGTH;
- if (byteSize % BYTE_LENGTH != 0) {
- wsize++;
- }
- return wsize;
- }
-
-}
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
index 9ddefd6..6ee4169 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
@@ -18,8 +18,8 @@
package org.apache.carbondata.core.metadata;
public enum ColumnarFormatVersion {
- V1((short)1),
- V2((short)2),
+ V1((short)1), // deprecated
+ V2((short)2), // deprecated
V3((short)3);
private short version;
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index aebae47..ac828a1 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -17,13 +17,9 @@
package org.apache.carbondata.core.metadata.blocklet;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@@ -31,7 +27,6 @@ import java.util.List;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-import org.apache.commons.io.input.ClassLoaderObjectInputStream;
import org.apache.hadoop.io.Writable;
/**
@@ -224,7 +219,6 @@ public class BlockletInfo implements Serializable, Writable
{
for (int i = 0; i < mSize; i++) {
output.writeInt(measureChunksLength.get(i));
}
- writeChunkInfoForOlderVersions(output);
boolean isSortedPresent = (isSorted != null);
output.writeBoolean(isSortedPresent);
@@ -242,48 +236,6 @@ public class BlockletInfo implements Serializable,
Writable {
}
}
- /**
- * Serialize datachunks as well for older versions like V1 and V2
- */
- private void writeChunkInfoForOlderVersions(DataOutput output) throws
IOException {
- int dimChunksSize = dimensionColumnChunk != null ?
dimensionColumnChunk.size() : 0;
- output.writeShort(dimChunksSize);
- for (int i = 0; i < dimChunksSize; i++) {
- byte[] bytes = serializeDataChunk(dimensionColumnChunk.get(i));
- output.writeInt(bytes.length);
- output.write(bytes);
- }
- int msrChunksSize = measureColumnChunk != null ? measureColumnChunk.size()
: 0;
- output.writeShort(msrChunksSize);
- for (int i = 0; i < msrChunksSize; i++) {
- byte[] bytes = serializeDataChunk(measureColumnChunk.get(i));
- output.writeInt(bytes.length);
- output.write(bytes);
- }
- }
-
- private byte[] serializeDataChunk(DataChunk chunk) throws IOException {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- ObjectOutputStream outputStream = new ObjectOutputStream(stream);
- outputStream.writeObject(chunk);
- outputStream.close();
- return stream.toByteArray();
- }
-
- private DataChunk deserializeDataChunk(byte[] bytes) throws IOException {
- ByteArrayInputStream stream = new ByteArrayInputStream(bytes);
- ObjectInputStream inputStream =
- new
ClassLoaderObjectInputStream(Thread.currentThread().getContextClassLoader(),
stream);
- DataChunk dataChunk = null;
- try {
- dataChunk = (DataChunk) inputStream.readObject();
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- inputStream.close();
- return dataChunk;
- }
-
@Override
public void readFields(DataInput input) throws IOException {
dimensionOffset = input.readLong();
@@ -307,7 +259,6 @@ public class BlockletInfo implements Serializable, Writable
{
for (int i = 0; i < measureChunkOffsetsSize; i++) {
measureChunksLength.add(input.readInt());
}
- readChunkInfoForOlderVersions(input);
final boolean isSortedPresent = input.readBoolean();
if (isSortedPresent) {
this.isSorted = input.readBoolean();
@@ -322,26 +273,6 @@ public class BlockletInfo implements Serializable,
Writable {
}
}
- /**
- * Deserialize datachunks as well for older versions like V1 and V2
- */
- private void readChunkInfoForOlderVersions(DataInput input) throws
IOException {
- short dimChunksSize = input.readShort();
- dimensionColumnChunk = new ArrayList<>(dimChunksSize);
- for (int i = 0; i < dimChunksSize; i++) {
- byte[] bytes = new byte[input.readInt()];
- input.readFully(bytes);
- dimensionColumnChunk.add(deserializeDataChunk(bytes));
- }
- short msrChunksSize = input.readShort();
- measureColumnChunk = new ArrayList<>(msrChunksSize);
- for (int i = 0; i < msrChunksSize; i++) {
- byte[] bytes = new byte[input.readInt()];
- input.readFully(bytes);
- measureColumnChunk.add(deserializeDataChunk(bytes));
- }
- }
-
public int[] getNumberOfRowsPerPage() {
return numberOfRowsPerPage;
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index 4f282e3..2a5be78 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -112,8 +112,6 @@ public class DataType implements Serializable {
return DataTypes.DOUBLE;
case BIG_DECIMAL_MEASURE_CHAR:
return DataTypes.createDefaultDecimalType();
- case 'l':
- return DataTypes.LEGACY_LONG;
default:
throw new RuntimeException("Unexpected type: " + type);
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index c073fa0..d4ad8d0 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -45,9 +45,6 @@ public class DataTypes {
// internal use only, for value compression from integer/long to 3 bytes
value
public static final DataType SHORT_INT = ShortIntType.SHORT_INT;
- // Only for internal use for backward compatability. It is only used for V1
version
- public static final DataType LEGACY_LONG = LegacyLongType.LEGACY_LONG;
-
public static final DataType VARCHAR = VarcharType.VARCHAR;
// these IDs are used within this package only
@@ -94,8 +91,6 @@ public class DataTypes {
return INT;
} else if (id == LONG.getId()) {
return LONG;
- } else if (id == LEGACY_LONG.getId()) {
- return LEGACY_LONG;
} else if (id == FLOAT.getId()) {
return FLOAT;
} else if (id == DOUBLE.getId()) {
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LegacyLongType.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LegacyLongType.java
deleted file mode 100644
index 610f2cc..0000000
---
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LegacyLongType.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.metadata.datatype;
-
-class LegacyLongType extends DataType {
-
- static final DataType LEGACY_LONG =
- new LegacyLongType(DataTypes.LEGACY_LONG_TYPE_ID, 15, "LEGACYBIGINT", 8);
-
- private LegacyLongType(int id, int precedenceOrder, String name, int
sizeInBytes) {
- super(id, precedenceOrder, name, sizeInBytes);
- }
-
- // this function is needed to ensure singleton pattern while supporting java
serialization
- private Object readResolve() {
- return DataTypes.LEGACY_LONG;
- }
-}
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8a41594..b3c7093 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -69,7 +69,6 @@ import
org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import
org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
-import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import
org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator;
import
org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
import org.apache.carbondata.core.locks.ICarbonLock;
@@ -533,20 +532,6 @@ public final class CarbonUtil {
return ++currentIndex;
}
- public static int[] getUnCompressColumnIndex(int totalLength, byte[]
columnIndexData,
- NumberCompressor numberCompressor, int offset) {
- ByteBuffer buffer = ByteBuffer.wrap(columnIndexData, offset, totalLength);
- int indexDataLength = buffer.getInt();
- byte[] indexData = new byte[indexDataLength];
- byte[] indexMap =
- new byte[totalLength - indexDataLength -
CarbonCommonConstants.INT_SIZE_IN_BYTE];
- buffer.get(indexData);
- buffer.get(indexMap);
- return UnBlockIndexer
- .uncompressIndex(numberCompressor.unCompress(indexData, 0,
indexData.length),
- numberCompressor.unCompress(indexMap, 0, indexMap.length));
- }
-
public static int[] getUnCompressColumnIndex(int totalLength, ByteBuffer
buffer, int offset) {
buffer.position(offset);
int indexDataLength = buffer.getInt();
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
index aeb7c99..7b3af05 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
@@ -55,13 +55,10 @@ public class DataFileFooterConverterFactory {
public AbstractDataFileFooterConverter getDataFileFooterConverter(
final ColumnarFormatVersion version) {
switch (version) {
- case V1:
- return new DataFileFooterConverter();
- case V2:
- return new DataFileFooterConverter2();
case V3:
- default:
return new DataFileFooterConverterV3();
+ default:
+ throw new UnsupportedOperationException("Unsupported file version: " +
version);
}
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 5471420..8422c78 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -1014,8 +1014,6 @@ public final class DataTypeUtil {
return DataTypes.INT;
} else if (DataTypes.LONG.getName().equalsIgnoreCase(name)) {
return DataTypes.LONG;
- } else if (DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(name)) {
- return DataTypes.LEGACY_LONG;
} else if (DataTypes.FLOAT.getName().equalsIgnoreCase(name)) {
return DataTypes.FLOAT;
} else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(name)) {
@@ -1064,8 +1062,6 @@ public final class DataTypeUtil {
return DataTypes.INT;
} else if (DataTypes.LONG.getName().equalsIgnoreCase(dataType.getName())) {
return DataTypes.LONG;
- } else if
(DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(dataType.getName())) {
- return DataTypes.LEGACY_LONG;
} else if (DataTypes.FLOAT.getName().equalsIgnoreCase(dataType.getName()))
{
return DataTypes.FLOAT;
} else if
(DataTypes.DOUBLE.getName().equalsIgnoreCase(dataType.getName())) {
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 0e0aac2..5e42c56 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -249,13 +249,13 @@ public class CarbonTablePath {
public static String getCarbonIndexFilePath(String tablePath, String taskId,
String segmentId,
String bucketNumber, String timeStamp, ColumnarFormatVersion
columnarFormatVersion) {
switch (columnarFormatVersion) {
- case V1:
- case V2:
- return getCarbonIndexFilePath(tablePath, taskId, segmentId,
bucketNumber);
- default:
+ case V3:
String segmentDir = getSegmentPath(tablePath, segmentId);
return segmentDir + CarbonCommonConstants.FILE_SEPARATOR +
getCarbonIndexFileName(taskId,
Integer.parseInt(bucketNumber), timeStamp, segmentId);
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported file version: " + columnarFormatVersion);
}
}
diff --git
a/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressorUnitTest.java
b/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressorUnitTest.java
deleted file mode 100644
index b64aa44..0000000
---
a/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressorUnitTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.keygenerator.mdkey;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
-
-import org.junit.Test;
-
-public class NumberCompressorUnitTest {
-
- private NumberCompressor numberCompressor;
-
- @Test public void testCompress() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- byte[] expected_result = new byte[] { 2, 86, 115 };
- int[] keys = new int[] { 2, 5, 6, 7, 3 };
- byte[] result = numberCompressor.compress(keys);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testGetWithIntKeysAndSameIndexes() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- long[] expected_result = new long[] { 153203, 0 };
- int[] keys = new int[] { 2, 5, 6, 7, 3 };
- int size = 2;
- long[] result = numberCompressor.get(keys, size);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testGetWithIntKeysAndDifferentIndexes() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- long[] expected_result = new long[] { 2695178248884938548L, 0 };
- int[] keys = new int[] { 2, 5, 6, 7, 3, 2, 5, 6, 7, 3, 2, 5, 6, 7, 3, 4 };
- int size = 2;
- long[] result = numberCompressor.get(keys, size);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void
testGetWithIntKeysAndDifferentIndexesAndConsideredBitsLessThanBitsLength()
- throws Exception {
- int cardinality = 1000;
- numberCompressor = new NumberCompressor(cardinality);
- long[] expected_result = new long[] { 2311479113337014277L, 0 };
- int[] keys = new int[] { 2, 5, 6, 7, 3, 2, 5 };
- int size = 2;
- long[] result = numberCompressor.get(keys, size);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testUnCompress() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- int[] expected_result = new int[] { 6, 4, 0, 2 };
- byte[] keys = new byte[] { 100, 2 };
- int[] result = numberCompressor.unCompress(keys, 0, keys.length);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testUnCompressWithTenKeys() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- int[] expected_result =
- new int[] { 0, 10, 0, 2, 0, 10, 0, 2, 0, 10, 0, 2, 0, 10, 0, 2, 0, 10,
0, 2 };
- byte[] keys = new byte[] { 10, 2, 10, 2, 10, 2, 10, 2, 10, 2 };
- int[] result = numberCompressor.unCompress(keys, 0, keys.length);
- System.out.println(result);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testUnCompressWithPOSGreaterThanZero() throws Exception {
- int cardinality = 100;
- numberCompressor = new NumberCompressor(cardinality);
- int[] expected_result = new int[] { 16, 4, 10, 1, 2, 64, 32, 80, 8, 20, 11
};
- byte[] keys = new byte[] { 100, 2, 10, 2, 10, 2, 10, 2, 10, 11 };
- int[] result = numberCompressor.unCompress(keys, 0, keys.length);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testCompressWithWordsSizeFromBytesSize() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- byte[] expected_result = new byte[] { 2, 86, 115, 37, 103, 50, 86, 115, 86
};
- int[] keys = new int[] { 2, 5, 6, 7, 3, 2, 5, 6, 7, 3, 2, 5, 6, 7, 3, 5, 6
};
- byte[] result = numberCompressor.compress(keys);
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testCompressWithIntMaxValue() throws Exception {
- int cardinality = 10;
- numberCompressor = new NumberCompressor(cardinality);
- byte[] expected_result = new byte[] { -35, -52, -52 };
- int[] keys = new int[] { 214748364, 5456, 214748364, 214748364, 214748364
};
- byte[] result = numberCompressor.compress(keys);
- assertThat(result, is(equalTo(expected_result)));
- }
-}
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/CarbonTestUtil.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonTestUtil.java
index 3812c01..6080d7e 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonTestUtil.java
@@ -28,9 +28,6 @@ public class CarbonTestUtil {
CarbonProperties.getInstance().getFormatVersion();
switch (version) {
- case V1:
- case V2:
- return new ValueEncoderMeta();
case V3:
return new ColumnPageEncoderMeta();
default:
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index c03e32d..baf4149 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -568,25 +568,25 @@ public class CarbonUtilTest {
}
@Test public void testToReadMetadataFile() throws IOException {
- new MockUp<DataFileFooterConverter>() {
+ new MockUp<DataFileFooterConverterV3>() {
@SuppressWarnings("unused") @Mock
public DataFileFooter readDataFileFooter(TableBlockInfo info) {
DataFileFooter fileFooter = new DataFileFooter();
- fileFooter.setVersionId(ColumnarFormatVersion.V1);
+ fileFooter.setVersionId(ColumnarFormatVersion.V3);
return fileFooter;
}
};
TableBlockInfo info =
- new TableBlockInfo("file:/", 1, "0", new String[0], 1,
ColumnarFormatVersion.V1, null);
+ new TableBlockInfo("file:/", 1, "0", new String[0], 1,
ColumnarFormatVersion.V3, null);
- assertEquals(CarbonUtil.readMetadataFile(info).getVersionId().number(), 1);
+ assertEquals(CarbonUtil.readMetadataFile(info).getVersionId().number(), 3);
}
@Test(expected = IOException.class)
public void testToReadMetadataFileWithException()
throws Exception {
TableBlockInfo info =
- new TableBlockInfo("file:/", 1, "0", new String[0], 1,
ColumnarFormatVersion.V1, null);
+ new TableBlockInfo("file:/", 1, "0", new String[0], 1,
ColumnarFormatVersion.V3, null);
CarbonUtil.readMetadataFile(info);
}
diff --git
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
deleted file mode 100644
index 900c658..0000000
---
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.cluster.sdv.generated
-
-import org.apache.spark.sql.common.util.QueryTest
-import org.apache.spark.sql.test.TestQueryExecutor
-import org.apache.spark.sql.{CarbonEnv, CarbonSession, Row, SparkSession}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * V1 to V3 compatability test. This test has to be at last
- */
-class CarbonV1toV3CompatabilityTestCase extends QueryTest with
BeforeAndAfterAll {
-
- var localspark: CarbonSession = null
- val storeLocation = s"${
- TestQueryExecutor
- .integrationPath
- }/spark-common-test/src/test/resources/Data/v1_version/store"
- val metaLocation = s"${
- TestQueryExecutor
- .integrationPath
- }/spark-common-test/src/test/resources/Data/v1_version"
-
- override def beforeAll {
- sqlContext.sparkSession.stop()
- CarbonEnv.carbonEnvMap.clear()
-
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION,
storeLocation)
- import org.apache.spark.sql.CarbonSession._
- println(s"store path for CarbonV1toV3CompatabilityTestCase is
$storeLocation and metastore is" +
- s" $metaLocation")
- localspark = SparkSession
- .builder()
- .master("local")
- .appName("CarbonV1toV3CompatabilityTestCase")
- .config("spark.driver.host", "localhost")
- .getOrCreateCarbonSession(storeLocation,
metaLocation).asInstanceOf[CarbonSession]
- println("store path : " + CarbonProperties.getStorePath)
- localspark.sparkContext.setLogLevel("WARN")
- hiveClient.runSqlHive(
- s"ALTER TABLE default.t3 SET SERDEPROPERTIES" +
- s"('tablePath'='$storeLocation/default/t3', 'dbname'='default',
'tablename'='t3')")
- localspark.sql("show tables").show()
- }
-
- test("test v1 to v3 compatabilty reading all with out fail") {
- val length = localspark.sql("select
ID,date,country,name,phonetype,serialname,salary from t3")
- .collect().length
- assert(length == 1000)
- }
-
- test("test v1 to v3 compatabilty groupy by query") {
- val dataFrame = localspark
- .sql(s"SELECT country, count(salary) AS amount FROM t3 WHERE country IN
('china','france') " +
- s"GROUP BY country")
- checkAnswer(dataFrame, Seq(Row("china", 849), Row("france", 101)))
- }
-
- test("test v1 to v3 compatabilty filter on measure with int measure") {
- val dataFrame = localspark
- .sql(s"SELECT sum(salary) FROM t3 where salary > 15408")
- checkAnswer(dataFrame, Seq(Row(9281064)))
- }
-
- test("test v1 to v3 compatabilty filter on measure with decimal dimension") {
- val dataFrame = localspark
- .sql(s"SELECT sum(salary2) FROM t3 where salary2 > 15408")
- checkAnswer(dataFrame, Seq(Row(9281064)))
- }
-
- test("test v1 to v3 compatabilty filter on measure with double dimension") {
- val dataFrame = localspark
- .sql(s"SELECT sum(salary1) FROM t3 where salary1 > 15408")
- checkAnswer(dataFrame, Seq(Row(9281064)))
- }
-
- override def afterAll {
- localspark.stop()
- }
-}
diff --git
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
index a1c6887..02d4a29 100644
---
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
+++
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
@@ -29,7 +29,7 @@ import
org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.TableBlockInfo
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk
import
org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.DimensionChunkReaderV3
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile,
CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -278,7 +278,7 @@ class LoadTableWithLocalDictionaryTestCase extends
QueryTest with BeforeAndAfter
blockletInfo,
dataFileFooter.getSegmentInfo.getColumnCardinality,
carbonDataFiles.getAbsolutePath,
- false).asInstanceOf[CompressedDimensionChunkFileBasedReaderV3]
+ false).asInstanceOf[DimensionChunkReaderV3]
dimensionRawColumnChunks
.add(dimensionColumnChunkReader.readRawDimensionChunk(fileReader,
blockIndex))
}
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index d8a8825..3c441bf 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -45,7 +45,7 @@ import
org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.TableBlockInfo
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk
import
org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.DimensionChunkReaderV3
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile,
CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -2674,7 +2674,7 @@ object testUtil{
blockletInfo,
dataFileFooter.getSegmentInfo.getColumnCardinality,
carbonDataFiles.getAbsolutePath,
- false).asInstanceOf[CompressedDimensionChunkFileBasedReaderV3]
+ false).asInstanceOf[DimensionChunkReaderV3]
dimensionRawColumnChunks
.add(dimensionColumnChunkReader.readRawDimensionChunk(fileReader,
blockIndex))
}
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
index eb88c36..8712316 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
@@ -30,7 +30,7 @@ import
org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.TableBlockInfo
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk
import
org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
-import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3
+import
org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.DimensionChunkReaderV3
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile,
CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -294,7 +294,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest
with BeforeAndAfterA
blockletInfo,
dataFileFooter.getSegmentInfo.getColumnCardinality,
carbonDataFiles.getAbsolutePath,
- false).asInstanceOf[CompressedDimensionChunkFileBasedReaderV3]
+ false).asInstanceOf[DimensionChunkReaderV3]
dimensionRawColumnChunks
.add(dimensionColumnChunkReader.readRawDimensionChunk(fileReader,
blockIndex))
}
diff --git
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index f385606..2b80985 100644
---
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -1013,7 +1013,7 @@ object CarbonDataRDDFactory {
val fileSplit = inputSplit.asInstanceOf[FileSplit]
new TableBlockInfo(fileSplit.getPath.toString,
fileSplit.getStart, "1",
- fileSplit.getLocations, fileSplit.getLength, ColumnarFormatVersion.V1,
null
+ fileSplit.getLocations, fileSplit.getLength, ColumnarFormatVersion.V3,
null
).asInstanceOf[Distributable]
}
// group blocks to nodes, tasks
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
index 0adabbe..3f74d91 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
@@ -58,13 +58,10 @@ class CarbonDataWriterFactory {
public CarbonFactDataWriter getFactDataWriter(final ColumnarFormatVersion
version,
final CarbonFactDataHandlerModel model) {
switch (version) {
- case V1:
- case V2:
- throw new UnsupportedOperationException("V1 and V2 CarbonData Writer
is not supported");
case V3:
return new CarbonFactDataWriterImplV3(model);
default:
- return new CarbonFactDataWriterImplV3(model);
+ throw new UnsupportedOperationException("V1 and V2 CarbonData Writer
is not supported");
}
}
