http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java deleted file mode 100644 index 6629d31..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datastore.chunk.impl; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; -import org.apache.carbondata.core.metadata.datatype.DataType; -import org.apache.carbondata.core.metadata.datatype.DataTypes; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; -import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; - -/** - * This class is gives access to fixed length dimension data chunk store - */ -public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk { - - /** - * Constructor - * - * @param dataChunk data chunk - * @param invertedIndex inverted index - * @param invertedIndexReverse reverse inverted index - * @param numberOfRows number of rows - * @param columnValueSize size of each column value - */ - public FixedLengthDimensionDataChunk(byte[] dataChunk, int[] invertedIndex, - int[] invertedIndexReverse, int numberOfRows, int columnValueSize) { - long totalSize = null != invertedIndex ? - dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) : - dataChunk.length; - dataChunkStore = DimensionChunkStoreFactory.INSTANCE - .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize, - DimensionStoreType.FIXEDLENGTH); - dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk); - } - - /** - * Below method will be used to fill the data based on offset and row id - * - * @param data data to filed - * @param offset offset from which data need to be filed - * @param index row id of the chunk - * @param keyStructureInfo define the structure of the key - * @return how many bytes was copied - */ - @Override public int fillChunkData(byte[] data, int offset, int index, - KeyStructureInfo keyStructureInfo) { - dataChunkStore.fillRow(index, data, offset); - return dataChunkStore.getColumnValueSize(); - } - - /** - * Converts to column dictionary integer value - * - * @param rowId - * @param columnIndex - * @param row - * @param restructuringInfo - * @return - */ - @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row, - KeyStructureInfo restructuringInfo) { - row[columnIndex] = dataChunkStore.getSurrogate(rowId); - return columnIndex + 1; - } - - /** - * Fill the data to vector - * - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column, - KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = columnVectorInfo.size + offset; - CarbonColumnVector vector = columnVectorInfo.vector; - for (int j = offset; j < len; j++) { - int dict = dataChunkStore.getSurrogate(j); - if (columnVectorInfo.directDictionaryGenerator == null) { - vector.putInt(vectorOffset++, dict); - } else { - Object valueFromSurrogate = - columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict); - if (valueFromSurrogate == null) { - vector.putNull(vectorOffset++); - } else { - DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType(); - if (dataType == DataTypes.INT) { - vector.putInt(vectorOffset++, (int) valueFromSurrogate); - } else if (dataType == DataTypes.LONG) { - vector.putLong(vectorOffset++, (long) valueFromSurrogate); - } else { - throw new IllegalArgumentException("unsupported data type: " + - columnVectorInfo.directDictionaryGenerator.getReturnType()); - } - } - } - } - return column + 1; - } - - /** - * Fill the data to vector - * - * @param rowMapping - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, - int column, KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = columnVectorInfo.size + offset; - CarbonColumnVector vector = columnVectorInfo.vector; - for (int j = offset; j < len; j++) { - int dict = dataChunkStore.getSurrogate(rowMapping[j]); - if (columnVectorInfo.directDictionaryGenerator == null) { - vector.putInt(vectorOffset++, dict); - } else { - Object valueFromSurrogate = - columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict); - if (valueFromSurrogate == null) { - vector.putNull(vectorOffset++); - } else { - DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType(); - if (dataType == DataTypes.INT) { - vector.putInt(vectorOffset++, (int) valueFromSurrogate); - } else if (dataType == DataTypes.LONG) { - vector.putLong(vectorOffset++, (long) valueFromSurrogate); - } else { - throw new IllegalArgumentException("unsupported data type: " + - columnVectorInfo.directDictionaryGenerator.getReturnType()); - } - } - } - } - return column + 1; - } -}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java index fa0777b..ebca6a5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java @@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.impl; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -28,7 +28,7 @@ import org.apache.carbondata.core.memory.MemoryException; /** * Contains raw measure data * 1. The read uncompressed raw data of column chunk with all pages is stored in this instance. - * 2. The raw data can be converted to processed chunk using convertToColumnPage method + * 2. The raw data can be converted to processed chunk using decodeColumnPage method * by specifying page number. */ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { @@ -37,25 +37,28 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { private MeasureColumnChunkReader chunkReader; - private FileHolder fileReader; + // TODO: this is a hack for CompressedMsrChunkFileBasedPageLevelReaderV3, refactor it + private FileReader fileReader; - public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length, + public MeasureRawColumnChunk(FileReader fileReader, + int columnIndex, ByteBuffer rawData, long offSet, int length, MeasureColumnChunkReader chunkReader) { super(columnIndex, rawData, offSet, length); + this.fileReader = fileReader; this.chunkReader = chunkReader; } /** * Convert all raw data with all pages to processed ColumnPage */ - public ColumnPage[] convertToColumnPage() { + public ColumnPage[] decodeAllColumnPages() { if (columnPages == null) { columnPages = new ColumnPage[pagesCount]; } for (int i = 0; i < pagesCount; i++) { try { if (columnPages[i] == null) { - columnPages[i] = chunkReader.convertToColumnPage(this, i); + columnPages[i] = chunkReader.decodeColumnPage(this, i); } } catch (Exception e) { throw new RuntimeException(e); @@ -68,21 +71,21 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { /** * Convert raw data with specified `columnIndex` processed to ColumnPage */ - public ColumnPage convertToColumnPage(int columnIndex) { - assert columnIndex < pagesCount; + public ColumnPage decodeColumnPage(int pageNumber) { + assert pageNumber < pagesCount; if (columnPages == null) { columnPages = new ColumnPage[pagesCount]; } try { - if (columnPages[columnIndex] == null) { - columnPages[columnIndex] = chunkReader.convertToColumnPage(this, columnIndex); + if (columnPages[pageNumber] == null) { + columnPages[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber); } } catch (IOException | MemoryException e) { throw new RuntimeException(e); } - return columnPages[columnIndex]; + return columnPages[pageNumber]; } /** @@ -95,7 +98,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { assert index < pagesCount; try { - return chunkReader.convertToColumnPage(this, index); + return chunkReader.decodeColumnPage(this, index); } catch (IOException | MemoryException e) { throw new RuntimeException(e); } @@ -111,11 +114,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { } } - public void setFileReader(FileHolder fileReader) { - this.fileReader = fileReader; - } - - public FileHolder getFileReader() { + public FileReader getFileReader() { return fileReader; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java new file mode 100644 index 0000000..d03b2de --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datastore.chunk.impl; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; +import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; +import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; +import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; + +/** + * This class is gives access to variable length dimension data chunk store + */ +public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPage { + + /** + * Constructor for this class + * @param dataChunks + * @param invertedIndex + * @param invertedIndexReverse + * @param numberOfRows + */ + public VariableLengthDimensionColumnPage(byte[] dataChunks, int[] invertedIndex, + int[] invertedIndexReverse, int numberOfRows) { + long totalSize = null != invertedIndex ? + (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + ( + numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) : + (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + dataChunkStore = DimensionChunkStoreFactory.INSTANCE + .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize, + DimensionStoreType.VARIABLELENGTH); + dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks); + } + + /** + * Below method will be used to fill the data based on offset and row id + * + * @param rowId row id of the chunk + * @param offset offset from which data need to be filed + * @param data data to filed + * @param restructuringInfo define the structure of the key + * @return how many bytes was copied + */ + @Override public int fillRawData(int rowId, int offset, byte[] data, + KeyStructureInfo restructuringInfo) { + // no required in this case because this column chunk is not the part if + // mdkey + return 0; + } + + /** + * Converts to column dictionary integer value + * + * @param rowId + * @param chunkIndex + * @param outputSurrogateKey + * @param restructuringInfo + * @return + */ + @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey, + KeyStructureInfo restructuringInfo) { + return chunkIndex + 1; + } + + /** + * @return whether column is dictionary column or not + */ + @Override public boolean isNoDicitionaryColumn() { + return true; + } + + /** + * Fill the data to vector + * + * @param vectorInfo + * @param chunkIndex + * @param restructuringInfo + * @return next column index + */ + @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex, + KeyStructureInfo restructuringInfo) { + ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex]; + CarbonColumnVector vector = columnVectorInfo.vector; + int offset = columnVectorInfo.offset; + int vectorOffset = columnVectorInfo.vectorOffset; + int len = offset + columnVectorInfo.size; + for (int i = offset; i < len; i++) { + // Considering only String case now as we support only + // string in no dictionary case at present. + dataChunkStore.fillRow(i, vector, vectorOffset++); + } + return chunkIndex + 1; + } + + /** + * Fill the data to vector + * + * @param filteredRowId + * @param vectorInfo + * @param chunkIndex + * @param restructuringInfo + * @return next column index + */ + @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, + int chunkIndex, KeyStructureInfo restructuringInfo) { + ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex]; + CarbonColumnVector vector = columnVectorInfo.vector; + int offset = columnVectorInfo.offset; + int vectorOffset = columnVectorInfo.vectorOffset; + int len = offset + columnVectorInfo.size; + for (int i = offset; i < len; i++) { + // Considering only String case now as we support only + // string in no dictionary case at present. + dataChunkStore.fillRow(filteredRowId[i], vector, vectorOffset++); + } + return chunkIndex + 1; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java deleted file mode 100644 index 6c47bf5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datastore.chunk.impl; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; -import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; - -/** - * This class is gives access to variable length dimension data chunk store - */ -public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk { - - /** - * Constructor for this class - * @param dataChunks - * @param invertedIndex - * @param invertedIndexReverse - * @param numberOfRows - */ - public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex, - int[] invertedIndexReverse, int numberOfRows) { - long totalSize = null != invertedIndex ? - (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + ( - numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) : - (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - dataChunkStore = DimensionChunkStoreFactory.INSTANCE - .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize, - DimensionStoreType.VARIABLELENGTH); - dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks); - } - - /** - * Below method will be used to fill the data based on offset and row id - * - * @param data data to filed - * @param offset offset from which data need to be filed - * @param index row id of the chunk - * @param restructuringInfo define the structure of the key - * @return how many bytes was copied - */ - @Override public int fillChunkData(byte[] data, int offset, int index, - KeyStructureInfo restructuringInfo) { - // no required in this case because this column chunk is not the part if - // mdkey - return 0; - } - - /** - * Converts to column dictionary integer value - * - * @param rowId - * @param columnIndex - * @param row - * @param restructuringInfo - * @return - */ - @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row, - KeyStructureInfo restructuringInfo) { - return columnIndex + 1; - } - - /** - * @return whether column is dictionary column or not - */ - @Override public boolean isNoDicitionaryColumn() { - return true; - } - - /** - * @return length of each column - */ - @Override public int getColumnValueSize() { - return -1; - } - - /** - * Fill the data to vector - * - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column, - KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - CarbonColumnVector vector = columnVectorInfo.vector; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = offset + columnVectorInfo.size; - for (int i = offset; i < len; i++) { - // Considering only String case now as we support only - // string in no dictionary case at present. - dataChunkStore.fillRow(i, vector, vectorOffset++); - } - return column + 1; - } - - /** - * Fill the data to vector - * - * @param rowMapping - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, - int column, KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - CarbonColumnVector vector = columnVectorInfo.vector; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = offset + columnVectorInfo.size; - for (int i = offset; i < len; i++) { - // Considering only String case now as we support only - // string in no dictionary case at present. - dataChunkStore.fillRow(rowMapping[i], vector, vectorOffset++); - } - return column + 1; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/ColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/ColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/ColumnChunkReader.java new file mode 100644 index 0000000..f18faa4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/ColumnChunkReader.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datastore.chunk.reader; + +public abstract class ColumnChunkReader { + /** + * file path from which blocks will be read + */ + protected String filePath; + + /** + * number of rows for blocklet + */ + protected int numberOfRows; + + public ColumnChunkReader(String filePath, int numberOfRows) { + this.filePath = filePath; + this.numberOfRows = numberOfRows; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java index 7b5b9c8..1e637f4 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java @@ -18,8 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.memory.MemoryException; @@ -37,7 +37,7 @@ public interface DimensionColumnChunkReader { * @param blockletIndexes blocklets to be read * @return dimension column chunks */ - DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, int[][] blockletIndexes) + DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, int[][] blockletIndexes) throws IOException; /** @@ -47,7 +47,7 @@ public interface DimensionColumnChunkReader { * @param blockletIndex block to be read * @return dimension column chunk */ - DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int blockletIndex) + DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int blockletIndex) throws IOException; /** @@ -58,6 +58,6 @@ public interface DimensionColumnChunkReader { * @return * @throws IOException */ - DimensionColumnDataChunk convertToDimensionChunk(DimensionRawColumnChunk dimensionRawColumnChunk, + DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException, MemoryException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java index 02dc6a2..a726c66 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java @@ -18,7 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.memory.MemoryException; @@ -35,7 +35,7 @@ public interface MeasureColumnChunkReader { * @param blockIndexes blocks to be read * @return measure data chunks */ - MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes) + MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, int[][] blockIndexes) throws IOException; /** @@ -45,17 +45,17 @@ public interface MeasureColumnChunkReader { * @param blockIndex block to be read * @return measure data chunk */ - MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int blockIndex) + MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int blockIndex) throws IOException; /** - * Covert raw data to measure chunk + * decode raw data to column page * @param measureRawColumnChunk * @param pageNumber * @return * @throws IOException */ - ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java index b08fad2..a3a217a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java @@ -17,6 +17,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.chunk.reader.ColumnChunkReader; import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -27,7 +28,8 @@ import org.apache.carbondata.core.util.CarbonProperties; * Class which will have all the common properties and behavior among all type * of reader */ -public abstract class AbstractChunkReader implements DimensionColumnChunkReader { +public abstract class AbstractChunkReader extends ColumnChunkReader implements + DimensionColumnChunkReader { /** * compressor will be used to uncompress the data @@ -41,23 +43,12 @@ public abstract class AbstractChunkReader implements DimensionColumnChunkReader protected int[] eachColumnValueSize; /** - * full qualified path of the data file from - * which data will be read - */ - protected String filePath; - - /** * this will be used to uncompress the * row id and rle chunk */ protected NumberCompressor numberComressor; /** - * number of element in each chunk - */ - protected int numberOfRows; - - /** * Constructor to get minimum parameter to create * instance of this class * @@ -66,8 +57,8 @@ public abstract class AbstractChunkReader implements DimensionColumnChunkReader */ public AbstractChunkReader(final int[] eachColumnValueSize, final String filePath, int numberOfRows) { + super(filePath, numberOfRows); this.eachColumnValueSize = eachColumnValueSize; - this.filePath = filePath; int numberOfElement = 0; try { numberOfElement = Integer.parseInt(CarbonProperties.getInstance() @@ -77,7 +68,6 @@ public abstract class AbstractChunkReader implements DimensionColumnChunkReader numberOfElement = Integer.parseInt(CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL); } this.numberComressor = new NumberCompressor(numberOfElement); - this.numberOfRows = numberOfRows; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java index f083612..e4abe7f 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java @@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension; import java.io.IOException; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.format.Encoding; @@ -55,43 +55,44 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader * For last column read is separately and process * * @param fileReader file reader to read the blocks from file - * @param blockletIndexes blocks range to be read + * @param columnIndexRange column chunk range to be read * @return dimension column chunks */ - @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileHolder fileReader, - final int[][] blockletIndexes) throws IOException { + @Override + public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, + final int[][] columnIndexRange) throws IOException { // read the column chunk based on block index and add DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionChunksOffset.size()]; // if blocklet index is empty then return empry data chunk - if (blockletIndexes.length == 0) { + if (columnIndexRange.length == 0) { return dataChunks; } DimensionRawColumnChunk[] groupChunk = null; int index = 0; // iterate till block indexes -1 as block index will be in sorted order, so to avoid // the last column reading in group - for (int i = 0; i < blockletIndexes.length - 1; i++) { + for (int i = 0; i < columnIndexRange.length - 1; i++) { index = 0; groupChunk = - readRawDimensionChunksInGroup(fileReader, blockletIndexes[i][0], blockletIndexes[i][1]); - for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) { + readRawDimensionChunksInGroup(fileReader, columnIndexRange[i][0], columnIndexRange[i][1]); + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = groupChunk[index++]; } } // check last index is present in block index, if it is present then read separately - if (blockletIndexes[blockletIndexes.length - 1][0] == dimensionChunksOffset.size() - 1) { - dataChunks[blockletIndexes[blockletIndexes.length - 1][0]] = - readRawDimensionChunk(fileReader, blockletIndexes[blockletIndexes.length - 1][0]); + if (columnIndexRange[columnIndexRange.length - 1][0] == dimensionChunksOffset.size() - 1) { + dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] = + readRawDimensionChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]); } // otherwise read the data in group else { - groupChunk = - readRawDimensionChunksInGroup(fileReader, blockletIndexes[blockletIndexes.length - 1][0], - blockletIndexes[blockletIndexes.length - 1][1]); + groupChunk = readRawDimensionChunksInGroup( + fileReader, columnIndexRange[columnIndexRange.length - 1][0], + columnIndexRange[columnIndexRange.length - 1][1]); index = 0; - for (int j = blockletIndexes[blockletIndexes.length - 1][0]; - j <= blockletIndexes[blockletIndexes.length - 1][1]; j++) { + for (int j = columnIndexRange[columnIndexRange.length - 1][0]; + j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) { dataChunks[j] = groupChunk[index++]; } } @@ -109,7 +110,7 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader * @return measure raw chunkArray * @throws IOException */ - protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException; /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java index 27a4d89..d8007d5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java @@ -20,12 +20,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -61,14 +61,15 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead * Below method will be used to read the raw chunk based on block indexes * * @param fileReader file reader to read the blocks from file - * @param blockletIndexes blocks to be read + * @param columnIndexRange blocks to be read * @return dimension column chunks */ - @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, - int[][] blockletIndexes) throws IOException { + @Override + public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionColumnChunk.size()]; - for (int i = 0; i < blockletIndexes.length; i++) { - for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) { + for (int i = 0; i < columnIndexRange.length; i++) { + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = readRawDimensionChunk(fileReader, j); } } @@ -82,7 +83,8 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead * @param columnIndex column to be read * @return dimension column chunk */ - @Override public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, + @Override + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex) throws IOException { DataChunk dataChunk = dimensionColumnChunk.get(columnIndex); ByteBuffer buffer = null; @@ -92,20 +94,21 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead } DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0, dataChunk.getDataPageLength(), this); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; } - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override + public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException { int blockIndex = dimensionRawColumnChunk.getColumnIndex(); byte[] dataPage = null; int[] invertedIndexes = null; int[] invertedIndexesReverse = null; int[] rlePage = null; - FileHolder fileReader = dimensionRawColumnChunk.getFileReader(); + FileReader fileReader = dimensionRawColumnChunk.getFileReader(); ByteBuffer rawData = dimensionRawColumnChunk.getRawData(); dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(), @@ -145,23 +148,23 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead rlePage = null; } // fill chunk attributes - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; if (dataChunk.isRowMajor()) { // to store fixed length column chunk values - columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex], - numberOfRows); + columnDataChunk = new ColumnGroupDimensionColumnPage( + dataPage, eachColumnValueSize[blockIndex], numberOfRows); } // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance else if (!CarbonUtil .hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows, eachColumnValueSize[blockIndex]); } return columnDataChunk; http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java index b43f89c..83d0578 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java @@ -19,12 +19,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -56,7 +56,8 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead * @param columnIndex column to be read * @return dimension column chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int columnIndex) + @Override + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex) throws IOException { int length = 0; if (dimensionChunksOffset.size() - 1 == columnIndex) { @@ -73,7 +74,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead } DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0, length, this); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; @@ -90,7 +91,8 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead * @return measure raw chunkArray * @throws IOException */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + @Override + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { long currentDimensionOffset = dimensionChunksOffset.get(startColumnBlockletIndex); ByteBuffer buffer = null; @@ -106,7 +108,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i)); dataChunks[index] = new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this); - dataChunks[index].setFileHolder(fileReader); + dataChunks[index].setFileReader(fileReader); dataChunks[index].setPagesCount(1); dataChunks[index].setRowCount(new int[] { numberOfRows }); runningLength += currentLength; @@ -115,7 +117,8 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead return dataChunks; } - public DimensionColumnDataChunk convertToDimensionChunk( + @Override + public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException { byte[] dataPage = null; int[] invertedIndexes = null; @@ -169,23 +172,23 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]); } // fill chunk attributes - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; if (dimensionColumnChunk.isRowMajor()) { // to store fixed length column chunk values - columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex], - numberOfRows); + columnDataChunk = new ColumnGroupDimensionColumnPage( + dataPage, eachColumnValueSize[blockIndex], numberOfRows); } // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance else if (!hasEncoding(dimensionColumnChunk.encoders, Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows, eachColumnValueSize[blockIndex]); } return columnDataChunk; http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java index 1edfd09..0bcad15 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java @@ -20,8 +20,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -70,7 +70,8 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param blockletColumnIndex blocklet index of the column in carbon data file * @return dimension raw chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, + @Override + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int blockletColumnIndex) throws IOException { // get the current dimension offset long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex); @@ -116,7 +117,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param endBlockletColumnIndex blocklet index of the last dimension column * @ DimensionRawColumnChunk array */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException { // create raw chunk for each dimension column DimensionRawColumnChunk[] dimensionDataChunks = @@ -136,7 +137,8 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param pageNumber number * @return DimensionColumnDataChunk */ - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override + public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException, MemoryException { // data chunk of page http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java index 566e9b7..7be8b31 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java @@ -20,11 +20,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; @@ -77,24 +77,24 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * 4. Get the data chunk object from data read * 5. Create the raw chunk object and fill the details * - * @param fileReader reader for reading the column from carbon data file - * @param blockletColumnIndex blocklet index of the column in carbon data file + * @param columnIndex blocklet index of the column in carbon data file * @return dimension raw chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, - int blockletColumnIndex) throws IOException { + @Override + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, + int columnIndex) throws IOException { // get the current dimension offset - long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex); + long currentDimensionOffset = dimensionChunksOffset.get(columnIndex); int length = 0; // to calculate the length of the data to be read // column other than last column we can subtract the offset of current column with // next column and get the total length. // but for last column we need to use lastDimensionOffset which is the end position // of the last dimension, we can subtract current dimension offset from lastDimesionOffset - if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) { + if (dimensionChunksOffset.size() - 1 == columnIndex) { length = (int) (lastDimensionOffsets - currentDimensionOffset); } else { - length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset); + length = (int) (dimensionChunksOffset.get(columnIndex + 1) - currentDimensionOffset); } ByteBuffer buffer = null; // read the data from carbon data file @@ -103,15 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead } // get the data chunk which will have all the details about the data pages DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length); - return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer, + return getDimensionRawColumnChunk(fileReader, columnIndex, 0, length, buffer, dataChunk); } - protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader, - int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) { + DimensionRawColumnChunk getDimensionRawColumnChunk(FileReader fileReader, int columnIndex, + long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) { // creating a raw chunks instance and filling all the details DimensionRawColumnChunk rawColumnChunk = - new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this); + new DimensionRawColumnChunk(columnIndex, buffer, offset, length, this); int numberOfPages = dataChunk.getPage_length().size(); byte[][] maxValueOfEachPage = new byte[numberOfPages][]; byte[][] minValueOfEachPage = new byte[numberOfPages][]; @@ -124,7 +124,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage(); } rawColumnChunk.setDataChunkV3(dataChunk); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(dataChunk.getPage_length().size()); rawColumnChunk.setMaxValues(maxValueOfEachPage); rawColumnChunk.setMinValues(minValueOfEachPage); @@ -153,7 +153,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * blocklet index of the last dimension column * @ DimensionRawColumnChunk array */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException { // to calculate the length of the data to be read // column we can subtract the offset of start column offset with @@ -188,9 +188,9 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * * @param rawColumnPage dimension raw chunk * @param pageNumber number - * @return DimensionColumnDataChunk + * @return DimensionColumnPage */ - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); @@ -228,20 +228,19 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead return false; } - protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage, - ByteBuffer pageData, DataChunk2 pageMetadata, int offset) + DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, + DataChunk2 pageMetadata, int offset) throws IOException, MemoryException { if (isEncodedWithMeta(pageMetadata)) { ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, offset); - return new ColumnPageWrapper(decodedPage, - eachColumnValueSize[rawColumnPage.getColumnIndex()]); + return new ColumnPageWrapper(decodedPage); } else { // following code is for backward compatibility return decodeDimensionLegacy(rawColumnPage, pageData, pageMetadata, offset); } } - private DimensionColumnDataChunk decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage, + private DimensionColumnPage decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, DataChunk2 pageMetadata, int offset) { byte[] dataPage; int[] rlePage; @@ -267,18 +266,18 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead eachColumnValueSize[rawColumnPage.getColumnIndex()]); } - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage()); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage(), eachColumnValueSize[rawColumnPage.getColumnIndex()]); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java index d781cea..c600860 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java @@ -16,6 +16,7 @@ */ package org.apache.carbondata.core.datastore.chunk.reader.measure; +import org.apache.carbondata.core.datastore.chunk.reader.ColumnChunkReader; import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader; import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory; import org.apache.carbondata.core.datastore.page.encoding.EncodingFactory; @@ -23,27 +24,18 @@ import org.apache.carbondata.core.datastore.page.encoding.EncodingFactory; /** * Measure block reader abstract class */ -public abstract class AbstractMeasureChunkReader implements MeasureColumnChunkReader { +public abstract class AbstractMeasureChunkReader extends ColumnChunkReader + implements MeasureColumnChunkReader { protected EncodingFactory encodingFactory = DefaultEncodingFactory.getInstance(); /** - * file path from which blocks will be read - */ - protected String filePath; - - /** - * number of rows for blocklet - */ - protected int numberOfRows; - - /** * Constructor to get minimum parameter to create instance of this class * * @param filePath file from which data will be read */ public AbstractMeasureChunkReader(String filePath, int numberOfRows) { - this.filePath = filePath; - this.numberOfRows = numberOfRows; + super(filePath, numberOfRows); } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java index 2239a2b..162b7f3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.BitSet; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -56,36 +56,39 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu * separately and process * * @param fileReader file reader to read the blocks from file - * @param blockIndexes blocks range to be read + * @param columnIndexRange column chunk range to be read * @return measure column chunks * @throws IOException */ - public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes) - throws IOException { + @Override + public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { // read the column chunk based on block index and add MeasureRawColumnChunk[] dataChunks = new MeasureRawColumnChunk[measureColumnChunkOffsets.size()]; - if (blockIndexes.length == 0) { + if (columnIndexRange.length == 0) { return dataChunks; } MeasureRawColumnChunk[] groupChunk = null; int index = 0; - for (int i = 0; i < blockIndexes.length - 1; i++) { + for (int i = 0; i < columnIndexRange.length - 1; i++) { index = 0; - groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[i][0], blockIndexes[i][1]); - for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) { + groupChunk = readRawMeasureChunksInGroup( + fileReader, columnIndexRange[i][0], columnIndexRange[i][1]); + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = groupChunk[index++]; } } - if (blockIndexes[blockIndexes.length - 1][0] == measureColumnChunkOffsets.size() - 1) { - dataChunks[blockIndexes[blockIndexes.length - 1][0]] = - readRawMeasureChunk(fileReader, blockIndexes[blockIndexes.length - 1][0]); + if (columnIndexRange[columnIndexRange.length - 1][0] == measureColumnChunkOffsets.size() - 1) { + dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] = + readRawMeasureChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]); } else { - groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[blockIndexes.length - 1][0], - blockIndexes[blockIndexes.length - 1][1]); + groupChunk = readRawMeasureChunksInGroup( + fileReader, columnIndexRange[columnIndexRange.length - 1][0], + columnIndexRange[columnIndexRange.length - 1][1]); index = 0; - for (int j = blockIndexes[blockIndexes.length - 1][0]; - j <= blockIndexes[blockIndexes.length - 1][1]; j++) { + for (int j = columnIndexRange[columnIndexRange.length - 1][0]; + j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) { dataChunks[j] = groupChunk[index++]; } } @@ -112,12 +115,12 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu * data from * * @param fileReader file reader to read the data - * @param startColumnBlockletIndex first column blocklet index to be read - * @param endColumnBlockletIndex end column blocklet index to be read + * @param startColumnIndex first column index to be read + * @param endColumnIndex end column index to be read * @return measure raw chunkArray * @throws IOException */ - protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException; + protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java index ae55375..0487947 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -55,15 +55,15 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun /** * Method to read the blocks data based on block indexes * - * @param fileReader file reader to read the blocks - * @param blockIndexes blocks to be read + * @param columnIndexRange blocks to be read * @return measure data chunks */ - @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, - int[][] blockIndexes) throws IOException { + @Override + public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { MeasureRawColumnChunk[] datChunk = new MeasureRawColumnChunk[measureColumnChunks.size()]; - for (int i = 0; i < blockIndexes.length; i++) { - for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) { + for (int i = 0; i < columnIndexRange.length; i++) { + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { datChunk[j] = readRawMeasureChunk(fileReader, j); } } @@ -77,21 +77,21 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun * @param columnIndex column to be read * @return measure data chunk */ - @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex) + @Override + public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { DataChunk dataChunk = measureColumnChunks.get(columnIndex); ByteBuffer buffer = fileReader .readByteBuffer(filePath, dataChunk.getDataPageOffset(), dataChunk.getDataPageLength()); - MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(columnIndex, buffer, 0, - dataChunk.getDataPageLength(), this); - rawColumnChunk.setFileReader(fileReader); + MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(fileReader, + columnIndex, buffer, 0, dataChunk.getDataPageLength(), this); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; } @Override - public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException { int blockIndex = measureRawColumnChunk.getColumnIndex(); DataChunk dataChunk = measureColumnChunks.get(blockIndex); http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java index 2ddc202..4be7b31 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -48,7 +48,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun } @Override - public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex) + public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { int dataLength = 0; if (measureColumnChunkOffsets.size() - 1 == columnIndex) { @@ -63,8 +63,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun .readByteBuffer(filePath, measureColumnChunkOffsets.get(columnIndex), dataLength); } MeasureRawColumnChunk rawColumnChunk = - new MeasureRawColumnChunk(columnIndex, buffer, 0, dataLength, this); - rawColumnChunk.setFileReader(fileReader); + new MeasureRawColumnChunk(fileReader, columnIndex, buffer, 0, dataLength, this); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; @@ -76,29 +75,29 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun * data from * * @param fileReader file reader to read the data - * @param startColumnBlockletIndex first column blocklet index to be read - * @param endColumnBlockletIndex end column blocklet index to be read + * @param startColumnIndex first column blocklet index to be read + * @param endColumnIndex end column blocklet index to be read * @return measure raw chunkArray * @throws IOException */ - protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { - long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex); + @Override + protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException { + long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex); ByteBuffer buffer = null; synchronized (fileReader) { buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, - (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset)); + (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset)); } MeasureRawColumnChunk[] dataChunks = - new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1]; + new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1]; int runningLength = 0; int index = 0; - for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) { + for (int i = startColumnIndex; i <= endColumnIndex; i++) { int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i)); MeasureRawColumnChunk measureRawColumnChunk = - new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this); - measureRawColumnChunk.setFileReader(fileReader); + new MeasureRawColumnChunk(fileReader, i, buffer, runningLength, currentLength, this); measureRawColumnChunk.setRowCount(new int[] { numberOfRows }); measureRawColumnChunk.setPagesCount(1); dataChunks[index] = measureRawColumnChunk; @@ -108,7 +107,8 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun return dataChunks; } - public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + @Override + public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException { int copyPoint = (int) measureRawColumnChunk.getOffSet(); int blockIndex = measureRawColumnChunk.getColumnIndex(); @@ -124,7 +124,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun return page; } - protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk, + private ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk, DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException, IOException { assert (measureColumnChunk.getEncoder_meta().size() > 0); List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/35564379/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java index 4f6987b..91752c3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -53,7 +53,8 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun */ private long measureOffsets; - public CompressedMeasureChunkFileBasedReaderV3(BlockletInfo blockletInfo, String filePath) { + public CompressedMeasureChunkFileBasedReaderV3(BlockletInfo blockletInfo, + String filePath) { super(blockletInfo, filePath); measureOffsets = blockletInfo.getMeasureOffsets(); } @@ -70,7 +71,8 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun * @param columnIndex column to be read * @return measure raw chunk */ - @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, + @Override + public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { int dataLength = 0; // to calculate the length of the data to be read @@ -99,12 +101,11 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun dataChunk); } - protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader, - int columnIndex, long offset, int dataLength, ByteBuffer buffer, - DataChunk3 dataChunk) { + MeasureRawColumnChunk getMeasureRawColumnChunk(FileReader fileReader, int columnIndex, + long offset, int dataLength, ByteBuffer buffer, DataChunk3 dataChunk) { // creating a raw chunks instance and filling all the details MeasureRawColumnChunk rawColumnChunk = - new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this); + new MeasureRawColumnChunk(fileReader, columnIndex, buffer, offset, dataLength, this); int numberOfPages = dataChunk.getPage_length().size(); byte[][] maxValueOfEachPage = new byte[numberOfPages][]; byte[][] minValueOfEachPage = new byte[numberOfPages][]; @@ -117,7 +118,6 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage(); } rawColumnChunk.setDataChunkV3(dataChunk); - rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(dataChunk.getPage_length().size()); rawColumnChunk.setMaxValues(maxValueOfEachPage); rawColumnChunk.setMinValues(minValueOfEachPage); @@ -140,30 +140,31 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun * * @param fileReader * reader which will be used to read the measure columns data from file - * @param startColumnBlockletIndex - * blocklet index of the first measure column - * @param endColumnBlockletIndex - * blocklet index of the last measure column + * @param startColumnIndex + * column index of the first measure column + * @param endColumnIndex + * column index of the last measure column * @return MeasureRawColumnChunk array */ - protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { + @Override + protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException { // to calculate the length of the data to be read // column we can subtract the offset of start column offset with // end column+1 offset and get the total length. - long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex); + long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex); ByteBuffer buffer = null; // read the data from carbon data file synchronized (fileReader) { buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, - (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset)); + (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset)); } // create raw chunk for each measure column MeasureRawColumnChunk[] measureDataChunk = - new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1]; + new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1]; int runningLength = 0; int index = 0; - for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) { + for (int i = startColumnIndex; i <= endColumnIndex; i++) { int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i)); DataChunk3 dataChunk = @@ -180,25 +181,25 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun /** * Below method will be used to convert the compressed measure chunk raw data to actual data * - * @param rawColumnPage measure raw chunk + * @param rawColumnChunk measure raw chunk * @param pageNumber number - * @return DimensionColumnDataChunk + * @return DimensionColumnPage */ @Override - public ColumnPage convertToColumnPage( - MeasureRawColumnChunk rawColumnPage, int pageNumber) + public ColumnPage decodeColumnPage( + MeasureRawColumnChunk rawColumnChunk, int pageNumber) throws IOException, MemoryException { // data chunk of blocklet column - DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); + DataChunk3 dataChunk3 = rawColumnChunk.getDataChunkV3(); // data chunk of page DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset - int offset = (int) rawColumnPage.getOffSet() + - measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) + + int offset = (int) rawColumnChunk.getOffSet() + + measureColumnChunkLength.get(rawColumnChunk.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); - ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset); + ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnChunk.getRawData(), offset); decodedPage.setNullBits(getNullBitSet(pageMetadata.presence)); return decodedPage; } @@ -206,7 +207,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun /** * Decode measure column page with page header and raw data starting from offset */ - protected ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset) + ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset) throws MemoryException, IOException { List<Encoding> encodings = pageMetadata.getEncoders(); List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
