Github user jackylk commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/584#discussion_r102621173
--- Diff:
core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
---
@@ -56,51 +58,74 @@ public CompressedDimensionChunkFileBasedReaderV1(final
BlockletInfo blockletInfo
}
/**
- * Below method will be used to read the chunk based on block indexes
+ * Below method will be used to read the raw chunk based on block indexes
*
* @param fileReader file reader to read the blocks from file
* @param blockIndexes blocks to be read
* @return dimension column chunks
*/
- @Override public DimensionColumnDataChunk[]
readDimensionChunks(FileHolder fileReader,
+ @Override public DimensionRawColumnChunk[]
readRawDimensionChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- // read the column chunk based on block index and add
- DimensionColumnDataChunk[] dataChunks =
- new DimensionColumnDataChunk[dimensionColumnChunk.size()];
+ DimensionRawColumnChunk[] dataChunks = new
DimensionRawColumnChunk[dimensionColumnChunk.size()];
for (int i = 0; i < blockIndexes.length; i++) {
for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
- dataChunks[j] = readDimensionChunk(fileReader, j);
+ dataChunks[j] = readRawDimensionChunk(fileReader, j);
}
}
return dataChunks;
}
/**
- * Below method will be used to read the chunk based on block index
+ * Below method will be used to read the raw chunk based on block index
*
* @param fileReader file reader to read the blocks from file
* @param blockIndex block to be read
* @return dimension column chunk
*/
- @Override public DimensionColumnDataChunk readDimensionChunk(FileHolder
fileReader,
+ @Override public DimensionRawColumnChunk
readRawDimensionChunk(FileHolder fileReader,
int blockIndex) throws IOException {
+ ByteBuffer buffer =
+
ByteBuffer.allocateDirect(dimensionColumnChunk.get(blockIndex).getDataPageLength());
+ synchronized (fileReader) {
+ fileReader.readByteBuffer(filePath, buffer,
+ dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
+ dimensionColumnChunk.get(blockIndex).getDataPageLength());
+ }
+ DimensionRawColumnChunk rawColumnChunk = new
DimensionRawColumnChunk(blockIndex, buffer, 0,
+ dimensionColumnChunk.get(blockIndex).getDataPageLength(), this);
+ rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setPagesCount(1);
+ rawColumnChunk.setRowCount(new int[] { numberOfRows });
+ return rawColumnChunk;
+ }
+
+ @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber)
throws IOException {
+ int blockIndex = dimensionRawColumnChunk.getBlockId();
byte[] dataPage = null;
int[] invertedIndexes = null;
int[] invertedIndexesReverse = null;
int[] rlePage = null;
+ FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
+
+ ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
+ rawData.position(dimensionRawColumnChunk.getOffSet());
+ byte[] data = new byte[dimensionRawColumnChunk.getLength()];
+ rawData.get(data);
+ dataPage = COMPRESSOR.unCompressByte(data);
- // first read the data and uncompressed it
- dataPage = COMPRESSOR.unCompressByte(fileReader
- .readByteArray(filePath,
dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
- dimensionColumnChunk.get(blockIndex).getDataPageLength()));
// if row id block is present then read the row id chunk and
uncompress it
if
(CarbonUtil.hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(),
Encoding.INVERTED_INDEX)) {
+ byte[] columnIndexData;
+ synchronized (fileReader) {
--- End diff --
why this is needed?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
