http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java index e5312f3..51dfbf2 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java @@ -308,7 +308,8 @@ public abstract class ColumnPage { private static ColumnPage newDecimalPage(ColumnPageEncoderMeta meta, byte[] lvEncodedByteArray) throws MemoryException { - return VarLengthColumnPageBase.newDecimalColumnPage(meta, lvEncodedByteArray); + return VarLengthColumnPageBase + .newDecimalColumnPage(meta, lvEncodedByteArray, lvEncodedByteArray.length); } private static ColumnPage newLVBytesPage(TableSpec.ColumnSpec columnSpec,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java index 9bed89f..1de8201 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java @@ -88,7 +88,8 @@ public class DecoderBasedFallbackEncoder implements Callable<FallbackEncodedColu CarbonUtil.getIntArray(data, offset, encodedColumnPage.getPageMetadata().rle_page_length); // uncompress the data with rle indexes bytes = UnBlockIndexer - .uncompressData(bytes, rlePage, CarbonCommonConstants.LOCAL_DICT_ENCODED_BYTEARRAY_SIZE); + .uncompressData(bytes, rlePage, CarbonCommonConstants.LOCAL_DICT_ENCODED_BYTEARRAY_SIZE, + bytes.length); } // disable encoding using local dictionary http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java index 81bb1b5..0f409f6 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java @@ -125,7 +125,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { * Create a new column page for decimal page */ public static ColumnPage newDecimalColumnPage(ColumnPageEncoderMeta meta, - byte[] lvEncodedBytes) throws MemoryException { + byte[] lvEncodedBytes, int actualDataLength) throws MemoryException { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(columnSpec.getPrecision(), @@ -137,7 +137,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { CarbonCommonConstants.INT_SIZE_IN_BYTE, meta.getCompressorName()); } else { // Here the size is always fixed. - return getDecimalColumnPage(meta, lvEncodedBytes, size); + return getDecimalColumnPage(meta, lvEncodedBytes, size, actualDataLength); } } @@ -160,7 +160,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { } private static ColumnPage getDecimalColumnPage(ColumnPageEncoderMeta meta, - byte[] lvEncodedBytes, int size) throws MemoryException { + byte[] lvEncodedBytes, int size, int actualDataLength) throws MemoryException { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); String compressorName = meta.getCompressorName(); TableSpec.ColumnSpec spec = TableSpec.ColumnSpec @@ -171,7 +171,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { int offset; int rowId = 0; int counter = 0; - for (offset = 0; offset < lvEncodedBytes.length; offset += size) { + for (offset = 0; offset < actualDataLength; offset += size) { rowOffset.putInt(counter, offset); rowId++; counter++; http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java index 6f36c08..b5dc502 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java @@ -20,6 +20,7 @@ package org.apache.carbondata.core.datastore.page.encoding; import java.io.IOException; import java.util.BitSet; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; @@ -29,13 +30,15 @@ public interface ColumnPageDecoder { /** * Apply decoding algorithm on input byte array and return decoded column page */ - ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException; + ColumnPage decode(byte[] input, int offset, int length) + throws MemoryException, IOException; /** * Apply decoding algorithm on input byte array and fill the vector here. */ void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, - BitSet nullBits, boolean isLVEncoded, int pageSize) throws MemoryException, IOException; + BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) + throws MemoryException, IOException; ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) throws MemoryException, IOException; http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java index 735847e..fb53dba 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java @@ -23,6 +23,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -123,20 +124,26 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec { @Override public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) { return new ColumnPageDecoder() { - @Override - public ColumnPage decode(byte[] input, int offset, int length) + @Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false); return LazyColumnPage.newPage(page, converter); } - @Override - public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) + @Override public void decodeAndFillVector(byte[] input, int offset, int length, + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); - byte[] unCompressData = compressor.unCompressByte(input, offset, length); + byte[] unCompressData; + if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { + int uncompressedLength = compressor.unCompressedLength(input, offset, length); + unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); + compressor.rawUncompress(input, offset, length, unCompressData); + } else { + unCompressData = compressor.unCompressByte(input, offset, length); + } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java index 578945b..9ed21f4 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java @@ -24,6 +24,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -130,8 +131,7 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec { @Override public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) { return new ColumnPageDecoder() { - @Override - public ColumnPage decode(byte[] input, int offset, int length) + @Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { ColumnPage page = null; if (DataTypes.isDecimal(meta.getSchemaDataType())) { @@ -142,13 +142,20 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec { return LazyColumnPage.newPage(page, converter); } - @Override - public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) + @Override public void decodeAndFillVector(byte[] input, int offset, int length, + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); - byte[] unCompressData = compressor.unCompressByte(input, offset, length); + byte[] unCompressData; + if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { + int uncompressedLength = compressor.unCompressedLength(input, offset, length); + unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); + compressor.rawUncompress(input, offset, length, unCompressData); + } else { + unCompressData = compressor.unCompressByte(input, offset, length); + } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = @@ -160,8 +167,7 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec { pageSize); } - @Override - public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) + @Override public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) throws MemoryException, IOException { return decode(input, offset, length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java index c66c065..b16e57d 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java @@ -23,6 +23,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -111,26 +112,31 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec { @Override public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) { return new ColumnPageDecoder() { - @Override - public ColumnPage decode(byte[] input, int offset, int length) + @Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false); return LazyColumnPage.newPage(page, converter); } - @Override - public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) + @Override public void decodeAndFillVector(byte[] input, int offset, int length, + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); - byte[] unCompressData = compressor.unCompressByte(input, offset, length); + byte[] unCompressData; + if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { + int uncompressedLength = compressor.unCompressedLength(input, offset, length); + unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); + compressor.rawUncompress(input, offset, length, unCompressData); + } else { + unCompressData = compressor.unCompressByte(input, offset, length); + } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); } - @Override - public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) + @Override public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) throws MemoryException, IOException { return decode(input, offset, length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java index d9db437..66639fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java @@ -23,6 +23,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -107,8 +108,7 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec { @Override public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) { return new ColumnPageDecoder() { - @Override - public ColumnPage decode(byte[] input, int offset, int length) + @Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { ColumnPage page = null; if (DataTypes.isDecimal(meta.getSchemaDataType())) { @@ -119,13 +119,19 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec { return LazyColumnPage.newPage(page, converter); } - @Override - public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) - throws MemoryException, IOException { + @Override public void decodeAndFillVector(byte[] input, int offset, int length, + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); - byte[] unCompressData = compressor.unCompressByte(input, offset, length); + byte[] unCompressData; + if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { + int uncompressedLength = compressor.unCompressedLength(input, offset, length); + unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); + compressor.rawUncompress(input, offset, length, unCompressData); + } else { + unCompressData = compressor.unCompressByte(input, offset, length); + } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java index 7b7c0b6..60344d6 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java @@ -23,6 +23,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -93,8 +94,8 @@ public class DirectCompressCodec implements ColumnPageCodec { public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) { return new ColumnPageDecoder() { - @Override - public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException { + @Override public ColumnPage decode(byte[] input, int offset, int length) + throws MemoryException { ColumnPage decodedPage; if (DataTypes.isDecimal(dataType)) { decodedPage = ColumnPage.decompressDecimalPage(meta, input, offset, length); @@ -106,11 +107,21 @@ public class DirectCompressCodec implements ColumnPageCodec { @Override public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); - byte[] unCompressData = compressor.unCompressByte(input, offset, length); + int uncompressedLength; + byte[] unCompressData; + if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { + uncompressedLength = compressor.unCompressedLength(input, offset, length); + unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); + compressor.rawUncompress(input, offset, length, unCompressData); + } else { + unCompressData = compressor.unCompressByte(input, offset, length); + uncompressedLength = unCompressData.length; + } if (DataTypes.isDecimal(dataType)) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = @@ -118,8 +129,8 @@ public class DirectCompressCodec implements ColumnPageCodec { .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; if (DataTypes.isDecimal(meta.getStoreDataType())) { - ColumnPage decimalColumnPage = - VarLengthColumnPageBase.newDecimalColumnPage(meta, unCompressData); + ColumnPage decimalColumnPage = VarLengthColumnPageBase + .newDecimalColumnPage(meta, unCompressData, uncompressedLength); decimalConverter.fillVector(decimalColumnPage.getByteArrayPage(), pageSize, vectorInfo, nullBits, meta.getStoreDataType()); } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java index 2c940bb..836a8b1 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java @@ -27,6 +27,7 @@ import java.util.BitSet; import java.util.List; import java.util.Map; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec; @@ -295,8 +296,7 @@ public class RLECodec implements ColumnPageCodec { this.compressorName = compressorName; } - @Override - public ColumnPage decode(byte[] input, int offset, int length) + @Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { DataType dataType = columnSpec.getSchemaDataType(); DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length)); @@ -316,9 +316,9 @@ public class RLECodec implements ColumnPageCodec { return resultPage; } - @Override - public void decodeAndFillVector(byte[] input, int offset, int length, - ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize) + @Override public void decodeAndFillVector(byte[] input, int offset, int length, + ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, + ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { throw new UnsupportedOperationException("Not supposed to be called here"); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index 710fbe3..40d04e2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -35,6 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datastore.IndexKey; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.block.AbstractIndex; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.block.TableBlockInfo; @@ -404,19 +405,33 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { // fill all the block execution infos for all the blocks selected in // query // and query will be executed based on that infos + ReusableDataBuffer[] dimensionReusableDataBuffers = null; + ReusableDataBuffer[] measureReusableDataBuffers = null; + for (int i = 0; i < queryProperties.dataBlocks.size(); i++) { AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i); BlockletDataRefNode dataRefNode = (BlockletDataRefNode) abstractIndex.getDataRefNode(); - blockExecutionInfoList.add( - getBlockExecutionInfoForBlock( - queryModel, - abstractIndex, + final BlockExecutionInfo blockExecutionInfoForBlock = + getBlockExecutionInfoForBlock(queryModel, abstractIndex, dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(), - dataRefNode.numberOfNodes(), - dataRefNode.getBlockInfos().get(0).getFilePath(), + dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(), dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(), - dataRefNode.getBlockInfos().get(0).getSegment())); + dataRefNode.getBlockInfos().get(0).getSegment()); + if (null == dimensionReusableDataBuffers || null == measureReusableDataBuffers) { + dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionResusableDataBuffer(); + measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureResusableDataBuffer(); + } else { + if (dimensionReusableDataBuffers.length == blockExecutionInfoForBlock + .getDimensionResusableDataBuffer().length) { + blockExecutionInfoForBlock.setDimensionResusableDataBuffer(dimensionReusableDataBuffers); + } + if (measureReusableDataBuffers.length == blockExecutionInfoForBlock + .getMeasureResusableDataBuffer().length) { + blockExecutionInfoForBlock.setMeasureResusableDataBuffer(measureReusableDataBuffers); + } + } + blockExecutionInfoList.add(blockExecutionInfoForBlock); } if (null != queryModel.getStatisticsRecorder()) { QueryStatistic queryStatistic = new QueryStatistic(); @@ -534,6 +549,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), currentBlockFilterDimensions, allProjectionListDimensionIdexes); + int reusableBufferSize = segmentProperties.getDimensionOrdinalToChunkMapping().size() + < projectDimensions.size() ? + projectDimensions.size() : + segmentProperties.getDimensionOrdinalToChunkMapping().size(); + ReusableDataBuffer[] dimensionBuffer = + new ReusableDataBuffer[reusableBufferSize]; + for (int i = 0; i < dimensionBuffer.length; i++) { + dimensionBuffer[i] = new ReusableDataBuffer(); + } + blockExecutionInfo.setDimensionResusableDataBuffer(dimensionBuffer); int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance() .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE)); @@ -558,6 +583,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { currentBlockQueryMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures, allProjectionListMeasureIndexes); + ReusableDataBuffer[] measureBuffer = + new ReusableDataBuffer[segmentProperties.getMeasuresOrdinalToChunkMapping().size()]; + for (int i = 0; i < measureBuffer.length; i++) { + measureBuffer[i] = new ReusableDataBuffer(); + } + blockExecutionInfo.setMeasureResusableDataBuffer(measureBuffer); if (measureChunkIndexes.length > 0) { numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1] http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java index f0ef23b..f19b96b 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.carbondata.core.datastore.DataRefNode; import org.apache.carbondata.core.datastore.IndexKey; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.block.AbstractIndex; import org.apache.carbondata.core.mutate.DeleteDeltaVo; import org.apache.carbondata.core.scan.filter.GenericQueryType; @@ -221,6 +222,10 @@ public class BlockExecutionInfo { */ private boolean isDirectVectorFill; + private ReusableDataBuffer[] dimensionResusableDataBuffer; + + private ReusableDataBuffer[] measureResusableDataBuffer; + /** * @param blockIndex the tableBlock to set */ @@ -638,4 +643,20 @@ public class BlockExecutionInfo { public void setDirectVectorFill(boolean directVectorFill) { isDirectVectorFill = directVectorFill; } + + public ReusableDataBuffer[] getDimensionResusableDataBuffer() { + return dimensionResusableDataBuffer; + } + + public void setDimensionResusableDataBuffer(ReusableDataBuffer[] dimensionResusableDataBuffer) { + this.dimensionResusableDataBuffer = dimensionResusableDataBuffer; + } + + public ReusableDataBuffer[] getMeasureResusableDataBuffer() { + return measureResusableDataBuffer; + } + + public void setMeasureResusableDataBuffer(ReusableDataBuffer[] measureResusableDataBuffer) { + this.measureResusableDataBuffer = measureResusableDataBuffer; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java index 8217487..bb373eb 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; @@ -149,8 +150,14 @@ public abstract class BlockletScannedResult { protected LazyBlockletLoader lazyBlockletLoader; + private ReusableDataBuffer[] dimensionReusableBuffer; + + private ReusableDataBuffer[] measureReusableBuffer; + public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo, QueryStatisticsModel queryStatisticsModel) { + this.dimensionReusableBuffer = blockExecutionInfo.getDimensionResusableDataBuffer(); + this.measureReusableBuffer = blockExecutionInfo.getMeasureResusableDataBuffer(); this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize(); this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes(); this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex(); @@ -382,15 +389,15 @@ public abstract class BlockletScannedResult { long startTime = System.currentTimeMillis(); for (int i = 0; i < dimensionColumnPages.length; i++) { if (dimensionColumnPages[i][pageCounter] == null && dimRawColumnChunks[i] != null) { - dimensionColumnPages[i][pageCounter] = - dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter); + dimensionColumnPages[i][pageCounter] = dimRawColumnChunks[i] + .convertToDimColDataChunkWithOutCache(pageCounter, null); } } for (int i = 0; i < measureColumnPages.length; i++) { if (measureColumnPages[i][pageCounter] == null && msrRawColumnChunks[i] != null) { - measureColumnPages[i][pageCounter] = - msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter); + measureColumnPages[i][pageCounter] = msrRawColumnChunks[i] + .convertToColumnPageWithOutCache(pageCounter, null); } } QueryStatistic pageUncompressTime = queryStatisticsModel.getStatisticsTypeAndObjMap() @@ -412,18 +419,20 @@ public abstract class BlockletScannedResult { for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) { dictionaryInfo[i].vector.setLazyPage( new LazyPageLoader(lazyBlockletLoader, dictionaryColumnChunkIndexes[i], false, - pageIdFiltered[pageCounter], dictionaryInfo[i])); + pageIdFiltered[pageCounter], dictionaryInfo[i], dimensionReusableBuffer[i])); } + int startIndex = dictionaryColumnChunkIndexes.length; for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) { noDictionaryInfo[i].vector.setLazyPage( new LazyPageLoader(lazyBlockletLoader, noDictionaryColumnChunkIndexes[i], false, - pageIdFiltered[pageCounter], noDictionaryInfo[i])); + pageIdFiltered[pageCounter], noDictionaryInfo[i], + dimensionReusableBuffer[startIndex++])); } for (int i = 0; i < measuresOrdinal.length; i++) { msrVectorInfo[i].vector.setLazyPage( new LazyPageLoader(lazyBlockletLoader, measuresOrdinal[i], true, - pageIdFiltered[pageCounter], msrVectorInfo[i])); + pageIdFiltered[pageCounter], msrVectorInfo[i], measureReusableBuffer[i])); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java index 84d52a6..298eadc 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java @@ -22,6 +22,10 @@ public interface CarbonDictionary { int getDictionarySize(); + boolean isDictionaryUsed(); + + void setDictionaryUsed(); + byte[] getDictionaryValue(int index); byte[][] getAllDictionaryValues(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java index 962a772..135ec3f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java @@ -30,6 +30,8 @@ public class CarbonDictionaryImpl implements CarbonDictionary { private int actualSize; + private boolean isDictUsed; + public CarbonDictionaryImpl(byte[][] dictionary, int actualSize) { this.dictionary = dictionary; this.actualSize = actualSize; @@ -43,6 +45,14 @@ public class CarbonDictionaryImpl implements CarbonDictionary { return this.dictionary.length; } + @Override public boolean isDictionaryUsed() { + return this.isDictUsed; + } + + @Override public void setDictionaryUsed() { + this.isDictUsed = true; + } + @Override public byte[] getDictionaryValue(int index) { return dictionary[index]; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java index 42af6c3..0aca79a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.scan.scanner; import java.io.IOException; +import org.apache.carbondata.core.datastore.ReusableDataBuffer; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; @@ -43,14 +44,17 @@ public class LazyPageLoader { private QueryStatisticsModel queryStatisticsModel; + private ReusableDataBuffer reusableDataBuffer; + public LazyPageLoader(LazyBlockletLoader lazyBlockletLoader, int index, boolean isMeasure, - int pageNumber, ColumnVectorInfo vectorInfo) { + int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) { this.lazyBlockletLoader = lazyBlockletLoader; this.lazyChunkWrapper = lazyBlockletLoader.getLazyChunkWrapper(index, isMeasure); this.isMeasure = isMeasure; this.pageNumber = pageNumber; this.vectorInfo = vectorInfo; this.queryStatisticsModel = lazyBlockletLoader.getQueryStatisticsModel(); + this.reusableDataBuffer = reusableDataBuffer; } public void loadPage() { @@ -64,10 +68,10 @@ public class LazyPageLoader { long startTime = System.currentTimeMillis(); if (isMeasure) { ((MeasureRawColumnChunk) lazyChunkWrapper.getRawColumnChunk()) - .convertToColumnPageAndFillVector(pageNumber, vectorInfo); + .convertToColumnPageAndFillVector(pageNumber, vectorInfo, reusableDataBuffer); } else { ((DimensionRawColumnChunk) lazyChunkWrapper.getRawColumnChunk()) - .convertToDimColDataChunkAndFillVector(pageNumber, vectorInfo); + .convertToDimColDataChunkAndFillVector(pageNumber, vectorInfo, reusableDataBuffer); } if (queryStatisticsModel.isEnabled()) { QueryStatistic pageUncompressTime = queryStatisticsModel.getStatisticsTypeAndObjMap() http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java index 8dabe4c..e6f3d9a 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java @@ -38,7 +38,8 @@ public class FixedLengthDimensionDataChunkTest { int invertedIndexReverse[] = { 1, 0, 5, 7, 8 }; fixedLengthDimensionDataChunk = - new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4); + new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4, + data.length); } @Test public void fillChunkDataTest() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java index 2662cee..ee3bc16 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java @@ -184,7 +184,7 @@ public class IncludeFilterExecuterImplTest extends TestCase { dim.setFilterKeys(filterKeys); dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunkSize, dimColumnSize); + dataChunkSize, dimColumnSize, dataChunk.length); // repeat query and compare 2 result between old code and new optimized code for (int j = 0; j < queryTimes; j++) { @@ -304,7 +304,7 @@ public class IncludeFilterExecuterImplTest extends TestCase { dim.setFilterKeys(filterKeys); dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / dimColumnSize, dimColumnSize); + dataChunk.length / dimColumnSize, dimColumnSize, dataChunk.length); // initial to run BitSet bitOld = this.setFilterdIndexToBitSetWithColumnIndexOld(dimensionColumnDataChunk, dataChunkSize, filterKeys); http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index 4a5d1b3..a82a8aa 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -247,7 +247,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 5, 6, 7, 8, 9 }; byte[] compareValues = { 7 }; FixedLengthDimensionColumnPage fixedLengthDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length); int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues); assertEquals(result, 1); } @@ -256,7 +256,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 7, 7, 7, 8, 9 }; byte[] compareValues = { 7 }; FixedLengthDimensionColumnPage fixedLengthDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length); int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues); assertEquals(result, -1); } @@ -265,7 +265,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 5, 6, 7, 8, 9 }; byte[] compareValues = { 7 }; FixedLengthDimensionColumnPage fixedLengthDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length); int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5); assertEquals(result, 3); } @@ -282,7 +282,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 5, 6, 7, 7, 7 }; byte[] compareValues = { 7 }; FixedLengthDimensionColumnPage fixedLengthDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length); int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5); assertEquals(result, 5); } @@ -772,7 +772,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 10, 20, 30, 40, 50, 60 }; byte[] compareValue = { 5 }; FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length); int result = CarbonUtil .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false); assertEquals(-2, result); @@ -782,7 +782,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 10, 20, 30, 40, 50, 60 }; byte[] compareValue = { 30 }; FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length); int result = CarbonUtil .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false); assertEquals(2, result); @@ -792,7 +792,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 10, 10, 10, 40, 50, 60 }; byte[] compareValue = { 10 }; FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length); int result = CarbonUtil .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false); assertEquals(0, result); @@ -802,7 +802,7 @@ public class CarbonUtilTest { byte[] dataChunks = { 10, 10, 10, 40, 50, 60 }; byte[] compareValue = { 10 }; FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = - new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1); + new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length); int result = CarbonUtil .getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true); assertEquals(2, result); @@ -819,7 +819,7 @@ public class CarbonUtilTest { dataChunk = "abbcccddddeffgggh".getBytes(); byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length]; fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) { dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii); @@ -851,7 +851,7 @@ public class CarbonUtilTest { dataChunk = "ab".getBytes(); fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); keyWord[0] = Byte.valueOf("97"); range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord); @@ -865,7 +865,7 @@ public class CarbonUtilTest { dataChunk = "aabb".getBytes(); fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); keyWord[0] = Byte.valueOf("97"); range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord); @@ -879,7 +879,7 @@ public class CarbonUtilTest { dataChunk = "a".getBytes(); fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); keyWord[0] = Byte.valueOf("97"); range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord); @@ -888,7 +888,7 @@ public class CarbonUtilTest { dataChunk = "aa".getBytes(); fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); keyWord[0] = Byte.valueOf("97"); range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord); @@ -897,7 +897,7 @@ public class CarbonUtilTest { dataChunk = "aabbbbbbbbbbcc".getBytes(); fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); keyWord[0] = Byte.valueOf("98"); range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord); assertEquals(2, range[0]); @@ -917,7 +917,7 @@ public class CarbonUtilTest { byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length]; fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) { dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii); @@ -959,7 +959,7 @@ public class CarbonUtilTest { byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length]; fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, - dataChunk.length / keyWord.length, keyWord.length); + dataChunk.length / keyWord.length, keyWord.length, dataChunk.length); for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) { dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii); http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala index f5153fc..6ec44e7 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala @@ -147,6 +147,21 @@ class CustomizeCompressor extends Compressor { override def supportUnsafe(): Boolean = { false } + + override def unCompressedLength(data: Array[Byte], + offset: Int, + length: Int): Int = { + throw new RuntimeException("Unsupported operation Exception") + } + + override def rawUncompress(data: Array[Byte], + offset: Int, + length: Int, + output: Array[Byte]): Int = { + throw new RuntimeException("Unsupported operation Exception") + } + + override def supportReusableBuffer(): Boolean = false } class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll { http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java ---------------------------------------------------------------------- diff --git a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java index 8082511..1f28b8c 100644 --- a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java +++ b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java @@ -365,7 +365,6 @@ public class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> { for (int i = 0; i < isNoDictStringField.length; i++) { if (isNoDictStringField[i]) { vectorProxy.resetDictionaryIds(i); - vectorProxy.column(i).setDictionary(null); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java ---------------------------------------------------------------------- diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java index 12384fc..07682ed 100644 --- a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java +++ b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java @@ -205,7 +205,7 @@ class ScanBenchmark implements Command { DimensionColumnPage[] pages = new DimensionColumnPage[numPages]; for (int i = 0; i < pages.length; i++) { pages[i] = dimensionColumnChunkReader.decodeColumnPage( - (DimensionRawColumnChunk) rawColumnChunk, i); + (DimensionRawColumnChunk) rawColumnChunk, i, null); } return pages; } @@ -215,7 +215,7 @@ class ScanBenchmark implements Command { ColumnPage[] pages = new ColumnPage[numPages]; for (int i = 0; i < pages.length; i++) { pages[i] = measureColumnChunkReader.decodeColumnPage( - (MeasureRawColumnChunk) rawColumnChunk, i); + (MeasureRawColumnChunk) rawColumnChunk, i, null); } return pages; }