http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java index 9391ebd..44f7c07 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java @@ -16,20 +16,10 @@ */ package org.apache.carbondata.core.scan.filter.executer; -import java.util.ArrayList; import java.util.BitSet; -import java.util.List; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.executor.util.QueryUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.util.ByteUtil; /** * It checks if filter is required on given block and if required, it does @@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil; public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl { /** - * LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName()); - - /** * @param dimColResolvedFilterInfo * @param segmentProperties */ @@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl } /** - * It fills BitSet with row index which matches filter key - */ - protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, - int numerOfRows) { - BitSet bitSet = new BitSet(numerOfRows); - bitSet.flip(0, numerOfRows); - try { - KeyStructureInfo keyStructureInfo = getKeyStructureInfo(); - byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys(); - for (int i = 0; i < filterValues.length; i++) { - byte[] filterVal = filterValues[i]; - for (int rowId = 0; rowId < numerOfRows; rowId++) { - byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length]; - dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo); - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) { - bitSet.flip(rowId); - } - } - } - - } catch (Exception e) { - LOGGER.error(e); - } - - return bitSet; - } - - /** - * It is required for extracting column data from columngroup chunk - * - * @return - * @throws KeyGenException - */ - private KeyStructureInfo getKeyStructureInfo() throws KeyGenException { - int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex()); - KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); - List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); - mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId)); - int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); - byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); - KeyStructureInfo restructureInfos = new KeyStructureInfo(); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - return restructureInfos; - } - - /** * Check if scan is required on given block based on min and max value */ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { @@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl return bitSet; } - private int getMdkeyOrdinal(int ordinal, int colGrpId) { - return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal); - } - - private int getColumnGroupId(int ordinal) { - int[][] columnGroups = segmentProperties.getColumnGroups(); - int colGrpId = -1; - for (int i = 0; i < columnGroups.length; i++) { - if (columnGroups[i].length > 1) { - colGrpId++; - if (QueryUtil.searchInArray(columnGroups[i], ordinal)) { - break; - } - } - } - return colGrpId; - } - - public KeyGenerator getKeyGenerator(int colGrpId) { - return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); - } } \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index 465bee6..5974666 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.BitSet; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.CarbonUtil; @@ -40,13 +40,13 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator; public class ExcludeFilterExecuterImpl implements FilterExecuter { - protected DimColumnResolvedFilterInfo dimColEvaluatorInfo; - protected DimColumnExecuterFilterInfo dimColumnExecuterInfo; - protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo; - protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo; + private DimColumnResolvedFilterInfo dimColEvaluatorInfo; + private DimColumnExecuterFilterInfo dimColumnExecuterInfo; + private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo; + private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo; protected SegmentProperties segmentProperties; - protected boolean isDimensionPresentInCurrentBlock = false; - protected boolean isMeasurePresentInCurrentBlock = false; + private boolean isDimensionPresentInCurrentBlock = false; + private boolean isMeasurePresentInCurrentBlock = false; private SerializableComparator comparator; /** * is dimension column data is natural sorted @@ -82,48 +82,50 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws IOException { + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { if (isDimensionPresentInCurrentBlock) { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } DimensionRawColumnChunk dimensionRawColumnChunk = - blockChunkHolder.getDimensionRawDataChunk()[blockIndex]; - DimensionColumnDataChunk[] dimensionColumnDataChunks = - dimensionRawColumnChunk.convertToDimColDataChunks(); + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; + DimensionColumnPage[] dimensionColumnPages = + dimensionRawColumnChunk.decodeAllColumnPages(); BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount()); - for (int i = 0; i < dimensionColumnDataChunks.length; i++) { - BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i], + for (int i = 0; i < dimensionColumnPages.length; i++) { + BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i], dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i); + rawBlockletColumnChunks.getBitSetGroup(), i); bitSetGroup.setBitSet(bitSet, i); } return bitSetGroup; } else if (isMeasurePresentInCurrentBlock) { - int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } MeasureRawColumnChunk measureRawColumnChunk = - blockChunkHolder.getMeasureRawDataChunk()[blockIndex]; + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]; ColumnPage[] ColumnPages = - measureRawColumnChunk.convertToColumnPage(); + measureRawColumnChunk.decodeAllColumnPages(); BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount()); DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo); for (int i = 0; i < ColumnPages.length; i++) { BitSet bitSet = getFilteredIndexesForMeasure( - measureRawColumnChunk.convertToColumnPage(i), + measureRawColumnChunk.decodeColumnPage(i), measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), + rawBlockletColumnChunks.getBitSetGroup(), i, msrType); bitSetGroup.setBitSet(bitSet, i); @@ -133,7 +135,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { return null; } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) { + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) { if (isDimensionPresentInCurrentBlock) { byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal()); @@ -275,43 +278,41 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { /** * Below method will be used to apply filter on dimension column - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numberOfRows * @param useBitsetPipeLine * @param prvBitSetGroup * @param pageNumber * @return filtered indexes bitset */ - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) { // check whether applying filtered based on previous bitset will be optimal if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber, dimColumnExecuterInfo.getExcludeFilterKeys().length)) { - return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber, - numberOfRows); + return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber); } else { - return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows); + return getFilteredIndexes(dimensionColumnPage, numberOfRows); } } - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows) { - if (dimensionColumnDataChunk.isExplicitSorted()) { - return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows); + if (dimensionColumnPage.isExplicitSorted()) { + return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows); } - return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows); + return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows); } /** * Below method will be used to apply filter based on previous filtered bitset - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param prvBitSetGroup * @param pageNumber - * @param numberOfRows * @return filtered indexes bitset */ - private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk, - BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) { + private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage, + BitSetGroup prvBitSetGroup, int pageNumber) { BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber); BitSet bitSet = new BitSet(); bitSet.or(prvPageBitSet); @@ -319,11 +320,11 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { int compareResult = 0; // if dimension data was natural sorted then get the index from previous bitset // and use the same in next column data, otherwise use the inverted index reverse - if (!dimensionColumnDataChunk.isExplicitSorted()) { + if (!dimensionColumnPage.isExplicitSorted()) { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index); + .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index); if (compareResult != 0) { bitSet.set(index); } else { @@ -336,8 +337,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, - dimensionColumnDataChunk.getInvertedReverseIndex(index)); + .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, + dimensionColumnPage.getInvertedReverseIndex(index)); if (compareResult != 0) { bitSet.set(index); } else { @@ -351,7 +352,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } private BitSet setFilterdIndexToBitSetWithColumnIndex( - DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) { + DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); int startIndex = 0; @@ -361,10 +362,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { break; } int[] rangeIndex = CarbonUtil - .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[i]); for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) { - bitSet.flip(dimensionColumnDataChunk.getInvertedIndex(j)); + bitSet.flip(dimensionColumnPage.getInvertedIndex(j)); } if (rangeIndex[1] >= 0) { startIndex = rangeIndex[1] + 1; @@ -373,7 +374,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { return bitSet; } - private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); @@ -390,7 +391,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { break; } int[] rangeIndex = CarbonUtil - .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[i]); for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) { bitSet.flip(j); @@ -403,14 +404,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { if (filterValues.length > 1) { for (int i = 0; i < numerOfRows; i++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimensionColumnDataChunk.getChunkData(i)); + dimensionColumnPage.getChunkData(i)); if (index >= 0) { bitSet.flip(i); } } } else { for (int j = 0; j < numerOfRows; j++) { - if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) { + if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) { bitSet.flip(j); } } @@ -419,26 +420,30 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { return bitSet; } - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); bitSet.flip(0, 1); return bitSet; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { if (isDimensionPresentInCurrentBlock) { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock) { - int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java index 85891dc..aac5e63 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java @@ -21,7 +21,7 @@ import java.util.BitSet; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowIntf; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public interface FilterExecuter { @@ -32,8 +32,8 @@ public interface FilterExecuter { * @return * @throws FilterUnsupportedException */ - BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException; + BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException; boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException; @@ -50,7 +50,7 @@ public interface FilterExecuter { /** * It just reads necessary block for filter executor, it does not uncompress the data. - * @param blockChunkHolder + * @param rawBlockletColumnChunks */ - void readBlocks(BlocksChunkHolder blockChunkHolder)throws IOException; + void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)throws IOException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java index 9c4c7ba..b683fd6 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java @@ -23,7 +23,7 @@ import java.util.BitSet; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -41,26 +41,31 @@ public class ImplicitIncludeFilterExecutorImpl } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeline) - throws FilterUnsupportedException { - BitSetGroup bitSetGroup = new BitSetGroup(blockChunkHolder.getDataBlock().numberOfPages()); - for (int i = 0; i < blockChunkHolder.getDataBlock().numberOfPages(); i++) { + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeline) { + BitSetGroup bitSetGroup = new BitSetGroup( + rawBlockletColumnChunks.getDataBlock().numberOfPages()); + for (int i = 0; i < rawBlockletColumnChunks.getDataBlock().numberOfPages(); i++) { bitSetGroup.setBitSet( - setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().getPageRowCount(i)), i); + setBitSetForCompleteDimensionData( + rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)), i); } return bitSetGroup; } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { return false; } - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { return null; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java index 1cbc0bc..e4da26f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java @@ -24,14 +24,14 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.keygenerator.KeyGenerator; import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.executor.util.QueryUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; @@ -59,7 +59,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl /** * It fills BitSet with row index which matches filter key */ - protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); @@ -70,7 +70,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl byte[] filterVal = filterValues[i]; for (int rowId = 0; rowId < numerOfRows; rowId++) { byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length]; - dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo); + dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo); if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) { bitSet.set(rowId); } @@ -85,20 +85,21 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws IOException { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } DimensionRawColumnChunk dimensionRawColumnChunk = - blockChunkHolder.getDimensionRawDataChunk()[blockIndex]; + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount()); for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) { if (dimensionRawColumnChunk.getMaxValues() != null) { - BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i), dimensionRawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } @@ -133,10 +134,10 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl BitSet bitSet = new BitSet(1); byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); int columnIndex = dimColumnEvaluatorInfo.getColumnIndex(); - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex); + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex); int[] cols = getAllColumns(columnIndex); - byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex); - byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex); + byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex); + byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex); boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // filter value should be in range of max and min value i.e http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index fe1421c..05328f3 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.BitSet; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.CarbonUtil; @@ -41,12 +41,12 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator; public class IncludeFilterExecuterImpl implements FilterExecuter { protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo; - protected DimColumnExecuterFilterInfo dimColumnExecuterInfo; - protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo; - protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo; + DimColumnExecuterFilterInfo dimColumnExecuterInfo; + private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo; + private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo; protected SegmentProperties segmentProperties; - protected boolean isDimensionPresentInCurrentBlock = false; - protected boolean isMeasurePresentInCurrentBlock = false; + private boolean isDimensionPresentInCurrentBlock = false; + private boolean isMeasurePresentInCurrentBlock = false; protected SerializableComparator comparator; /** * is dimension column data is natural sorted @@ -86,44 +86,46 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws IOException { - if (isDimensionPresentInCurrentBlock == true) { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { + if (isDimensionPresentInCurrentBlock) { + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } DimensionRawColumnChunk dimensionRawColumnChunk = - blockChunkHolder.getDimensionRawDataChunk()[blockIndex]; + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount()); for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) { if (dimensionRawColumnChunk.getMaxValues() != null) { if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i], dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) { - BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i), dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i); + rawBlockletColumnChunks.getBitSetGroup(), i); bitSetGroup.setBitSet(bitSet, i); } } else { - BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i), dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i); + rawBlockletColumnChunks.getBitSetGroup(), i); bitSetGroup.setBitSet(bitSet, i); } } return bitSetGroup; } else if (isMeasurePresentInCurrentBlock) { - int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } MeasureRawColumnChunk measureRawColumnChunk = - blockChunkHolder.getMeasureRawDataChunk()[blockIndex]; + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount()); DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo); for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) { @@ -132,16 +134,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(), msrColumnEvaluatorInfo.getType())) { BitSet bitSet = - getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i), + getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i), measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i, msrType); + rawBlockletColumnChunks.getBitSetGroup(), i, msrType); bitSetGroup.setBitSet(bitSet, i); } } else { BitSet bitSet = - getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i), + getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i), measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i, msrType); + rawBlockletColumnChunks.getBitSetGroup(), i, msrType); bitSetGroup.setBitSet(bitSet, i); } } @@ -293,43 +295,43 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { /** * Below method will be used to apply filter on dimension column - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numberOfRows * @param useBitsetPipeLine * @param prvBitSetGroup * @param pageNumber * @return filtered indexes bitset */ - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) { // check whether previous indexes can be optimal to apply filter on dimension column if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber, dimColumnExecuterInfo.getFilterKeys().length)) { - return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber, + return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber, numberOfRows); } else { - return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows); + return getFilteredIndexes(dimensionColumnPage, numberOfRows); } } - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows) { - if (dimensionColumnDataChunk.isExplicitSorted()) { - return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows); + if (dimensionColumnPage.isExplicitSorted()) { + return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows); } - return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows); + return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows); } /** * Below method will be used to apply filter on dimension * column based on previous filtered indexes - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param prvBitSetGroup * @param pageNumber * @param numberOfRows * @return filtered bitset */ - private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage, BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) { BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber); BitSet bitSet = new BitSet(numberOfRows); @@ -337,11 +339,11 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { int compareResult = 0; // if dimension data was natural sorted then get the index from previous bitset // and use the same in next column data, otherwise use the inverted index reverse - if (!dimensionColumnDataChunk.isExplicitSorted()) { + if (!dimensionColumnPage.isExplicitSorted()) { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index); + .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index); if (compareResult == 0) { bitSet.set(index); } @@ -350,8 +352,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, - dimensionColumnDataChunk.getInvertedReverseIndex(index)); + .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, + dimensionColumnPage.getInvertedReverseIndex(index)); if (compareResult == 0) { bitSet.set(index); } @@ -360,7 +362,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return bitSet; } private BitSet setFilterdIndexToBitSetWithColumnIndex( - DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) { + DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); int startIndex = 0; byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); @@ -369,10 +371,10 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { break; } int[] rangeIndex = CarbonUtil - .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[i]); for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) { - bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j)); + bitSet.set(dimensionColumnPage.getInvertedIndex(j)); } if (rangeIndex[1] >= 0) { startIndex = rangeIndex[1] + 1; @@ -381,7 +383,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return bitSet; } - private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); @@ -394,7 +396,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { break; } int[] rangeIndex = CarbonUtil - .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[i]); for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) { bitSet.set(j); @@ -407,14 +409,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { if (filterValues.length > 1) { for (int i = 0; i < numerOfRows; i++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimensionColumnDataChunk.getChunkData(i)); + dimensionColumnPage.getChunkData(i)); if (index >= 0) { bitSet.set(i); } } } else { for (int j = 0; j < numerOfRows; j++) { - if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) { + if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) { bitSet.set(j); } } @@ -423,26 +425,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return bitSet; } + @Override public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { BitSet bitSet = new BitSet(1); byte[][] filterValues = null; int columnIndex = 0; - int blockIndex = 0; + int chunkIndex = 0; boolean isScanRequired = false; if (isDimensionPresentInCurrentBlock) { filterValues = dimColumnExecuterInfo.getFilterKeys(); columnIndex = dimColumnEvaluatorInfo.getColumnIndex(); - blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex); - isScanRequired = - isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues); - + chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex); + isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues); } else if (isMeasurePresentInCurrentBlock) { columnIndex = msrColumnEvaluatorInfo.getColumnIndex(); - blockIndex = - segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties - .getLastDimensionColOrdinal(); - isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], + chunkIndex = + segmentProperties.getMeasuresOrdinalToChunkMapping().get(columnIndex) + + segmentProperties.getLastDimensionColOrdinal(); + isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], msrColumnExecutorInfo.getFilterKeys(), msrColumnEvaluatorInfo.getType()); } @@ -492,20 +493,23 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return false; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { - if (isDimensionPresentInCurrentBlock == true) { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { + if (isDimensionPresentInCurrentBlock) { + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } - } else if (isMeasurePresentInCurrentBlock == true) { - int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() + } else if (isMeasurePresentInCurrentBlock) { + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java index 87273bb..9f0afb5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java @@ -21,7 +21,7 @@ import java.util.BitSet; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowIntf; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public class OrFilterExecuterImpl implements FilterExecuter { @@ -35,30 +35,33 @@ public class OrFilterExecuterImpl implements FilterExecuter { } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException { - BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false); - BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false); + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { + BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, false); + BitSetGroup rightFilters = rightExecuter.applyFilter(rawBlockletColumnChunks, false); leftFilters.or(rightFilters); - blockChunkHolder.setBitSetGroup(leftFilters); + rawBlockletColumnChunks.setBitSetGroup(leftFilters); return leftFilters; } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { return leftExecuter.applyFilter(value, dimOrdinalMax) || rightExecuter.applyFilter(value, dimOrdinalMax); } - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue); BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue); leftFilters.or(rightFilters); return leftFilters; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { - leftExecuter.readBlocks(blockChunkHolder); - rightExecuter.readBlocks(blockChunkHolder); + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { + leftExecuter.readColumnChunks(rawBlockletColumnChunks); + rightExecuter.readColumnChunks(rawBlockletColumnChunks); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index ee373c5..4a9716c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -22,11 +22,10 @@ import java.util.BitSet; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; @@ -38,8 +37,7 @@ import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.CarbonUtil; @@ -59,25 +57,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { /** * it has index at which given dimension is stored in file */ - protected int dimensionBlocksIndex; + private int dimensionChunkIndex; /** * flag to check whether the filter dimension is present in current block list of dimensions. * Applicable for restructure scenarios */ - protected boolean isDimensionPresentInCurrentBlock; - boolean lessThanExp; - boolean lessThanEqualExp; - boolean greaterThanExp; - boolean greaterThanEqualExp; - boolean startBlockMinIsDefaultStart; - boolean endBlockMaxisDefaultEnd; - boolean isRangeFullyCoverBlock; + private boolean isDimensionPresentInCurrentBlock; + private boolean lessThanExp; + private boolean lessThanEqualExp; + private boolean greaterThanExp; + private boolean greaterThanEqualExp; + private boolean startBlockMinIsDefaultStart; + private boolean endBlockMaxisDefaultEnd; + private boolean isRangeFullyCoverBlock; public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, - MeasureColumnResolvedFilterInfo msrColEvaluatorInfo, Expression exp, - AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, - SegmentProperties segmentProperties) { + Expression exp, byte[][] filterRangeValues, SegmentProperties segmentProperties) { this.dimColEvaluatorInfo = dimColEvaluatorInfo; this.exp = exp; @@ -90,7 +86,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { startBlockMinIsDefaultStart = false; endBlockMaxisDefaultEnd = false; isRangeFullyCoverBlock = false; - initDimensionBlockIndexes(); + initDimensionChunkIndexes(); ifDefaultValueMatchesFilter(); } @@ -99,13 +95,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * This method will initialize the dimension info for the current block to be * used for filtering the data */ - private void initDimensionBlockIndexes() { + private void initDimensionChunkIndexes() { // find the dimension in the current block dimensions list CarbonDimension dimensionFromCurrentBlock = segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension()); if (null != dimensionFromCurrentBlock) { dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); - this.dimensionBlocksIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + this.dimensionChunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimensionFromCurrentBlock.getOrdinal()); isDimensionPresentInCurrentBlock = true; } @@ -135,14 +131,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { /** * Method to apply the filter. - * @param blockChunkHolder + * @param rawBlockletColumnChunks * @return * @throws FilterUnsupportedException * @throws IOException */ - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException { - return applyNoAndDirectFilter(blockChunkHolder); + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { + return applyNoAndDirectFilter(rawBlockletColumnChunks); } /** @@ -243,32 +239,32 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // Filter Min <-----------------------------------------------> Filter Max if (isDimensionPresentInCurrentBlock) { - if (((lessThanExp == true) && ( + if (((lessThanExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || ( - (lessThanEqualExp == true) && ( + (lessThanEqualExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || ( - (greaterThanExp == true) && ( + (greaterThanExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || ( - (greaterThanEqualExp == true) && ( + (greaterThanEqualExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) { // completely out of block boundary isScanRequired = false; } else { - if (((greaterThanExp == true) && ( + if (((greaterThanExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || ( - (greaterThanEqualExp == true) && ( + (greaterThanEqualExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) { startBlockMinIsDefaultStart = true; } - if (((lessThanExp == true) && ( + if (((lessThanExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || ( - (lessThanEqualExp == true) && ( + (lessThanEqualExp) && ( ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) { endBlockMaxisDefaultEnd = true; } - if (startBlockMinIsDefaultStart == true && endBlockMaxisDefaultEnd == true) { + if (startBlockMinIsDefaultStart && endBlockMaxisDefaultEnd) { isRangeFullyCoverBlock = true; } } @@ -284,7 +280,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * @param blockMinValue * @return */ - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); byte[][] filterValues = this.filterRangesValues; int columnIndex = this.dimColEvaluatorInfo.getColumnIndex(); @@ -300,50 +297,49 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * Method to apply the Range Filter. * @param blockChunkHolder * @return - * @throws FilterUnsupportedException * @throws IOException */ - public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder) - throws FilterUnsupportedException, IOException { + private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder) + throws IOException { // In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be // false, in that scenario the default values of the column should be shown. // select all rows if dimension does not exists in the current block if (!isDimensionPresentInCurrentBlock) { - int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); - return FilterUtil - .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), - numberOfRows, true); + int numberOfRows = blockChunkHolder.getDataBlock().numRows(); + return FilterUtil.createBitSetGroupWithDefaultValue( + blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true); } - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]) { + blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex] = + blockChunkHolder.getDataBlock().readDimensionChunk( + blockChunkHolder.getFileReader(), chunkIndex); } DimensionRawColumnChunk rawColumnChunk = - blockChunkHolder.getDimensionRawDataChunk()[blockIndex]; + blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i], this.filterRangesValues)) { - if (isRangeFullyCoverBlock == true) { + if (isRangeFullyCoverBlock) { // Set all the bits in this case as filter Min Max values cover the whole block. BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]); bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } } } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } @@ -351,12 +347,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { return bitSetGroup; } - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numerOfRows) { - if (dimensionColumnDataChunk.isExplicitSorted()) { - return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows); + if (dimensionColumnPage.isExplicitSorted()) { + return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows); } - return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows); + return setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows); } /** @@ -365,12 +361,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * column is not supported by default so column index mapping will be present for * accesing the members from the block. * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numerOfRows * @return BitSet. */ private BitSet setFilterdIndexToBitSetWithColumnIndex( - DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) { + DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); int start = 0; int startIndex = 0; @@ -381,13 +377,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // For Range expression we expect two values. The First is the Min Value and Second is the // Max value. // Get the Min Value - if (startBlockMinIsDefaultStart == false) { + if (!startBlockMinIsDefaultStart) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[0], greaterThanExp); - if (greaterThanExp == true && start >= 0) { + if (greaterThanExp && start >= 0) { start = CarbonUtil - .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0], + .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0], numerOfRows); } @@ -399,8 +395,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // Method will compare the tentative index value after binary search, this tentative // index needs to be compared by the filter member if its >= filter then from that // index the bitset will be considered for filtering process. - if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk - .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) { + if ((ByteUtil.compare(filterValues[0], dimensionColumnPage + .getChunkData(dimensionColumnPage.getInvertedIndex(start)))) > 0) { start = start + 1; } } @@ -411,14 +407,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { } // Get the Max value - if (endBlockMaxisDefaultEnd == false) { + if (!endBlockMaxisDefaultEnd) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[1], lessThanEqualExp); - if (lessThanExp == true && start >= 0) { + if (lessThanExp && start >= 0) { start = - CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]); + CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]); } if (start < 0) { @@ -429,7 +425,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // In case the start is less than 0, then positive value of start is pointing to the next // value of the searched key. So move to the previous one. if ((ByteUtil.compare(filterValues[1], - dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start))) + dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start))) < 0)) { start = start - 1; } @@ -440,18 +436,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { } for (int j = startMin; j <= endMax; j++) { - bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j)); + bitSet.set(dimensionColumnPage.getInvertedIndex(j)); } // Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on // matching row. - if (dimensionColumnDataChunk.isNoDicitionaryColumn()) { - updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet); + if (dimensionColumnPage.isNoDicitionaryColumn()) { + updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet); } return bitSet; } - private void updateForNoDictionaryColumn(int start, int end, DimensionColumnDataChunk dataChunk, + private void updateForNoDictionaryColumn(int start, int end, DimensionColumnPage dataChunk, BitSet bitset) { for (int j = start; j <= end; j++) { if (dataChunk.compareTo(j, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) == 0 @@ -467,31 +463,31 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * be called if the column is sorted default so column index * mapping will be present for accesaing the members from the block. * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numerOfRows * @return BitSet. */ - private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - // if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) { + // if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) { byte[][] filterValues = this.filterRangesValues; - if (dimensionColumnDataChunk.isExplicitSorted()) { + if (dimensionColumnPage.isExplicitSorted()) { int start = 0; int startMin = 0; int endMax = 0; int startIndex = 0; // For Range expression we expect two values. The First is the Min Value and Second is the // Max value. - if (startBlockMinIsDefaultStart == false) { + if (!startBlockMinIsDefaultStart) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[0], greaterThanExp); - if (greaterThanExp == true && start >= 0) { + if (greaterThanExp && start >= 0) { start = CarbonUtil - .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0], + .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0], numerOfRows); } @@ -503,7 +499,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // Method will compare the tentative index value after binary search, this tentative // index needs to be compared by the filter member if its >= filter then from that // index the bitset will be considered for filtering process. - if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start))) + if ((ByteUtil.compare(filterValues[0], dimensionColumnPage.getChunkData(start))) > 0) { start = start + 1; } @@ -513,14 +509,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { startMin = startIndex; } - if (endBlockMaxisDefaultEnd == false) { + if (!endBlockMaxisDefaultEnd) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[1], lessThanEqualExp); - if (lessThanExp == true && start >= 0) { + if (lessThanExp && start >= 0) { start = - CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]); + CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]); } if (start < 0) { @@ -530,7 +526,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { } // In case the start is less than 0, then positive value of start is pointing to the next // value of the searched key. So move to the previous one. - if ((ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start)) + if ((ByteUtil.compare(filterValues[1], dimensionColumnPage.getChunkData(start)) < 0)) { start = start - 1; } @@ -546,8 +542,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { // Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on // matching row. - if (dimensionColumnDataChunk.isNoDicitionaryColumn()) { - updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet); + if (dimensionColumnPage.isNoDicitionaryColumn()) { + updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet); } } else { byte[] defaultValue = null; @@ -556,7 +552,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { .getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType()); int key = directDictionaryGenerator.generateDirectSurrogateKey(null); CarbonDimension currentBlockDimension = - segmentProperties.getDimensions().get(dimensionBlocksIndex); + segmentProperties.getDimensions().get(dimensionChunkIndex); if (currentBlockDimension.isSortColumn()) { defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, this.segmentProperties.getSortColumnsGenerator()); @@ -572,55 +568,37 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { } // evaluate result for lower range value first and then perform and operation in the // upper range value in order to compute the final result - bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[0], + bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[0], numerOfRows); BitSet upperRangeBitSet = - evaluateLessThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[1], + evaluateLessThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[1], numerOfRows); bitSet.and(upperRangeBitSet); - FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet, defaultValue); + FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue); } return bitSet; } /** - * This method will compare the selected data against null values and - * flip the bitSet if any null value is found - * - * @param dimensionColumnDataChunk - * @param bitSet - */ - private void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk, BitSet bitSet) { - if (!bitSet.isEmpty()) { - for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { - if (dimensionColumnDataChunk.compareTo(i, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) - == 0) { - bitSet.flip(i); - } - } - } - } - - /** * This method will evaluate the result for filter column based on the lower range value * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param filterValue * @param numberOfRows * @return */ private BitSet evaluateGreaterThanFilterForUnsortedColumn( - DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) { + DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) { BitSet bitSet = new BitSet(numberOfRows); if (greaterThanExp) { for (int i = 0; i < numberOfRows; i++) { - if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) > 0)) { + if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) > 0)) { bitSet.set(i); } } } else if (greaterThanEqualExp) { for (int i = 0; i < numberOfRows; i++) { - if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) >= 0)) { + if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) >= 0)) { bitSet.set(i); } } @@ -631,23 +609,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { /** * This method will evaluate the result for filter column based on the upper range value * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param filterValue * @param numberOfRows * @return */ private BitSet evaluateLessThanFilterForUnsortedColumn( - DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) { + DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) { BitSet bitSet = new BitSet(numberOfRows); if (lessThanExp) { for (int i = 0; i < numberOfRows; i++) { - if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) < 0)) { + if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) < 0)) { bitSet.set(i); } } } else if (lessThanEqualExp) { for (int i = 0; i < numberOfRows; i++) { - if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) <= 0)) { + if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) <= 0)) { bitSet.set(i); } } @@ -657,16 +635,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { /** * Method to read the blocks. - * @param blockChunkHolder + * @param rawBlockletColumnChunks * @throws IOException */ - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { - if (isDimensionPresentInCurrentBlock == true) { - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { + if (isDimensionPresentInCurrentBlock) { + int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColEvaluatorInfo.getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java index 5707eb4..d7bec7e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java @@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl { @@ -34,7 +34,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm * flag to check whether filter values contain the default value applied on the dimension column * which does not exist in the current block */ - protected boolean isDefaultValuePresentInFilterValues; + private boolean isDefaultValuePresentInFilterValues; public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) { @@ -48,26 +48,29 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws IOException { - int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { + int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); return FilterUtil - .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), + .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, !isDefaultValuePresentInFilterValues); } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException { throw new FilterUnsupportedException("Unsupported RestructureExcludeFilterExecutorImpl on row"); } - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); bitSet.flip(0, 1); return bitSet; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) { } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7b79ff18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java index 8bcc53f..c874fc5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java @@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl { @@ -33,7 +33,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm * flag to check whether filter values contain the default value applied on the dimension column * which does not exist in the current block */ - protected boolean isDefaultValuePresentInFilterValues; + private boolean isDefaultValuePresentInFilterValues; public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) { @@ -47,15 +47,16 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws IOException { - int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); - return FilterUtil - .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), - numberOfRows, isDefaultValuePresentInFilterValues); + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { + int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); + return FilterUtil.createBitSetGroupWithDefaultValue( + rawBlockletColumnChunks.getDataBlock().numberOfPages(), + numberOfRows, isDefaultValuePresentInFilterValues); } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException { throw new FilterUnsupportedException("Unsupported RestructureIncludeFilterExecutorImpl on row"); } @@ -66,7 +67,8 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm return bitSet; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) { }