Repository: carbondata Updated Branches: refs/heads/master f781337ea -> d2e70a464
[CARBONDATA-1353] Fixed measure filter tests with null SDV tests are failing on measure filter http://144.76.159.231:8080/job/ApacheSDVTests/32/, now those are fixed This closes #1222 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d2e70a46 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d2e70a46 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d2e70a46 Branch: refs/heads/master Commit: d2e70a464c342d8ef51331bd5bd195c3d76658a1 Parents: f781337 Author: Ravindra Pesala <[email protected]> Authored: Tue Aug 1 15:35:13 2017 +0530 Committer: chenliang613 <[email protected]> Committed: Wed Aug 2 07:21:49 2017 +0800 ---------------------------------------------------------------------- .../statistics/PrimitivePageStatsCollector.java | 2 +- .../executer/ExcludeFilterExecuterImpl.java | 24 ++++++++++--------- .../executer/IncludeFilterExecuterImpl.java | 25 ++++++++++---------- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 25 +++++++++++--------- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 23 ++++++++++-------- ...velRangeLessThanEqualFilterExecuterImpl.java | 18 +++++++------- .../RowLevelRangeLessThanFiterExecuterImpl.java | 18 +++++++------- .../apache/carbondata/core/util/CarbonUtil.java | 8 ------- 8 files changed, 71 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java index 294d699..918039a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java @@ -259,7 +259,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si isFirst = false; } else { maxDecimal = (decimalValue.compareTo(maxDecimal) > 0) ? decimalValue : maxDecimal; - minDecimal = (decimalValue.compareTo(maxDecimal) < 0) ? decimalValue : minDecimal; + minDecimal = (decimalValue.compareTo(minDecimal) < 0) ? decimalValue : minDecimal; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index 6601797..20217b7 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -53,7 +53,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, SegmentProperties segmentProperties, boolean isMeasure) { this.segmentProperties = segmentProperties; - if (isMeasure == false) { + if (!isMeasure) { this.dimColEvaluatorInfo = dimColEvaluatorInfo; dimColumnExecuterInfo = new DimColumnExecuterFilterInfo(); @@ -75,7 +75,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException { - if (isDimensionPresentInCurrentBlock == true) { + if (isDimensionPresentInCurrentBlock) { int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() .get(dimColEvaluatorInfo.getColumnIndex()); if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { @@ -94,7 +94,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } return bitSetGroup; - } else if (isMeasurePresentInCurrentBlock == true) { + } else if (isMeasurePresentInCurrentBlock) { int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { @@ -143,22 +143,24 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { Object[] filterValues = msrColumnExecutorInfo.getFilterKeys(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); for (int i = 0; i < filterValues.length; i++) { + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.flip(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { - // Check if filterValue[i] matches with measure Values. - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColumnEvaluatorInfo.getMeasure()); + if (!nullBitSet.get(startIndex)) { + // Check if filterValue[i] matches with measure Values. + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColumnEvaluatorInfo.getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) == 0) { - // This is a match. - bitSet.flip(startIndex); + if (comparator.compare(msrValue, filterValues[i]) == 0) { + // This is a match. + bitSet.flip(startIndex); + } } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index f848e07..9a5e754 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -107,7 +107,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { } } return bitSetGroup; - } else if (isMeasurePresentInCurrentBlock == true) { + } else if (isMeasurePresentInCurrentBlock) { int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { @@ -164,23 +164,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { Object[] filterValues = msrColumnExecutorInfo.getFilterKeys(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < rowsInPage; startIndex++) { - // Check if filterValue[i] matches with measure Values. - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColumnEvaluatorInfo.getMeasure()); + if (!nullBitSet.get(startIndex)) { + // Check if filterValue[i] matches with measure Values. + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColumnEvaluatorInfo.getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) == 0) { - // This is a match. - bitSet.set(startIndex); + if (comparator.compare(msrValue, filterValues[i]) == 0) { + // This is a match. + bitSet.set(startIndex); + } } } } @@ -266,7 +268,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { int blockIndex = 0; boolean isScanRequired = false; - if (isDimensionPresentInCurrentBlock == true) { + if (isDimensionPresentInCurrentBlock) { filterValues = dimColumnExecuterInfo.getFilterKeys(); columnIndex = dimColumnEvaluatorInfo.getColumnIndex(); blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex); @@ -275,9 +277,6 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { } else if (isMeasurePresentInCurrentBlock) { columnIndex = msrColumnEvaluatorInfo.getColumnIndex(); - // blockIndex = - // segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex) + segmentProperties - // .getLastDimensionColOrdinal(); blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties .getLastDimensionColOrdinal(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index a1774dd..ca6df0c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -223,13 +223,16 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil .getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], msrColEvalutorInfoList.get(0).getType())); - if (compare < 0) { + MeasureColumnDataChunk measureColumnDataChunk = + rawColumnChunk.convertToMeasureColDataChunk(i); + if (compare < 0 && + measureColumnDataChunk.getNullValueIndexHolder().getBitSet().isEmpty()) { BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]); bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } else { BitSet bitSet = - getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i), + getFilteredIndexesForMeasures(measureColumnDataChunk, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } @@ -252,27 +255,27 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColEvalutorInfoList.get(0).getMeasure()); + if (!nullBitSet.get(startIndex)) { + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) > 0) { - // This is a match. - bitSet.set(startIndex); + if (comparator.compare(msrValue, filterValues[i]) > 0) { + // This is a match. + bitSet.set(startIndex); + } } } } - CarbonUtil - .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet); return bitSet; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index 7823e34..e20f4a6 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -222,7 +222,10 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil .getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], msrColEvalutorInfoList.get(0).getType())); - if (compare <= 0) { + MeasureColumnDataChunk measureColumnDataChunk = + rawColumnChunk.convertToMeasureColDataChunk(i); + if (compare <= 0 && + measureColumnDataChunk.getNullValueIndexHolder().getBitSet().isEmpty()) { BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]); bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); @@ -251,27 +254,27 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColEvalutorInfoList.get(0).getMeasure()); + if (!nullBitSet.get(startIndex)) { + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) >= 0) { - // This is a match. - bitSet.set(startIndex); + if (comparator.compare(msrValue, filterValues[i]) >= 0) { + // This is a match. + bitSet.set(startIndex); + } } } } - CarbonUtil - .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet); return bitSet; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index 422a099..7b9e7d2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -234,27 +234,27 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColEvalutorInfoList.get(0).getMeasure()); + if (!nullBitSet.get(startIndex)) { + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) <= 0) { - // This is a match. - bitSet.set(startIndex); + if (comparator.compare(msrValue, filterValues[i]) <= 0) { + // This is a match. + bitSet.set(startIndex); + } } } } - CarbonUtil - .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet); return bitSet; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java index 4f5f1e1..4681e4e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java @@ -236,27 +236,27 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); + BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { - BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { - Object msrValue = DataTypeUtil - .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, - msrType, msrColEvalutorInfoList.get(0).getMeasure()); + if (!nullBitSet.get(startIndex)) { + Object msrValue = DataTypeUtil + .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex, + msrType, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filterValues[i]) < 0) { - // This is a match. - bitSet.set(startIndex); + if (comparator.compare(msrValue, filterValues[i]) < 0) { + // This is a match. + bitSet.set(startIndex); + } } } } - CarbonUtil - .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet); return bitSet; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2e70a46/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index b5cfc53..7628415 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -30,7 +30,6 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -1741,13 +1740,6 @@ public final class CarbonUtil { return !(null == badRecordsLocation || badRecordsLocation.length() == 0); } - public static void updateBitSetForNull(BitSet nullBitSet, BitSet filterBitSet) { - for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { - filterBitSet.flip(j); - } - } - - public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator, String quote, String prefix) { Gson gson = new Gson();
