Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1079#discussion_r125154226
--- Diff:
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
---
@@ -91,67 +131,167 @@ private void ifDefaultValueMatchesFilter() {
private boolean isScanRequired(byte[] blockMaxValue, byte[][]
filterValues) {
boolean isScanRequired = false;
- if (isDimensionPresentInCurrentBlock[0]) {
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare =
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
- // if any filter value is in range than this block needs to be
- // scanned less than equal to max range.
- if (maxCompare <= 0) {
- isScanRequired = true;
- break;
- }
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare =
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare <= 0) {
+ isScanRequired = true;
+ break;
}
- } else {
- isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
+ private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+ DataType dataType) {
+ for (int i = 0; i < filterValue.length; i++) {
+ if (filterValue[i].length == 0 || maxValue.length == 0) {
+ return isScanRequired(maxValue, filterValue);
+ }
+ switch (dataType) {
+ case DOUBLE:
+ double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble();
+ double filterValueDouble =
ByteBuffer.wrap(filterValue[i]).getDouble();
+ if (filterValueDouble <= maxValueDouble) {
+ return true;
+ }
+ break;
+ case INT:
+ case SHORT:
+ case LONG:
+ long maxValueLong = ByteBuffer.wrap(maxValue).getLong();
+ long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+ if (filterValueLong <= maxValueLong) {
+ return true;
+ }
+ break;
+ case DECIMAL:
+ BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue);
+ BigDecimal filterDecimal =
DataTypeUtil.byteToBigDecimal(filterValue[i]);
+ if (filterDecimal.compareTo(maxDecimal) <= 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override public BitSetGroup applyFilter(BlocksChunkHolder
blockChunkHolder)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
- if (!isDimensionPresentInCurrentBlock[0]) {
+ if (!isDimensionPresentInCurrentBlock[0] &&
!isMeasurePresentInCurrentBlock[0]) {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
numberOfRows, true);
}
- int blockIndex =
-
segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] =
blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- BitSetGroup bitSetGroup = new
BitSetGroup(rawColumnChunk.getPagesCount());
- for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
- if (rawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(rawColumnChunk.getMaxValues()[i],
this.filterRangeValues)) {
- int compare = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(filterRangeValues[0],
rawColumnChunk.getMinValues()[i]);
- if (compare <= 0) {
- BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
- bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
- } else {
- BitSet bitSet =
getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
+
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int blockIndex =
+
segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex])
{
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] =
blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(),
blockIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new
BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i],
this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0],
rawColumnChunk.getMinValues()[i]);
+ if (compare <= 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet =
getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
}
+ } else {
+ BitSet bitSet =
getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex =
+
segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] =
blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk =
blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new
BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i],
this.filterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
--- End diff --
Binary comparison cannot be used here
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
