Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2820#discussion_r226863869
--- Diff:
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
---
@@ -179,6 +167,75 @@ public BitSetGroup applyFilter(RawBlockletColumnChunks
rawBlockletColumnChunks,
return null;
}
+ private boolean isScanRequired(DimensionRawColumnChunk
dimensionRawColumnChunk, int i) {
+ boolean scanRequired;
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if
(DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
+ &&
!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
+ dimensionRawColumnChunk.getMinValues()[i],
dimColumnExecuterInfo.getFilterKeys(),
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else {
+ scanRequired =
isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
+ dimensionRawColumnChunk.getMinValues()[i],
dimColumnExecuterInfo.getFilterKeys(),
+ dimensionRawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ return scanRequired;
+ }
+
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex =
segmentProperties.getDimensionOrdinalToChunkMapping()
+ .get(dimColumnEvaluatorInfo.getColumnIndex());
+ if (null ==
rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+
.readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk dimensionRawColumnChunk =
+
rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ filterValues = dimColumnExecuterInfo.getFilterKeys();
+ BitSet bitSet = new BitSet(dimensionRawColumnChunk.getPagesCount());
+ for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
+ if (dimensionRawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(dimensionRawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else if (isMeasurePresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColumnEvaluatorInfo.getColumnIndex());
+ if (null ==
rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(),
chunkIndex);
+ }
+ MeasureRawColumnChunk measureRawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(measureRawColumnChunk.getPagesCount());
+ for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) {
+ if (measureRawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(measureRawColumnChunk.getMaxValues()[i],
+ measureRawColumnChunk.getMinValues()[i],
msrColumnExecutorInfo.getFilterKeys(),
+ msrColumnEvaluatorInfo.getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+ return null;
--- End diff --
this case not supposed to happen, even in applyFilter also return null.
---
