This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1cb172aae7 Proper null handling in equality, inequality and membership
operators for all SV column data types (#9173)
1cb172aae7 is described below
commit 1cb172aae7f1d9cd5855c74b9f004405aeaedeaa
Author: nizarhejazi <[email protected]>
AuthorDate: Fri Aug 5 18:08:10 2022 -0700
Proper null handling in equality, inequality and membership operators for
all SV column data types (#9173)
1. Proper null handling in equality, inequality and membership operators
for all SV column data types.
2. Fix a bug in retrieving dictionary id set of the values in the given
IN/NOT_IN predicate
---
.../dociditerators/SVScanDocIdIterator.java | 259 ++++++++++++++++++++-
.../core/operator/docidsets/SVScanDocIdSet.java | 10 +-
.../core/operator/filter/FilterOperatorUtils.java | 14 +-
.../filter/RangeIndexBasedFilterOperator.java | 3 +-
.../operator/filter/ScanBasedFilterOperator.java | 7 +-
.../operator/filter/predicate/PredicateUtils.java | 20 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 6 +-
.../core/query/reduce/GapfillFilterHandler.java | 3 +-
.../core/query/reduce/GroupByDataTableReducer.java | 3 +-
.../core/query/reduce/HavingFilterHandler.java | 5 +-
.../core/query/reduce/filter/AndRowMatcher.java | 6 +-
.../core/query/reduce/filter/NotRowMatcher.java | 5 +-
.../core/query/reduce/filter/OrRowMatcher.java | 6 +-
.../query/reduce/filter/PredicateRowMatcher.java | 7 +-
.../query/reduce/filter/RowMatcherFactory.java | 11 +-
.../core/query/reduce/HavingFilterHandlerTest.java | 8 +-
.../apache/pinot/queries/AllNullQueriesTest.java | 31 +++
.../pinot/queries/BigDecimalQueriesTest.java | 48 ++++
.../queries/BooleanNullEnabledQueriesTest.java | 194 +++++++++++++--
.../pinot/perf/BenchmarkScanDocIdIterators.java | 2 +-
20 files changed, 590 insertions(+), 58 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
index 61e0138f3a..16da5704a1 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
@@ -18,10 +18,12 @@
*/
package org.apache.pinot.core.operator.dociditerators;
+import javax.annotation.Nullable;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.segment.spi.Constants;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.roaringbitmap.BatchIterator;
import org.roaringbitmap.RoaringBitmapWriter;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
@@ -48,12 +50,17 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
private int _nextDocId = 0;
private long _numEntriesScanned = 0L;
- public SVScanDocIdIterator(PredicateEvaluator predicateEvaluator,
ForwardIndexReader reader, int numDocs) {
+ public SVScanDocIdIterator(PredicateEvaluator predicateEvaluator,
ForwardIndexReader reader, int numDocs,
+ @Nullable NullValueVectorReader nullValueReader) {
_predicateEvaluator = predicateEvaluator;
_reader = reader;
_readerContext = reader.createContext();
_numDocs = numDocs;
- _valueMatcher = getValueMatcher();
+ ImmutableRoaringBitmap nullBitmap = nullValueReader != null ?
nullValueReader.getNullBitmap() : null;
+ if (nullBitmap != null && nullBitmap.isEmpty()) {
+ nullBitmap = null;
+ }
+ _valueMatcher = getValueMatcher(nullBitmap);
}
@Override
@@ -122,25 +129,25 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
return _numEntriesScanned;
}
- private ValueMatcher getValueMatcher() {
+ private ValueMatcher getValueMatcher(@Nullable ImmutableRoaringBitmap
nullBitmap) {
if (_reader.isDictionaryEncoded()) {
- return new DictIdMatcher();
+ return nullBitmap == null ? new DictIdMatcher() : new
DictIdMatcherAndNullHandler(nullBitmap);
} else {
switch (_reader.getStoredType()) {
case INT:
- return new IntMatcher();
+ return nullBitmap == null ? new IntMatcher() : new
IntMatcherAndNullHandler(nullBitmap);
case LONG:
- return new LongMatcher();
+ return nullBitmap == null ? new LongMatcher() : new
LongMatcherAndNullHandler(nullBitmap);
case FLOAT:
- return new FloatMatcher();
+ return nullBitmap == null ? new FloatMatcher() : new
FloatMatcherAndNullHandler(nullBitmap);
case DOUBLE:
- return new DoubleMatcher();
+ return nullBitmap == null ? new DoubleMatcher() : new
DoubleMatcherAndNullHandler(nullBitmap);
case BIG_DECIMAL:
- return new BigDecimalMatcher();
+ return nullBitmap == null ? new BigDecimalMatcher() : new
BigDecimalMatcherAndNullHandler(nullBitmap);
case STRING:
- return new StringMatcher();
+ return nullBitmap == null ? new StringMatcher() : new
StringMatcherAndNullHandler(nullBitmap);
case BYTES:
- return new BytesMatcher();
+ return nullBitmap == null ? new BytesMatcher() : new
BytesMatcherAndNullHandler(nullBitmap);
default:
throw new UnsupportedOperationException();
}
@@ -172,6 +179,57 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private static class MatcherUtils {
+ public static int removeNullDocs(int[] docIds, int[] values, int limit,
ImmutableRoaringBitmap nullBitmap) {
+ assert !nullBitmap.isEmpty();
+ int copyToIdx = 0;
+ for (int i = 0; i < limit; i++) {
+ if (!nullBitmap.contains(docIds[i])) {
+ // Compact non-null entries into the prefix of the docIds and values
arrays.
+ docIds[copyToIdx] = docIds[i];
+ values[copyToIdx++] = values[i];
+ }
+ }
+ return copyToIdx;
+ }
+
+ public static int removeNullDocs(int[] docIds, long[] values, int limit,
ImmutableRoaringBitmap nullBitmap) {
+ assert !nullBitmap.isEmpty();
+ int copyToIdx = 0;
+ for (int i = 0; i < limit; i++) {
+ if (!nullBitmap.contains(docIds[i])) {
+ docIds[copyToIdx] = docIds[i];
+ values[copyToIdx++] = values[i];
+ }
+ }
+ return copyToIdx;
+ }
+
+ public static int removeNullDocs(int[] docIds, float[] values, int limit,
ImmutableRoaringBitmap nullBitmap) {
+ assert !nullBitmap.isEmpty();
+ int copyToIdx = 0;
+ for (int i = 0; i < limit; i++) {
+ if (!nullBitmap.contains(docIds[i])) {
+ docIds[copyToIdx] = docIds[i];
+ values[copyToIdx++] = values[i];
+ }
+ }
+ return copyToIdx;
+ }
+
+ public static int removeNullDocs(int[] docIds, double[] values, int limit,
ImmutableRoaringBitmap nullBitmap) {
+ assert !nullBitmap.isEmpty();
+ int copyToIdx = 0;
+ for (int i = 0; i < limit; i++) {
+ if (!nullBitmap.contains(docIds[i])) {
+ docIds[copyToIdx] = docIds[i];
+ values[copyToIdx++] = values[i];
+ }
+ }
+ return copyToIdx;
+ }
+ }
+
private class DictIdMatcher implements ValueMatcher {
private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
@@ -188,6 +246,34 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class DictIdMatcherAndNullHandler implements ValueMatcher {
+
+ private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final ImmutableRoaringBitmap _nullBitmap;
+
+ public DictIdMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ // Any comparison (equality, inequality, or membership) with null
results in false (similar to Presto) even if
+ // the compared with value is null, and comparison is equality.
+ // To consider nulls, use: IS NULL, or IS NOT NULL operators.
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getDictId(docId,
_readerContext));
+ }
+
+ @Override
+ public int matchValues(int limit, int[] docIds) {
+ _reader.readDictIds(docIds, limit, _buffer, _readerContext);
+ int newLimit = MatcherUtils.removeNullDocs(docIds, _buffer, limit,
_nullBitmap);
+ return _predicateEvaluator.applySV(newLimit, docIds, _buffer);
+ }
+ }
+
private class IntMatcher implements ValueMatcher {
private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
@@ -204,6 +290,31 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class IntMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+ private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+
+ public IntMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getInt(docId,
_readerContext));
+ }
+
+ @Override
+ public int matchValues(int limit, int[] docIds) {
+ _reader.readValuesSV(docIds, limit, _buffer, _readerContext);
+ int newLimit = MatcherUtils.removeNullDocs(docIds, _buffer, limit,
_nullBitmap);
+ return _predicateEvaluator.applySV(newLimit, docIds, _buffer);
+ }
+ }
+
private class LongMatcher implements ValueMatcher {
private final long[] _buffer = new long[OPTIMAL_ITERATOR_BATCH_SIZE];
@@ -220,6 +331,31 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class LongMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+ private final long[] _buffer = new long[OPTIMAL_ITERATOR_BATCH_SIZE];
+
+ public LongMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getLong(docId,
_readerContext));
+ }
+
+ @Override
+ public int matchValues(int limit, int[] docIds) {
+ _reader.readValuesSV(docIds, limit, _buffer, _readerContext);
+ int newLimit = MatcherUtils.removeNullDocs(docIds, _buffer, limit,
_nullBitmap);
+ return _predicateEvaluator.applySV(newLimit, docIds, _buffer);
+ }
+ }
+
private class FloatMatcher implements ValueMatcher {
private final float[] _buffer = new float[OPTIMAL_ITERATOR_BATCH_SIZE];
@@ -236,6 +372,31 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class FloatMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+ private final float[] _buffer = new float[OPTIMAL_ITERATOR_BATCH_SIZE];
+
+ public FloatMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getFloat(docId,
_readerContext));
+ }
+
+ @Override
+ public int matchValues(int limit, int[] docIds) {
+ _reader.readValuesSV(docIds, limit, _buffer, _readerContext);
+ int newLimit = MatcherUtils.removeNullDocs(docIds, _buffer, limit,
_nullBitmap);
+ return _predicateEvaluator.applySV(newLimit, docIds, _buffer);
+ }
+ }
+
private class DoubleMatcher implements ValueMatcher {
private final double[] _buffer = new double[OPTIMAL_ITERATOR_BATCH_SIZE];
@@ -252,6 +413,31 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class DoubleMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+ private final double[] _buffer = new double[OPTIMAL_ITERATOR_BATCH_SIZE];
+
+ public DoubleMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getDouble(docId,
_readerContext));
+ }
+
+ @Override
+ public int matchValues(int limit, int[] docIds) {
+ _reader.readValuesSV(docIds, limit, _buffer, _readerContext);
+ int newLimit = MatcherUtils.removeNullDocs(docIds, _buffer, limit,
_nullBitmap);
+ return _predicateEvaluator.applySV(newLimit, docIds, _buffer);
+ }
+ }
+
private class BigDecimalMatcher implements ValueMatcher {
@Override
@@ -260,6 +446,23 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class BigDecimalMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+
+ public BigDecimalMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getBigDecimal(docId,
_readerContext));
+ }
+ }
+
private class StringMatcher implements ValueMatcher {
@Override
@@ -268,6 +471,23 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
}
}
+ private class StringMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+
+ public StringMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getString(docId,
_readerContext));
+ }
+ }
+
private class BytesMatcher implements ValueMatcher {
@Override
@@ -275,4 +495,21 @@ public final class SVScanDocIdIterator implements
ScanBasedDocIdIterator {
return _predicateEvaluator.applySV(_reader.getBytes(docId,
_readerContext));
}
}
+
+ private class BytesMatcherAndNullHandler implements ValueMatcher {
+
+ private final ImmutableRoaringBitmap _nullBitmap;
+
+ public BytesMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
+ _nullBitmap = nullBitmap;
+ }
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ if (_nullBitmap.contains(docId)) {
+ return false;
+ }
+ return _predicateEvaluator.applySV(_reader.getBytes(docId,
_readerContext));
+ }
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/SVScanDocIdSet.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/SVScanDocIdSet.java
index 919d6525bb..e55998ca40 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/SVScanDocIdSet.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/SVScanDocIdSet.java
@@ -20,14 +20,18 @@ package org.apache.pinot.core.operator.docidsets;
import org.apache.pinot.core.operator.dociditerators.SVScanDocIdIterator;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
public final class SVScanDocIdSet implements FilterBlockDocIdSet {
private final SVScanDocIdIterator _docIdIterator;
- public SVScanDocIdSet(PredicateEvaluator predicateEvaluator,
ForwardIndexReader<?> reader, int numDocs) {
- _docIdIterator = new SVScanDocIdIterator(predicateEvaluator, reader,
numDocs);
+ public SVScanDocIdSet(PredicateEvaluator predicateEvaluator, DataSource
dataSource, int numDocs,
+ boolean nullHandlingEnabled) {
+ NullValueVectorReader nullValueVector = nullHandlingEnabled ?
dataSource.getNullValueVector() : null;
+ _docIdIterator = new SVScanDocIdIterator(
+ predicateEvaluator, dataSource.getForwardIndex(), numDocs,
nullValueVector);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index b0c4a13856..3aa1dfafcd 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -36,6 +36,14 @@ public class FilterOperatorUtils {
*/
public static BaseFilterOperator getLeafFilterOperator(PredicateEvaluator
predicateEvaluator, DataSource dataSource,
int numDocs) {
+ return getLeafFilterOperator(predicateEvaluator, dataSource, numDocs,
false);
+ }
+
+ /**
+ * Returns the leaf filter operator (i.e. not {@link AndFilterOperator} or
{@link OrFilterOperator}).
+ */
+ public static BaseFilterOperator getLeafFilterOperator(PredicateEvaluator
predicateEvaluator, DataSource dataSource,
+ int numDocs, boolean nullHandlingEnabled) {
if (predicateEvaluator.isAlwaysFalse()) {
return EmptyFilterOperator.getInstance();
} else if (predicateEvaluator.isAlwaysTrue()) {
@@ -56,7 +64,7 @@ public class FilterOperatorUtils {
if (dataSource.getRangeIndex() != null) {
return new RangeIndexBasedFilterOperator(predicateEvaluator,
dataSource, numDocs);
}
- return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs);
+ return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs, nullHandlingEnabled);
} else if (predicateType == Predicate.Type.REGEXP_LIKE) {
if (dataSource.getFSTIndex() != null &&
dataSource.getDataSourceMetadata().isSorted()) {
return new SortedIndexBasedFilterOperator(predicateEvaluator,
dataSource, numDocs);
@@ -64,7 +72,7 @@ public class FilterOperatorUtils {
if (dataSource.getFSTIndex() != null && dataSource.getInvertedIndex() !=
null) {
return new BitmapBasedFilterOperator(predicateEvaluator, dataSource,
numDocs);
}
- return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs);
+ return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs, nullHandlingEnabled);
} else {
if (dataSource.getDataSourceMetadata().isSorted() &&
dataSource.getDictionary() != null) {
return new SortedIndexBasedFilterOperator(predicateEvaluator,
dataSource, numDocs);
@@ -72,7 +80,7 @@ public class FilterOperatorUtils {
if (dataSource.getInvertedIndex() != null) {
return new BitmapBasedFilterOperator(predicateEvaluator, dataSource,
numDocs);
}
- return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs);
+ return new ScanBasedFilterOperator(predicateEvaluator, dataSource,
numDocs, nullHandlingEnabled);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
index 66f6ab667d..4b968ba2ed 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
@@ -78,9 +78,10 @@ public class RangeIndexBasedFilterOperator extends
BaseFilterOperator {
if (partialMatches == null) {
return new FilterBlock(new BitmapDocIdSet(matches == null ? new
MutableRoaringBitmap() : matches, _numDocs));
}
+ // TODO: support proper null handling in range index.
// Need to scan the first and last range as they might be partially matched
ScanBasedFilterOperator scanBasedFilterOperator =
- new ScanBasedFilterOperator(_rangePredicateEvaluator, _dataSource,
_numDocs);
+ new ScanBasedFilterOperator(_rangePredicateEvaluator, _dataSource,
_numDocs, false);
FilterBlockDocIdSet scanBasedDocIdSet =
scanBasedFilterOperator.getNextBlock().getBlockDocIdSet();
MutableRoaringBitmap docIds = ((ScanBasedDocIdIterator)
scanBasedDocIdSet.iterator()).applyAnd(partialMatches);
if (matches != null) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ScanBasedFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ScanBasedFilterOperator.java
index ec7c0e2a8c..ac08c9337c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ScanBasedFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ScanBasedFilterOperator.java
@@ -35,18 +35,21 @@ public class ScanBasedFilterOperator extends
BaseFilterOperator {
private final PredicateEvaluator _predicateEvaluator;
private final DataSource _dataSource;
private final int _numDocs;
+ private final boolean _nullHandlingEnabled;
- ScanBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource
dataSource, int numDocs) {
+ ScanBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource
dataSource, int numDocs,
+ boolean nullHandlingEnabled) {
_predicateEvaluator = predicateEvaluator;
_dataSource = dataSource;
_numDocs = numDocs;
+ _nullHandlingEnabled = nullHandlingEnabled;
}
@Override
protected FilterBlock getNextBlock() {
DataSourceMetadata dataSourceMetadata =
_dataSource.getDataSourceMetadata();
if (dataSourceMetadata.isSingleValue()) {
- return new FilterBlock(new SVScanDocIdSet(_predicateEvaluator,
_dataSource.getForwardIndex(), _numDocs));
+ return new FilterBlock(new SVScanDocIdSet(_predicateEvaluator,
_dataSource, _numDocs, _nullHandlingEnabled));
} else {
return new FilterBlock(new MVScanDocIdSet(_predicateEvaluator,
_dataSource.getForwardIndex(), _numDocs,
dataSourceMetadata.getMaxNumValuesPerMVEntry()));
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateUtils.java
index 19ca4c1176..a487d36654 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateUtils.java
@@ -74,7 +74,7 @@ public class PredicateUtils {
List<String> values = inPredicate.getValues();
int hashSetSize = Integer.min(HashUtil.getMinHashSetSize(values.size()),
MAX_INITIAL_DICT_ID_SET_SIZE);
IntSet dictIdSet = new IntOpenHashSet(hashSetSize);
- switch (dataType.getStoredType()) {
+ switch (dataType) {
case INT:
int[] intValues = inPredicate.getIntValues();
for (int value : intValues) {
@@ -120,6 +120,24 @@ public class PredicateUtils {
}
}
break;
+ case BOOLEAN:
+ int[] booleanValues = inPredicate.getBooleanValues();
+ for (int value : booleanValues) {
+ int dictId = dictionary.indexOf(value);
+ if (dictId >= 0) {
+ dictIdSet.add(dictId);
+ }
+ }
+ break;
+ case TIMESTAMP:
+ long[] timestampValues = inPredicate.getTimestampValues();
+ for (long value : timestampValues) {
+ int dictId = dictionary.indexOf(value);
+ if (dictId >= 0) {
+ dictIdSet.add(dictId);
+ }
+ }
+ break;
case STRING:
for (String value : values) {
int dictId = dictionary.indexOf(value);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index 6e24267bf9..b04ae67e72 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
@@ -275,7 +275,8 @@ public class FilterPlanNode implements PlanNode {
dataSource.getDataSourceMetadata().getDataType());
}
_predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
- return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs);
+ return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs,
+ _queryContext.isNullHandlingEnabled());
case JSON_MATCH:
JsonIndexReader jsonIndex = dataSource.getJsonIndex();
Preconditions.checkState(jsonIndex != null, "Cannot apply
JSON_MATCH on column: %s without json index",
@@ -300,7 +301,8 @@ public class FilterPlanNode implements PlanNode {
PredicateEvaluatorProvider.getPredicateEvaluator(predicate,
dataSource.getDictionary(),
dataSource.getDataSourceMetadata().getDataType());
_predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
- return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs);
+ return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs,
+ _queryContext.isNullHandlingEnabled());
}
}
default:
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GapfillFilterHandler.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GapfillFilterHandler.java
index 919f99e7bd..8ec110a0cf 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GapfillFilterHandler.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GapfillFilterHandler.java
@@ -48,7 +48,8 @@ public class GapfillFilterHandler implements
ValueExtractorFactory {
// TODO: Please refer to {@link PostAggregationHandler} on how to handle
the index for aggregation queries.
_indexes.put(_dataSchema.getColumnName(i), i);
}
- _rowMatcher = RowMatcherFactory.getRowMatcher(filter, this);
+ // TODO: support proper null handling in GapfillFilterHandler.
+ _rowMatcher = RowMatcherFactory.getRowMatcher(filter, this, false);
}
/**
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
index 45144556fe..f167ca8a50 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
@@ -148,7 +148,8 @@ public class GroupByDataTableReducer implements
DataTableReducer {
FilterContext havingFilter = _queryContext.getHavingFilter();
if (havingFilter != null) {
rows = new ArrayList<>();
- HavingFilterHandler havingFilterHandler = new
HavingFilterHandler(havingFilter, postAggregationHandler);
+ HavingFilterHandler havingFilterHandler = new
HavingFilterHandler(havingFilter, postAggregationHandler,
+ _queryContext.isNullHandlingEnabled());
while (rows.size() < limit && sortedIterator.hasNext()) {
Object[] row = sortedIterator.next().getValues();
extractFinalAggregationResults(row);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/HavingFilterHandler.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/HavingFilterHandler.java
index 885a8f16d7..07d30ebc3e 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/HavingFilterHandler.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/HavingFilterHandler.java
@@ -29,8 +29,9 @@ import
org.apache.pinot.core.query.reduce.filter.RowMatcherFactory;
public class HavingFilterHandler {
private final RowMatcher _rowMatcher;
- public HavingFilterHandler(FilterContext havingFilter,
PostAggregationHandler postAggregationHandler) {
- _rowMatcher = RowMatcherFactory.getRowMatcher(havingFilter,
postAggregationHandler);
+ public HavingFilterHandler(FilterContext havingFilter,
PostAggregationHandler postAggregationHandler,
+ boolean nullHandlingEnabled) {
+ _rowMatcher = RowMatcherFactory.getRowMatcher(havingFilter,
postAggregationHandler, nullHandlingEnabled);
}
/**
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/AndRowMatcher.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/AndRowMatcher.java
index 0991952748..3913e60e38 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/AndRowMatcher.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/AndRowMatcher.java
@@ -28,11 +28,13 @@ import
org.apache.pinot.common.request.context.FilterContext;
public class AndRowMatcher implements RowMatcher {
private final RowMatcher[] _childMatchers;
- public AndRowMatcher(List<FilterContext> childFilters, ValueExtractorFactory
valueExtractorFactory) {
+ public AndRowMatcher(List<FilterContext> childFilters, ValueExtractorFactory
valueExtractorFactory,
+ boolean nullHandlingEnabled) {
int numChildren = childFilters.size();
_childMatchers = new RowMatcher[numChildren];
for (int i = 0; i < numChildren; i++) {
- _childMatchers[i] = RowMatcherFactory.getRowMatcher(childFilters.get(i),
valueExtractorFactory);
+ _childMatchers[i] = RowMatcherFactory.getRowMatcher(childFilters.get(i),
valueExtractorFactory,
+ nullHandlingEnabled);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/NotRowMatcher.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/NotRowMatcher.java
index 4b6b51d64f..c594a74780 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/NotRowMatcher.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/NotRowMatcher.java
@@ -27,8 +27,9 @@ import org.apache.pinot.common.request.context.FilterContext;
public class NotRowMatcher implements RowMatcher {
private final RowMatcher _childMatcher;
- public NotRowMatcher(FilterContext childFilter, ValueExtractorFactory
valueExtractorFactory) {
- _childMatcher = RowMatcherFactory.getRowMatcher(childFilter,
valueExtractorFactory);
+ public NotRowMatcher(FilterContext childFilter, ValueExtractorFactory
valueExtractorFactory,
+ boolean nullHandlingEnabled) {
+ _childMatcher = RowMatcherFactory.getRowMatcher(childFilter,
valueExtractorFactory, nullHandlingEnabled);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/OrRowMatcher.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/OrRowMatcher.java
index 543d518271..0b1ae88bbc 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/OrRowMatcher.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/OrRowMatcher.java
@@ -28,11 +28,13 @@ import
org.apache.pinot.common.request.context.FilterContext;
public class OrRowMatcher implements RowMatcher {
private final RowMatcher[] _childMatchers;
- public OrRowMatcher(List<FilterContext> childFilters, ValueExtractorFactory
valueExtractorFactory) {
+ public OrRowMatcher(List<FilterContext> childFilters, ValueExtractorFactory
valueExtractorFactory,
+ boolean nullHandlingEnabled) {
int numChildren = childFilters.size();
_childMatchers = new RowMatcher[numChildren];
for (int i = 0; i < numChildren; i++) {
- _childMatchers[i] = RowMatcherFactory.getRowMatcher(childFilters.get(i),
valueExtractorFactory);
+ _childMatchers[i] = RowMatcherFactory.getRowMatcher(childFilters.get(i),
valueExtractorFactory,
+ nullHandlingEnabled);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/PredicateRowMatcher.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/PredicateRowMatcher.java
index d2eddb8d17..edd5c6caeb 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/PredicateRowMatcher.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/PredicateRowMatcher.java
@@ -33,16 +33,21 @@ public class PredicateRowMatcher implements RowMatcher {
private final ValueExtractor _valueExtractor;
private final DataType _valueType;
private final PredicateEvaluator _predicateEvaluator;
+ private final boolean _nullHandlingEnabled;
- public PredicateRowMatcher(Predicate predicate, ValueExtractor
valueExtractor) {
+ public PredicateRowMatcher(Predicate predicate, ValueExtractor
valueExtractor, boolean nullHandlingEnabled) {
_valueExtractor = valueExtractor;
_valueType = _valueExtractor.getColumnDataType().toDataType();
_predicateEvaluator =
PredicateEvaluatorProvider.getPredicateEvaluator(predicate, null, _valueType);
+ _nullHandlingEnabled = nullHandlingEnabled;
}
@Override
public boolean isMatch(Object[] row) {
Object value = _valueExtractor.extract(row);
+ if (_nullHandlingEnabled && value == null) {
+ return false;
+ }
switch (_valueType) {
case INT:
return _predicateEvaluator.applySV((int) value);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/RowMatcherFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/RowMatcherFactory.java
index d7f04213d2..575b3f4575 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/RowMatcherFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/filter/RowMatcherFactory.java
@@ -31,18 +31,19 @@ public class RowMatcherFactory {
/**
* Helper method to construct a RowMatcher based on the given filter.
*/
- public static RowMatcher getRowMatcher(FilterContext filter,
ValueExtractorFactory valueExtractorFactory) {
+ public static RowMatcher getRowMatcher(FilterContext filter,
ValueExtractorFactory valueExtractorFactory,
+ boolean nullHandlingEnabled) {
switch (filter.getType()) {
case AND:
- return new AndRowMatcher(filter.getChildren(), valueExtractorFactory);
+ return new AndRowMatcher(filter.getChildren(), valueExtractorFactory,
nullHandlingEnabled);
case OR:
- return new OrRowMatcher(filter.getChildren(), valueExtractorFactory);
+ return new OrRowMatcher(filter.getChildren(), valueExtractorFactory,
nullHandlingEnabled);
case NOT:
assert filter.getChildren().size() == 1;
- return new NotRowMatcher(filter.getChildren().get(0),
valueExtractorFactory);
+ return new NotRowMatcher(filter.getChildren().get(0),
valueExtractorFactory, nullHandlingEnabled);
case PREDICATE:
return new PredicateRowMatcher(filter.getPredicate(),
-
valueExtractorFactory.getValueExtractor(filter.getPredicate().getLhs()));
+
valueExtractorFactory.getValueExtractor(filter.getPredicate().getLhs()),
nullHandlingEnabled);
default:
throw new IllegalStateException();
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/reduce/HavingFilterHandlerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/reduce/HavingFilterHandlerTest.java
index d75db62a7d..6c9b47b7cc 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/reduce/HavingFilterHandlerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/reduce/HavingFilterHandlerTest.java
@@ -40,7 +40,7 @@ public class HavingFilterHandlerTest {
new DataSchema(new String[]{"d1", "count(*)"}, new
ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG});
PostAggregationHandler postAggregationHandler = new
PostAggregationHandler(queryContext, dataSchema);
HavingFilterHandler havingFilterHandler =
- new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler);
+ new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler, false);
assertFalse(havingFilterHandler.isMatch(new Object[]{1, 5L}));
assertTrue(havingFilterHandler.isMatch(new Object[]{2, 10L}));
assertFalse(havingFilterHandler.isMatch(new Object[]{3, 3L}));
@@ -55,7 +55,7 @@ public class HavingFilterHandlerTest {
new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.DOUBLE,
ColumnDataType.DOUBLE});
PostAggregationHandler postAggregationHandler = new
PostAggregationHandler(queryContext, dataSchema);
HavingFilterHandler havingFilterHandler =
- new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler);
+ new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler, false);
assertFalse(havingFilterHandler.isMatch(new Object[]{1, 15.5, 13.0}));
assertTrue(havingFilterHandler.isMatch(new Object[]{2, 15.0, 3.0}));
assertFalse(havingFilterHandler.isMatch(new Object[]{3, 20.0, 7.5}));
@@ -69,7 +69,7 @@ public class HavingFilterHandlerTest {
new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.DOUBLE,
ColumnDataType.DOUBLE});
PostAggregationHandler postAggregationHandler = new
PostAggregationHandler(queryContext, dataSchema);
HavingFilterHandler havingFilterHandler =
- new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler);
+ new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler, false);
assertFalse(havingFilterHandler.isMatch(new Object[]{1, 15.5, 13.0}));
assertTrue(havingFilterHandler.isMatch(new Object[]{2, 15.0, 3.0}));
assertFalse(havingFilterHandler.isMatch(new Object[]{3, 20.0, 10.0}));
@@ -87,7 +87,7 @@ public class HavingFilterHandlerTest {
});
PostAggregationHandler postAggregationHandler = new
PostAggregationHandler(queryContext, dataSchema);
HavingFilterHandler havingFilterHandler =
- new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler);
+ new HavingFilterHandler(queryContext.getHavingFilter(),
postAggregationHandler, false);
assertTrue(havingFilterHandler.isMatch(new Object[]{11, 11L, 10.5f,
10.5, "11", new byte[]{17}, 5}));
assertFalse(havingFilterHandler.isMatch(new Object[]{10, 11L, 10.5f,
10.5, "11", new byte[]{17}, 5}));
assertFalse(havingFilterHandler.isMatch(new Object[]{11, 10L, 10.5f,
10.5, "11", new byte[]{17}, 5}));
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
index 305ed783a5..ee4274cc53 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
@@ -301,6 +301,17 @@ public class AllNullQueriesTest extends BaseQueriesTest {
assertNull(row[0]);
}
}
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s is not
null limit 5000",
+ COLUMN_NAME, COLUMN_NAME);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{COLUMN_NAME}, new
ColumnDataType[]{columnDataType}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 0);
+ }
if (columnDataType != ColumnDataType.STRING) {
{
String query = String.format(
@@ -495,6 +506,26 @@ public class AllNullQueriesTest extends BaseQueriesTest {
assertEquals(rows.size(), 0);
}
if (columnDataType != ColumnDataType.STRING) {
+ {
+ String query = String.format("SELECT COUNT(%s) AS count, MIN(%s) AS
min, MAX(%s) AS max, SUM(%s) AS sum" + " "
+ + "FROM testTable GROUP BY %s ORDER BY max", COLUMN_NAME,
COLUMN_NAME, COLUMN_NAME, COLUMN_NAME,
+ COLUMN_NAME);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema, new DataSchema(new String[]{"count", "min",
"max", "sum"}, new ColumnDataType[]{
+ ColumnDataType.LONG, ColumnDataType.DOUBLE, ColumnDataType.DOUBLE,
ColumnDataType.DOUBLE
+ }));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 1);
+ Object[] row = rows.get(0);
+ assertEquals(row.length, 4);
+ // Count(column) return 0 if all values are nulls.
+ assertEquals(row[0], 0L);
+ assertNull(row[1]);
+ assertNull(row[2]);
+ assertNull(row[3]);
+ }
{
String query = String.format(
"SELECT AVG(%s) AS avg FROM testTable GROUP BY %s ORDER BY avg
LIMIT 20", COLUMN_NAME, COLUMN_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
index a600a4d8e6..05c2ed7480 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
@@ -401,6 +401,54 @@ public class BigDecimalQueriesTest extends BaseQueriesTest
{
// List<Object[]> rows = resultTable.getRows();
// assertEquals(rows.size(), 1);
}
+ {
+ String query = String.format(
+ "SELECT MAX(%s) AS maxValue FROM testTable GROUP BY %s HAVING
maxValue < %s ORDER BY maxValue",
+ BIG_DECIMAL_COLUMN, BIG_DECIMAL_COLUMN,
BASE_BIG_DECIMAL.add(BigDecimal.valueOf(5)));
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{"maxValue"}, new
ColumnDataType[]{ColumnDataType.DOUBLE}));
+ List<Object[]> rows = resultTable.getRows();
+ // The default null ordering is: 'NULLS LAST'. This is why the number of
returned value is 4 and not 5.
+ assertEquals(rows.size(), 4);
+ int i = 0;
+ for (int index = 0; index < 4; index++) {
+ if (i % 4 == 3) {
+ // Null values are inserted at: index % 4 == 3.
+ i++;
+ }
+ Object[] row = rows.get(index);
+ assertEquals(row.length, 1);
+ assertEquals(row[0],
BASE_BIG_DECIMAL.add(BigDecimal.valueOf(i)).doubleValue());
+ i++;
+ }
+ }
+ {
+ int lowerLimit = 991;
+ String query = String.format(
+ "SELECT MAX(%s) AS maxValue FROM testTable GROUP BY %s HAVING
maxValue > %s ORDER BY maxValue",
+ BIG_DECIMAL_COLUMN, BIG_DECIMAL_COLUMN,
BASE_BIG_DECIMAL.add(BigDecimal.valueOf(lowerLimit)));
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{"maxValue"}, new
ColumnDataType[]{ColumnDataType.DOUBLE}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 6);
+ int i = lowerLimit;
+ for (int index = 0; index < 6; index++) {
+ if (i % 4 == 3) {
+ // Null values are inserted at: index % 4 == 3.
+ i++;
+ }
+ Object[] row = rows.get(index);
+ assertEquals(row.length, 1);
+ assertEquals(row[0],
BASE_BIG_DECIMAL.add(BigDecimal.valueOf(i)).doubleValue());
+ i++;
+ }
+ }
DataTableFactory.setDataTableVersion(DataTableFactory.VERSION_3);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
index 8f080ecc92..4c3fdd40ba 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
@@ -73,6 +73,10 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
private IndexSegment _indexSegment;
private List<IndexSegment> _indexSegments;
+ private int _trueValuesCount;
+ private int _falseValuesCount;
+ private int _nullValuesCount;
+
@Override
protected String getFilter() {
return "";
@@ -100,24 +104,31 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
switch (i % 7) {
case 0:
record.putValue(BOOLEAN_COLUMN, false);
+ _falseValuesCount++;
break;
case 1:
record.putValue(BOOLEAN_COLUMN, 1);
+ _trueValuesCount++;
break;
case 2:
record.putValue(BOOLEAN_COLUMN, 0L);
+ _falseValuesCount++;
break;
case 3:
record.putValue(BOOLEAN_COLUMN, 0.1f);
+ _trueValuesCount++;
break;
case 4:
record.putValue(BOOLEAN_COLUMN, 0.0);
+ _falseValuesCount++;
break;
case 5:
record.putValue(BOOLEAN_COLUMN, "true");
+ _trueValuesCount++;
break;
case 6:
record.putValue(BOOLEAN_COLUMN, null);
+ _nullValuesCount++;
break;
default:
break;
@@ -173,6 +184,145 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
HashSet<Integer> trueIndices = new HashSet<Integer>(Arrays.asList(1, 3,
5));
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s is null
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _nullValuesCount * 4);
+ for (Object[] row : rows) {
+ assertNull(row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s = false
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _falseValuesCount * 4);
+ for (Object[] row : rows) {
+ assertFalse((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s != false
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _trueValuesCount * 4);
+ for (Object[] row : rows) {
+ assertTrue((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s = true
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _trueValuesCount * 4);
+ for (Object[] row : rows) {
+ assertTrue((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s in
(true) LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _trueValuesCount * 4);
+ for (Object[] row : rows) {
+ assertTrue((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s not in
(true) LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _falseValuesCount * 4);
+ for (Object[] row : rows) {
+ assertFalse((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s in (1)
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _trueValuesCount * 4);
+ for (Object[] row : rows) {
+ assertTrue((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s in
(false) LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _falseValuesCount * 4);
+ for (Object[] row : rows) {
+ assertFalse((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s != true
LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ // Note: comparison w/ nulls always return fasle including inequality.
To be able to return rows w/ both nulls
+ // and false values, we should introduce IS DISTINCT FROM, and IS NOT
DISTINCT FROM operators.
+ assertEquals(rows.size(), _falseValuesCount * 4);
+ for (Object[] row : rows) {
+ assertFalse((boolean) row[0]);
+ }
+ }
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s is not
null LIMIT 5000",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), _trueValuesCount * 4 + _falseValuesCount * 4);
+ }
{
String query = "SELECT * FROM testTable";
BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
@@ -194,26 +344,41 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
}
}
}
+ {
+ String query = "SELECT booleanColumn FROM testTable WHERE
booleanColumn";
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ Object[] row = rows.get(i);
+ assertEquals(row.length, 1);
+ assertEquals(row[0], true);
+ }
+ }
{
String query = "SELECT * FROM testTable ORDER BY booleanColumn DESC
LIMIT 4000";
BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
ResultTable resultTable = brokerResponse.getResultTable();
DataSchema dataSchema = resultTable.getDataSchema();
assertEquals(dataSchema,
- new DataSchema(new String[]{"booleanColumn"}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
List<Object[]> rows = resultTable.getRows();
assertEquals(rows.size(), 4000);
- for (int i = 0; i < 1716; i++) {
+ for (int i = 0; i < _trueValuesCount * 4; i++) {
Object[] row = rows.get(i);
assertEquals(row.length, 1);
assertTrue((boolean) row[0]);
}
- for (int i = 1716; i < 3432; i++) {
+ for (int i = _trueValuesCount * 4; i < _trueValuesCount * 4 +
_falseValuesCount * 4; i++) {
Object[] row = rows.get(i);
assertEquals(row.length, 1);
assertFalse((boolean) row[0]);
}
- for (int i = 3432; i < 4000; i++) {
+ for (int i = _trueValuesCount * 4 + _falseValuesCount * 4; i < 4000;
i++) {
Object[] row = rows.get(i);
assertEquals(row.length, 1);
// Note 2: The default null ordering is 'NULLS LAST', regardless of
the ordering direction.
@@ -242,12 +407,13 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
}
}
{
- String query = "SELECT DISTINCT booleanColumn FROM testTable ORDER BY
booleanColumn DESC";
+ String query = String.format("SELECT DISTINCT %s FROM testTable ORDER BY
%s DESC",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN);
BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
ResultTable resultTable = brokerResponse.getResultTable();
DataSchema dataSchema = resultTable.getDataSchema();
assertEquals(dataSchema,
- new DataSchema(new String[]{"booleanColumn"}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ new DataSchema(new String[]{BOOLEAN_COLUMN}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
List<Object[]> rows = resultTable.getRows();
assertEquals(rows.size(), 3);
Object[] firstRow = rows.get(0);
@@ -261,26 +427,26 @@ public class BooleanNullEnabledQueriesTest extends
BaseQueriesTest {
assertEquals(thirdRow[0], false);
}
{
- String query =
- "SELECT COUNT(*) AS count, booleanColumn FROM testTable GROUP BY
booleanColumn ORDER BY booleanColumn";
+ String query = String.format("SELECT COUNT(*) AS count, %s FROM
testTable GROUP BY %s ORDER BY %s",
+ BOOLEAN_COLUMN, BOOLEAN_COLUMN, BOOLEAN_COLUMN);
BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
ResultTable resultTable = brokerResponse.getResultTable();
DataSchema dataSchema = resultTable.getDataSchema();
- assertEquals(dataSchema, new DataSchema(new String[]{"count",
"booleanColumn"},
+ assertEquals(dataSchema, new DataSchema(new String[]{"count",
BOOLEAN_COLUMN},
new ColumnDataType[]{ColumnDataType.LONG, ColumnDataType.BOOLEAN}));
List<Object[]> rows = resultTable.getRows();
assertEquals(rows.size(), 3);
Object[] firstRow = rows.get(0);
assertEquals(firstRow.length, 2);
- assertEquals(firstRow[0], (long) 1716);
- assertEquals(firstRow[1], false);
+ assertEquals(firstRow[0], (long) _falseValuesCount * 4);
+ assertFalse((boolean) firstRow[1]);
Object[] secondRow = rows.get(1);
assertEquals(secondRow.length, 2);
- assertEquals(secondRow[0], (long) 1716);
- assertEquals(secondRow[1], true);
+ assertEquals(secondRow[0], (long) _trueValuesCount * 4);
+ assertTrue((boolean) secondRow[1]);
Object[] thirdRow = rows.get(2);
assertEquals(thirdRow.length, 2);
- assertEquals(thirdRow[0], (long) 568);
+ assertEquals(thirdRow[0], (long) _nullValuesCount * 4);
assertNull(thirdRow[1]);
}
DataTableFactory.setDataTableVersion(DataTableFactory.VERSION_3);
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
index 718d99e9ba..3e061c0d2d 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
@@ -123,7 +123,7 @@ public class BenchmarkScanDocIdIterators {
@Benchmark
public MutableRoaringBitmap benchmarkSVLong() {
- return new SVScanDocIdIterator(_predicateEvaluator, _readerV2,
_numDocs).applyAnd(_bitmap);
+ return new SVScanDocIdIterator(_predicateEvaluator, _readerV2, _numDocs,
null).applyAnd(_bitmap);
}
public static class DummyPredicateEvaluator implements PredicateEvaluator {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
