This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 189e8b9d18 add NumericRangeIndex interface and BoundFilter support
(#12830)
189e8b9d18 is described below
commit 189e8b9d18b1f9266215ff4a6811b9b571e78aa0
Author: Clint Wylie <[email protected]>
AuthorDate: Fri Jul 29 18:58:49 2022 -0700
add NumericRangeIndex interface and BoundFilter support (#12830)
add NumericRangeIndex interface and BoundFilter support
changes:
* NumericRangeIndex interface, like LexicographicalRangeIndex but for
numbers
* BoundFilter now uses NumericRangeIndex if comparator is numeric and there
is no extractionFn
* NestedFieldLiteralColumnIndexSupplier.java now supports supplying
NumericRangeIndex for single typed numeric nested literal columns
* better faster stronger and (ever so slightly) more understandable
* more tests, fix bug
* fix style
---
.../benchmark/query/SqlNestedDataBenchmark.java | 14 +-
.../org/apache/druid/query/filter/InDimFilter.java | 2 +-
.../druid/segment/column/NumericRangeIndex.java | 42 +
.../apache/druid/segment/filter/BoundFilter.java | 135 +-
.../NestedFieldLiteralColumnIndexSupplier.java | 276 ++--
.../NestedFieldLiteralColumnIndexSupplierTest.java | 1517 +++++++++++++++++++-
.../sql/calcite/CalciteNestedDataQueryTest.java | 4 +-
7 files changed, 1758 insertions(+), 232 deletions(-)
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
index d41589142d..d719da7dba 100644
---
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
@@ -165,7 +165,13 @@ public class SqlNestedDataBenchmark
"SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo
WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21,
23, 25, 26, 46)",
// 24, 25
"SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46) GROUP
BY 1",
- "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo
WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21,
23, 25, 26, 46) GROUP BY 1"
+ "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo
WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21,
23, 25, 26, 46) GROUP BY 1",
+ // 26, 27
+ "SELECT SUM(long1) FROM foo WHERE double3 < 1005.0 AND double3 > 1000.0",
+ "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo
WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 1005.0 AND
JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+ // 28, 29
+ "SELECT SUM(long1) FROM foo WHERE double3 < 2000.0 AND double3 > 1000.0",
+ "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo
WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND
JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0"
);
@Param({"5000000"})
@@ -203,7 +209,11 @@ public class SqlNestedDataBenchmark
"22",
"23",
"24",
- "25"
+ "25",
+ "26",
+ "27",
+ "28",
+ "29"
})
private String query;
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
index 74270b65c5..60e07ee09b 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
@@ -523,7 +523,7 @@ public class InDimFilter extends
AbstractOptimizableDimFilter implements Filter
private final Supplier<DruidFloatPredicate> floatPredicateSupplier;
private final Supplier<DruidDoublePredicate> doublePredicateSupplier;
- InFilterDruidPredicateFactory(
+ public InFilterDruidPredicateFactory(
final ExtractionFn extractionFn,
final ValuesSet values
)
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java
b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java
new file mode 100644
index 0000000000..bb9bf5c625
--- /dev/null
+++
b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.column;
+
+import javax.annotation.Nullable;
+
+/**
+ * An optimized column value {@link BitmapColumnIndex} provider for
specialized processing of numeric value ranges.
+ * This index does not match null values, union the results of this index with
{@link NullValueIndex} if null values
+ * should be considered part of the value range.
+ */
+public interface NumericRangeIndex
+{
+ /**
+ * Get a {@link BitmapColumnIndex} corresponding to the values supplied in
the specified range. If supplied starting
+ * value is null, the range will begin at the first non-null value in the
underlying value dictionary. If the end
+ * value is null, the range will extend to the last value in the underlying
value dictionary.
+ */
+ BitmapColumnIndex forRange(
+ @Nullable Number startValue,
+ boolean startStrict,
+ @Nullable Number endValue,
+ boolean endStrict
+ );
+}
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
index 0c00d31086..b4c1463b07 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
@@ -48,6 +48,7 @@ import
org.apache.druid.segment.column.ColumnIndexCapabilities;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.NullValueIndex;
+import org.apache.druid.segment.column.NumericRangeIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
import javax.annotation.Nullable;
@@ -75,71 +76,107 @@ public class BoundFilter implements Filter
if (!Filters.checkFilterTuningUseIndex(boundDimFilter.getDimension(),
selector, filterTuning)) {
return null;
}
- if (supportShortCircuit()) {
+ if (supportStringShortCircuit()) {
final ColumnIndexSupplier indexSupplier =
selector.getIndexSupplier(boundDimFilter.getDimension());
if (indexSupplier == null) {
return Filters.makeNullIndex(doesMatchNull(), selector);
}
final LexicographicalRangeIndex rangeIndex =
indexSupplier.as(LexicographicalRangeIndex.class);
- if (rangeIndex == null) {
- // column
- return null;
+ if (rangeIndex != null) {
+ final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange(
+ boundDimFilter.getLower(),
+ boundDimFilter.isLowerStrict(),
+ boundDimFilter.getUpper(),
+ boundDimFilter.isUpperStrict()
+ );
+ // preserve sad backwards compatible behavior where bound filter
matches 'null' if the lower bound is not set
+ if (boundDimFilter.hasLowerBound() &&
!NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) {
+ return rangeBitmaps;
+ } else {
+ return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps);
+ }
}
- final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange(
- boundDimFilter.getLower(),
- boundDimFilter.isLowerStrict(),
- boundDimFilter.getUpper(),
- boundDimFilter.isUpperStrict()
- );
- // preserve sad backwards compatible behavior where bound filter matches
'null' if the lower bound is not set
- if (boundDimFilter.hasLowerBound() &&
!NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) {
- return rangeBitmaps;
- } else {
- final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class);
- if (nulls == null) {
- return null;
+ }
+ if (supportNumericShortCircuit()) {
+ final ColumnIndexSupplier indexSupplier =
selector.getIndexSupplier(boundDimFilter.getDimension());
+ if (indexSupplier == null) {
+ return Filters.makeNullIndex(doesMatchNull(), selector);
+ }
+ final NumericRangeIndex rangeIndex =
indexSupplier.as(NumericRangeIndex.class);
+ if (rangeIndex != null) {
+ final Number lower = boundDimFilter.hasLowerBound() ?
Double.parseDouble(boundDimFilter.getLower()) : null;
+ final Number upper = boundDimFilter.hasUpperBound() ?
Double.parseDouble(boundDimFilter.getUpper()) : null;
+ final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange(
+ lower,
+ boundDimFilter.isLowerStrict(),
+ upper,
+ boundDimFilter.isUpperStrict()
+ );
+ // preserve sad backwards compatible behavior where bound filter
matches 'null' if the lower bound is not set
+ if (boundDimFilter.hasLowerBound() &&
!NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) {
+ return rangeBitmaps;
+ } else {
+ return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps);
}
- final BitmapColumnIndex nullBitmap = nulls.forNull();
- return new BitmapColumnIndex()
- {
- @Override
- public ColumnIndexCapabilities getIndexCapabilities()
- {
- return
rangeBitmaps.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities());
- }
-
- @Override
- public double estimateSelectivity(int totalRows)
- {
- return Math.min(
- 1.0,
- rangeBitmaps.estimateSelectivity(totalRows) +
nullBitmap.estimateSelectivity(totalRows)
- );
- }
-
- @Override
- public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory)
- {
- return bitmapResultFactory.union(
- ImmutableList.of(
- rangeBitmaps.computeBitmapResult(bitmapResultFactory),
- nullBitmap.computeBitmapResult(bitmapResultFactory)
- )
- );
- }
- };
}
- } else {
- return Filters.makePredicateIndex(boundDimFilter.getDimension(),
selector, getPredicateFactory());
}
+ // fall back to predicate based index if it is available
+ return Filters.makePredicateIndex(boundDimFilter.getDimension(), selector,
getPredicateFactory());
+ }
+
+ @Nullable
+ private BitmapColumnIndex wrapRangeIndexWithNullValueIndex(
+ ColumnIndexSupplier indexSupplier,
+ BitmapColumnIndex rangeIndex
+ )
+ {
+ final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class);
+ if (nulls == null) {
+ return null;
+ }
+ final BitmapColumnIndex nullBitmap = nulls.forNull();
+ return new BitmapColumnIndex()
+ {
+ @Override
+ public ColumnIndexCapabilities getIndexCapabilities()
+ {
+ return
rangeIndex.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities());
+ }
+
+ @Override
+ public double estimateSelectivity(int totalRows)
+ {
+ return Math.min(
+ 1.0,
+ rangeIndex.estimateSelectivity(totalRows) +
nullBitmap.estimateSelectivity(totalRows)
+ );
+ }
+
+ @Override
+ public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory)
+ {
+ return bitmapResultFactory.union(
+ ImmutableList.of(
+ rangeIndex.computeBitmapResult(bitmapResultFactory),
+ nullBitmap.computeBitmapResult(bitmapResultFactory)
+ )
+ );
+ }
+ };
}
- private boolean supportShortCircuit()
+ private boolean supportStringShortCircuit()
{
// Optimization for lexicographic bounds with no extractionFn => binary
search through the index
return
boundDimFilter.getOrdering().equals(StringComparators.LEXICOGRAPHIC) &&
extractionFn == null;
}
+ private boolean supportNumericShortCircuit()
+ {
+ // Optimization for numeric bounds with no extractionFn => binary search
through the index
+ return boundDimFilter.getOrdering().equals(StringComparators.NUMERIC) &&
extractionFn == null;
+ }
+
@Override
public ValueMatcher makeMatcher(ColumnSelectorFactory factory)
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
index 99df4d99d5..e67ac8ac52 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
@@ -35,23 +35,27 @@ import it.unimi.dsi.fastutil.longs.LongIterator;
import it.unimi.dsi.fastutil.longs.LongSet;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
+import org.apache.druid.common.config.NullHandling;
import org.apache.druid.common.guava.GuavaUtils;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.DruidDoublePredicate;
import org.apache.druid.query.filter.DruidLongPredicate;
import org.apache.druid.query.filter.DruidPredicateFactory;
+import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.NullValueIndex;
+import org.apache.druid.segment.column.NumericRangeIndex;
import org.apache.druid.segment.column.SimpleBitmapColumnIndex;
import org.apache.druid.segment.column.SimpleImmutableBitmapIndex;
import org.apache.druid.segment.column.SimpleImmutableBitmapIterableIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.data.FixedIndexed;
import org.apache.druid.segment.data.GenericIndexed;
+import org.apache.druid.segment.data.Indexed;
import javax.annotation.Nullable;
import java.util.Iterator;
@@ -120,6 +124,8 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
case LONG:
if (clazz.equals(StringValueSetIndex.class)) {
return (T) new NestedLongLiteralValueSetIndex();
+ } else if (clazz.equals(NumericRangeIndex.class)) {
+ return (T) new NestedLongLiteralNumericRangeIndex();
} else if (clazz.equals(DruidPredicateIndex.class)) {
return (T) new NestedLongLiteralPredicateIndex();
}
@@ -127,6 +133,8 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
case DOUBLE:
if (clazz.equals(StringValueSetIndex.class)) {
return (T) new NestedDoubleLiteralValueSetIndex();
+ } else if (clazz.equals(NumericRangeIndex.class)) {
+ return (T) new NestedDoubleLiteralNumericRangeIndex();
} else if (clazz.equals(DruidPredicateIndex.class)) {
return (T) new NestedDoubleLiteralPredicateIndex();
}
@@ -153,47 +161,106 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() : bitmap;
}
- private IntIntPair getGlobalRange(
- @Nullable String startValue,
+ /**
+ * Gets a value range from a global dictionary and maps it to a range on the
local {@link #dictionary}.
+ * The starting index of the resulting range is inclusive, while the
endpoint is exclusive [start, end)
+ */
+ private <T> IntIntPair getLocalRangeFromDictionary(
+ @Nullable T startValue,
boolean startStrict,
- @Nullable String endValue,
+ @Nullable T endValue,
boolean endStrict,
- int rangeStart,
- int rangeEnd,
- GlobalIndexGetFunction getFn
+ Indexed<T> globalDictionary,
+ int adjust
)
{
- int startIndex, endIndex;
+ int globalStartIndex, globalEndIndex;
+ int localStartIndex, localEndIndex;
if (startValue == null) {
- startIndex = rangeStart;
+ globalStartIndex = adjust == 0 ? 1 : adjust; // global index 0 is always
the null value
} else {
- final int found = getFn.indexOf(startValue);
+ final int found = globalDictionary.indexOf(startValue);
if (found >= 0) {
- startIndex = startStrict ? found + 1 : found;
+ globalStartIndex = adjust + (startStrict ? found + 1 : found);
} else {
- startIndex = -(found + 1);
+ globalStartIndex = adjust + (-(found + 1));
}
}
+ // with starting global index settled, now lets find starting local index
+ int localFound = dictionary.indexOf(globalStartIndex);
+ if (localFound < 0) {
+ // the first valid global index is not within the local dictionary, so
the insertion point is where we begin
+ localStartIndex = -(localFound + 1);
+ } else {
+ // valid global index in local dictionary, start here
+ localStartIndex = localFound;
+ }
if (endValue == null) {
- endIndex = rangeEnd;
+ globalEndIndex = globalDictionary.size() + adjust;
} else {
- final int found = getFn.indexOf(endValue);
+ final int found = globalDictionary.indexOf(endValue);
if (found >= 0) {
- endIndex = endStrict ? found : found + 1;
+ globalEndIndex = adjust + (endStrict ? found : found + 1);
} else {
- endIndex = -(found + 1);
+ globalEndIndex = adjust + (-(found + 1));
}
}
+ globalEndIndex = Math.max(globalStartIndex, globalEndIndex);
+ // end index is not inclusive, so we find the last value in the local
dictionary that falls within the range
+ int localEndFound = dictionary.indexOf(globalEndIndex - 1);
+ if (localEndFound < 0) {
+ localEndIndex = -localEndFound;
+ } else {
+ // add 1 because the last valid global end value is in the local
dictionary, and end index is exclusive
+ localEndIndex = localEndFound + 1;
+ }
- endIndex = Math.max(startIndex, endIndex);
- return new IntIntImmutablePair(startIndex, endIndex);
+ return new IntIntImmutablePair(localStartIndex,
Math.min(dictionary.size(), localEndIndex));
}
- @FunctionalInterface
- interface GlobalIndexGetFunction
+ private <T> BitmapColumnIndex makeRangeIndex(
+ @Nullable T startValue,
+ boolean startStrict,
+ @Nullable T endValue,
+ boolean endStrict,
+ Indexed<T> globalDictionary,
+ int adjust
+ )
{
- int indexOf(String value);
+ final IntIntPair localRange = getLocalRangeFromDictionary(
+ startValue,
+ startStrict,
+ endValue,
+ endStrict,
+ globalDictionary,
+ adjust
+ );
+ final int startIndex = localRange.leftInt();
+ final int endIndex = localRange.rightInt();
+ return new SimpleImmutableBitmapIterableIndex()
+ {
+ @Override
+ public Iterable<ImmutableBitmap> getBitmapIterable()
+ {
+ return () -> new Iterator<ImmutableBitmap>()
+ {
+ final IntIterator rangeIterator = IntListUtils.fromTo(startIndex,
endIndex).iterator();
+
+ @Override
+ public boolean hasNext()
+ {
+ return rangeIterator.hasNext();
+ }
+
+ @Override
+ public ImmutableBitmap next()
+ {
+ return getBitmap(rangeIterator.nextInt());
+ }
+ };
+ }
+ };
}
private class NestedStringLiteralValueSetIndex implements StringValueSetIndex
@@ -274,7 +341,6 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
private class NestedStringLiteralLexicographicalRangeIndex implements
LexicographicalRangeIndex
{
-
@Override
public BitmapColumnIndex forRange(
@Nullable String startValue,
@@ -283,70 +349,14 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
boolean endStrict
)
{
- return new SimpleImmutableBitmapIterableIndex()
- {
- @Override
- public Iterable<ImmutableBitmap> getBitmapIterable()
- {
- final IntIntPair range = getGlobalRange(
- startValue,
- startStrict,
- endValue,
- endStrict,
- 0,
- globalDictionary.size(),
- globalDictionary::indexOf
- );
- final int start = range.leftInt(), end = range.rightInt();
- // iterates over the range of values in the global dictionary,
mapping to relevant range in the local
- // dictionary, skipping duplicates
- return () -> new Iterator<ImmutableBitmap>()
- {
- int currentGlobalIndex = start;
- // initialize to -1 because findNext uses this field to check for
duplicates, and could legitimately find
- // 0 for the first candidate
- @SuppressWarnings("UnusedAssignment")
- int currentLocalIndex = -1;
- {
- currentLocalIndex = findNext();
- }
-
- private int findNext()
- {
- int candidateLocalIndex =
Math.abs(dictionary.indexOf(currentGlobalIndex));
- while (currentGlobalIndex < end && candidateLocalIndex ==
currentLocalIndex) {
- currentGlobalIndex++;
- candidateLocalIndex =
Math.abs(dictionary.indexOf(currentGlobalIndex));
- }
- if (currentGlobalIndex < end) {
- currentGlobalIndex++;
- return candidateLocalIndex;
- } else {
- return -1;
- }
- }
-
- @Override
- public boolean hasNext()
- {
- return currentLocalIndex != -1;
- }
-
- @Override
- public ImmutableBitmap next()
- {
- int cur = currentLocalIndex;
-
- if (cur == -1) {
- throw new NoSuchElementException();
- }
-
- currentLocalIndex = findNext();
- return getBitmap(cur);
- }
- };
- }
- };
+ return makeRangeIndex(
+ NullHandling.emptyToNullIfNeeded(startValue),
+ startStrict,
+ NullHandling.emptyToNullIfNeeded(endValue),
+ endStrict,
+ globalDictionary,
+ 0
+ );
}
@Override
@@ -363,67 +373,53 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
@Override
public Iterable<ImmutableBitmap> getBitmapIterable()
{
- final IntIntPair stringsRange = getGlobalRange(
+ final IntIntPair range = getLocalRangeFromDictionary(
startValue,
startStrict,
endValue,
endStrict,
- 0,
- globalDictionary.size(),
- globalDictionary::indexOf
+ globalDictionary,
+ 0
);
- // iterates over the range of values in the global dictionary,
mapping to relevant range in the local
- // dictionary, skipping duplicates
+ final int start = range.leftInt(), end = range.rightInt();
return () -> new Iterator<ImmutableBitmap>()
{
- int currentGlobalIndex = stringsRange.leftInt();
- final int end = stringsRange.rightInt();
- // initialize to -1 because findNext uses this field to check for
duplicates, and could legitimately find
- // 0 for the first candidate
- @SuppressWarnings("UnusedAssignment")
- int currentLocalIndex = -1;
+ int currIndex = start;
+ int found;
+
{
- currentLocalIndex = findNext();
+ found = findNext();
}
private int findNext()
{
- int candidateLocalIndex =
Math.abs(dictionary.indexOf(currentGlobalIndex));
- while (currentGlobalIndex < end &&
shouldSkipGlobal(candidateLocalIndex)) {
- currentGlobalIndex++;
- candidateLocalIndex =
Math.abs(dictionary.indexOf(currentGlobalIndex));
+ while (currIndex < end &&
!matcher.apply(globalDictionary.get(dictionary.get(currIndex)))) {
+ currIndex++;
}
- if (currentGlobalIndex < end) {
- currentGlobalIndex++;
- return candidateLocalIndex;
+ if (currIndex < end) {
+ return currIndex++;
} else {
return -1;
}
}
- private boolean shouldSkipGlobal(int candidate)
- {
- return currentLocalIndex == candidate ||
!matcher.apply(globalDictionary.get(currentGlobalIndex));
- }
-
-
@Override
public boolean hasNext()
{
- return currentLocalIndex != -1;
+ return found != -1;
}
@Override
public ImmutableBitmap next()
{
- int cur = currentLocalIndex;
+ int cur = found;
if (cur == -1) {
throw new NoSuchElementException();
}
- currentLocalIndex = findNext();
+ found = findNext();
return getBitmap(cur);
}
};
@@ -492,10 +488,8 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
}
}
-
private class NestedLongLiteralValueSetIndex implements StringValueSetIndex
{
-
@Override
public BitmapColumnIndex forValue(@Nullable String value)
{
@@ -592,6 +586,27 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
}
}
+ private class NestedLongLiteralNumericRangeIndex implements NumericRangeIndex
+ {
+ @Override
+ public BitmapColumnIndex forRange(
+ @Nullable Number startValue,
+ boolean startStrict,
+ @Nullable Number endValue,
+ boolean endStrict
+ )
+ {
+ return makeRangeIndex(
+ startValue != null ? startValue.longValue() : null,
+ startStrict,
+ endValue != null ? endValue.longValue() : null,
+ endStrict,
+ globalLongDictionary,
+ adjustLongId
+ );
+ }
+ }
+
private class NestedLongLiteralPredicateIndex implements DruidPredicateIndex
{
@Override
@@ -658,7 +673,6 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
private class NestedDoubleLiteralValueSetIndex implements StringValueSetIndex
{
-
@Override
public BitmapColumnIndex forValue(@Nullable String value)
{
@@ -755,6 +769,27 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
}
}
+ private class NestedDoubleLiteralNumericRangeIndex implements
NumericRangeIndex
+ {
+ @Override
+ public BitmapColumnIndex forRange(
+ @Nullable Number startValue,
+ boolean startStrict,
+ @Nullable Number endValue,
+ boolean endStrict
+ )
+ {
+ return makeRangeIndex(
+ startValue != null ? startValue.doubleValue() : null,
+ startStrict,
+ endValue != null ? endValue.doubleValue() : null,
+ endStrict,
+ globalDoubleDictionary,
+ adjustDoubleId
+ );
+ }
+ }
+
private class NestedDoubleLiteralPredicateIndex implements
DruidPredicateIndex
{
@Override
@@ -818,7 +853,7 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
}
}
- private abstract class NestedAnyLiteralIndex
+ private abstract class NestedVariantLiteralIndex
{
IntList getIndexes(@Nullable String value)
{
@@ -858,7 +893,7 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
/**
* {@link StringValueSetIndex} but for variant typed nested literal columns
*/
- private class NestedVariantLiteralValueSetIndex extends
NestedAnyLiteralIndex implements StringValueSetIndex
+ private class NestedVariantLiteralValueSetIndex extends
NestedVariantLiteralIndex implements StringValueSetIndex
{
@Override
public BitmapColumnIndex forValue(@Nullable String value)
@@ -937,9 +972,8 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
/**
* {@link DruidPredicateIndex} but for variant typed nested literal columns
*/
- private class NestedVariantLiteralPredicateIndex extends
NestedAnyLiteralIndex implements DruidPredicateIndex
+ private class NestedVariantLiteralPredicateIndex extends
NestedVariantLiteralIndex implements DruidPredicateIndex
{
-
@Override
public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory)
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
index c72afd9009..13a451428d 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
@@ -19,80 +19,1483 @@
package org.apache.druid.segment.nested;
-import junit.framework.TestCase;
+import com.google.common.collect.ImmutableSet;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
-import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
+import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.filter.DruidPredicateFactory;
+import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
+import org.apache.druid.segment.column.NullValueIndex;
+import org.apache.druid.segment.column.NumericRangeIndex;
+import org.apache.druid.segment.column.StringValueSetIndex;
+import org.apache.druid.segment.column.TypeStrategies;
+import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.FixedIndexed;
+import org.apache.druid.segment.data.FixedIndexedWriter;
import org.apache.druid.segment.data.GenericIndexed;
-import org.easymock.EasyMock;
+import org.apache.druid.segment.data.GenericIndexedWriter;
+import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
+import org.apache.druid.segment.serde.Serializer;
+import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium;
+import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.roaringbitmap.IntIterator;
-public class NestedFieldLiteralColumnIndexSupplierTest extends TestCase
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.WritableByteChannel;
+import java.util.TreeSet;
+
+public class NestedFieldLiteralColumnIndexSupplierTest extends
InitializedNullHandlingTest
{
+ BitmapSerdeFactory roaringFactory = new RoaringBitmapSerdeFactory(null);
+ BitmapResultFactory<ImmutableBitmap> bitmapResultFactory = new
DefaultBitmapResultFactory(
+ roaringFactory.getBitmapFactory()
+ );
+ GenericIndexed<String> globalStrings;
+ FixedIndexed<Long> globalLongs;
+ FixedIndexed<Double> globalDoubles;
+
+ @Before
+ public void setup() throws IOException
+ {
+ ByteBuffer stringBuffer = ByteBuffer.allocate(1 << 12);
+ ByteBuffer longBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer doubleBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+
+ GenericIndexedWriter<String> stringWriter = new GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "strings",
+ GenericIndexed.STRING_STRATEGY
+ );
+ stringWriter.open();
+ stringWriter.write(null);
+ stringWriter.write("a");
+ stringWriter.write("b");
+ stringWriter.write("fo");
+ stringWriter.write("foo");
+ stringWriter.write("fooo");
+ stringWriter.write("z");
+ writeToBuffer(stringBuffer, stringWriter);
+
+ FixedIndexedWriter<Long> longWriter = new FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ TypeStrategies.LONG,
+ ByteOrder.nativeOrder(),
+ Long.BYTES,
+ true
+ );
+ longWriter.open();
+ longWriter.write(1L);
+ longWriter.write(2L);
+ longWriter.write(3L);
+ longWriter.write(5L);
+ longWriter.write(100L);
+ longWriter.write(300L);
+ longWriter.write(9000L);
+ writeToBuffer(longBuffer, longWriter);
+
+ FixedIndexedWriter<Double> doubleWriter = new FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ TypeStrategies.DOUBLE,
+ ByteOrder.nativeOrder(),
+ Double.BYTES,
+ true
+ );
+ doubleWriter.open();
+ doubleWriter.write(1.0);
+ doubleWriter.write(1.1);
+ doubleWriter.write(1.2);
+ doubleWriter.write(2.0);
+ doubleWriter.write(2.5);
+ doubleWriter.write(3.3);
+ doubleWriter.write(6.6);
+ doubleWriter.write(9.9);
+ writeToBuffer(doubleBuffer, doubleWriter);
+
+ globalStrings = GenericIndexed.read(stringBuffer,
GenericIndexed.STRING_STRATEGY);
+ globalLongs = FixedIndexed.read(longBuffer, TypeStrategies.LONG,
ByteOrder.nativeOrder(), Long.BYTES);
+ globalDoubles = FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE,
ByteOrder.nativeOrder(), Double.BYTES);
+ }
+
@Test
- public void testRangeValueSkipping()
- {
- FixedIndexed<Integer> localDictionary =
EasyMock.createMock(FixedIndexed.class);
- GenericIndexed<String> stringDictionary =
EasyMock.createMock(GenericIndexed.class);
- FixedIndexed<Long> longDictionary =
EasyMock.createMock(FixedIndexed.class);
- FixedIndexed<Double> doubleDictionary =
EasyMock.createMock(FixedIndexed.class);
- GenericIndexed<ImmutableBitmap> bitmaps =
EasyMock.createMock(GenericIndexed.class);
-
- RoaringBitmapFactory bitmapFactory = new RoaringBitmapFactory();
- MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap();
- bitmap.add(1);
- ImmutableBitmap immutableBitmap =
bitmapFactory.makeImmutableBitmap(bitmap);
- MutableBitmap bitmap2 = bitmapFactory.makeEmptyMutableBitmap();
- bitmap2.add(2);
- ImmutableBitmap immutableBitmap2 =
bitmapFactory.makeImmutableBitmap(bitmap2);
-
- EasyMock.expect(stringDictionary.size()).andReturn(10).times(3);
- EasyMock.expect(longDictionary.size()).andReturn(0).times(1);
- EasyMock.expect(stringDictionary.indexOf("fo")).andReturn(3).times(2);
- EasyMock.expect(stringDictionary.indexOf("fooo")).andReturn(5).times(2);
- EasyMock.expect(stringDictionary.get(3)).andReturn("fo").times(1);
- EasyMock.expect(stringDictionary.get(4)).andReturn("foo").times(1);
- EasyMock.expect(stringDictionary.get(5)).andReturn("fooo").times(1);
- EasyMock.expect(localDictionary.indexOf(3)).andReturn(0).times(2);
- EasyMock.expect(localDictionary.indexOf(4)).andReturn(0).times(2);
- EasyMock.expect(localDictionary.indexOf(5)).andReturn(1).times(2);
- EasyMock.expect(localDictionary.indexOf(6)).andReturn(-2).times(2);
- EasyMock.expect(bitmaps.get(0)).andReturn(immutableBitmap).times(1);
- EasyMock.expect(bitmaps.get(1)).andReturn(immutableBitmap2).times(2);
-
- EasyMock.replay(localDictionary, stringDictionary, longDictionary,
doubleDictionary, bitmaps);
-
- NestedFieldLiteralColumnIndexSupplier indexSupplier = new
NestedFieldLiteralColumnIndexSupplier(
- new NestedLiteralTypeInfo.TypeSet(new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue()),
- bitmapFactory,
- bitmaps,
- localDictionary,
- stringDictionary,
- longDictionary,
- doubleDictionary
+ public void testSingleTypeStringColumnValueSetIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringSupplier();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [b, foo, fooo, z]
+ // column: [foo, b, fooo, b, z, fooo, z, b, b, foo]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("b");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 7, 8);
+
+ // non-existent in local column
+ columnIndex = valueSetIndex.forValue("fo");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("b", "fooo", "z")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.8, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 2, 3, 4, 5, 6, 7, 8);
+ }
+
+ @Test
+ public void testSingleTypeStringColumnRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringSupplier();
+
+ LexicographicalRangeIndex rangeIndex =
indexSupplier.as(LexicographicalRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [b, foo, fooo, z]
+ // column: [foo, b, fooo, b, z, fooo, z, b, b, foo]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 5, 9);
+
+ forRange = rangeIndex.forRange(null, false, "g", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9);
+
+ forRange = rangeIndex.forRange("f", false, null, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange("b", true, "fooo", true);
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 9);
+
+ forRange = rangeIndex.forRange("b", true, "fooo", false);
+ Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 5, 9);
+
+ forRange = rangeIndex.forRange(null, true, "fooo", true);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 3, 7, 8, 9);
+
+ forRange = rangeIndex.forRange("b", true, null, false);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange("b", false, null, true);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, true, "fooo", false);
+ Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ }
+
+ @Test
+ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws
IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringSupplier();
+
+ LexicographicalRangeIndex rangeIndex =
indexSupplier.as(LexicographicalRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [b, foo, fooo, z]
+ // column: [foo, b, fooo, b, z, fooo, z, b, b, foo]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange(
+ "f",
+ true,
+ "g",
+ true,
+ s -> !"fooo".equals(s)
+ );
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 9);
+
+ forRange = rangeIndex.forRange(
+ "f",
+ true,
+ "g",
+ true,
+ s -> "fooo".equals(s)
+ );
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 5);
+
+ forRange = rangeIndex.forRange(
+ null,
+ false,
+ "z",
+ false,
+ s -> !"fooo".equals(s)
+ );
+ Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(
+ null,
+ false,
+ "z",
+ true,
+ s -> !"fooo".equals(s)
+ );
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 3, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(
+ "f",
+ true,
+ null,
+ true,
+ s -> true
+ );
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 6, 9);
+ }
+
+ @Test
+ public void testSingleTypeStringColumnPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringSupplier();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("b", "z"))
+ );
+
+ // 10 rows
+ // local: [b, foo, fooo, z]
+ // column: [foo, b, fooo, b, z, fooo, z, b, b, foo]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 6, 7, 8);
+ }
+
+ @Test
+ public void testSingleTypeStringColumnWithNullValueIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringWithNullsSupplier();
+
+ NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class);
+ Assert.assertNotNull(nullIndex);
+
+ // 10 rows
+ // local: [null, b, foo, fooo, z]
+ // column: [foo, null, fooo, b, z, fooo, z, null, null, foo]
+
+ BitmapColumnIndex columnIndex = nullIndex.forNull();
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 7, 8);
+ }
+
+ @Test
+ public void testSingleTypeStringColumnWithNullValueSetIndex() throws
IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringWithNullsSupplier();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [null, b, foo, fooo, z]
+ // column: [foo, null, fooo, b, z, fooo, z, null, null, foo]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("b");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 3);
+
+ // non-existent in local column
+ columnIndex = valueSetIndex.forValue("fo");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("b", "fooo", "z")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 3, 4, 5, 6);
+ }
+
+ @Test
+ public void testSingleValueStringWithNullRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringWithNullsSupplier();
+
+ LexicographicalRangeIndex rangeIndex =
indexSupplier.as(LexicographicalRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [null, b, foo, fooo, z]
+ // column: [foo, null, fooo, b, z, fooo, z, null, null, foo]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
+
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 5, 9);
+
+ forRange = rangeIndex.forRange(null, false, "g", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 3, 5, 9);
+
+ forRange = rangeIndex.forRange("f", false, null, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange("b", true, "fooo", true);
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 9);
+
+ forRange = rangeIndex.forRange("b", true, "fooo", false);
+ Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 5, 9);
+
+ forRange = rangeIndex.forRange(null, true, "fooo", true);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 3, 9);
+
+ forRange = rangeIndex.forRange("b", true, null, false);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange("b", false, null, true);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange(null, true, "fooo", false);
+ Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 3, 5, 9);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9);
+ }
+
+ @Test
+ public void testSingleValueStringWithNullPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeStringWithNullsSupplier();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("b", "z"))
+ );
+
+ // 10 rows
+ // local: [null, b, foo, fooo, z]
+ // column: [foo, null, fooo, b, z, fooo, z, null, null, foo]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 3, 4, 6);
+ }
+
+ @Test
+ public void testSingleTypeLongColumnValueSetIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplier();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [1, 3, 100, 300]
+ // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("1");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 9);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("1", "300", "700")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 2, 3, 7, 8, 9);
+ }
+
+ @Test
+ public void testSingleTypeLongColumnRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplier();
+
+ NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [1, 3, 100, 300]
+ // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange(10L, true, 400L, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0);
+
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 6, 7, 8);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ }
+
+ @Test
+ public void testSingleTypeLongColumnPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplier();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("1", "3"))
+ );
+
+ // 10 rows
+ // local: [1, 3, 100, 300]
+ // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 5, 9);
+ }
+
+ @Test
+ public void testSingleTypeLongColumnWithNullValueIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplierWithNull();
+
+ NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class);
+ Assert.assertNotNull(nullIndex);
+
+ // 10 rows
+ // local: [null, 1, 3, 100, 300]
+ // column: [100, 1, null, 1, 3, null, 100, 300, null, 1]
+
+ BitmapColumnIndex columnIndex = nullIndex.forNull();
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 5, 8);
+ }
+
+ @Test
+ public void testSingleTypeLongColumnWithNullValueSetIndex() throws
IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplierWithNull();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [null, 1, 3, 100, 300]
+ // column: [100, 1, null, 1, 3, null, 100, 300, null, 1]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("3");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 4);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("1", "3", "300")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 7, 9);
+ }
+
+ @Test
+ public void testSingleValueLongWithNullRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplierWithNull();
+
+ NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [null, 1, 3, 100, 300]
+ // column: [100, 1, null, 1, 3, null, 100, 300, null, 1]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange(100, false, 700, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 6, 7);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9);
+ }
+
+ @Test
+ public void testSingleValueLongWithNullPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeLongSupplierWithNull();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("3", "100"))
+ );
+
+ // 10 rows
+ // local: [null, 1, 3, 100, 300]
+ // column: [100, 1, null, 1, 3, null, 100, 300, null, 1]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 4, 6);
+ }
+
+ @Test
+ public void testSingleTypeDoubleColumnValueSetIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplier();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("1.2");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 4, 7);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("1.2", "3.3", "6.6")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.7, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 3, 4, 5, 6, 7, 9);
+ }
+
+ @Test
+ public void testSingleTypeDoubleColumnRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplier();
+
+ NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange(1.0, true, 5.0, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0);
+
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(1.1, false, 3.3, false);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0);
+
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(1.1, true, 3.3, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 3, 4, 6, 7, 9);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ }
+
+ @Test
+ public void testSingleTypeDoubleColumnPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplier();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("1.2", "3.3", "5.0"))
+ );
+
+ // 10 rows
+ // local: [1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 3, 4, 6, 7, 9);
+ }
+
+ @Test
+ public void testSingleTypeDoubleColumnWithNullValueIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplierWithNull();
+
+ NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class);
+ Assert.assertNotNull(nullIndex);
+
+ // 10 rows
+ // local: [null, 1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex columnIndex = nullIndex.forNull();
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 6);
+ }
+
+ @Test
+ public void testSingleTypeDoubleColumnWithNullValueSetIndex() throws
IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplierWithNull();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [null, 1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("6.6");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 5);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("1.2", "3.3", "7.7")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 4, 7, 9);
+ }
+
+ @Test
+ public void testSingleValueDoubleWithNullRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplierWithNull();
+
+ NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class);
+ Assert.assertNotNull(rangeIndex);
+
+ // 10 rows
+ // local: [null, 1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex forRange = rangeIndex.forRange(1.1, false, 5.0, true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
+
+ ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, true, null, true);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, false, null, false);
+ Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9);
+ }
+
+ @Test
+ public void testSingleValueDoubleWithNullPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeSingleTypeDoubleSupplierWithNull();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("1.2", "3.3"))
);
+ // 10 rows
+ // local: [null, 1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 4, 7, 9);
+ }
+
+ @Test
+ public void testVariantNullValueIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeVariantSupplierWithNull();
+
+ NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class);
+ Assert.assertNotNull(nullIndex);
+
+ // 10 rows
+ // local: [null, b, z, 1, 300, 1.1, 9.9]
+ // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b]
+
+ BitmapColumnIndex columnIndex = nullIndex.forNull();
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 7);
+ }
+
+ @Test
+ public void testVariantValueSetIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeVariantSupplierWithNull();
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 10 rows
+ // local: [null, b, z, 1, 300, 1.1, 9.9]
+ // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("b");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 9);
+
+ columnIndex = valueSetIndex.forValue("1");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 5);
+
+ columnIndex = valueSetIndex.forValue("1.1");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 8);
+
+ // set index
+ columnIndex = valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of("b", "300", "9.9", "1.6")));
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 9);
+ }
+
+ @Test
+ public void testVariantRangeIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeVariantSupplierWithNull();
+
LexicographicalRangeIndex rangeIndex =
indexSupplier.as(LexicographicalRangeIndex.class);
+ Assert.assertNull(rangeIndex);
+
+ NumericRangeIndex numericRangeIndex =
indexSupplier.as(NumericRangeIndex.class);
+ Assert.assertNull(numericRangeIndex);
+ }
+
+ @Test
+ public void testVariantPredicateIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeVariantSupplierWithNull();
+
+ DruidPredicateIndex predicateIndex =
indexSupplier.as(DruidPredicateIndex.class);
+ Assert.assertNotNull(predicateIndex);
+ DruidPredicateFactory predicateFactory = new
InDimFilter.InFilterDruidPredicateFactory(
+ null,
+ new InDimFilter.ValuesSet(ImmutableSet.of("b", "z", "9.9", "300"))
+ );
+
+ // 10 rows
+ // local: [null, b, z, 1, 300, 1.1, 9.9]
+ // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b]
+
+ BitmapColumnIndex columnIndex =
predicateIndex.forPredicate(predicateFactory);
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 6, 9);
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringSupplier()
throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [b, foo, fooo, z]
+ // column: [foo, b, fooo, b, z, fooo, z, b, b, foo]
+
+ // b
+ localDictionaryWriter.write(2);
+ bitmapWriter.write(fillBitmap(1, 3, 7, 8));
+
+ // foo
+ localDictionaryWriter.write(4);
+ bitmapWriter.write(fillBitmap(0, 9));
+
+ // fooo
+ localDictionaryWriter.write(5);
+ bitmapWriter.write(fillBitmap(2, 5));
+
+ // z
+ localDictionaryWriter.write(6);
+ bitmapWriter.write(fillBitmap(4, 6));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier
makeSingleTypeStringWithNullsSupplier() throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [null, b, foo, fooo, z]
+ // column: [foo, null, fooo, b, z, fooo, z, null, null, foo]
+
+ // null
+ localDictionaryWriter.write(0);
+ bitmapWriter.write(fillBitmap(1, 7, 8));
+
+ // b
+ localDictionaryWriter.write(2);
+ bitmapWriter.write(fillBitmap(3));
+
+ // foo
+ localDictionaryWriter.write(4);
+ bitmapWriter.write(fillBitmap(0, 9));
+
+ // fooo
+ localDictionaryWriter.write(5);
+ bitmapWriter.write(fillBitmap(2, 5));
+
+ // z
+ localDictionaryWriter.write(6);
+ bitmapWriter.write(fillBitmap(4, 6));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeLongSupplier()
throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [1, 3, 100, 300]
+ // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1]
+
+ // 1
+ localDictionaryWriter.write(7);
+ bitmapWriter.write(fillBitmap(1, 3, 9));
+
+ // 3
+ localDictionaryWriter.write(9);
+ bitmapWriter.write(fillBitmap(4, 5));
+
+ // 100
+ localDictionaryWriter.write(11);
+ bitmapWriter.write(fillBitmap(0, 6));
+
+ // 300
+ localDictionaryWriter.write(12);
+ bitmapWriter.write(fillBitmap(2, 7, 8));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier
makeSingleTypeLongSupplierWithNull() throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [null, 1, 3, 100, 300]
+ // column: [100, 1, null, 1, 3, null, 100, 300, null, 1]
+
+ // null
+ localDictionaryWriter.write(0);
+ bitmapWriter.write(fillBitmap(2, 5, 8));
+
+ // 1
+ localDictionaryWriter.write(7);
+ bitmapWriter.write(fillBitmap(1, 3, 9));
+
+ // 3
+ localDictionaryWriter.write(9);
+ bitmapWriter.write(fillBitmap(4));
+
+ // 100
+ localDictionaryWriter.write(11);
+ bitmapWriter.write(fillBitmap(0, 6));
+
+ // 300
+ localDictionaryWriter.write(12);
+ bitmapWriter.write(fillBitmap(7));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier makeSingleTypeDoubleSupplier()
throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3]
+
+ // 1.1
+ localDictionaryWriter.write(15);
+ bitmapWriter.write(fillBitmap(0, 1, 8));
+
+ // 1.2
+ localDictionaryWriter.write(16);
+ bitmapWriter.write(fillBitmap(2, 4, 7));
+
+ // 3.3
+ localDictionaryWriter.write(19);
+ bitmapWriter.write(fillBitmap(3, 6, 9));
+
+ // 6.6
+ localDictionaryWriter.write(20);
+ bitmapWriter.write(fillBitmap(5));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier
makeSingleTypeDoubleSupplierWithNull() throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [null, 1.1, 1.2, 3.3, 6.6]
+ // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3]
+
+ // null
+ localDictionaryWriter.write(0);
+ bitmapWriter.write(fillBitmap(1, 3, 6));
+
+ // 1.1
+ localDictionaryWriter.write(15);
+ bitmapWriter.write(fillBitmap(0, 8));
+
+ // 1.2
+ localDictionaryWriter.write(16);
+ bitmapWriter.write(fillBitmap(2, 4, 7));
+
+ // 3.3
+ localDictionaryWriter.write(19);
+ bitmapWriter.write(fillBitmap(9));
+
+ // 6.6
+ localDictionaryWriter.write(20);
+ bitmapWriter.write(fillBitmap(5));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new
NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private NestedFieldLiteralColumnIndexSupplier makeVariantSupplierWithNull()
throws IOException
+ {
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, a, b, fo, foo, fooo, z],
+ // [1, 2, 3, 5, 100, 300, 9000],
+ // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9]
+ // ]
+ // local: [null, b, z, 1, 300, 1.1, 9.9]
+ // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b]
+
+ // null
+ localDictionaryWriter.write(0);
+ bitmapWriter.write(fillBitmap(2, 7));
+
+ // b
+ localDictionaryWriter.write(2);
+ bitmapWriter.write(fillBitmap(1, 9));
+
+ // z
+ localDictionaryWriter.write(6);
+ bitmapWriter.write(fillBitmap(6));
+
+ // 1
+ localDictionaryWriter.write(7);
+ bitmapWriter.write(fillBitmap(0, 5));
+
+ // 300
+ localDictionaryWriter.write(12);
+ bitmapWriter.write(fillBitmap(4));
+
+ // 1.1
+ localDictionaryWriter.write(15);
+ bitmapWriter.write(fillBitmap(8));
+
+ // 9.9
+ localDictionaryWriter.write(21);
+ bitmapWriter.write(fillBitmap(3));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ FixedIndexed<Integer> dictionary = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ return new NestedFieldLiteralColumnIndexSupplier(
+ new NestedLiteralTypeInfo.TypeSet(
+ new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING)
+ .add(ColumnType.LONG)
+ .add(ColumnType.DOUBLE)
+ .getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionary,
+ globalStrings,
+ globalLongs,
+ globalDoubles
+ );
+ }
+
+ private ImmutableBitmap fillBitmap(int... rows)
+ {
+ MutableBitmap bitmap =
roaringFactory.getBitmapFactory().makeEmptyMutableBitmap();
+ for (int i : rows) {
+ bitmap.add(i);
+ }
+ return roaringFactory.getBitmapFactory().makeImmutableBitmap(bitmap);
+ }
+
+ void checkBitmap(ImmutableBitmap bitmap, int... expectedRows)
+ {
+ IntIterator iterator = bitmap.iterator();
+ for (int i : expectedRows) {
+ Assert.assertTrue(iterator.hasNext());
+ Assert.assertEquals(i, iterator.next());
+ }
+ Assert.assertFalse(iterator.hasNext());
+ }
+
+ static void writeToBuffer(ByteBuffer buffer, Serializer serializer) throws
IOException
+ {
+ WritableByteChannel channel = new WritableByteChannel()
+ {
+ @Override
+ public int write(ByteBuffer src)
+ {
+ int size = src.remaining();
+ buffer.put(src);
+ return size;
+ }
+
+ @Override
+ public boolean isOpen()
+ {
+ return true;
+ }
+
+ @Override
+ public void close()
+ {
+ }
+ };
- BitmapColumnIndex columnIndex = rangeIndex.forRange("fo", false, "fooo",
false);
- DefaultBitmapResultFactory defaultBitmapResultFactory = new
DefaultBitmapResultFactory(bitmapFactory);
- ImmutableBitmap result =
columnIndex.computeBitmapResult(defaultBitmapResultFactory);
- Assert.assertEquals(2, result.size());
- Assert.assertTrue(result.get(1));
- Assert.assertTrue(result.get(2));
-
- // predicate skips first index
- columnIndex = rangeIndex.forRange("fo", false, "fooo", false,
"fooo"::equals);
- result = columnIndex.computeBitmapResult(defaultBitmapResultFactory);
- Assert.assertEquals(1, result.size());
- Assert.assertTrue(result.get(2));
- EasyMock.verify(localDictionary, stringDictionary, longDictionary,
doubleDictionary, bitmaps);
+ serializer.writeTo(channel, null);
+ buffer.position(0);
}
}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index ea3c66d11d..182ef9ab95 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -852,7 +852,6 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
.build()
),
ImmutableList.of(
- new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"100", 2L}
)
);
@@ -954,6 +953,7 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
.build()
),
ImmutableList.of(
+ new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"100", 2L}
)
);
@@ -1052,7 +1052,6 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
.build()
),
ImmutableList.of(
- new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"2.02", 2L}
)
);
@@ -1154,6 +1153,7 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
.build()
),
ImmutableList.of(
+ new Object[]{NullHandling.defaultStringValue(), 4L},
new Object[]{"2.02", 2L}
)
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
