This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch 25.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/25.0.0 by this push:
new 1b009edcf8 Fix cool nested column bug caused by not properly
validating that global id is present in global dictionary before lookup up
local id (#13561) (#13572)
1b009edcf8 is described below
commit 1b009edcf8ace3365cfff6de5caebb0e3bc863a4
Author: Kashif Faraz <[email protected]>
AuthorDate: Fri Dec 16 08:26:07 2022 +0530
Fix cool nested column bug caused by not properly validating that global id
is present in global dictionary before lookup up local id (#13561) (#13572)
This commit fixes a bug with nested column "value set" indexes caused by
not properly
validating that the globalId looked up for value is present in the global
dictionary prior to
looking it up in the local dictionary, which when "adjusting" the global
ids for value type
can cause incorrect selection of value indexes.
To use an example of a variant typed nested column with 3 values `["1",
null, -2]`.
The string dictionary is `[null, "1"]`, the long dictionary is `[-2]` and
our local dictionary is `[0, 1, 2]`.
The code for variant typed indexes checks if the value is present in all
global dictionaries
and returns indexes for all matches. So in this case, we first lookup "1"
in the string dictionary,
find it at global id 1, all is good. Now, we check the long dictionary for
`1`, which due to
`-(insertionpoint + 1)` gives us `-(1 + 2) = -2`. Since the global id space
is actually stacked
dictionaries, global ids for long and double values must be "adjusted" by
the size of string
dictionary, and size of string + size of long for doubles.
Prior to this patch we were not checking that the globalId is 0 or larger,
we then immediately
looked up the `localDictionary.indexOf(-2 + adjustLong) =
localDictionary.indexOf(-2 + 2) = localDictionary.indexOf(0)` ... which is an
actual value contained in the dictionary! The fix is
to skip the longs completely since there were no global matches.
On to doubles, `-(insertionPoint + 1)` gives us `-(0 + 1) = -1`. The double
adjust value is '3'
since 2 strings and 1 long, so `localDictionary.indexOf(-1 + 3)` =
`localDictionary.indexOf(2)`
which is also a real value in our local dictionary that is definitely not
'1'.
So in this one case, looking for '1' incorrectly ended up matching every
row.
Co-authored-by: Clint Wylie <[email protected]>
---
.../NestedFieldLiteralColumnIndexSupplier.java | 98 +++++---
.../NestedFieldLiteralDictionaryEncodedColumn.java | 145 ++++++++++-
.../druid/query/scan/NestedDataScanQueryTest.java | 9 +-
.../nested/NestedDataColumnSupplierTest.java | 134 +++++++---
.../NestedFieldLiteralColumnIndexSupplierTest.java | 274 +++++++++++++++++++++
5 files changed, 579 insertions(+), 81 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
index 36c39f251b..22ba59835e 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
@@ -336,17 +336,21 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
@Override
public double estimateSelectivity(int totalRows)
{
- return (double) getBitmap(
-
localDictionary.indexOf(stringDictionary.indexOf(StringUtils.toUtf8ByteBuffer(value)))
- ).size() / totalRows;
+ final int globalId =
stringDictionary.indexOf(StringUtils.toUtf8ByteBuffer(value));
+ if (globalId < 0) {
+ return 0.0;
+ }
+ return (double) getBitmap(localDictionary.indexOf(globalId)).size()
/ totalRows;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory)
{
- return bitmapResultFactory.wrapDimensionValue(
-
getBitmap(localDictionary.indexOf(stringDictionary.indexOf(StringUtils.toUtf8ByteBuffer(value))))
- );
+ final int globalId =
stringDictionary.indexOf(StringUtils.toUtf8ByteBuffer(value));
+ if (globalId < 0) {
+ return
bitmapResultFactory.wrapDimensionValue(bitmapFactory.makeEmptyImmutableBitmap());
+ }
+ return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(globalId)));
}
};
}
@@ -564,6 +568,7 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
@Override
public BitmapColumnIndex forValue(@Nullable String value)
{
+ final boolean inputNull = value == null;
final Long longValue = GuavaUtils.tryParseLong(value);
return new SimpleBitmapColumnIndex()
{
@@ -574,22 +579,34 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
public double estimateSelectivity(int totalRows)
{
if (longValue == null) {
- return (double) getBitmap(localDictionary.indexOf(0)).size() /
totalRows;
+ if (inputNull) {
+ return (double) getBitmap(localDictionary.indexOf(0)).size() /
totalRows;
+ } else {
+ return 0.0;
+ }
+ }
+ final int globalId = longDictionary.indexOf(longValue);
+ if (globalId < 0) {
+ return 0.0;
}
- return (double) getBitmap(
- localDictionary.indexOf(longDictionary.indexOf(longValue) +
adjustLongId)
- ).size() / totalRows;
+ return (double) getBitmap(localDictionary.indexOf(globalId +
adjustLongId)).size() / totalRows;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory)
{
if (longValue == null) {
- return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(0)));
+ if (inputNull) {
+ return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(0)));
+ } else {
+ return
bitmapResultFactory.wrapDimensionValue(bitmapFactory.makeEmptyImmutableBitmap());
+ }
}
- return bitmapResultFactory.wrapDimensionValue(
-
getBitmap(localDictionary.indexOf(longDictionary.indexOf(longValue) +
adjustLongId))
- );
+ final int globalId = longDictionary.indexOf(longValue);
+ if (globalId < 0) {
+ return
bitmapResultFactory.wrapDimensionValue(bitmapFactory.makeEmptyImmutableBitmap());
+ }
+ return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(globalId
+ adjustLongId)));
}
};
}
@@ -759,6 +776,7 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
@Override
public BitmapColumnIndex forValue(@Nullable String value)
{
+ final boolean inputNull = value == null;
final Double doubleValue = Strings.isNullOrEmpty(value) ? null :
Doubles.tryParse(value);
return new SimpleBitmapColumnIndex()
{
@@ -768,22 +786,34 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
public double estimateSelectivity(int totalRows)
{
if (doubleValue == null) {
- return (double) getBitmap(localDictionary.indexOf(0)).size() /
totalRows;
+ if (inputNull) {
+ return (double) getBitmap(localDictionary.indexOf(0)).size() /
totalRows;
+ } else {
+ return 0.0;
+ }
+ }
+ final int globalId = doubleDictionary.indexOf(doubleValue);
+ if (globalId < 0) {
+ return 0.0;
}
- return (double) getBitmap(
- localDictionary.indexOf(doubleDictionary.indexOf(doubleValue) +
adjustDoubleId)
- ).size() / totalRows;
+ return (double) getBitmap(localDictionary.indexOf(globalId +
adjustDoubleId)).size() / totalRows;
}
@Override
public <T> T computeBitmapResult(BitmapResultFactory<T>
bitmapResultFactory)
{
if (doubleValue == null) {
- return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(0)));
+ if (inputNull) {
+ return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(0)));
+ } else {
+ return
bitmapResultFactory.wrapDimensionValue(bitmapFactory.makeEmptyImmutableBitmap());
+ }
}
- return bitmapResultFactory.wrapDimensionValue(
-
getBitmap(localDictionary.indexOf(doubleDictionary.indexOf(doubleValue) +
adjustDoubleId))
- );
+ final int globalId = doubleDictionary.indexOf(doubleValue);
+ if (globalId < 0) {
+ return
bitmapResultFactory.wrapDimensionValue(bitmapFactory.makeEmptyImmutableBitmap());
+ }
+ return
bitmapResultFactory.wrapDimensionValue(getBitmap(localDictionary.indexOf(globalId
+ adjustDoubleId)));
}
};
}
@@ -964,25 +994,31 @@ public class
NestedFieldLiteralColumnIndexSupplier<TStringDictionary extends Ind
// multi-type, return all that match
int globalId =
stringDictionary.indexOf(StringUtils.toUtf8ByteBuffer(value));
- int localId = localDictionary.indexOf(globalId);
- if (localId >= 0) {
- intList.add(localId);
+ if (globalId >= 0) {
+ int localId = localDictionary.indexOf(globalId);
+ if (localId >= 0) {
+ intList.add(localId);
+ }
}
Long someLong = GuavaUtils.tryParseLong(value);
if (someLong != null) {
globalId = longDictionary.indexOf(someLong);
- localId = localDictionary.indexOf(globalId + adjustLongId);
- if (localId >= 0) {
- intList.add(localId);
+ if (globalId >= 0) {
+ int localId = localDictionary.indexOf(globalId + adjustLongId);
+ if (localId >= 0) {
+ intList.add(localId);
+ }
}
}
Double someDouble = Doubles.tryParse(value);
if (someDouble != null) {
globalId = doubleDictionary.indexOf(someDouble);
- localId = localDictionary.indexOf(globalId + adjustDoubleId);
- if (localId >= 0) {
- intList.add(localId);
+ if (globalId >= 0) {
+ int localId = localDictionary.indexOf(globalId + adjustDoubleId);
+ if (localId >= 0) {
+ intList.add(localId);
+ }
}
}
return intList;
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralDictionaryEncodedColumn.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralDictionaryEncodedColumn.java
index 855112e6b4..6f794b61db 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralDictionaryEncodedColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralDictionaryEncodedColumn.java
@@ -25,7 +25,6 @@ import com.google.common.base.Predicates;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
-import org.apache.druid.common.config.NullHandling;
import org.apache.druid.common.guava.GuavaUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.UOE;
@@ -146,7 +145,7 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
final int globalId = dictionary.get(id);
if (globalId < globalDictionary.size()) {
return StringUtils.fromUtf8Nullable(globalDictionary.get(globalId));
- } else if (globalId < adjustLongId + globalLongDictionary.size()) {
+ } else if (globalId < globalDictionary.size() +
globalLongDictionary.size()) {
return String.valueOf(globalLongDictionary.get(globalId - adjustLongId));
} else {
return String.valueOf(globalDoubleDictionary.get(globalId -
adjustDoubleId));
@@ -156,7 +155,11 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
@Override
public int lookupId(String name)
{
- return dictionary.indexOf(getIdFromGlobalDictionary(name));
+ final int globalId = getIdFromGlobalDictionary(name);
+ if (globalId < 0) {
+ return -1;
+ }
+ return dictionary.indexOf(globalId);
}
@Override
@@ -179,9 +182,17 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
if (singleType != null) {
switch (singleType.getType()) {
case LONG:
- return globalLongDictionary.indexOf(GuavaUtils.tryParseLong(val));
+ final int globalLong =
globalLongDictionary.indexOf(GuavaUtils.tryParseLong(val));
+ if (globalLong < 0) {
+ return -1;
+ }
+ return globalLong + adjustLongId;
case DOUBLE:
- return globalDoubleDictionary.indexOf(Doubles.tryParse(val));
+ final int globalDouble =
globalDoubleDictionary.indexOf(Doubles.tryParse(val));
+ if (globalDouble < 0) {
+ return -1;
+ }
+ return globalDouble + adjustDoubleId;
default:
return globalDictionary.indexOf(StringUtils.toUtf8ByteBuffer(val));
}
@@ -189,9 +200,15 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
int candidate =
globalDictionary.indexOf(StringUtils.toUtf8ByteBuffer(val));
if (candidate < 0) {
candidate = globalLongDictionary.indexOf(GuavaUtils.tryParseLong(val));
+ if (candidate >= 0) {
+ candidate += adjustLongId;
+ }
}
if (candidate < 0) {
candidate = globalDoubleDictionary.indexOf(Doubles.tryParse(val));
+ if (candidate >= 0) {
+ candidate += adjustDoubleId;
+ }
}
return candidate;
}
@@ -228,7 +245,6 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
final int globalId = dictionary.get(localId);
if (globalId == 0) {
// zero
- assert NullHandling.replaceWithDefault();
return 0f;
} else if (globalId < adjustLongId) {
// try to convert string to float
@@ -248,7 +264,6 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
final int globalId = dictionary.get(localId);
if (globalId == 0) {
// zero
- assert NullHandling.replaceWithDefault();
return 0.0;
} else if (globalId < adjustLongId) {
// try to convert string to double
@@ -268,7 +283,6 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
final int globalId = dictionary.get(localId);
if (globalId == 0) {
// zero
- assert NullHandling.replaceWithDefault();
return 0L;
} else if (globalId < adjustLongId) {
// try to convert string to long
@@ -509,6 +523,121 @@ public class
NestedFieldLiteralDictionaryEncodedColumn<TStringDictionary extends
};
}
}
+ if (singleType == null) {
+ return new ColumnValueSelector<Object>()
+ {
+
+ private PeekableIntIterator nullIterator =
nullBitmap.peekableIterator();
+ private int nullMark = -1;
+ private int offsetMark = -1;
+
+ @Nullable
+ @Override
+ public Object getObject()
+ {
+ final int localId = column.get(offset.getOffset());
+ final int globalId = dictionary.get(localId);
+ if (globalId < adjustLongId) {
+ return
StringUtils.fromUtf8Nullable(globalDictionary.get(globalId));
+ } else if (globalId < adjustDoubleId) {
+ return globalLongDictionary.get(globalId - adjustLongId);
+ } else {
+ return globalDoubleDictionary.get(globalId - adjustDoubleId);
+ }
+ }
+
+ @Override
+ public float getFloat()
+ {
+ final int localId = column.get(offset.getOffset());
+ final int globalId = dictionary.get(localId);
+ if (globalId == 0) {
+ // zero
+ return 0f;
+ } else if (globalId < adjustLongId) {
+ // try to convert string to float
+ Float f =
Floats.tryParse(StringUtils.fromUtf8(globalDictionary.get(globalId)));
+ return f == null ? 0f : f;
+ } else if (globalId < adjustDoubleId) {
+ return globalLongDictionary.get(globalId -
adjustLongId).floatValue();
+ } else {
+ return globalDoubleDictionary.get(globalId -
adjustDoubleId).floatValue();
+ }
+ }
+
+ @Override
+ public double getDouble()
+ {
+ final int localId = column.get(offset.getOffset());
+ final int globalId = dictionary.get(localId);
+ if (globalId == 0) {
+ // zero
+ return 0.0;
+ } else if (globalId < adjustLongId) {
+ // try to convert string to double
+ Double d =
Doubles.tryParse(StringUtils.fromUtf8(globalDictionary.get(globalId)));
+ return d == null ? 0.0 : d;
+ } else if (globalId < adjustDoubleId) {
+ return globalLongDictionary.get(globalId -
adjustLongId).doubleValue();
+ } else {
+ return globalDoubleDictionary.get(globalId - adjustDoubleId);
+ }
+ }
+
+ @Override
+ public long getLong()
+ {
+ final int localId = column.get(offset.getOffset());
+ final int globalId = dictionary.get(localId);
+ if (globalId == 0) {
+ // zero
+ return 0L;
+ } else if (globalId < adjustLongId) {
+ // try to convert string to long
+ Long l =
GuavaUtils.tryParseLong(StringUtils.fromUtf8(globalDictionary.get(globalId)));
+ return l == null ? 0L : l;
+ } else if (globalId < adjustDoubleId) {
+ return globalLongDictionary.get(globalId - adjustLongId);
+ } else {
+ return globalDoubleDictionary.get(globalId -
adjustDoubleId).longValue();
+ }
+ }
+
+ @Override
+ public boolean isNull()
+ {
+ final int i = offset.getOffset();
+ if (i < offsetMark) {
+ // offset was reset, reset iterator state
+ nullMark = -1;
+ nullIterator = nullBitmap.peekableIterator();
+ }
+ offsetMark = i;
+ if (nullMark < i) {
+ nullIterator.advanceIfNeeded(offsetMark);
+ if (nullIterator.hasNext()) {
+ nullMark = nullIterator.next();
+ }
+ }
+ return nullMark == offsetMark;
+ }
+
+ @Override
+ public Class<?> classOfObject()
+ {
+ return Object.class;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("longColumn", longsColumn);
+ inspector.visit("nullBitmap", nullBitmap);
+ }
+ };
+ }
+
+ // single type strings, use a dimension selector
return makeDimensionSelector(offset, null);
}
diff --git
a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
index 4436d9c95c..defca9cb17 100644
---
a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
@@ -43,7 +43,6 @@ import
org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.Cursor;
-import org.apache.druid.segment.DimensionDictionarySelector;
import org.apache.druid.segment.DoubleColumnSelector;
import org.apache.druid.segment.LongColumnSelector;
import org.apache.druid.segment.Segment;
@@ -438,13 +437,13 @@ public class NestedDataScanQueryTest extends
InitializedNullHandlingTest
NESTED_MIXED_NUMERIC_FIELD
);
Assert.assertNotNull(mixedNumericValueSelector);
- Assert.assertTrue(mixedNumericValueSelector instanceof
DimensionDictionarySelector);
+ Assert.assertTrue(mixedNumericValueSelector instanceof
ColumnValueSelector);
ColumnValueSelector mixedValueSelector =
columnSelectorFactory.makeColumnValueSelector(
NESTED_MIXED_FIELD
);
Assert.assertNotNull(mixedValueSelector);
- Assert.assertTrue(mixedValueSelector instanceof
DimensionDictionarySelector);
+ Assert.assertTrue(mixedValueSelector instanceof ColumnValueSelector);
ColumnValueSelector sparseLongValueSelector =
columnSelectorFactory.makeColumnValueSelector(
@@ -463,13 +462,13 @@ public class NestedDataScanQueryTest extends
InitializedNullHandlingTest
NESTED_SPARSE_MIXED_NUMERIC_FIELD
);
Assert.assertNotNull(sparseMixedNumericValueSelector);
- Assert.assertTrue(sparseMixedNumericValueSelector instanceof
DimensionDictionarySelector);
+ Assert.assertTrue(sparseMixedNumericValueSelector instanceof
ColumnValueSelector);
ColumnValueSelector sparseMixedValueSelector =
columnSelectorFactory.makeColumnValueSelector(
NESTED_SPARSE_MIXED_FIELD
);
Assert.assertNotNull(sparseMixedValueSelector);
- Assert.assertTrue(sparseMixedValueSelector instanceof
DimensionDictionarySelector);
+ Assert.assertTrue(sparseMixedValueSelector instanceof ColumnValueSelector);
//CHECKSTYLE.ON: Regexp
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
index 98dc97731f..774ac51b58 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
@@ -33,9 +33,11 @@ import
org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
import org.apache.druid.java.util.common.io.smoosh.SmooshedWriter;
import org.apache.druid.query.DefaultBitmapResultFactory;
+import org.apache.druid.query.filter.SelectorPredicateFactory;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.BaseProgressIndicator;
import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.IndexSpec;
import org.apache.druid.segment.NestedDataColumnIndexer;
import org.apache.druid.segment.ObjectColumnSelector;
@@ -44,6 +46,7 @@ import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.column.ColumnBuilder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.NullValueIndex;
import org.apache.druid.segment.column.StringValueSetIndex;
import org.apache.druid.segment.column.TypeStrategy;
@@ -64,8 +67,10 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
@@ -75,18 +80,20 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
{
private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
+ private static final String NO_MATCH = "no";
+
@Rule
public final TemporaryFolder tempFolder = new TemporaryFolder();
DefaultBitmapResultFactory resultFactory = new
DefaultBitmapResultFactory(new RoaringBitmapFactory());
List<Map<String, Object>> data = ImmutableList.of(
- ImmutableMap.of("x", 1L, "y", 1.0, "z", "a"),
- ImmutableMap.of("y", 3.0, "z", "d"),
+ ImmutableMap.of("x", 1L, "y", 1.0, "z", "a", "v", "100"),
+ ImmutableMap.of("y", 3.0, "z", "d", "v", 1000L),
ImmutableMap.of("x", 5L, "y", 5.0, "z", "b"),
- ImmutableMap.of("x", 3L, "y", 4.0, "z", "c"),
- ImmutableMap.of("x", 2L),
- ImmutableMap.of("x", 4L, "y", 2.0, "z", "e")
+ ImmutableMap.of("x", 3L, "y", 4.0, "z", "c", "v", 3000.333),
+ ImmutableMap.of("x", 2L, "v", "40000"),
+ ImmutableMap.of("x", 4L, "y", 2.0, "z", "e", "v", 11111L)
);
Closer closer = Closer.create();
@@ -216,30 +223,47 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
Assert.assertEquals(ImmutableSet.of(ColumnType.LONG),
column.getColumnTypes(xPath));
Assert.assertEquals(ColumnType.LONG,
column.getColumnHolder(xPath).getCapabilities().toColumnType());
ColumnValueSelector<?> xSelector = column.makeColumnValueSelector(xPath,
offset);
+ DimensionSelector xDimSelector = column.makeDimensionSelector(xPath,
offset, null);
ColumnIndexSupplier xIndexSupplier = column.getColumnIndexSupplier(xPath);
Assert.assertNotNull(xIndexSupplier);
StringValueSetIndex xValueIndex =
xIndexSupplier.as(StringValueSetIndex.class);
+ DruidPredicateIndex xPredicateIndex =
xIndexSupplier.as(DruidPredicateIndex.class);
NullValueIndex xNulls = xIndexSupplier.as(NullValueIndex.class);
final List<NestedPathPart> yPath = NestedPathFinder.parseJsonPath("$.y");
Assert.assertEquals(ImmutableSet.of(ColumnType.DOUBLE),
column.getColumnTypes(yPath));
Assert.assertEquals(ColumnType.DOUBLE,
column.getColumnHolder(yPath).getCapabilities().toColumnType());
ColumnValueSelector<?> ySelector = column.makeColumnValueSelector(yPath,
offset);
+ DimensionSelector yDimSelector = column.makeDimensionSelector(yPath,
offset, null);
ColumnIndexSupplier yIndexSupplier = column.getColumnIndexSupplier(yPath);
Assert.assertNotNull(yIndexSupplier);
StringValueSetIndex yValueIndex =
yIndexSupplier.as(StringValueSetIndex.class);
+ DruidPredicateIndex yPredicateIndex =
yIndexSupplier.as(DruidPredicateIndex.class);
NullValueIndex yNulls = yIndexSupplier.as(NullValueIndex.class);
final List<NestedPathPart> zPath = NestedPathFinder.parseJsonPath("$.z");
Assert.assertEquals(ImmutableSet.of(ColumnType.STRING),
column.getColumnTypes(zPath));
Assert.assertEquals(ColumnType.STRING,
column.getColumnHolder(zPath).getCapabilities().toColumnType());
ColumnValueSelector<?> zSelector = column.makeColumnValueSelector(zPath,
offset);
+ DimensionSelector zDimSelector = column.makeDimensionSelector(zPath,
offset, null);
ColumnIndexSupplier zIndexSupplier = column.getColumnIndexSupplier(zPath);
Assert.assertNotNull(zIndexSupplier);
StringValueSetIndex zValueIndex =
zIndexSupplier.as(StringValueSetIndex.class);
+ DruidPredicateIndex zPredicateIndex =
zIndexSupplier.as(DruidPredicateIndex.class);
NullValueIndex zNulls = zIndexSupplier.as(NullValueIndex.class);
- Assert.assertEquals(ImmutableList.of(xPath, yPath, zPath),
column.getNestedFields());
+ final List<NestedPathPart> vPath = NestedPathFinder.parseJsonPath("$.v");
+ Assert.assertEquals(ImmutableSet.of(ColumnType.STRING, ColumnType.LONG,
ColumnType.DOUBLE), column.getColumnTypes(vPath));
+ Assert.assertEquals(ColumnType.STRING,
column.getColumnHolder(vPath).getCapabilities().toColumnType());
+ ColumnValueSelector<?> vSelector = column.makeColumnValueSelector(vPath,
offset);
+ DimensionSelector vDimSelector = column.makeDimensionSelector(vPath,
offset, null);
+ ColumnIndexSupplier vIndexSupplier = column.getColumnIndexSupplier(vPath);
+ Assert.assertNotNull(vIndexSupplier);
+ StringValueSetIndex vValueIndex =
vIndexSupplier.as(StringValueSetIndex.class);
+ DruidPredicateIndex vPredicateIndex =
vIndexSupplier.as(DruidPredicateIndex.class);
+ NullValueIndex vNulls = vIndexSupplier.as(NullValueIndex.class);
+
+ Assert.assertEquals(ImmutableList.of(vPath, xPath, yPath, zPath),
column.getNestedFields());
for (int i = 0; i < data.size(); i++) {
Map row = data.get(i);
@@ -247,41 +271,77 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
JSON_MAPPER.writeValueAsString(row),
JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject()))
);
- if (row.containsKey("x")) {
- Assert.assertEquals(row.get("x"), xSelector.getObject());
- Assert.assertEquals(row.get("x"), xSelector.getLong());
-
Assert.assertTrue(xValueIndex.forValue(String.valueOf(row.get("x"))).computeBitmapResult(resultFactory).get(i));
-
Assert.assertFalse(xNulls.forNull().computeBitmapResult(resultFactory).get(i));
- } else {
- Assert.assertNull(xSelector.getObject());
- Assert.assertTrue(xSelector.isNull());
-
Assert.assertTrue(xValueIndex.forValue(null).computeBitmapResult(resultFactory).get(i));
-
Assert.assertTrue(xNulls.forNull().computeBitmapResult(resultFactory).get(i));
- }
- if (row.containsKey("y")) {
- Assert.assertEquals(row.get("y"), ySelector.getObject());
- Assert.assertEquals(row.get("y"), ySelector.getDouble());
-
Assert.assertTrue(yValueIndex.forValue(String.valueOf(row.get("y"))).computeBitmapResult(resultFactory).get(i));
-
Assert.assertFalse(yNulls.forNull().computeBitmapResult(resultFactory).get(i));
- } else {
- Assert.assertNull(ySelector.getObject());
- Assert.assertTrue(ySelector.isNull());
-
Assert.assertTrue(yValueIndex.forValue(null).computeBitmapResult(resultFactory).get(i));
-
Assert.assertTrue(yNulls.forNull().computeBitmapResult(resultFactory).get(i));
- }
- if (row.containsKey("z")) {
- Assert.assertEquals(row.get("z"), zSelector.getObject());
- Assert.assertTrue(zValueIndex.forValue((String)
row.get("z")).computeBitmapResult(resultFactory).get(i));
-
Assert.assertFalse(zNulls.forNull().computeBitmapResult(resultFactory).get(i));
- } else {
- Assert.assertNull(zSelector.getObject());
-
Assert.assertTrue(zValueIndex.forValue(null).computeBitmapResult(resultFactory).get(i));
-
Assert.assertTrue(zNulls.forNull().computeBitmapResult(resultFactory).get(i));
- }
+
+ testPath(row, i, "v", vSelector, vDimSelector, vValueIndex,
vPredicateIndex, vNulls, null);
+ testPath(row, i, "x", xSelector, xDimSelector, xValueIndex,
xPredicateIndex, xNulls, ColumnType.LONG);
+ testPath(row, i, "y", ySelector, yDimSelector, yValueIndex,
yPredicateIndex, yNulls, ColumnType.DOUBLE);
+ testPath(row, i, "z", zSelector, zDimSelector, zValueIndex,
zPredicateIndex, zNulls, ColumnType.STRING);
+
offset.increment();
}
}
+ private void testPath(
+ Map row,
+ int rowNumber,
+ String path,
+ ColumnValueSelector<?> valueSelector,
+ DimensionSelector dimSelector,
+ StringValueSetIndex valueSetIndex,
+ DruidPredicateIndex predicateIndex,
+ NullValueIndex nullValueIndex,
+ @Nullable ColumnType singleType
+ )
+ {
+ if (row.containsKey(path) && row.get(path) != null) {
+ Assert.assertEquals(row.get(path), valueSelector.getObject());
+ if (ColumnType.LONG.equals(singleType)) {
+ Assert.assertEquals(row.get(path), valueSelector.getLong());
+ } else if (ColumnType.DOUBLE.equals(singleType)) {
+ Assert.assertEquals((double) row.get(path), valueSelector.getDouble(),
0.0);
+ }
+ Assert.assertFalse(valueSelector.isNull());
+
+ final String theString = String.valueOf(row.get(path));
+ Assert.assertEquals(theString, dimSelector.getObject());
+ String dimSelectorLookupVal =
dimSelector.lookupName(dimSelector.getRow().get(0));
+ Assert.assertEquals(theString, dimSelectorLookupVal);
+
Assert.assertEquals(dimSelector.idLookup().lookupId(dimSelectorLookupVal),
dimSelector.getRow().get(0));
+
+
Assert.assertTrue(valueSetIndex.forValue(theString).computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertTrue(valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of(theString))).computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertTrue(predicateIndex.forPredicate(new
SelectorPredicateFactory(theString)).computeBitmapResult(resultFactory).get(rowNumber));
+
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertFalse(valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of(NO_MATCH))).computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertFalse(predicateIndex.forPredicate(new
SelectorPredicateFactory(NO_MATCH)).computeBitmapResult(resultFactory).get(rowNumber));
+
Assert.assertFalse(nullValueIndex.forNull().computeBitmapResult(resultFactory).get(rowNumber));
+
+ Assert.assertTrue(dimSelector.makeValueMatcher(theString).matches());
+ Assert.assertFalse(dimSelector.makeValueMatcher(NO_MATCH).matches());
+ Assert.assertTrue(dimSelector.makeValueMatcher(x -> Objects.equals(x,
theString)).matches());
+ Assert.assertFalse(dimSelector.makeValueMatcher(x -> Objects.equals(x,
NO_MATCH)).matches());
+ } else {
+ Assert.assertNull(valueSelector.getObject());
+ Assert.assertTrue(valueSelector.isNull());
+
+ Assert.assertEquals(0, dimSelector.getRow().get(0));
+ Assert.assertNull(dimSelector.getObject());
+ Assert.assertNull(dimSelector.lookupName(dimSelector.getRow().get(0)));
+
+
Assert.assertTrue(valueSetIndex.forValue(null).computeBitmapResult(resultFactory).get(rowNumber));
+
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory).get(rowNumber));
+
Assert.assertTrue(nullValueIndex.forNull().computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertTrue(predicateIndex.forPredicate(new
SelectorPredicateFactory(null)).computeBitmapResult(resultFactory).get(rowNumber));
+
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory).get(rowNumber));
+ Assert.assertFalse(predicateIndex.forPredicate(new
SelectorPredicateFactory(NO_MATCH)).computeBitmapResult(resultFactory).get(rowNumber));
+
+ Assert.assertTrue(dimSelector.makeValueMatcher((String) null).matches());
+ Assert.assertFalse(dimSelector.makeValueMatcher(NO_MATCH).matches());
+ Assert.assertTrue(dimSelector.makeValueMatcher(x -> x ==
null).matches());
+ Assert.assertFalse(dimSelector.makeValueMatcher(x -> Objects.equals(x,
NO_MATCH)).matches());
+ }
+ }
+
private static class SettableSelector extends
ObjectColumnSelector<StructuredData>
{
private StructuredData data;
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
index 25fec68bb8..a4dc7bf741 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
@@ -210,12 +210,37 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap);
+ forRange = rangeIndex.forRange(null, true, "a", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(null, false, "b", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
forRange = rangeIndex.forRange(null, false, "b", false);
Assert.assertNotNull(forRange);
Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 1, 3, 7, 8);
+
+ forRange = rangeIndex.forRange("a", false, "b", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange("a", true, "b", false);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 7, 8);
+
forRange = rangeIndex.forRange("b", false, "fon", false);
Assert.assertNotNull(forRange);
Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0);
@@ -491,6 +516,24 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 0, 2, 3, 5, 9);
+ forRange = rangeIndex.forRange(null, false, "a", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(null, false, "b", true);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(null, false, "b", false);
+ Assert.assertNotNull(forRange);
+ Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 3);
+
forRange = rangeIndex.forRange("f", false, null, true);
Assert.assertNotNull(forRange);
Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0);
@@ -609,6 +652,45 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 0, 2, 6, 7, 8);
+ forRange = rangeIndex.forRange(1, true, 3, true);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(1, false, 3, true);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 9);
+
+ forRange = rangeIndex.forRange(1, false, 3, false);
+ Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 4, 5, 9);
+
+
+ forRange = rangeIndex.forRange(100L, true, 300L, true);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+
+ forRange = rangeIndex.forRange(100L, true, 300L, false);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2, 7, 8);
+
+
+ forRange = rangeIndex.forRange(100L, false, 300L, true);
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 6);
+
+
+ forRange = rangeIndex.forRange(100L, false, 300L, false);
+ Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 2, 6, 7, 8);
+
forRange = rangeIndex.forRange(null, true, null, true);
Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0);
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
@@ -726,6 +808,26 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 0, 6, 7);
+ forRange = rangeIndex.forRange(100, true, 300, true);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(100, false, 300, true);
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 6);
+
+ forRange = rangeIndex.forRange(100, true, 300, false);
+ Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 7);
+
+ forRange = rangeIndex.forRange(100, false, 300, false);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 6, 7);
+
forRange = rangeIndex.forRange(null, true, null, true);
Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
@@ -735,6 +837,21 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9);
+
+ forRange = rangeIndex.forRange(null, false, 0, false);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(null, false, 1, false);
+ Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 1, 3, 9);
+
+ forRange = rangeIndex.forRange(null, false, 1, true);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
}
@Test
@@ -997,6 +1114,26 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0);
bitmap = forRange.computeBitmapResult(bitmapResultFactory);
checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9);
+
+ forRange = rangeIndex.forRange(null, true, 1.0, true);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+
+ forRange = rangeIndex.forRange(null, true, 1.1, false);
+ Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0, 8);
+
+ forRange = rangeIndex.forRange(6.6, false, null, false);
+ Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 5);
+
+ forRange = rangeIndex.forRange(6.6, true, null, false);
+ Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0);
+ bitmap = forRange.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
}
@Test
@@ -1164,6 +1301,143 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
checkBitmap(lowLevelIndex.getBitmap(-1));
}
+ @Test
+ public void testEnsureNoImproperSelectionFromAdjustedGlobals() throws
IOException
+ {
+ // make sure we only pick matching values, not "matching" values from not
validating that
+ // globalId actually exists before looking it up in local dictionary
+ ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
10).order(ByteOrder.nativeOrder());
+ ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 10);
+
+ ByteBuffer stringBuffer = ByteBuffer.allocate(1 << 10);
+ ByteBuffer longBuffer = ByteBuffer.allocate(1 <<
10).order(ByteOrder.nativeOrder());
+ ByteBuffer doubleBuffer = ByteBuffer.allocate(1 <<
10).order(ByteOrder.nativeOrder());
+
+ GenericIndexedWriter<String> stringWriter = new GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "strings",
+ GenericIndexed.STRING_STRATEGY
+ );
+ stringWriter.open();
+ stringWriter.write(null);
+ stringWriter.write("1");
+ writeToBuffer(stringBuffer, stringWriter);
+
+ FixedIndexedWriter<Long> longWriter = new FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ TypeStrategies.LONG,
+ ByteOrder.nativeOrder(),
+ Long.BYTES,
+ true
+ );
+ longWriter.open();
+ longWriter.write(-2L);
+ writeToBuffer(longBuffer, longWriter);
+
+ FixedIndexedWriter<Double> doubleWriter = new FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ TypeStrategies.DOUBLE,
+ ByteOrder.nativeOrder(),
+ Double.BYTES,
+ true
+ );
+ doubleWriter.open();
+ writeToBuffer(doubleBuffer, doubleWriter);
+
+ GenericIndexed<ByteBuffer> strings = GenericIndexed.read(stringBuffer,
GenericIndexed.UTF8_STRATEGY);
+ Supplier<Indexed<ByteBuffer>> stringIndexed = () ->
strings.singleThreaded();
+ Supplier<FixedIndexed<Long>> longIndexed = FixedIndexed.read(longBuffer,
TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES);
+ Supplier<FixedIndexed<Double>> doubleIndexed =
FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE, ByteOrder.nativeOrder(),
Double.BYTES);
+
+ FixedIndexedWriter<Integer> localDictionaryWriter = new
FixedIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES,
+ true
+ );
+ localDictionaryWriter.open();
+ GenericIndexedWriter<ImmutableBitmap> bitmapWriter = new
GenericIndexedWriter<>(
+ new OnHeapMemorySegmentWriteOutMedium(),
+ "bitmaps",
+ roaringFactory.getObjectStrategy()
+ );
+ bitmapWriter.setObjectsNotSorted();
+ bitmapWriter.open();
+
+ // 10 rows
+ // globals: [
+ // [null, '1'],
+ // [-2],
+ // []
+ // ]
+ // local: [null, '1', -2]
+ // column: ['1', null, -2]
+
+ // null
+ localDictionaryWriter.write(0);
+ bitmapWriter.write(fillBitmap(1));
+
+ // '1'
+ localDictionaryWriter.write(1);
+ bitmapWriter.write(fillBitmap(0));
+
+ // -2
+ localDictionaryWriter.write(2);
+ bitmapWriter.write(fillBitmap(2));
+
+ writeToBuffer(localDictionaryBuffer, localDictionaryWriter);
+ writeToBuffer(bitmapsBuffer, bitmapWriter);
+
+ Supplier<FixedIndexed<Integer>> dictionarySupplier = FixedIndexed.read(
+ localDictionaryBuffer,
+ NestedDataColumnSerializer.INT_TYPE_STRATEGY,
+ ByteOrder.nativeOrder(),
+ Integer.BYTES
+ );
+
+ GenericIndexed<ImmutableBitmap> bitmaps =
GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy());
+
+ NestedFieldLiteralColumnIndexSupplier<?> indexSupplier = new
NestedFieldLiteralColumnIndexSupplier<>(
+ new NestedLiteralTypeInfo.TypeSet(
+ new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING)
+ .add(ColumnType.LONG)
+ .getByteValue()
+ ),
+ roaringFactory.getBitmapFactory(),
+ bitmaps,
+ dictionarySupplier,
+ stringIndexed,
+ longIndexed,
+ doubleIndexed
+ );
+
+ StringValueSetIndex valueSetIndex =
indexSupplier.as(StringValueSetIndex.class);
+ Assert.assertNotNull(valueSetIndex);
+
+ // 3 rows
+ // local: [null, '1', -2]
+ // column: ['1', null, -2]
+
+ BitmapColumnIndex columnIndex = valueSetIndex.forValue("1");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3333, columnIndex.estimateSelectivity(3), 0.001);
+ ImmutableBitmap bitmap =
columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 0);
+
+ columnIndex = valueSetIndex.forValue("-2");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.3333, columnIndex.estimateSelectivity(3), 0.001);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap, 2);
+
+ columnIndex = valueSetIndex.forValue("2");
+ Assert.assertNotNull(columnIndex);
+ Assert.assertEquals(0.0, columnIndex.estimateSelectivity(3), 0.0);
+ bitmap = columnIndex.computeBitmapResult(bitmapResultFactory);
+ checkBitmap(bitmap);
+ }
+
private NestedFieldLiteralColumnIndexSupplier<?>
makeSingleTypeStringSupplier() throws IOException
{
ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
