This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6046a392b6 add DictionaryEncodedStringValueIndex implementation to
NestedFieldLiteralColumnIndexSupplier (#12837)
6046a392b6 is described below
commit 6046a392b61b94bbfa6f7a083e2958e8e942ab6e
Author: Clint Wylie <[email protected]>
AuthorDate: Mon Aug 1 21:40:35 2022 -0700
add DictionaryEncodedStringValueIndex implementation to
NestedFieldLiteralColumnIndexSupplier (#12837)
---
.../column/DictionaryEncodedStringValueIndex.java | 18 ++++--------
.../column/DictionaryEncodedValueIndex.java | 11 +++++--
.../NestedFieldLiteralColumnIndexSupplier.java | 34 ++++++++++++++++++++++
.../DictionaryEncodedStringIndexSupplier.java | 11 -------
.../segment/virtual/ListFilteredVirtualColumn.java | 12 --------
.../ColumnSelectorColumnIndexSelectorTest.java | 1 -
.../druid/segment/IndexMergerNullHandlingTest.java | 5 ----
.../NestedFieldLiteralColumnIndexSupplierTest.java | 24 +++++++++++++++
.../ListFilteredVirtualColumnSelectorTest.java | 10 -------
9 files changed, 71 insertions(+), 55 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java
b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java
index f8df78ca09..f75d9e2926 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java
@@ -22,17 +22,15 @@ package org.apache.druid.segment.column;
import javax.annotation.Nullable;
/**
- * This exposes a 'raw' view into a bitmap value indexes of a string {@link
DictionaryEncodedColumn}, allowing
- * operation via dictionaryIds, as well as access to lower level details of
such a column like value lookup and
- * value cardinality.
+ * This exposes a 'raw' view into bitmap value indexes of a string {@link
DictionaryEncodedColumn}. This allows callers
+ * to directly retrieve bitmaps via dictionary ids, as well as access to lower
level details of such a column like
+ * value lookup and value cardinality.
*
- * This interface should only be used when it is beneficial to operate in such
a manner, most filter implementations
- * should likely be using {@link StringValueSetIndex}, {@link
DruidPredicateIndex}, {@link LexicographicalRangeIndex} or
- * some other higher level index instead.
+ * Most filter implementations should likely be using higher level index
instead, such as {@link StringValueSetIndex},
+ * {@link LexicographicalRangeIndex}, {@link NumericRangeIndex}, or {@link
DruidPredicateIndex}
*/
public interface DictionaryEncodedStringValueIndex extends
DictionaryEncodedValueIndex
{
- boolean hasNulls();
/**
* Get the cardinality of the underlying value dictionary
*/
@@ -43,10 +41,4 @@ public interface DictionaryEncodedStringValueIndex extends
DictionaryEncodedValu
*/
@Nullable
String getValue(int index);
-
- /**
- * Returns the index of "value" in this DictionaryEncodedStringValueIndex,
or a negative value, if not present in
- * the underlying dictionary
- */
- int getIndex(@Nullable String value);
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedValueIndex.java
b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedValueIndex.java
index c088f7455b..b1f0115062 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedValueIndex.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedValueIndex.java
@@ -22,10 +22,15 @@ package org.apache.druid.segment.column;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
/**
- * This exposes a 'raw' view into a bitmap value indexes, allowing operation
via dictionaryIds
+ * This exposes a 'raw' view into bitmap value indexes for {@link
DictionaryEncodedColumn}. This allows callers
+ * to directly retrieve bitmaps via dictionary ids.
*
- * This interface should only be used when it is beneficial to operate in such
a manner, most filter implementations
- * should likely be using higher level index instead.
+ * This interface should only be used when it is beneficial to operate in such
a manner; callers of this class must
+ * either already know what value the dictionary id represents, not care at
all, or have some other means to know
+ * exactly which bitmaps to retrieve.
+ *
+ * Most filter implementations should likely be using higher level index
instead, such as {@link StringValueSetIndex},
+ * {@link LexicographicalRangeIndex}, {@link NumericRangeIndex}, or {@link
DruidPredicateIndex}.
*/
public interface DictionaryEncodedValueIndex
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
index e67ac8ac52..de7476bfbc 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java
@@ -45,6 +45,8 @@ import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
+import org.apache.druid.segment.column.DictionaryEncodedValueIndex;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.NullValueIndex;
@@ -108,6 +110,8 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
if (clazz.equals(NullValueIndex.class)) {
// null index is always 0 in the global dictionary, even if there are no
null rows in any of the literal columns
return (T) (NullValueIndex) () -> new
SimpleImmutableBitmapIndex(bitmaps.get(0));
+ } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) ||
clazz.equals(DictionaryEncodedValueIndex.class)) {
+ return (T) new NestedLiteralDictionaryEncodedStringValueIndex();
}
if (singleType != null) {
@@ -219,6 +223,7 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
return new IntIntImmutablePair(localStartIndex,
Math.min(dictionary.size(), localEndIndex));
}
+
private <T> BitmapColumnIndex makeRangeIndex(
@Nullable T startValue,
boolean startStrict,
@@ -263,6 +268,35 @@ public class NestedFieldLiteralColumnIndexSupplier
implements ColumnIndexSupplie
};
}
+ private class NestedLiteralDictionaryEncodedStringValueIndex implements
DictionaryEncodedStringValueIndex
+ {
+ @Override
+ public int getCardinality()
+ {
+ return dictionary.size();
+ }
+
+ @Nullable
+ @Override
+ public String getValue(int index)
+ {
+ int globalIndex = dictionary.get(index);
+ if (globalIndex < adjustLongId) {
+ return globalDictionary.get(globalIndex);
+ } else if (globalIndex < adjustDoubleId) {
+ return String.valueOf(globalLongDictionary.get(globalIndex -
adjustLongId));
+ } else {
+ return String.valueOf(globalDoubleDictionary.get(globalIndex -
adjustDoubleId));
+ }
+ }
+
+ @Override
+ public ImmutableBitmap getBitmap(int idx)
+ {
+ return NestedFieldLiteralColumnIndexSupplier.this.getBitmap(idx);
+ }
+ }
+
private class NestedStringLiteralValueSetIndex implements StringValueSetIndex
{
@Override
diff --git
a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java
b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java
index 85831d268a..b0dc0072cf 100644
---
a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java
@@ -156,12 +156,6 @@ public class DictionaryEncodedStringIndexSupplier
implements ColumnIndexSupplier
super(bitmapFactory, dictionary, bitmaps);
}
- @Override
- public boolean hasNulls()
- {
- return dictionary.indexOf(null) >= 0;
- }
-
@Override
public int getCardinality()
{
@@ -175,11 +169,6 @@ public class DictionaryEncodedStringIndexSupplier
implements ColumnIndexSupplier
return dictionary.get(index);
}
- @Override
- public int getIndex(@Nullable String value)
- {
- return dictionary.indexOf(value);
- }
}
public static final class GenericIndexedDictionaryEncodedStringValueSetIndex
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
index e273ab2bcd..f5f7e94d37 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
@@ -607,12 +607,6 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
super(delegate, idMapping);
}
- @Override
- public boolean hasNulls()
- {
- return delegate.hasNulls();
- }
-
@Override
public int getCardinality()
{
@@ -626,12 +620,6 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
return delegate.getValue(idMapping.getReverseId(index));
}
- @Override
- public int getIndex(@Nullable String value)
- {
- return getReverseIndex(value);
- }
-
@Override
public ImmutableBitmap getBitmap(int idx)
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java
index b2c8544ea2..f875be0ce3 100644
---
a/processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java
@@ -79,7 +79,6 @@ public class ColumnSelectorColumnIndexSelectorTest
EasyMock.expect(indexSupplier.as(DictionaryEncodedStringValueIndex.class)).andReturn(valueIndex).anyTimes();
BitmapColumnIndex columnIndex =
EasyMock.createMock(BitmapColumnIndex.class);
ImmutableBitmap someBitmap = EasyMock.createMock(ImmutableBitmap.class);
- EasyMock.expect(valueIndex.getIndex("foo")).andReturn(0).anyTimes();
EasyMock.expect(valueIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
EasyMock.expect(someIndex.forValue("foo")).andReturn(columnIndex).anyTimes();
diff --git
a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
index b5da898fae..44948bbbd8 100644
---
a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
@@ -198,11 +198,8 @@ public class IndexMergerNullHandlingTest
}
}
- Assert.assertEquals(subsetList.toString(), expectedNullRows.size()
> 0, valueIndex.hasNulls());
if (expectedNullRows.size() > 0) {
- Assert.assertEquals(subsetList.toString(), 0,
valueIndex.getIndex(null));
-
final ImmutableBitmap nullBitmap = valueSetIndex.forValue(null)
.computeBitmapResult(
new
DefaultBitmapResultFactory(
@@ -217,8 +214,6 @@ public class IndexMergerNullHandlingTest
}
Assert.assertEquals(subsetList.toString(), expectedNullRows,
actualNullRows);
- } else {
- Assert.assertEquals(-1, valueIndex.getIndex(null));
}
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
index 13a451428d..b4d576ac04 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java
@@ -28,6 +28,8 @@ import org.apache.druid.query.filter.DruidPredicateFactory;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.segment.column.BitmapColumnIndex;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex;
+import org.apache.druid.segment.column.DictionaryEncodedValueIndex;
import org.apache.druid.segment.column.DruidPredicateIndex;
import org.apache.druid.segment.column.LexicographicalRangeIndex;
import org.apache.druid.segment.column.NullValueIndex;
@@ -929,6 +931,28 @@ public class NestedFieldLiteralColumnIndexSupplierTest
extends InitializedNullHa
checkBitmap(bitmap, 1, 3, 4, 6, 9);
}
+ @Test
+ public void testDictionaryEncodedStringValueIndex() throws IOException
+ {
+ NestedFieldLiteralColumnIndexSupplier indexSupplier =
makeVariantSupplierWithNull();
+
+ DictionaryEncodedStringValueIndex lowLevelIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
+ Assert.assertNotNull(lowLevelIndex);
+ Assert.assertNotNull(indexSupplier.as(DictionaryEncodedValueIndex.class));
+
+ // 10 rows
+ // local: [null, b, z, 1, 300, 1.1, 9.9]
+ // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b]
+
+ Assert.assertNull(lowLevelIndex.getValue(0));
+ Assert.assertEquals("b", lowLevelIndex.getValue(1));
+ Assert.assertEquals("z", lowLevelIndex.getValue(2));
+ Assert.assertEquals("1", lowLevelIndex.getValue(3));
+ Assert.assertEquals("300", lowLevelIndex.getValue(4));
+ Assert.assertEquals("1.1", lowLevelIndex.getValue(5));
+ Assert.assertEquals("9.9", lowLevelIndex.getValue(6));
+ }
+
private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringSupplier()
throws IOException
{
ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 <<
12).order(ByteOrder.nativeOrder());
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
index d2c146b193..4b7e1f640b 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
@@ -196,7 +196,6 @@ public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandli
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(2)).andReturn(bitmap).once();
- EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, timeHolder, indexSupplier, index,
bitmap, bitmapFactory);
@@ -211,14 +210,10 @@ public class ListFilteredVirtualColumnSelectorTest
extends InitializedNullHandli
DictionaryEncodedStringValueIndex listFilteredIndex =
bitmapIndexSelector.getIndexSupplier(ALLOW_VIRTUAL_NAME).as(DictionaryEncodedStringValueIndex.class);
- Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
- Assert.assertEquals(0, listFilteredIndex.getIndex("b"));
- Assert.assertEquals(1, listFilteredIndex.getIndex("c"));
Assert.assertEquals(2, listFilteredIndex.getCardinality());
Assert.assertEquals("b", listFilteredIndex.getValue(0));
Assert.assertEquals("c", listFilteredIndex.getValue(1));
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
- Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, timeHolder, indexSupplier, index,
bitmap, bitmapFactory);
}
@@ -260,7 +255,6 @@ public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandli
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
- EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, timeHolder, indexSupplier, index,
bitmap, bitmapFactory);
@@ -275,12 +269,8 @@ public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandli
DictionaryEncodedStringValueIndex listFilteredIndex =
bitmapIndexSelector.getIndexSupplier(DENY_VIRTUAL_NAME).as(DictionaryEncodedStringValueIndex.class);
- Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
- Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
- Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
Assert.assertEquals(1, listFilteredIndex.getCardinality());
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
- Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, timeHolder, indexSupplier, index,
bitmap, bitmapFactory);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
