This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch 0.23.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/0.23.0 by this push:
new dec5456f1d Revert "push value range and set index get operations into
BitmapIndex (#12315)" (#12533)
dec5456f1d is described below
commit dec5456f1dbff67e1927af77cf67dae410c2b6cb
Author: Clint Wylie <[email protected]>
AuthorDate: Thu May 19 04:27:03 2022 -0700
Revert "push value range and set index get operations into BitmapIndex
(#12315)" (#12533)
This reverts commit 9cfb23935ffc3eace79b526dd84d912d129a9b1c.
---
.../ExpressionVectorSelectorBenchmark.java | 14 +-
.../druid/benchmark/MockBitmapIndexSelector.java | 11 +-
.../PredicateFilteredDimensionSelector.java | 7 +-
.../druid/query/filter/BitmapIndexSelector.java | 6 +-
.../org/apache/druid/query/filter/InDimFilter.java | 29 +++-
.../apache/druid/query/filter/LikeDimFilter.java | 9 +-
.../druid/query/metadata/SegmentAnalyzer.java | 2 +-
.../org/apache/druid/segment/ColumnInspector.java | 9 --
.../org/apache/druid/segment/ColumnSelector.java | 14 +-
.../segment/ColumnSelectorBitmapIndexSelector.java | 16 +-
.../QueryableIndexColumnSelectorFactory.java | 7 +-
.../segment/QueryableIndexIndexableAdapter.java | 2 +-
.../segment/QueryableIndexStorageAdapter.java | 17 ++-
.../org/apache/druid/segment/VirtualColumns.java | 13 +-
.../apache/druid/segment/column/BitmapIndex.java | 67 +-------
.../apache/druid/segment/column/BitmapIndexes.java | 65 --------
.../apache/druid/segment/data/GenericIndexed.java | 24 +--
.../org/apache/druid/segment/data/Indexed.java | 35 +++++
.../apache/druid/segment/filter/BoundFilter.java | 66 ++++++--
.../druid/segment/filter/ExpressionFilter.java | 11 +-
.../org/apache/druid/segment/filter/Filters.java | 1 -
.../apache/druid/segment/filter/LikeFilter.java | 103 +++++++++++--
.../serde/StringBitmapIndexColumnPartSupplier.java | 168 +--------------------
.../QueryableIndexVectorColumnSelectorFactory.java | 8 +-
.../druid/segment/virtual/ExpressionPlan.java | 1 -
.../druid/segment/virtual/ExpressionSelectors.java | 9 +-
.../segment/virtual/ListFilteredVirtualColumn.java | 166 +-------------------
.../ColumnSelectorBitmapIndexSelectorTest.java | 1 -
.../druid/segment/IndexMergerNullHandlingTest.java | 2 +-
.../druid/segment/filter/BaseFilterTest.java | 8 +-
.../druid/segment/filter/BoundFilterTest.java | 39 -----
.../segment/filter/ExtractionDimFilterTest.java | 9 --
.../apache/druid/segment/filter/FiltersTest.java | 38 -----
.../druid/segment/filter/SelectorFilterTest.java | 14 --
.../BaseHashJoinSegmentStorageAdapterTest.java | 3 +-
.../segment/virtual/ExpressionPlannerTest.java | 4 +-
.../segment/virtual/ExpressionSelectorsTest.java | 31 +---
.../virtual/ExpressionVectorSelectorsTest.java | 13 +-
.../ListFilteredVirtualColumnSelectorTest.java | 16 +-
39 files changed, 322 insertions(+), 736 deletions(-)
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
index a262aea7ed..a0e06d019f 100644
---
a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
@@ -30,11 +30,13 @@ import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
import org.apache.druid.segment.generator.SegmentGenerator;
@@ -124,7 +126,17 @@ public class ExpressionVectorSelectorBenchmark
);
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
- outputType = parsed.getOutputType(index);
+ outputType = parsed.getOutputType(
+ new ColumnInspector()
+ {
+ @Nullable
+ @Override
+ public ColumnCapabilities getColumnCapabilities(String column)
+ {
+ return QueryableIndexStorageAdapter.getColumnCapabilities(index,
column);
+ }
+ }
+ );
checkSanity();
}
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java
index 5d6dd23228..55934fa67f 100644
---
a/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java
@@ -28,8 +28,6 @@ import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.GenericIndexed;
-import javax.annotation.Nullable;
-
public class MockBitmapIndexSelector implements BitmapIndexSelector
{
private final GenericIndexed<String> dictionary;
@@ -73,7 +71,7 @@ public class MockBitmapIndexSelector implements
BitmapIndexSelector
@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value)
{
- return bitmapIndex.getBitmapForValue(value);
+ return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
}
@Override
@@ -87,11 +85,4 @@ public class MockBitmapIndexSelector implements
BitmapIndexSelector
{
throw new UnsupportedOperationException();
}
-
- @Nullable
- @Override
- public ColumnCapabilities getColumnCapabilities(String column)
- {
- return null;
- }
}
diff --git
a/processing/src/main/java/org/apache/druid/query/dimension/PredicateFilteredDimensionSelector.java
b/processing/src/main/java/org/apache/druid/query/dimension/PredicateFilteredDimensionSelector.java
index 3aaf66d248..535442bdcc 100644
---
a/processing/src/main/java/org/apache/druid/query/dimension/PredicateFilteredDimensionSelector.java
+++
b/processing/src/main/java/org/apache/druid/query/dimension/PredicateFilteredDimensionSelector.java
@@ -64,7 +64,6 @@ final class PredicateFilteredDimensionSelector extends
AbstractDimensionSelector
@Override
public ValueMatcher makeValueMatcher(final String value)
{
- final boolean matchNull = predicate.apply(null);
return new ValueMatcher()
{
@Override
@@ -82,8 +81,8 @@ final class PredicateFilteredDimensionSelector extends
AbstractDimensionSelector
nullRow = false;
}
}
- // null should match empty rows in multi-value columns if predicate
matches null
- return nullRow && value == null && matchNull;
+ // null should match empty rows in multi-value columns
+ return nullRow && value == null;
}
@Override
@@ -98,7 +97,7 @@ final class PredicateFilteredDimensionSelector extends
AbstractDimensionSelector
@Override
public ValueMatcher makeValueMatcher(final Predicate<String>
matcherPredicate)
{
- final boolean matchNull = predicate.apply(null) &&
matcherPredicate.apply(null);
+ final boolean matchNull = predicate.apply(null);
return new ValueMatcher()
{
@Override
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java
b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java
index 6a55aa34ef..fd90e7412b 100644
---
a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java
+++
b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java
@@ -23,7 +23,6 @@ import com.google.errorprone.annotations.MustBeClosed;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.spatial.ImmutableRTree;
-import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.data.CloseableIndexed;
@@ -32,15 +31,12 @@ import javax.annotation.Nullable;
/**
*/
-public interface BitmapIndexSelector extends ColumnInspector
+public interface BitmapIndexSelector
{
@MustBeClosed
@Nullable
CloseableIndexed<String> getDimensionValues(String dimension);
-
- @Deprecated
ColumnCapabilities.Capable hasMultipleValues(String dimension);
-
int getNumRows();
BitmapFactory getBitmapFactory();
@Nullable
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
index 839a7dfe8c..0cae9809a8 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
@@ -38,6 +38,8 @@ import com.google.common.collect.TreeRangeSet;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntIterable;
+import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
@@ -58,6 +60,7 @@ import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.DimensionHandlerUtils;
+import org.apache.druid.segment.IntIteratorUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -68,6 +71,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -301,7 +305,8 @@ public class InDimFilter extends
AbstractOptimizableDimFilter implements Filter
if (extractionFn == null) {
final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension);
return Filters.estimateSelectivity(
- bitmapIndex.getBitmapsForValues(values).iterator(),
+ bitmapIndex,
+ IntIteratorUtils.toIntList(getBitmapIndexIterable(values,
bitmapIndex).iterator()),
indexSelector.getNumRows()
);
} else {
@@ -498,7 +503,27 @@ public class InDimFilter extends
AbstractOptimizableDimFilter implements Filter
private static Iterable<ImmutableBitmap> getBitmapIterable(final Set<String>
values, final BitmapIndex bitmapIndex)
{
- return bitmapIndex.getBitmapsForValues(values);
+ return Filters.bitmapsFromIndexes(getBitmapIndexIterable(values,
bitmapIndex), bitmapIndex);
+ }
+
+ private static IntIterable getBitmapIndexIterable(final Set<String> values,
final BitmapIndex bitmapIndex)
+ {
+ return () -> new IntIterator()
+ {
+ final Iterator<String> iterator = values.iterator();
+
+ @Override
+ public boolean hasNext()
+ {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public int nextInt()
+ {
+ return bitmapIndex.getIndex(iterator.next());
+ }
+ };
}
@SuppressWarnings("ReturnValueIgnored")
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java
index 50a06b2392..a92bd44fa7 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java
@@ -32,6 +32,7 @@ import com.google.common.primitives.Chars;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.extraction.ExtractionFn;
+import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.filter.LikeFilter;
import javax.annotation.Nullable;
@@ -298,15 +299,17 @@ public class LikeDimFilter extends
AbstractOptimizableDimFilter implements DimFi
* of s are ignored. This method is useful if you've already independently
verified the prefix. This method
* evalutes strings.get(i) lazily to save time when it isn't necessary to
actually look at the string.
*/
- public boolean matchesSuffixOnly(@Nullable String value)
+ public boolean matchesSuffixOnly(final Indexed<String> strings, final int
i)
{
if (suffixMatch == SuffixMatch.MATCH_ANY) {
return true;
} else if (suffixMatch == SuffixMatch.MATCH_EMPTY) {
- return value == null ? matches(null) : value.length() ==
prefix.length();
+ final String s = strings.get(i);
+ return s == null ? matches(null) : s.length() == prefix.length();
} else {
// suffixMatch is MATCH_PATTERN
- return matches(value);
+ final String s = strings.get(i);
+ return matches(s);
}
}
diff --git
a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
index 00f244d5ec..29b05f3e0e 100644
---
a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
+++
b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
@@ -209,7 +209,7 @@ public class SegmentAnalyzer
String value = bitmapIndex.getValue(i);
if (value != null) {
size += StringUtils.estimatedBinaryLengthAsUTF8(value) *
- ((long) bitmapIndex.getBitmapForValue(value).size());
+ ((long)
bitmapIndex.getBitmap(bitmapIndex.getIndex(value)).size());
}
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java
b/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java
index 7d91d50e85..d8b4aaf083 100644
--- a/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java
+++ b/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java
@@ -37,15 +37,6 @@ public interface ColumnInspector extends
Expr.InputBindingInspector
@Nullable
ColumnCapabilities getColumnCapabilities(String column);
- default ColumnCapabilities getColumnCapabilitiesWithDefault(String column,
ColumnCapabilities defaultCapabilites)
- {
- final ColumnCapabilities capabilities = getColumnCapabilities(column);
- if (capabilities != null) {
- return capabilities;
- }
- return defaultCapabilites;
- }
-
@Nullable
@Override
default ExpressionType getType(String name)
diff --git
a/processing/src/main/java/org/apache/druid/segment/ColumnSelector.java
b/processing/src/main/java/org/apache/druid/segment/ColumnSelector.java
index 38bfc0da60..d73b2dacc4 100644
--- a/processing/src/main/java/org/apache/druid/segment/ColumnSelector.java
+++ b/processing/src/main/java/org/apache/druid/segment/ColumnSelector.java
@@ -19,7 +19,6 @@
package org.apache.druid.segment;
-import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import javax.annotation.Nullable;
@@ -27,21 +26,10 @@ import java.util.List;
/**
*/
-public interface ColumnSelector extends ColumnInspector
+public interface ColumnSelector
{
List<String> getColumnNames();
@Nullable
ColumnHolder getColumnHolder(String columnName);
-
- @Nullable
- @Override
- default ColumnCapabilities getColumnCapabilities(String column)
- {
- final ColumnHolder columnHolder = getColumnHolder(column);
- if (columnHolder == null) {
- return null;
- }
- return columnHolder.getCapabilities();
- }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java
b/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java
index 24eacffe47..2f62e9f3e1 100644
---
a/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java
+++
b/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java
@@ -171,7 +171,10 @@ public class ColumnSelectorBitmapIndexSelector implements
BitmapIndexSelector
VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(dimension);
ColumnCapabilities virtualCapabilities = null;
if (virtualColumn != null) {
- virtualCapabilities = virtualColumn.capabilities(index, dimension);
+ virtualCapabilities = virtualColumn.capabilities(
+ QueryableIndexStorageAdapter.getColumnInspectorForIndex(index),
+ dimension
+ );
}
return virtualCapabilities != null ?
virtualCapabilities.hasMultipleValues() : ColumnCapabilities.Capable.FALSE;
}
@@ -231,7 +234,7 @@ public class ColumnSelectorBitmapIndexSelector implements
BitmapIndexSelector
if (idx == null) {
return null;
}
- return idx.getBitmapForValue(value);
+ return idx.getBitmap(idx.getIndex(value));
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
@@ -250,7 +253,7 @@ public class ColumnSelectorBitmapIndexSelector implements
BitmapIndexSelector
}
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
- return bitmapIndex.getBitmapForValue(value);
+ return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
}
@Override
@@ -272,11 +275,4 @@ public class ColumnSelectorBitmapIndexSelector implements
BitmapIndexSelector
{
return virtualColumns.getVirtualColumn(columnName) != null;
}
-
- @Nullable
- @Override
- public ColumnCapabilities getColumnCapabilities(String column)
- {
- return virtualColumns.getColumnCapabilities(index, column);
- }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
index 72859024ec..28c8b6a0c8 100644
---
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
+++
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java
@@ -198,9 +198,12 @@ public class QueryableIndexColumnSelectorFactory
implements ColumnSelectorFactor
public ColumnCapabilities getColumnCapabilities(String columnName)
{
if (virtualColumns.exists(columnName)) {
- return virtualColumns.getColumnCapabilities(index, columnName);
+ return virtualColumns.getColumnCapabilities(
+ QueryableIndexStorageAdapter.getColumnInspectorForIndex(index),
+ columnName
+ );
}
- return index.getColumnCapabilities(columnName);
+ return QueryableIndexStorageAdapter.getColumnCapabilities(index,
columnName);
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
index b6b6cfd495..801e19a0d0 100644
---
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
@@ -397,7 +397,7 @@ public class QueryableIndexIndexableAdapter implements
IndexableAdapter
return BitmapValues.EMPTY;
}
- return new ImmutableBitmapValues(bitmaps.getBitmapForValue(value));
+ return new
ImmutableBitmapValues(bitmaps.getBitmap(bitmaps.getIndex(value)));
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
index 5025ef693b..cb070e1e49 100644
---
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
@@ -168,7 +168,7 @@ public class QueryableIndexStorageAdapter implements
StorageAdapter
@Nullable
public ColumnCapabilities getColumnCapabilities(String column)
{
- return index.getColumnCapabilities(column);
+ return getColumnCapabilities(index, column);
}
@Override
@@ -280,6 +280,21 @@ public class QueryableIndexStorageAdapter implements
StorageAdapter
);
}
+ @Nullable
+ public static ColumnCapabilities getColumnCapabilities(ColumnSelector index,
String columnName)
+ {
+ final ColumnHolder columnHolder = index.getColumnHolder(columnName);
+ if (columnHolder == null) {
+ return null;
+ }
+ return columnHolder.getCapabilities();
+ }
+
+ public static ColumnInspector getColumnInspectorForIndex(ColumnSelector
index)
+ {
+ return column -> getColumnCapabilities(index, column);
+ }
+
@Override
public Metadata getMetadata()
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
index ba8f033310..5a0b0d9398 100644
--- a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
+++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
@@ -174,9 +174,10 @@ public class VirtualColumns implements Cacheable
public BitmapIndex getBitmapIndex(String columnName, ColumnSelector
columnSelector)
{
final VirtualColumn virtualColumn =
getVirtualColumnForSelector(columnName);
- return virtualColumn.capabilities(columnSelector,
columnName).hasBitmapIndexes()
- ? virtualColumn.getBitmapIndex(columnName, columnSelector)
- : null;
+ return virtualColumn.capabilities(columnName).hasBitmapIndexes() ?
virtualColumn.getBitmapIndex(
+ columnName,
+ columnSelector
+ ) : null;
}
/**
@@ -386,7 +387,11 @@ public class VirtualColumns implements Cacheable
{
final VirtualColumn virtualColumn = getVirtualColumn(columnName);
if (virtualColumn != null) {
- return virtualColumn.capabilities(column ->
getColumnCapabilitiesWithFallback(inspector, column), columnName);
+ return Preconditions.checkNotNull(
+ virtualColumn.capabilities(column ->
getColumnCapabilitiesWithFallback(inspector, column), columnName),
+ "capabilities for column[%s]",
+ columnName
+ );
} else {
return null;
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java
b/processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java
index fb028aa60e..0d68216056 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java
@@ -23,85 +23,28 @@ import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import javax.annotation.Nullable;
-import java.util.Set;
-import java.util.function.Predicate;
/**
- * Provides a mechanism to get {@link ImmutableBitmap} for a value, or {@link
Iterable} of {@link ImmutableBitmap}
- * for a range or exact set of values, the set bits of which correspond to
which rows contain the matching values.
- *
- * Used to power {@link org.apache.druid.segment.BitmapOffset} and
- * {@link org.apache.druid.segment.vector.BitmapVectorOffset} which are used
with column cursors for fast filtering
- * of indexed values.
- *
- * The column must be ingested with a bitmap index for filters to use them and
to participate in this "pre-filtering"
- * step when scanning segments
- *
- * @see org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter
*/
public interface BitmapIndex
{
- /**
- * Get the cardinality of the underlying value dictionary
- */
int getCardinality();
- /**
- * Get the value in the underlying value dictionary of the specified
dictionary id
- */
@Nullable
String getValue(int index);
- /**
- * Returns true if the underlying value dictionary has nulls
- */
boolean hasNulls();
BitmapFactory getBitmapFactory();
/**
- * Returns the index of "value" in this BitmapIndex, or a negative value, if
not present in the underlying dictionary
+ * Returns the index of "value" in this BitmapIndex, or (-(insertion point)
- 1) if the value is not
+ * present, in the manner of Arrays.binarySearch.
+ *
+ * @param value value to search for
+ * @return index of value, or negative number equal to (-(insertion point) -
1).
*/
int getIndex(@Nullable String value);
- /**
- * Get the {@link ImmutableBitmap} for dictionary id of the underlying
dictionary
- */
ImmutableBitmap getBitmap(int idx);
-
- /**
- * Get the {@link ImmutableBitmap} corresponding to the supplied value
- */
- ImmutableBitmap getBitmapForValue(@Nullable String value);
-
- /**
- * Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the
values supplied in the specified range
- */
- default Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict
- )
- {
- return getBitmapsInRange(startValue, startStrict, endValue, endStrict,
(index) -> true);
- }
-
- /**
- * Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the
values supplied in the specified range
- * whose dictionary ids also match some predicate
- */
- Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict,
- Predicate<String> matcher
- );
-
- /**
- * Get an {@link Iterable} of {@link ImmutableBitmap} corresponding to the
specified set of values (if they are
- * contained in the underlying column)
- */
- Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values);
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java
b/processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java
index 771dbf4a8c..b7670d7e9d 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java
@@ -21,17 +21,13 @@ package org.apache.druid.segment.column;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
-import com.google.common.collect.ImmutableList;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier;
import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.Set;
import java.util.function.IntSupplier;
-import java.util.function.Predicate;
public final class BitmapIndexes
{
@@ -100,67 +96,6 @@ public final class BitmapIndexes
return bitmapFactory.makeEmptyImmutableBitmap();
}
}
-
- @Override
- public ImmutableBitmap getBitmapForValue(@Nullable String value)
- {
- if (NullHandling.isNullOrEquivalent(value)) {
- return nullBitmapSupplier.get();
- } else {
- return bitmapFactory.makeEmptyImmutableBitmap();
- }
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict,
- Predicate<String> matcher
- )
- {
- final int startIndex; // inclusive
- int endIndex; // exclusive
-
- if (startValue == null) {
- startIndex = 0;
- } else {
- if (NullHandling.isNullOrEquivalent(startValue)) {
- startIndex = startStrict ? 1 : 0;
- } else {
- startIndex = 1;
- }
- }
-
- if (endValue == null) {
- endIndex = 1;
- } else {
- if (NullHandling.isNullOrEquivalent(endValue)) {
- endIndex = endStrict ? 0 : 1;
- } else {
- endIndex = 1;
- }
- }
-
- endIndex = Math.max(startIndex, endIndex);
- if (startIndex == endIndex) {
- return Collections.emptyList();
- }
- if (matcher.test(null)) {
- return ImmutableList.of(getBitmap(0));
- }
- return ImmutableList.of(bitmapFactory.makeEmptyImmutableBitmap());
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
- {
- if (values.contains(null) || (NullHandling.replaceWithDefault() &&
values.contains(""))) {
- return ImmutableList.of(getBitmap(0));
- }
- return ImmutableList.of(bitmapFactory.makeEmptyImmutableBitmap());
- }
};
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
index 982f7b0767..be1ed1cb40 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/GenericIndexed.java
@@ -347,33 +347,15 @@ public class GenericIndexed<T> implements
CloseableIndexed<T>, Serializer
@Override
public int indexOf(@Nullable T value)
{
- if (!allowReverseLookup) {
- throw new UnsupportedOperationException("Reverse lookup not allowed.");
- }
return indexOf(this, value);
}
private int indexOf(Indexed<T> indexed, @Nullable T value)
{
- int minIndex = 0;
- int maxIndex = size - 1;
- while (minIndex <= maxIndex) {
- int currIndex = (minIndex + maxIndex) >>> 1;
-
- T currValue = indexed.get(currIndex);
- int comparison = strategy.compare(currValue, value);
- if (comparison == 0) {
- return currIndex;
- }
-
- if (comparison < 0) {
- minIndex = currIndex + 1;
- } else {
- maxIndex = currIndex - 1;
- }
+ if (!allowReverseLookup) {
+ throw new UnsupportedOperationException("Reverse lookup not allowed.");
}
-
- return -(minIndex + 1);
+ return Indexed.indexOf(indexed::get, size, strategy, value);
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/segment/data/Indexed.java
b/processing/src/main/java/org/apache/druid/segment/data/Indexed.java
index b99fc79266..aae7c37776 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/Indexed.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/Indexed.java
@@ -24,6 +24,7 @@ import
org.apache.druid.query.monomorphicprocessing.CalledFromHotLoop;
import org.apache.druid.query.monomorphicprocessing.HotLoopCallee;
import javax.annotation.Nullable;
+import java.util.Comparator;
/**
* Indexed is a fixed-size, immutable, indexed set of values which allows
@@ -53,6 +54,40 @@ public interface Indexed<T> extends Iterable<T>,
HotLoopCallee
*/
int indexOf(@Nullable T value);
+ /**
+ * Returns the index of "value" in some object whose values are accessible
by index some {@link IndexedGetter}, or
+ * (-(insertion point) - 1) if the value is not present, in the manner of
Arrays.binarySearch.
+ *
+ * This is used by {@link GenericIndexed} to strengthen the contract of
{@link #indexOf(Object)}, which only
+ * guarantees that values-not-found will return some negative number.
+ *
+ * @param value value to search for
+ *
+ * @return index of value, or negative number equal to (-(insertion point) -
1).
+ */
+ static <T> int indexOf(IndexedGetter<T> indexed, int size, Comparator<T>
comparator, @Nullable T value)
+ {
+ int minIndex = 0;
+ int maxIndex = size - 1;
+ while (minIndex <= maxIndex) {
+ int currIndex = (minIndex + maxIndex) >>> 1;
+
+ T currValue = indexed.get(currIndex);
+ int comparison = comparator.compare(currValue, value);
+ if (comparison == 0) {
+ return currIndex;
+ }
+
+ if (comparison < 0) {
+ minIndex = currIndex + 1;
+ } else {
+ maxIndex = currIndex - 1;
+ }
+ }
+
+ return -(minIndex + 1);
+ }
+
@FunctionalInterface
interface IndexedGetter<T>
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
index 115b8d36d4..73c10e71a1 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
@@ -22,9 +22,11 @@ package org.apache.druid.segment.filter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
+import it.unimi.dsi.fastutil.ints.IntList;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.Pair;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.BitmapIndexSelector;
@@ -43,6 +45,7 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
+import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -99,12 +102,8 @@ public class BoundFilter implements Filter
}
return Filters.estimateSelectivity(
- bitmapIndex.getBitmapsInRange(
- boundDimFilter.getLower(),
- boundDimFilter.isLowerStrict(),
- boundDimFilter.getUpper(),
- boundDimFilter.isUpperStrict()
- ).iterator(),
+ bitmapIndex,
+ getBitmapIndexList(boundDimFilter, bitmapIndex),
indexSelector.getNumRows()
);
} else {
@@ -201,17 +200,60 @@ public class BoundFilter implements Filter
);
}
+ private static Pair<Integer, Integer> getStartEndIndexes(
+ final BoundDimFilter boundDimFilter,
+ final BitmapIndex bitmapIndex
+ )
+ {
+ final int startIndex; // inclusive
+ int endIndex; // exclusive
+
+ if (!boundDimFilter.hasLowerBound()) {
+ startIndex = 0;
+ } else {
+ final int found =
bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getLower()));
+ if (found >= 0) {
+ startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found;
+ } else {
+ startIndex = -(found + 1);
+ }
+ }
+
+ if (!boundDimFilter.hasUpperBound()) {
+ endIndex = bitmapIndex.getCardinality();
+ } else {
+ final int found =
bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getUpper()));
+ if (found >= 0) {
+ endIndex = boundDimFilter.isUpperStrict() ? found : found + 1;
+ } else {
+ endIndex = -(found + 1);
+ }
+ }
+
+ endIndex = startIndex > endIndex ? startIndex : endIndex;
+
+ return new Pair<>(startIndex, endIndex);
+ }
+
private static Iterable<ImmutableBitmap> getBitmapIterator(
final BoundDimFilter boundDimFilter,
final BitmapIndex bitmapIndex
)
{
- return bitmapIndex.getBitmapsInRange(
- boundDimFilter.getLower(),
- boundDimFilter.isLowerStrict(),
- boundDimFilter.getUpper(),
- boundDimFilter.isUpperStrict()
- );
+ return Filters.bitmapsFromIndexes(getBitmapIndexList(boundDimFilter,
bitmapIndex), bitmapIndex);
+ }
+
+ private static IntList getBitmapIndexList(
+ final BoundDimFilter boundDimFilter,
+ final BitmapIndex bitmapIndex
+ )
+ {
+ // search for start, end indexes in the bitmaps; then include all bitmaps
between those points
+ final Pair<Integer, Integer> indexes = getStartEndIndexes(boundDimFilter,
bitmapIndex);
+ final int startIndex = indexes.lhs;
+ final int endIndex = indexes.rhs;
+
+ return IntListUtils.fromTo(startIndex, endIndex);
}
private DruidPredicateFactory getPredicateFactory()
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
index f293713402..4ec947a23d 100644
---
a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
+++
b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
@@ -47,7 +47,6 @@ import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
-import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -230,14 +229,8 @@ public class ExpressionFilter implements Filter
// Single-column expression. We can use bitmap indexes if this column
has an index and the expression can
// map over the values of the index.
final String column =
Iterables.getOnlyElement(details.getRequiredBindings());
- // we use a default 'all false' capabilities here because if the column
has a bitmap index, but the capabilities
- // are null, it means that the column is missing and should take the
single valued path, while truly unknown
- // things will not have a bitmap index available
- final ColumnCapabilities capabilities =
selector.getColumnCapabilitiesWithDefault(
- column,
- ColumnCapabilitiesImpl.createDefault()
- );
- return selector.getBitmapIndex(column) != null &&
ExpressionSelectors.canMapOverDictionary(details, capabilities);
+ return selector.getBitmapIndex(column) != null
+ && ExpressionSelectors.canMapOverDictionary(details,
selector.hasMultipleValues(column));
} else {
// Multi-column expression.
return false;
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java
b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java
index 9032f8d99e..98a2c3f4f6 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java
@@ -247,7 +247,6 @@ public class Filters
// Missing dimension -> match all rows if the predicate matches null;
match no rows otherwise
try (final CloseableIndexed<String> dimValues =
selector.getDimensionValues(dimension)) {
if (dimValues == null || dimValues.size() == 0) {
-
return ImmutableList.of(predicate.apply(null) ? allTrue(selector) :
allFalse(selector));
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
index d03ac71d36..a0dac33a45 100644
--- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
+++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
@@ -21,6 +21,8 @@ package org.apache.druid.segment.filter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
+import it.unimi.dsi.fastutil.ints.IntIterable;
+import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
@@ -38,9 +40,14 @@ import org.apache.druid.segment.ColumnProcessors;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.column.BitmapIndex;
+import org.apache.druid.segment.data.CloseableIndexed;
+import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
@@ -168,18 +175,16 @@ public class LikeFilter implements Filter
return ImmutableList.of(likeMatcher.matches(null) ?
Filters.allTrue(selector) : Filters.allFalse(selector));
}
- final String lower =
NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix());
- final String upper =
NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE;
-
- // Union bitmaps for all matching dimension values in range.
- // Use lazy iterator to allow unioning bitmaps one by one and avoid
materializing all of them at once.
- return bitmapIndex.getBitmapsInRange(
- lower,
- false,
- upper,
- false,
- (value) -> likeMatcher.matchesSuffixOnly(value)
- );
+ // search for start, end indexes in the bitmaps; then include all
matching bitmaps between those points
+ try (final CloseableIndexed<String> dimValues =
selector.getDimensionValues(dimension)) {
+
+ // Union bitmaps for all matching dimension values in range.
+ // Use lazy iterator to allow unioning bitmaps one by one and avoid
materializing all of them at once.
+ return
Filters.bitmapsFromIndexes(getDimValueIndexIterableForPrefixMatch(bitmapIndex,
dimValues), bitmapIndex);
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
} else {
// fallback
return Filters.matchPredicateNoUnion(
@@ -206,6 +211,80 @@ public class LikeFilter implements Filter
return extractionFn == null && !likeMatcher.getPrefix().isEmpty();
}
+ private IntIterable getDimValueIndexIterableForPrefixMatch(
+ final BitmapIndex bitmapIndex,
+ final Indexed<String> dimValues
+ )
+ {
+
+ final String lower =
NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix());
+ final String upper =
NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE;
+
+ final int startIndex; // inclusive
+ final int endIndex; // exclusive
+
+ if (lower == null) {
+ // For Null values
+ startIndex = bitmapIndex.getIndex(null);
+ endIndex = startIndex + 1;
+ } else {
+ final int lowerFound = bitmapIndex.getIndex(lower);
+ startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1);
+
+ final int upperFound = bitmapIndex.getIndex(upper);
+ endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1);
+ }
+
+ return new IntIterable()
+ {
+ @Override
+ public IntIterator iterator()
+ {
+ return new IntIterator()
+ {
+ int currIndex = startIndex;
+ int found;
+
+ {
+ found = findNext();
+ }
+
+ private int findNext()
+ {
+ while (currIndex < endIndex &&
!likeMatcher.matchesSuffixOnly(dimValues, currIndex)) {
+ currIndex++;
+ }
+
+ if (currIndex < endIndex) {
+ return currIndex++;
+ } else {
+ return -1;
+ }
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ return found != -1;
+ }
+
+ @Override
+ public int nextInt()
+ {
+ int cur = found;
+
+ if (cur == -1) {
+ throw new NoSuchElementException();
+ }
+
+ found = findNext();
+ return cur;
+ }
+ };
+ }
+ };
+ }
+
@Override
public boolean equals(Object o)
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java
b/processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java
index 393b9ee64c..0a771f6700 100644
---
a/processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java
@@ -20,21 +20,12 @@
package org.apache.druid.segment.serde;
import com.google.common.base.Supplier;
-import it.unimi.dsi.fastutil.ints.IntIntImmutablePair;
-import it.unimi.dsi.fastutil.ints.IntIntPair;
-import it.unimi.dsi.fastutil.ints.IntIterator;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.segment.IntListUtils;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.data.GenericIndexed;
import javax.annotation.Nullable;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.function.Predicate;
/**
* Provides {@link BitmapIndex} for some dictionary encoded column, where the
dictionary and bitmaps are stored in some
@@ -89,6 +80,7 @@ public class StringBitmapIndexColumnPartSupplier implements
Supplier<BitmapIndex
@Override
public int getIndex(@Nullable String value)
{
+ // GenericIndexed.indexOf satisfies contract needed by
BitmapIndex.indexOf
return dictionary.indexOf(value);
}
@@ -102,164 +94,6 @@ public class StringBitmapIndexColumnPartSupplier
implements Supplier<BitmapIndex
final ImmutableBitmap bitmap = bitmaps.get(idx);
return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() :
bitmap;
}
-
- @Override
- public ImmutableBitmap getBitmapForValue(@Nullable String value)
- {
- final int idx = dictionary.indexOf(value);
- return getBitmap(idx);
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict
- )
- {
- final IntIntPair range = getRange(startValue, startStrict, endValue,
endStrict);
- final int start = range.leftInt(), end = range.rightInt();
- return () -> new Iterator<ImmutableBitmap>()
- {
- final IntIterator rangeIterator = IntListUtils.fromTo(start,
end).iterator();
-
- @Override
- public boolean hasNext()
- {
- return rangeIterator.hasNext();
- }
-
- @Override
- public ImmutableBitmap next()
- {
- return getBitmap(rangeIterator.nextInt());
- }
- };
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict,
- Predicate<String> indexMatcher
- )
- {
- final IntIntPair range = getRange(startValue, startStrict, endValue,
endStrict);
- final int start = range.leftInt(), end = range.rightInt();
- return () -> new Iterator<ImmutableBitmap>()
- {
- int currIndex = start;
- int found;
- {
- found = findNext();
- }
-
- private int findNext()
- {
- while (currIndex < end &&
!indexMatcher.test(dictionary.get(currIndex))) {
- currIndex++;
- }
-
- if (currIndex < end) {
- return currIndex++;
- } else {
- return -1;
- }
- }
-
- @Override
- public boolean hasNext()
- {
- return found != -1;
- }
-
- @Override
- public ImmutableBitmap next()
- {
- int cur = found;
-
- if (cur == -1) {
- throw new NoSuchElementException();
- }
-
- found = findNext();
- return getBitmap(cur);
- }
- };
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
- {
- return () -> new Iterator<ImmutableBitmap>()
- {
- final Iterator<String> iterator = values.iterator();
- int next = -1;
-
- @Override
- public boolean hasNext()
- {
- if (next < 0) {
- findNext();
- }
- return next >= 0;
- }
-
- @Override
- public ImmutableBitmap next()
- {
- if (next < 0) {
- findNext();
- if (next < 0) {
- throw new NoSuchElementException();
- }
- }
- final int swap = next;
- next = -1;
- return getBitmap(swap);
- }
-
- private void findNext()
- {
- while (next < 0 && iterator.hasNext()) {
- String nextValue = iterator.next();
- next = dictionary.indexOf(nextValue);
- }
- }
- };
- }
-
- private IntIntPair getRange(@Nullable String startValue, boolean
startStrict, @Nullable String endValue, boolean endStrict)
- {
- int startIndex, endIndex;
- if (startValue == null) {
- startIndex = 0;
- } else {
- final int found =
dictionary.indexOf(NullHandling.emptyToNullIfNeeded(startValue));
- if (found >= 0) {
- startIndex = startStrict ? found + 1 : found;
- } else {
- startIndex = -(found + 1);
- }
- }
-
- if (endValue == null) {
- endIndex = dictionary.size();
- } else {
- final int found =
dictionary.indexOf(NullHandling.emptyToNullIfNeeded(endValue));
- if (found >= 0) {
- endIndex = endStrict ? found : found + 1;
- } else {
- endIndex = -(found + 1);
- }
- }
-
- endIndex = Math.max(startIndex, endIndex);
- return new IntIntImmutablePair(startIndex, endIndex);
- }
};
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
index e300674cc5..af9e2832cb 100644
---
a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
+++
b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java
@@ -23,6 +23,7 @@ import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BaseColumn;
import org.apache.druid.segment.column.ColumnCapabilities;
@@ -265,8 +266,11 @@ public class QueryableIndexVectorColumnSelectorFactory
implements VectorColumnSe
public ColumnCapabilities getColumnCapabilities(final String columnName)
{
if (virtualColumns.exists(columnName)) {
- return virtualColumns.getColumnCapabilities(index, columnName);
+ return virtualColumns.getColumnCapabilities(
+ QueryableIndexStorageAdapter.getColumnInspectorForIndex(index),
+ columnName
+ );
}
- return index.getColumnCapabilities(columnName);
+ return QueryableIndexStorageAdapter.getColumnCapabilities(index,
columnName);
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java
index 86dbbd7826..b0b57e0917 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java
@@ -270,7 +270,6 @@ public class ExpressionPlan
.setType(ColumnType.STRING)
.setDictionaryValuesSorted(false)
.setDictionaryValuesUnique(false)
- .setHasBitmapIndexes(false)
.setHasNulls(true);
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
index a691f625c0..670fcf41cf 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
@@ -233,18 +233,15 @@ public class ExpressionSelectors
* and that single column has a dictionary.
*
* @param bindingAnalysis result of calling {@link
Expr#analyzeInputs()} on an expression
- * @param columnCapabilities {@link ColumnCapabilities} for the input
binding
+ * @param hasMultipleValues result of calling {@link
ColumnCapabilities#hasMultipleValues()}
*/
public static boolean canMapOverDictionary(
final Expr.BindingAnalysis bindingAnalysis,
- final ColumnCapabilities columnCapabilities
+ final ColumnCapabilities.Capable hasMultipleValues
)
{
Preconditions.checkState(bindingAnalysis.getRequiredBindings().size() ==
1, "requiredBindings.size == 1");
- return columnCapabilities != null &&
- !columnCapabilities.hasMultipleValues().isUnknown() &&
- !bindingAnalysis.hasInputArrays() &&
- !bindingAnalysis.isOutputArray();
+ return !hasMultipleValues.isUnknown() && !bindingAnalysis.hasInputArrays()
&& !bindingAnalysis.isOutputArray();
}
/**
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
index 96e0f1454e..91ffe4da0d 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
@@ -24,12 +24,11 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
-import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.dimension.ListFilteredDimensionSpec;
-import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnSelector;
import org.apache.druid.segment.ColumnSelectorFactory;
@@ -41,16 +40,13 @@ import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.data.Indexed;
import javax.annotation.Nullable;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
import java.util.List;
-import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
-import java.util.function.Predicate;
/**
* {@link VirtualColumn} form of {@link ListFilteredDimensionSpec}, powered by
@@ -256,18 +252,6 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
return delegate.getBitmapFactory();
}
- @Override
- public int getCardinality()
- {
- return idMapping.getValueCardinality();
- }
-
- @Override
- public int getIndex(@Nullable String value)
- {
- return getReverseIndex(value);
- }
-
@Override
public ImmutableBitmap getBitmap(int idx)
{
@@ -275,153 +259,15 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
}
@Override
- public ImmutableBitmap getBitmapForValue(@Nullable String value)
- {
- if (getReverseIndex(value) < 0) {
- return delegate.getBitmap(-1);
- }
- return delegate.getBitmapForValue(value);
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict,
- Predicate<String> matcher
- )
+ public int getCardinality()
{
- int startIndex, endIndex;
- if (startValue == null) {
- startIndex = 0;
- } else {
- final int found =
getReverseIndex(NullHandling.emptyToNullIfNeeded(startValue));
- if (found >= 0) {
- startIndex = startStrict ? found + 1 : found;
- } else {
- startIndex = -(found + 1);
- }
- }
-
- if (endValue == null) {
- endIndex = idMapping.getValueCardinality();
- } else {
- final int found =
getReverseIndex(NullHandling.emptyToNullIfNeeded(endValue));
- if (found >= 0) {
- endIndex = endStrict ? found : found + 1;
- } else {
- endIndex = -(found + 1);
- }
- }
-
- endIndex = startIndex > endIndex ? startIndex : endIndex;
- final int start = startIndex, end = endIndex;
- return () -> new Iterator<ImmutableBitmap>()
- {
- int currIndex = start;
- int found;
- {
- found = findNext();
- }
-
- private int findNext()
- {
- while (currIndex < end &&
!matcher.test(delegate.getValue(idMapping.getReverseId(currIndex)))) {
- currIndex++;
- }
-
- if (currIndex < end) {
- return currIndex++;
- } else {
- return -1;
- }
- }
-
- @Override
- public boolean hasNext()
- {
- return found != -1;
- }
-
- @Override
- public ImmutableBitmap next()
- {
- int cur = found;
-
- if (cur == -1) {
- throw new NoSuchElementException();
- }
-
- found = findNext();
- return getBitmap(cur);
- }
- };
+ return idMapping.getValueCardinality();
}
@Override
- public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
- {
- return () -> new Iterator<ImmutableBitmap>()
- {
- final Iterator<String> iterator = values.iterator();
- int next = -1;
-
- @Override
- public boolean hasNext()
- {
- if (next < 0) {
- findNext();
- }
- return next >= 0;
- }
-
- @Override
- public ImmutableBitmap next()
- {
- if (next < 0) {
- findNext();
- if (next < 0) {
- throw new NoSuchElementException();
- }
- }
- final int swap = next;
- next = -1;
- return getBitmap(swap);
- }
-
- private void findNext()
- {
- while (next < 0 && iterator.hasNext()) {
- String nextValue = iterator.next();
- next = getReverseIndex(nextValue);
- }
- }
- };
- }
-
- private int getReverseIndex(@Nullable String value)
+ public int getIndex(@Nullable String value)
{
- int minIndex = 0;
- int maxIndex = idMapping.getValueCardinality() - 1;
- final Comparator<String> comparator = StringComparators.LEXICOGRAPHIC;
- while (minIndex <= maxIndex) {
- int currIndex = (minIndex + maxIndex) >>> 1;
-
- String currValue =
delegate.getValue(idMapping.getReverseId(currIndex));
- int comparison = comparator.compare(currValue, value);
- if (comparison == 0) {
- return currIndex;
- }
-
- if (comparison < 0) {
- minIndex = currIndex + 1;
- } else {
- maxIndex = currIndex - 1;
- }
- }
-
- return -(minIndex + 1);
+ return Indexed.indexOf(this::getValue, getCardinality(),
Comparators.naturalNullsFirst(), value);
}
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java
index 92d51b5426..125f8ab428 100644
---
a/processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java
@@ -71,7 +71,6 @@ public class ColumnSelectorBitmapIndexSelectorTest
ImmutableBitmap someBitmap = EasyMock.createMock(ImmutableBitmap.class);
EasyMock.expect(someIndex.getIndex("foo")).andReturn(0).anyTimes();
EasyMock.expect(someIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
-
EasyMock.expect(someIndex.getBitmapForValue("foo")).andReturn(someBitmap).anyTimes();
ColumnHolder nonStringHolder = EasyMock.createMock(ColumnHolder.class);
diff --git
a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
index 13f344b795..a9b63b477a 100644
---
a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
@@ -196,7 +196,7 @@ public class IndexMergerNullHandlingTest
if (expectedNullRows.size() > 0) {
Assert.assertEquals(subsetList.toString(), 0,
bitmapIndex.getIndex(null));
- final ImmutableBitmap nullBitmap =
bitmapIndex.getBitmapForValue(null);
+ final ImmutableBitmap nullBitmap =
bitmapIndex.getBitmap(bitmapIndex.getIndex(null));
final List<Integer> actualNullRows = new ArrayList<>();
final IntIterator iterator = nullBitmap.iterator();
while (iterator.hasNext()) {
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
index 4297e5d852..ab4cca94fb 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
@@ -23,7 +23,6 @@ import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import org.apache.druid.common.config.NullHandling;
@@ -92,7 +91,6 @@ import org.apache.druid.segment.vector.VectorCursor;
import org.apache.druid.segment.vector.VectorObjectSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
-import org.apache.druid.segment.virtual.ListFilteredVirtualColumn;
import
org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory;
@@ -128,11 +126,7 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
new ExpressionVirtualColumn("vdim1", "dim1", ColumnType.STRING,
TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("vd0", "d0", ColumnType.DOUBLE,
TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("vf0", "f0", ColumnType.FLOAT,
TestExprMacroTable.INSTANCE),
- new ExpressionVirtualColumn("vl0", "l0", ColumnType.LONG,
TestExprMacroTable.INSTANCE),
- new ListFilteredVirtualColumn("allow-dim0",
DefaultDimensionSpec.of("dim0"), ImmutableSet.of("3", "4"), true),
- new ListFilteredVirtualColumn("deny-dim0",
DefaultDimensionSpec.of("dim0"), ImmutableSet.of("3", "4"), false),
- new ListFilteredVirtualColumn("allow-dim2",
DefaultDimensionSpec.of("dim2"), ImmutableSet.of("a"), true),
- new ListFilteredVirtualColumn("deny-dim2",
DefaultDimensionSpec.of("dim2"), ImmutableSet.of("a"), false)
+ new ExpressionVirtualColumn("vl0", "l0", ColumnType.LONG,
TestExprMacroTable.INSTANCE)
)
);
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/BoundFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/BoundFilterTest.java
index 033b5bd5db..1e6d188e40 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/BoundFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/BoundFilterTest.java
@@ -439,45 +439,6 @@ public class BoundFilterTest extends BaseFilterTest
);
}
- @Test
- public void testListFilteredVirtualColumn()
- {
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("allow-dim0", "0", "2", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of()
- );
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("allow-dim0", "0", "6", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of("3", "4")
- );
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("deny-dim0", "0", "6", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of("0", "1", "2", "5", "6")
- );
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("deny-dim0", "3", "4", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of()
- );
-
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("allow-dim2", "a", "c", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of("0", "3", "6")
- );
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("allow-dim2", "c", "z", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of()
- );
-
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("deny-dim2", "a", "b", false, true, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of()
- );
- assertFilterMatchesSkipVectorize(
- new BoundDimFilter("deny-dim2", "c", "z", false, false, false, null,
StringComparators.LEXICOGRAPHIC),
- ImmutableList.of("4", "7")
- );
- }
-
@Test
public void testNumericMatchExactlySingleValue()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
index c37efe024f..a6c0aec017 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
@@ -37,8 +37,6 @@ import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
-import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
-import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
@@ -97,13 +95,6 @@ public class ExtractionDimFilterTest
private final BitmapIndexSelector BITMAP_INDEX_SELECTOR = new
BitmapIndexSelector()
{
- @Nullable
- @Override
- public ColumnCapabilities getColumnCapabilities(String column)
- {
- return
ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING).setHasMultipleValues(true);
- }
-
@Override
public CloseableIndexed<String> getDimensionValues(String dimension)
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java
index d0f2009345..f9d493a167 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java
@@ -31,10 +31,7 @@ import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
-import javax.annotation.Nullable;
import java.util.List;
-import java.util.Set;
-import java.util.function.Predicate;
public class FiltersTest extends InitializedNullHandlingTest
{
@@ -88,41 +85,6 @@ public class FiltersTest extends InitializedNullHandlingTest
throw new UnsupportedOperationException();
}
- @Override
- public ImmutableBitmap getBitmapForValue(@Nullable String value)
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict
- )
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsInRange(
- @Nullable String startValue,
- boolean startStrict,
- @Nullable String endValue,
- boolean endStrict,
- Predicate<String> matcher
- )
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterable<ImmutableBitmap> getBitmapsForValues(Set<String> values)
- {
- throw new UnsupportedOperationException();
- }
-
@Override
public ImmutableBitmap getBitmap(int idx)
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/SelectorFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/SelectorFilterTest.java
index 3939cf8af6..81aa2efa4d 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/SelectorFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/SelectorFilterTest.java
@@ -115,20 +115,6 @@ public class SelectorFilterTest extends BaseFilterTest
assertFilterMatches(new SelectorDimFilter("vdim0", "1", null),
ImmutableList.of("1"));
}
- @Test
- public void testListFilteredVirtualColumn()
- {
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("allow-dim0", "1",
null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("allow-dim0", "4",
null), ImmutableList.of("4"));
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("deny-dim0", "0",
null), ImmutableList.of("0"));
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("deny-dim0", "4",
null), ImmutableList.of());
-
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("allow-dim2", "b",
null), ImmutableList.of());
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("allow-dim2", "a",
null), ImmutableList.of("0", "3"));
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("deny-dim2", "b",
null), ImmutableList.of("0"));
- assertFilterMatchesSkipVectorize(new SelectorDimFilter("deny-dim2", "a",
null), ImmutableList.of());
- }
-
@Test
public void testSingleValueStringColumnWithNulls()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
b/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
index d1de10fdaa..fed718de52 100644
---
a/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/join/BaseHashJoinSegmentStorageAdapterTest.java
@@ -39,7 +39,6 @@ import org.apache.druid.segment.join.lookup.LookupJoinable;
import org.apache.druid.segment.join.table.IndexedTable;
import org.apache.druid.segment.join.table.IndexedTableJoinable;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
-import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.SegmentId;
import org.junit.After;
import org.junit.Assert;
@@ -53,7 +52,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
-public class BaseHashJoinSegmentStorageAdapterTest extends
InitializedNullHandlingTest
+public class BaseHashJoinSegmentStorageAdapterTest
{
public static JoinFilterRewriteConfig DEFAULT_JOIN_FILTER_REWRITE_CONFIG =
new JoinFilterRewriteConfig(
true,
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
index ad7f1daab1..7966e9a565 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
@@ -369,7 +369,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
Assert.assertFalse(inferred.areDictionaryValuesSorted().isMaybeTrue());
Assert.assertFalse(inferred.areDictionaryValuesUnique().isMaybeTrue());
Assert.assertFalse(inferred.hasMultipleValues().isMaybeTrue());
- Assert.assertFalse(inferred.hasBitmapIndexes());
+ Assert.assertTrue(inferred.hasBitmapIndexes());
Assert.assertFalse(inferred.hasSpatialIndexes());
// multiple input columns
@@ -463,7 +463,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
Assert.assertFalse(inferred.areDictionaryValuesSorted().isMaybeTrue());
Assert.assertFalse(inferred.areDictionaryValuesUnique().isMaybeTrue());
Assert.assertTrue(inferred.hasMultipleValues().isTrue());
- Assert.assertFalse(inferred.hasBitmapIndexes());
+ Assert.assertTrue(inferred.hasBitmapIndexes());
Assert.assertFalse(inferred.hasSpatialIndexes());
thePlan = plan("concat(scalar_string, multi_dictionary_string_nonunique)");
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
index 67620a1d27..93e7b10727 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
@@ -52,8 +52,6 @@ import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.TestObjectColumnSelector;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnCapabilities;
-import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
-import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
import org.apache.druid.segment.generator.SegmentGenerator;
@@ -84,19 +82,6 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
private static IncrementalIndexStorageAdapter
INCREMENTAL_INDEX_STORAGE_ADAPTER;
private static List<StorageAdapter> ADAPTERS;
- private static final ColumnCapabilities SINGLE_VALUE = new
ColumnCapabilitiesImpl().setType(ColumnType.STRING)
-
.setDictionaryEncoded(true)
-
.setDictionaryValuesUnique(true)
-
.setDictionaryValuesSorted(true)
-
.setHasMultipleValues(false)
-
.setHasNulls(true);
- private static final ColumnCapabilities MULTI_VAL = new
ColumnCapabilitiesImpl().setType(ColumnType.STRING)
-
.setDictionaryEncoded(true)
-
.setDictionaryValuesUnique(true)
-
.setDictionaryValuesSorted(true)
-
.setHasMultipleValues(true)
-
.setHasNulls(true);
-
@BeforeClass
public static void setup()
{
@@ -409,7 +394,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertTrue(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("dim1 == 2", ExprMacroTable.nil()).analyzeInputs(),
- SINGLE_VALUE
+ ColumnCapabilities.Capable.FALSE
)
);
}
@@ -420,7 +405,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertTrue(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("concat(dim1, dim1) == 2",
ExprMacroTable.nil()).analyzeInputs(),
- SINGLE_VALUE
+ ColumnCapabilities.Capable.FALSE
)
);
}
@@ -431,7 +416,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertTrue(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("dim1 == 2", ExprMacroTable.nil()).analyzeInputs(),
- MULTI_VAL
+ ColumnCapabilities.Capable.TRUE
)
);
}
@@ -442,7 +427,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertFalse(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("dim1 == 2", ExprMacroTable.nil()).analyzeInputs(),
- new ColumnCapabilitiesImpl()
+ ColumnCapabilities.Capable.UNKNOWN
)
);
}
@@ -453,7 +438,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertFalse(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("array_contains(dim1, 2)",
ExprMacroTable.nil()).analyzeInputs(),
-
ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING_ARRAY)
+ ColumnCapabilities.Capable.FALSE
)
);
}
@@ -464,7 +449,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertFalse(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("array_contains(dim1, 2)",
ExprMacroTable.nil()).analyzeInputs(),
- MULTI_VAL
+ ColumnCapabilities.Capable.TRUE
)
);
}
@@ -475,7 +460,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertFalse(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("array_contains(dim1, 2)",
ExprMacroTable.nil()).analyzeInputs(),
- new ColumnCapabilitiesImpl()
+ ColumnCapabilities.Capable.UNKNOWN
)
);
}
@@ -486,7 +471,7 @@ public class ExpressionSelectorsTest extends
InitializedNullHandlingTest
Assert.assertTrue(
ExpressionSelectors.canMapOverDictionary(
Parser.parse("dim1 == 2", ExprMacroTable.nil()).analyzeInputs(),
- SINGLE_VALUE
+ ColumnCapabilities.Capable.FALSE
)
);
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
index 54fc20deed..e0295eb6fb 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
@@ -30,6 +30,7 @@ import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.QueryableIndex;
@@ -144,7 +145,17 @@ public class ExpressionVectorSelectorsTest
public void setup()
{
Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
- outputType = parsed.getOutputType(INDEX);
+ outputType = parsed.getOutputType(
+ new ColumnInspector()
+ {
+ @Nullable
+ @Override
+ public ColumnCapabilities getColumnCapabilities(String column)
+ {
+ return QueryableIndexStorageAdapter.getColumnCapabilities(INDEX,
column);
+ }
+ }
+ );
if (outputType == null) {
outputType = ExpressionType.STRING;
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
index 638e9e2413..9dbc25d431 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
@@ -36,7 +36,6 @@ import org.apache.druid.segment.RowBasedColumnSelectorFactory;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BitmapIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
-import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
@@ -173,13 +172,6 @@ public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandli
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
- EasyMock.expect(selector.getColumnCapabilities(COLUMN_NAME))
- .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.STRING)
- .setDictionaryEncoded(true)
-
.setDictionaryValuesUnique(true)
-
.setDictionaryValuesSorted(true)
- .setHasBitmapIndexes(true)
- ).anyTimes();
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
@@ -235,13 +227,7 @@ public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandli
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
- EasyMock.expect(selector.getColumnCapabilities(COLUMN_NAME))
- .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.STRING)
- .setDictionaryEncoded(true)
-
.setDictionaryValuesUnique(true)
-
.setDictionaryValuesSorted(true)
- .setHasBitmapIndexes(true)
- ).anyTimes();
+
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
