This is an automated email from the ASF dual-hosted git repository.
adarshsanjeev pushed a commit to branch 32.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/32.0.0 by this push:
new 532fda5e0ad [Backport] fix search query to check for the specific
type of indexes it requires instead of using capabilities
532fda5e0ad is described below
commit 532fda5e0ad7deb3bdf1687199ffc523f8a4d5fd
Author: Clint Wylie <[email protected]>
AuthorDate: Mon Jan 27 20:59:36 2025 -0800
[Backport] fix search query to check for the specific type of indexes it
requires instead of using capabilities
* fix search query to check for the specific type of indexes it requires
instead of using capabilities
---
.../druid/query/search/UseIndexesStrategy.java | 107 ++++++++++-----------
.../druid/query/search/SearchQueryRunnerTest.java | 29 ++++++
2 files changed, 80 insertions(+), 56 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
index bd33d7f9783..32895eeaf88 100644
---
a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
+++
b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
@@ -25,7 +25,7 @@ import it.unimi.dsi.fastutil.objects.Object2IntRBTreeMap;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
-import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.NonnullPair;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.Order;
import org.apache.druid.query.dimension.DimensionSpec;
@@ -35,23 +35,21 @@ import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.search.CursorOnlyStrategy.CursorBasedExecutor;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
-import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.Cursors;
import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.VirtualColumns;
-import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.NumericColumn;
import org.apache.druid.segment.index.BitmapColumnIndex;
import
org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
-import org.apache.druid.segment.virtual.VirtualizedColumnInspector;
import org.joda.time.Interval;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
public class UseIndexesStrategy extends SearchStrategy
@@ -74,67 +72,64 @@ public class UseIndexesStrategy extends SearchStrategy
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment
segment)
{
final ImmutableList.Builder<SearchQueryExecutor> builder =
ImmutableList.builder();
- final QueryableIndex index = segment.as(QueryableIndex.class);
- final CursorFactory cursorFactory = segment.asCursorFactory();
final List<DimensionSpec> searchDims = getDimsToSearch(segment,
query.getDimensions());
- if (index != null) {
- // pair of bitmap dims and non-bitmap dims
- final Pair<List<DimensionSpec>, List<DimensionSpec>> pair =
partitionDimensionList(
- segment,
- cursorFactory,
- query.getVirtualColumns(),
- searchDims
- );
- final List<DimensionSpec> bitmapSuppDims = pair.lhs;
- final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
+ final QueryableIndex index = segment.as(QueryableIndex.class);
+ if (index == null) {
+ return Collections.singletonList(new CursorBasedExecutor(query, segment,
searchDims));
+ }
- if (bitmapSuppDims.size() > 0) {
- final ColumnIndexSelector selector = new
ColumnSelectorColumnIndexSelector(
- index.getBitmapFactoryForDimensions(),
+ final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
+ index.getBitmapFactoryForDimensions(),
+ query.getVirtualColumns(),
+ new DeprecatedQueryableIndexColumnSelector(index)
+ );
+
+ // pair of bitmap dims and non-bitmap dims
+ final NonnullPair<List<DimensionSpec>, List<DimensionSpec>> pair =
partitionDimensionList(
+ segment,
+ selector,
+ searchDims
+ );
+ final List<DimensionSpec> bitmapSuppDims = pair.lhs;
+ final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
+
+ if (!bitmapSuppDims.isEmpty()) {
+ // Index-only plan is used only when any filter is not specified or the
filter supports bitmap indexes.
+ //
+ // Note: if some filters support bitmap indexes but others are not, the
current implementation always employs
+ // the cursor-based plan. This can be more optimized. One possible
optimization is generating a bitmap index
+ // from the non-bitmap-support filter, and then use it to compute the
filtered result by intersecting bitmaps.
+ if ((filter == null || filter.getBitmapColumnIndex(selector) != null)
+ && Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) {
+ final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(
+ index,
+ segment,
query.getVirtualColumns(),
- new DeprecatedQueryableIndexColumnSelector(index)
+ filter,
+ interval
);
-
- // Index-only plan is used only when any filter is not specified or
the filter supports bitmap indexes.
- //
- // Note: if some filters support bitmap indexes but others are not,
the current implementation always employs
- // the cursor-based plan. This can be more optimized. One possible
optimization is generating a bitmap index
- // from the non-bitmap-support filter, and then use it to compute the
filtered result by intersecting bitmaps.
- if ((filter == null || filter.getBitmapColumnIndex(selector) != null)
- && Cursors.getTimeOrdering(index.getOrdering()) ==
Order.ASCENDING) {
- final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(
- index,
- segment,
- query.getVirtualColumns(),
- filter,
- interval
- );
- builder.add(new IndexOnlyExecutor(query, segment,
timeFilteredBitmap, bitmapSuppDims));
- } else {
- // Fall back to cursor-based execution strategy
- nonBitmapSuppDims.addAll(bitmapSuppDims);
- }
+ builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap,
bitmapSuppDims));
+ } else {
+ // Fall back to cursor-based execution strategy
+ nonBitmapSuppDims.addAll(bitmapSuppDims);
}
+ }
- if (nonBitmapSuppDims.size() > 0) {
- builder.add(new CursorBasedExecutor(query, segment,
nonBitmapSuppDims));
- }
- } else {
- builder.add(new CursorBasedExecutor(query, segment, searchDims));
+ if (!nonBitmapSuppDims.isEmpty()) {
+ builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims));
}
return builder.build();
}
/**
- * Split the given dimensions list into bitmap-supporting dimensions and
non-bitmap supporting ones.
- * Note that the returned lists are free to modify.
+ * Split the given dimensions list into columns which provide {@link
DictionaryEncodedStringValueIndex} and those
+ * which do not. Note that the returned lists are free to modify.
*/
- private static Pair<List<DimensionSpec>, List<DimensionSpec>>
partitionDimensionList(
+ private static NonnullPair<List<DimensionSpec>, List<DimensionSpec>>
partitionDimensionList(
Segment segment,
- CursorFactory cursorFactory,
- VirtualColumns virtualColumns,
+ ColumnIndexSelector columnIndexSelector,
List<DimensionSpec> dimensions
)
{
@@ -144,22 +139,21 @@ public class UseIndexesStrategy extends SearchStrategy
segment,
dimensions
);
- VirtualizedColumnInspector columnInspector = new
VirtualizedColumnInspector(cursorFactory, virtualColumns);
-
for (DimensionSpec spec : dimsToSearch) {
- ColumnCapabilities capabilities =
columnInspector.getColumnCapabilities(spec.getDimension());
- if (capabilities == null) {
+ ColumnIndexSupplier indexSupplier =
columnIndexSelector.getIndexSupplier(spec.getDimension());
+ if (indexSupplier == null) {
+ // column doesn't exist, ignore it
continue;
}
- if (capabilities.hasBitmapIndexes()) {
+ if (indexSupplier.as(DictionaryEncodedStringValueIndex.class) != null) {
bitmapDims.add(spec);
} else {
nonBitmapDims.add(spec);
}
}
- return new Pair<>(bitmapDims, nonBitmapDims);
+ return new NonnullPair<>(bitmapDims, nonBitmapDims);
}
static ImmutableBitmap makeTimeFilteredBitmap(
@@ -304,6 +298,7 @@ public class UseIndexesStrategy extends SearchStrategy
}
}
} else {
+ // these were checked to be non-null in partitionDimensionList
final DictionaryEncodedStringValueIndex bitmapIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
diff --git
a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
index fa8fb02ffcb..1b0012560bb 100644
---
a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
+++
b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
@@ -40,6 +40,7 @@ import org.apache.druid.query.Result;
import org.apache.druid.query.context.ResponseContext;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.ExtractionDimensionSpec;
+import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.extraction.JavaScriptExtractionFn;
import org.apache.druid.query.extraction.MapLookupExtractor;
@@ -60,6 +61,7 @@ import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndex;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.virtual.ListFilteredVirtualColumn;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.SegmentId;
@@ -827,6 +829,33 @@ public class SearchQueryRunnerTest extends
InitializedNullHandlingTest
checkSearchQuery(searchQuery, expectedHits);
}
+ @Test
+ public void testSearchVirtualColumns()
+ {
+ List<SearchHit> expectedHits = new ArrayList<>();
+ expectedHits.add(new SearchHit("vquality", "a", 93));
+ expectedHits.add(new SearchHit("vquality", "not a", 1116));
+
+ checkSearchQuery(
+ Druids.newSearchQueryBuilder()
+ .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .granularity(QueryRunnerTestHelper.ALL_GRAN)
+ .virtualColumns(
+ new ExpressionVirtualColumn(
+ "vquality",
+ "case_searched(like(quality,'a%'), 'a', 'not a')",
+ ColumnType.STRING,
+ TestExprMacroTable.INSTANCE
+ )
+ )
+ .dimensions("vquality")
+ .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
+ .query("")
+ .build(),
+ expectedHits
+ );
+ }
+
private void checkSearchQuery(Query searchQuery, List<SearchHit>
expectedResults)
{
checkSearchQuery(searchQuery, runner, expectedResults);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
