This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new fd771c06c0e fixes for projection matching logic and unnest/join cursor
build spec translation (#18451)
fd771c06c0e is described below
commit fd771c06c0e523af27d7c747c4c921a047b96f85
Author: Clint Wylie <[email protected]>
AuthorDate: Wed Sep 3 17:28:35 2025 -0700
fixes for projection matching logic and unnest/join cursor build spec
translation (#18451)
---
.../org/apache/druid/segment/CursorBuildSpec.java | 21 ++-
.../segment/UnnestColumnValueSelectorCursor.java | 5 +-
.../apache/druid/segment/UnnestCursorFactory.java | 159 ++++++++++++-----
.../druid/segment/UnnestDimensionCursor.java | 7 +-
.../segment/join/HashJoinSegmentCursorFactory.java | 31 +++-
.../projections/ProjectionMatchBuilder.java | 20 ---
.../druid/segment/projections/Projections.java | 67 ++-----
.../apache/druid/segment/IndexMergerTestBase.java | 3 +
.../UnnestColumnValueSelectorCursorTest.java | 60 +++----
.../druid/segment/UnnestCursorFactoryTest.java | 197 ++++++++++++++++++++-
.../druid/segment/projections/ProjectionsTest.java | 5 +
11 files changed, 386 insertions(+), 189 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
index 35e847818e3..001805ca288 100644
--- a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
+++ b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java
@@ -117,7 +117,7 @@ public class CursorBuildSpec
/**
* {@link Filter} to supply to the {@link CursorHolder}. Only rows which
match will be available through the
- * selectors created from the {@link Cursor} or {@link
org.apache.druid.segment.vector.VectorCursor}
+ * selectors created from the {@link Cursor} or {@link VectorCursor}
*/
@Nullable
public Filter getFilter()
@@ -128,7 +128,7 @@ public class CursorBuildSpec
/**
* {@link Interval} filter to supply to the {@link CursorHolder}. Only rows
whose timestamps fall within this range
* will be available through the selectors created from the {@link Cursor} or
- * {@link org.apache.druid.segment.vector.VectorCursor}
+ * {@link VectorCursor}
*/
public Interval getInterval()
{
@@ -136,8 +136,8 @@ public class CursorBuildSpec
}
/**
- * Set of physical columns required from a cursor. If null, and {@link
#groupingColumns} is null or empty and
- * {@link #aggregators} is null or empty, then a {@link CursorHolder} must
assume that ALL columns are required.
+ * Set of physical columns required from a cursor. If null, then a {@link
CursorHolder} must assume that ALL columns
+ * are required.
*/
@Nullable
public Set<String> getPhysicalColumns()
@@ -156,7 +156,7 @@ public class CursorBuildSpec
/**
* Any columns which will be used for grouping by a query engine for the
{@link CursorHolder}, useful for
- * specializing the {@link Cursor} or {@link
org.apache.druid.segment.vector.VectorCursor} if any pre-aggregated
+ * specializing the {@link Cursor} or {@link VectorCursor} if any
pre-aggregated
* data is available.
*/
@Nullable
@@ -168,7 +168,7 @@ public class CursorBuildSpec
/**
* Any {@link AggregatorFactory} which will be used by a query engine for
the {@link CursorHolder}, useful
* to assist in determining if {@link CursorHolder#canVectorize()}, as well
as specializing the {@link Cursor} or
- * {@link org.apache.druid.segment.vector.VectorCursor} if any
pre-aggregated data is available.
+ * {@link VectorCursor} if any pre-aggregated data is available.
*/
@Nullable
public List<AggregatorFactory> getAggregators()
@@ -179,7 +179,7 @@ public class CursorBuildSpec
/**
* List of all {@link OrderBy} columns which a query engine will use to sort
its results to supply to the
* {@link CursorHolder}, which can allow optimization of the provided {@link
Cursor} or
- * {@link org.apache.druid.segment.vector.VectorCursor} if data matching the
preferred ordering is available.
+ * {@link VectorCursor} if data matching the preferred ordering is available.
* <p>
* If not specified, the cursor will advance in the native order of the
underlying data.
*/
@@ -190,7 +190,7 @@ public class CursorBuildSpec
/**
* {@link QueryContext} for the {@link CursorHolder} to provide a mechanism
to push various data into
- * {@link Cursor} and {@link org.apache.druid.segment.vector.VectorCursor}
such as
+ * {@link Cursor} and {@link VectorCursor} such as
* {@link org.apache.druid.query.QueryContexts#VECTORIZE_KEY} and
* {@link org.apache.druid.query.QueryContexts#VECTOR_SIZE_KEY}
*/
@@ -201,7 +201,7 @@ public class CursorBuildSpec
/**
* {@link QueryMetrics} to use for measuring things involved with {@link
Cursor} and
- * {@link org.apache.druid.segment.vector.VectorCursor} creation.
+ * {@link VectorCursor} creation.
*/
@Nullable
public QueryMetrics<?> getQueryMetrics()
@@ -381,8 +381,7 @@ public class CursorBuildSpec
* @see CursorBuildSpec#getPhysicalColumns() for usage. The backing value
is not automatically populated by calls to
* {@link #setFilter(Filter)}, {@link #setVirtualColumns(VirtualColumns)},
{@link #setAggregators(List)}, or
* {@link #setPreferredOrdering(List)}, so this must be explicitly set for
all required physical columns. If set to
- * null, and {@link #groupingColumns} is null or empty and {@link
#aggregators} is null or empty, then a
- * {@link CursorHolder} must assume that ALL columns are required
+ * null, then a {@link CursorHolder} must assume that ALL columns are
required
*/
public CursorBuildSpecBuilder setPhysicalColumns(@Nullable Set<String>
physicalColumns)
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
b/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
index 57aa5683c8d..260899df4d5 100644
---
a/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
+++
b/processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java
@@ -70,8 +70,7 @@ public class UnnestColumnValueSelectorCursor implements Cursor
public UnnestColumnValueSelectorCursor(
Cursor cursor,
ColumnSelectorFactory baseColumnSelectorFactory,
- VirtualColumn unnestColumn,
- String outputColumnName
+ VirtualColumn unnestColumn
)
{
this.baseCursor = cursor;
@@ -81,8 +80,8 @@ public class UnnestColumnValueSelectorCursor implements Cursor
this.baseColumnSelectorFactory
);
this.unnestColumn = unnestColumn;
+ this.outputName = unnestColumn.getOutputName();
this.index = 0;
- this.outputName = outputColumnName;
this.needInitialization = true;
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/UnnestCursorFactory.java
b/processing/src/main/java/org/apache/druid/segment/UnnestCursorFactory.java
index e316022421b..177f2c02416 100644
--- a/processing/src/main/java/org/apache/druid/segment/UnnestCursorFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/UnnestCursorFactory.java
@@ -22,8 +22,8 @@ package org.apache.druid.segment;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
-import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.Order;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.DimFilter;
@@ -51,7 +51,6 @@ import org.apache.druid.utils.CloseableUtils;
import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -79,43 +78,19 @@ public class UnnestCursorFactory implements CursorFactory
public CursorHolder makeCursorHolder(CursorBuildSpec spec)
{
final String input = getUnnestInputIfDirectAccess(unnestColumn);
- final Pair<Filter, Filter> filterPair = computeBaseAndPostUnnestFilters(
+ final UnnestFilterSplit filterSplit = computeBaseAndPostUnnestFilters(
spec.getFilter(),
filter != null ? filter.toFilter() : null,
spec.getVirtualColumns(),
+ unnestColumn,
input,
- input == null ? null : spec.getVirtualColumns()
-
.getColumnCapabilitiesWithFallback(baseCursorFactory, input)
+ input == null ? null :
spec.getVirtualColumns().getColumnCapabilitiesWithFallback(baseCursorFactory,
input)
+ );
+ final CursorBuildSpec unnestBuildSpec = transformCursorBuildSpec(
+ spec,
+ unnestColumn,
+ filterSplit.getBaseTableFilter()
);
- final Set<String> physicalColumns;
- if (spec.getPhysicalColumns() == null) {
- physicalColumns = null;
- } else {
- physicalColumns = new HashSet<>();
- for (String column : unnestColumn.requiredColumns()) {
- if (!spec.getVirtualColumns().exists(column)) {
- physicalColumns.add(column);
- }
- }
- if (filter != null) {
- for (String column : filter.getRequiredColumns()) {
- if (!spec.getVirtualColumns().exists(column)) {
- physicalColumns.add(column);
- }
- }
- }
- for (String column : spec.getPhysicalColumns()) {
- if (!column.equals(unnestColumn.getOutputName())) {
- physicalColumns.add(column);
- }
- }
- }
- final CursorBuildSpec unnestBuildSpec =
- CursorBuildSpec.builder(spec)
- .setFilter(filterPair.lhs)
- .setPhysicalColumns(physicalColumns)
-
.setVirtualColumns(VirtualColumns.create(Collections.singletonList(unnestColumn)))
- .build();
return new CursorHolder()
{
@@ -141,21 +116,19 @@ public class UnnestCursorFactory implements CursorFactory
unnestCursor = new UnnestDimensionCursor(
cursor,
cursor.getColumnSelectorFactory(),
- unnestColumn,
- unnestColumn.getOutputName()
+ unnestColumn
);
} else {
unnestCursor = new UnnestColumnValueSelectorCursor(
cursor,
cursor.getColumnSelectorFactory(),
- unnestColumn,
- unnestColumn.getOutputName()
+ unnestColumn
);
}
return PostJoinCursor.wrap(
unnestCursor,
spec.getVirtualColumns(),
- filterPair.rhs
+ filterSplit.getPostUnnestFilter()
);
}
@@ -235,10 +208,11 @@ public class UnnestCursorFactory implements CursorFactory
* @return pair of pre- and post-unnest filters
*/
@VisibleForTesting
- public Pair<Filter, Filter> computeBaseAndPostUnnestFilters(
+ static UnnestFilterSplit computeBaseAndPostUnnestFilters(
@Nullable final Filter queryFilter,
@Nullable final Filter unnestFilter,
final VirtualColumns queryVirtualColumns,
+ final VirtualColumn unnestColumn,
@Nullable final String inputColumn,
@Nullable final ColumnCapabilities inputColumnCapabilites
)
@@ -290,6 +264,7 @@ public class UnnestCursorFactory implements CursorFactory
if (queryFilter instanceof BooleanFilter) {
List<Filter> preFilterList = recursiveRewriteOnUnnestFilters(
(BooleanFilter) queryFilter,
+ unnestColumn,
inputColumn,
inputColumnCapabilites,
filterSplitter
@@ -316,7 +291,7 @@ public class UnnestCursorFactory implements CursorFactory
}
filterSplitter.addPostFilterWithPreFilterIfRewritePossible(unnestFilter,
false);
- return Pair.of(
+ return new UnnestFilterSplit(
Filters.maybeAnd(filterSplitter.filtersPushedDownToBaseCursor).orElse(null),
Filters.maybeAnd(filterSplitter.filtersForPostUnnestCursor).orElse(null)
);
@@ -343,8 +318,9 @@ public class UnnestCursorFactory implements CursorFactory
* @param inputColumn input column to unnest if it's a direct
access; otherwise null
* @param inputColumnCapabilites input column capabilities if known;
otherwise null
*/
- private List<Filter> recursiveRewriteOnUnnestFilters(
+ private static List<Filter> recursiveRewriteOnUnnestFilters(
BooleanFilter queryFilter,
+ VirtualColumn unnestColumn,
final String inputColumn,
final ColumnCapabilities inputColumnCapabilites,
final FilterSplitter filterSplitter
@@ -356,6 +332,7 @@ public class UnnestCursorFactory implements CursorFactory
if (filter instanceof AndFilter) {
List<Filter> andChildFilters = recursiveRewriteOnUnnestFilters(
(BooleanFilter) filter,
+ unnestColumn,
inputColumn,
inputColumnCapabilites,
filterSplitter
@@ -366,6 +343,7 @@ public class UnnestCursorFactory implements CursorFactory
} else if (filter instanceof OrFilter) {
List<Filter> orChildFilters = recursiveRewriteOnUnnestFilters(
(BooleanFilter) filter,
+ unnestColumn,
inputColumn,
inputColumnCapabilites,
filterSplitter
@@ -449,7 +427,7 @@ public class UnnestCursorFactory implements CursorFactory
* Computes the capabilities of {@link #unnestColumn}, after unnesting.
*/
@Nullable
- public static ColumnCapabilities computeOutputColumnCapabilities(
+ static ColumnCapabilities computeOutputColumnCapabilities(
final ColumnInspector baseColumnInspector,
final VirtualColumn unnestColumn
)
@@ -475,6 +453,74 @@ public class UnnestCursorFactory implements CursorFactory
}
}
+ /**
+ * Converts a {@link CursorBuildSpec} to the base table {@link
CursorBuildSpec}, ensuring that all required columns
+ * of the original spec are added to {@link
CursorBuildSpec#getPhysicalColumns()} and the unnest column is added to
+ * {@link CursorBuildSpec#getVirtualColumns()}
+ */
+ @VisibleForTesting
+ static CursorBuildSpec transformCursorBuildSpec(
+ CursorBuildSpec spec,
+ VirtualColumn unnestColumn,
+ @Nullable Filter baseTableFilter
+ )
+ {
+ final Set<String> physicalColumns;
+ if (spec.getPhysicalColumns() == null) {
+ physicalColumns = null;
+ } else {
+ physicalColumns = new HashSet<>();
+
+ // add all physical columns, skip unnest column if specified as a
physical column, we'll deal with it next
+ for (String column : spec.getPhysicalColumns()) {
+ if (!column.equals(unnestColumn.getOutputName())) {
+ physicalColumns.add(column);
+ }
+ }
+
+ // add all unnest input physical columns
+ for (String input : unnestColumn.requiredColumns()) {
+ if (!spec.getVirtualColumns().exists(input)) {
+ physicalColumns.add(input);
+ }
+ }
+
+ // add all the base table filter columns. this is probably unecessary -
while part of the filter might have been
+ // on the unnest rather than the query and so its required columns
missing from the physical column list, it is
+ // likely the unnest filter is only referencing the unnest column, which
is already covered by the physical
+ // column substitution... however just in case something wild happened
on the spec we add all these too
+ if (baseTableFilter != null) {
+ for (String column : baseTableFilter.getRequiredColumns()) {
+ if (!spec.getVirtualColumns().exists(column)) {
+ physicalColumns.add(column);
+ }
+ }
+ }
+ }
+
+ // trim off the grouping, aggregators, and ordering (other than time) to
turn this into a plain scan cursor build
+ // spec this could be improved in the future to be able to match
projections. in some cases, we could push this
+ // through as a grouping, but would need to push in all the query virtual
columns, and ensure that aggregators do
+ // not refer to the unnest column (the filter has already been
preprocessed and passed in as a separate argument to
+ // this method)
+ Order timeOrder = Cursors.getTimeOrdering(spec.getPreferredOrdering());
+ List<OrderBy> maybeOrderByTime = List.of();
+ if (timeOrder == Order.DESCENDING) {
+ maybeOrderByTime = Cursors.descendingTimeOrder();
+ } else if (timeOrder == Order.ASCENDING) {
+ maybeOrderByTime = Cursors.ascendingTimeOrder();
+ }
+ return CursorBuildSpec.builder()
+ .setInterval(spec.getInterval())
+ .setFilter(baseTableFilter)
+ .setPhysicalColumns(physicalColumns)
+
.setVirtualColumns(VirtualColumns.create(List.of(unnestColumn)))
+ .setPreferredOrdering(maybeOrderByTime)
+ .setQueryContext(spec.getQueryContext())
+ .setQueryMetrics(spec.getQueryMetrics())
+ .build();
+ }
+
/**
* Requirement for {@link #rewriteFilterOnUnnestColumnIfPossible}: filter
must support rewrites and also must map
* over multi-value strings. (Rather than treat them as arrays.) There isn't
a method on the Filter interface that
@@ -618,4 +664,31 @@ public class UnnestCursorFactory implements CursorFactory
return preFilterCount;
}
}
+
+ @VisibleForTesting
+ static final class UnnestFilterSplit
+ {
+ @Nullable
+ private final Filter baseTableFilter;
+ @Nullable
+ private final Filter postUnnestFilter;
+
+ private UnnestFilterSplit(@Nullable Filter baseTableFilter, @Nullable
Filter postUnnestFilter)
+ {
+ this.baseTableFilter = baseTableFilter;
+ this.postUnnestFilter = postUnnestFilter;
+ }
+
+ @Nullable
+ public Filter getBaseTableFilter()
+ {
+ return baseTableFilter;
+ }
+
+ @Nullable
+ public Filter getPostUnnestFilter()
+ {
+ return postUnnestFilter;
+ }
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
index 3012d31ff2a..cfd0e30e21d 100644
---
a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
+++
b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java
@@ -81,8 +81,7 @@ public class UnnestDimensionCursor implements Cursor
public UnnestDimensionCursor(
Cursor cursor,
ColumnSelectorFactory baseColumnSelectorFactory,
- VirtualColumn unnestColumn,
- String outputColumnName
+ VirtualColumn unnestColumn
)
{
this.baseCursor = cursor;
@@ -91,9 +90,9 @@ public class UnnestDimensionCursor implements Cursor
DefaultDimensionSpec.of(unnestColumn.getOutputName()),
this.baseColumnSelectorFactory
);
- this.unnestColumn = unnestColumn;
this.index = 0;
- this.outputName = outputColumnName;
+ this.unnestColumn = unnestColumn;
+ this.outputName = unnestColumn.getOutputName();
this.needInitialization = true;
// this shouldn't happen, but just in case...
final IdLookup lookup = Preconditions.checkNotNull(dimSelector.idLookup());
diff --git
a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentCursorFactory.java
b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentCursorFactory.java
index b608017d9a7..b5354de02ad 100644
---
a/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentCursorFactory.java
+++
b/processing/src/main/java/org/apache/druid/segment/join/HashJoinSegmentCursorFactory.java
@@ -24,12 +24,14 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.query.Order;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.Cursors;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnCapabilities;
@@ -74,10 +76,6 @@ public class HashJoinSegmentCursorFactory implements
CursorFactory
@Override
public CursorHolder makeCursorHolder(CursorBuildSpec spec)
{
- // make a copy of CursorBuildSpec with filters removed
- final CursorBuildSpec.CursorBuildSpecBuilder cursorBuildSpecBuilder =
CursorBuildSpec.builder(spec)
-
.setFilter(null);
-
final Filter combinedFilter = baseFilterAnd(spec.getFilter());
// for physical column tracking, we start by copying base spec physical
columns
@@ -95,12 +93,15 @@ public class HashJoinSegmentCursorFactory implements
CursorFactory
if (clauses.isEmpty()) {
// if there are no clauses, we can just use the base cursor directly if
we apply the combined filter
- final CursorBuildSpec newSpec =
cursorBuildSpecBuilder.setFilter(combinedFilter)
-
.setPhysicalColumns(physicalColumns)
- .build();
+ final CursorBuildSpec newSpec = CursorBuildSpec.builder(spec)
+ .setFilter(combinedFilter)
+
.setPhysicalColumns(physicalColumns)
+ .build();
return baseCursorFactory.makeCursorHolder(newSpec);
}
+ // else we need to wipe out the grouping, aggregations, and ordering
+
return new CursorHolder()
{
final Closer joinablesCloser = Closer.create();
@@ -152,6 +153,22 @@ public class HashJoinSegmentCursorFactory implements
CursorFactory
baseFilter
);
+ // start with a full scan clipped to interval
+ final CursorBuildSpec.CursorBuildSpecBuilder cursorBuildSpecBuilder =
+ CursorBuildSpec.builder()
+ .setInterval(spec.getInterval())
+ .setQueryContext(spec.getQueryContext())
+ .setQueryMetrics(spec.getQueryMetrics());
+
+ // retain time ordering if preferred
+ Order timeOrder = Cursors.getTimeOrdering(spec.getPreferredOrdering());
+ if (timeOrder == Order.DESCENDING) {
+
cursorBuildSpecBuilder.setPreferredOrdering(Cursors.descendingTimeOrder());
+ } else if (timeOrder == Order.ASCENDING) {
+
cursorBuildSpecBuilder.setPreferredOrdering(Cursors.ascendingTimeOrder());
+ }
+
+ // add pushdown filters if present
if (joinFilterSplit.getBaseTableFilter().isPresent()) {
cursorBuildSpecBuilder.setFilter(joinFilterSplit.getBaseTableFilter().get());
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java
b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java
index 8a704120ae5..3709a9fec5d 100644
---
a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java
+++
b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java
@@ -27,7 +27,6 @@ import org.apache.druid.segment.VirtualColumns;
import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -45,7 +44,6 @@ public final class ProjectionMatchBuilder
private final Set<VirtualColumn> referencedVirtualColumns;
private final Map<String, String> remapColumns;
private final List<AggregatorFactory> combiningFactories;
- private final Set<String> matchedQueryColumns;
@Nullable
private Filter rewriteFilter;
@@ -55,7 +53,6 @@ public final class ProjectionMatchBuilder
this.referencedVirtualColumns = new HashSet<>();
this.remapColumns = new HashMap<>();
this.combiningFactories = new ArrayList<>();
- this.matchedQueryColumns = new HashSet<>();
}
/**
@@ -103,18 +100,6 @@ public final class ProjectionMatchBuilder
return this;
}
- public ProjectionMatchBuilder addMatchedQueryColumn(String queryColumn)
- {
- matchedQueryColumns.add(queryColumn);
- return this;
- }
-
- public ProjectionMatchBuilder addMatchedQueryColumns(Collection<String>
queryColumns)
- {
- matchedQueryColumns.addAll(queryColumns);
- return this;
- }
-
public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter)
{
this.rewriteFilter = rewriteFilter;
@@ -131,11 +116,6 @@ public final class ProjectionMatchBuilder
return remapColumns;
}
- public Set<String> getMatchedQueryColumns()
- {
- return matchedQueryColumns;
- }
-
public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec)
{
return new ProjectionMatch(
diff --git
a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
index 7b280544abc..a91b17de58f 100644
---
a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
+++
b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
@@ -19,7 +19,6 @@
package org.apache.druid.segment.projections;
-import com.google.common.collect.Sets;
import org.apache.druid.data.input.impl.AggregateProjectionSpec;
import org.apache.druid.error.InvalidInput;
import org.apache.druid.java.util.common.granularity.Granularities;
@@ -115,6 +114,9 @@ public class Projections
if
(!queryCursorBuildSpec.isCompatibleOrdering(projection.getOrderingWithTimeColumnSubstitution()))
{
return null;
}
+ if
(CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getPhysicalColumns())) {
+ return null;
+ }
ProjectionMatchBuilder matchBuilder = new ProjectionMatchBuilder();
// match virtual columns first, which will populate the 'remapColumns' of
the match builder
@@ -123,23 +125,17 @@ public class Projections
return null;
}
-
- matchBuilder = matchGrouping(projection, queryCursorBuildSpec,
physicalColumnChecker, matchBuilder);
- if (matchBuilder == null) {
- return null;
- }
-
- matchBuilder = matchAggregators(projection, queryCursorBuildSpec,
matchBuilder);
+ matchBuilder = matchFilter(projection, queryCursorBuildSpec,
physicalColumnChecker, matchBuilder);
if (matchBuilder == null) {
return null;
}
- matchBuilder = matchFilter(projection, queryCursorBuildSpec,
physicalColumnChecker, matchBuilder);
+ matchBuilder = matchGrouping(projection, queryCursorBuildSpec,
physicalColumnChecker, matchBuilder);
if (matchBuilder == null) {
return null;
}
- matchBuilder = matchRemainingPhysicalColumns(projection,
queryCursorBuildSpec, physicalColumnChecker, matchBuilder);
+ matchBuilder = matchAggregators(projection, queryCursorBuildSpec,
matchBuilder);
if (matchBuilder == null) {
return null;
}
@@ -204,11 +200,9 @@ public class Projections
if (rewritten == ProjectionFilterMatch.INSTANCE) {
// we can remove the whole thing since the query filter exactly
matches the projection filter
matchBuilder.rewriteFilter(null);
- matchBuilder.addMatchedQueryColumns(originalRequired);
} else {
// otherwise, we partially rewrote the query filter to eliminate the
projection filter since it is baked in
matchBuilder.rewriteFilter(rewritten);
-
matchBuilder.addMatchedQueryColumns(Sets.difference(originalRequired,
rewritten.getRequiredColumns()));
}
} else {
// projection has a filter, but the query doesn't, no good
@@ -290,8 +284,7 @@ public class Projections
if (combining != null) {
matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName())
.addReferencedPhysicalColumn(projectionAgg.getName())
- .addPreAggregatedAggregator(combining)
- .addMatchedQueryColumns(queryAgg.requiredFields());
+ .addPreAggregatedAggregator(combining);
foundMatch = true;
break;
}
@@ -304,39 +297,6 @@ public class Projections
return null;
}
- @Nullable
- public static ProjectionMatchBuilder matchRemainingPhysicalColumns(
- AggregateProjectionMetadata.Schema projection,
- CursorBuildSpec queryCursorBuildSpec,
- PhysicalColumnChecker physicalColumnChecker,
- ProjectionMatchBuilder matchBuilder
- )
- {
- // validate physical and virtual columns have all been accounted for
- final Set<String> matchedQueryColumns =
matchBuilder.getMatchedQueryColumns();
- if (queryCursorBuildSpec.getPhysicalColumns() != null) {
- for (String queryColumn : queryCursorBuildSpec.getPhysicalColumns()) {
- // a projection always has a __time column, it just might be a
constant of the segment interval start if the
- // projection itself did not transform the base table __time column
- if (ColumnHolder.TIME_COLUMN_NAME.equals(queryColumn)) {
- continue;
- }
- if (!matchedQueryColumns.contains(queryColumn)) {
- matchBuilder = matchQueryPhysicalColumn(
- queryColumn,
- projection,
- physicalColumnChecker,
- matchBuilder
- );
- if (matchBuilder == null) {
- return null;
- }
- }
- }
- }
- return matchBuilder;
- }
-
/**
* Ensure that the projection has the specified column required by a {@link
CursorBuildSpec} in one form or another.
* If the column is a {@link VirtualColumn} on the build spec, ensure that
the projection has an equivalent virtual
@@ -370,9 +330,6 @@ public class Projections
ProjectionMatchBuilder matchBuilder
)
{
- if (matchBuilder.getMatchedQueryColumns().contains(column)) {
- return matchBuilder;
- }
final VirtualColumn virtualColumn =
queryCursorBuildSpec.getVirtualColumns().getVirtualColumn(column);
if (virtualColumn != null) {
return matchQueryVirtualColumn(
@@ -408,13 +365,10 @@ public class Projections
if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) {
matchBuilder.remapColumn(queryVirtualColumn.getOutputName(),
remapColumnName);
}
- return
matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName())
-
.addMatchedQueryColumns(queryVirtualColumn.requiredColumns())
- .addReferencedPhysicalColumn(remapColumnName);
+ return matchBuilder.addReferencedPhysicalColumn(remapColumnName);
}
- matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName())
- .addReferenceedVirtualColumn(queryVirtualColumn);
+ matchBuilder.addReferenceedVirtualColumn(queryVirtualColumn);
final List<String> requiredInputs = queryVirtualColumn.requiredColumns();
if (requiredInputs.size() == 1 &&
ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) {
// special handle time granularity. in the future this should be
reworked to push this concept into the
@@ -463,8 +417,7 @@ public class Projections
)
{
if (physicalColumnChecker.check(projection.getName(), column)) {
- return matchBuilder.addMatchedQueryColumn(column)
- .addReferencedPhysicalColumn(column);
+ return matchBuilder.addReferencedPhysicalColumn(column);
}
return null;
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
index b6a1dfa0390..98eb6cb7906 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
@@ -94,6 +94,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
public abstract class IndexMergerTestBase extends InitializedNullHandlingTest
@@ -3071,6 +3072,7 @@ public abstract class IndexMergerTestBase extends
InitializedNullHandlingTest
ImmutableMap.of(QueryContexts.USE_PROJECTION, "a_hourly_c_sum")
)
)
+ .setPhysicalColumns(Set.of("c",
ColumnHolder.TIME_COLUMN_NAME))
.setVirtualColumns(
VirtualColumns.create(
Granularities.toVirtualColumn(Granularities.HOUR, "gran")
@@ -3089,6 +3091,7 @@ public abstract class IndexMergerTestBase extends
InitializedNullHandlingTest
ImmutableMap.of(QueryContexts.USE_PROJECTION, "a_c_sum")
)
)
+ .setPhysicalColumns(Set.of("a",
"c"))
.setAggregators(
Collections.singletonList(
new
LongSumAggregatorFactory("c", "c")
diff --git
a/processing/src/test/java/org/apache/druid/segment/UnnestColumnValueSelectorCursorTest.java
b/processing/src/test/java/org/apache/druid/segment/UnnestColumnValueSelectorCursorTest.java
index 5e60b1e0496..b87a7faf029 100644
---
a/processing/src/test/java/org/apache/druid/segment/UnnestColumnValueSelectorCursorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/UnnestColumnValueSelectorCursorTest.java
@@ -58,8 +58,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -91,8 +90,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -123,8 +121,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -157,8 +154,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "mv_to_array(\"dummy\")",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "mv_to_array(\"dummy\")",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -191,8 +187,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING_ARRAY, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING_ARRAY, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -230,8 +225,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -275,8 +269,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING_ARRAY, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING_ARRAY, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -308,8 +301,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -337,8 +329,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -371,8 +362,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"", null,
ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"", null,
ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -404,14 +394,12 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor childCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"", null,
ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"", null,
ExprMacroTable.nil())
);
UnnestColumnValueSelectorCursor parentCursor = new
UnnestColumnValueSelectorCursor(
childCursor,
childCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"" + OUTPUT_NAME + "\"",
null, ExprMacroTable.nil()),
- "tmp-out"
+ new ExpressionVirtualColumn("tmp-out", "\"" + OUTPUT_NAME + "\"",
null, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
parentCursor.getColumnSelectorFactory()
.makeColumnValueSelector("tmp-out");
@@ -444,8 +432,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -481,8 +468,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -518,8 +504,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -551,8 +536,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -584,8 +568,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -620,8 +603,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"", null,
ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"", null,
ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
@@ -655,8 +637,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
// should return a column value selector for this case
BaseSingleValueDimensionSelector unnestDimSelector =
(BaseSingleValueDimensionSelector) unnestCursor.getColumnSelectorFactory()
@@ -697,8 +678,7 @@ public class UnnestColumnValueSelectorCursorTest extends
InitializedNullHandling
UnnestColumnValueSelectorCursor unnestCursor = new
UnnestColumnValueSelectorCursor(
listCursor,
listCursor.getColumnSelectorFactory(),
- new ExpressionVirtualColumn("__unnest__", "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil()),
- OUTPUT_NAME
+ new ExpressionVirtualColumn(OUTPUT_NAME, "\"dummy\"",
ColumnType.STRING, ExprMacroTable.nil())
);
ColumnValueSelector unnestColumnValueSelector =
unnestCursor.getColumnSelectorFactory()
.makeColumnValueSelector(OUTPUT_NAME);
diff --git
a/processing/src/test/java/org/apache/druid/segment/UnnestCursorFactoryTest.java
b/processing/src/test/java/org/apache/druid/segment/UnnestCursorFactoryTest.java
index ef26393a773..5d16e579bdd 100644
---
a/processing/src/test/java/org/apache/druid/segment/UnnestCursorFactoryTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/UnnestCursorFactoryTest.java
@@ -22,7 +22,6 @@ package org.apache.druid.segment;
import com.google.common.collect.ImmutableList;
import org.apache.druid.data.input.InputSource;
import org.apache.druid.data.input.ResourceInputSource;
-import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.java.util.common.io.Closer;
@@ -32,9 +31,11 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.EqualityFilter;
import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.RangeFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.query.filter.ValueMatcher;
import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
@@ -61,12 +62,14 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
import org.junit.rules.TemporaryFolder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.function.Function;
import java.util.function.ToLongFunction;
@@ -836,6 +839,190 @@ public class UnnestCursorFactoryTest extends
InitializedNullHandlingTest
}
}
+ @Test
+ public void testTransformCursor()
+ {
+ final String inputColumn = "col";
+ final String unnestColumn = "unnest";
+ ExpressionVirtualColumn identifier = new ExpressionVirtualColumn(
+ unnestColumn,
+ "\"" + inputColumn + "\"",
+ ColumnType.STRING_ARRAY,
+ ExprMacroTable.nil()
+ );
+ CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(unnestColumn))
+
.setGroupingColumns(List.of(unnestColumn))
+ .build();
+
+
+ CursorBuildSpec expected = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(inputColumn))
+
.setVirtualColumns(VirtualColumns.create(identifier))
+ .build();
+
+ CursorBuildSpec transformed = UnnestCursorFactory.transformCursorBuildSpec(
+ buildSpec,
+ identifier,
+ null
+ );
+
+ Assertions.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void testTransformCursorMoreGrouping()
+ {
+ final String unnestColumn = "unnest";
+ ExpressionVirtualColumn identifier = new ExpressionVirtualColumn(
+ unnestColumn,
+ "\"a\"",
+ ColumnType.STRING_ARRAY,
+ ExprMacroTable.nil()
+ );
+ CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(unnestColumn, "b", "c"))
+
.setGroupingColumns(List.of(unnestColumn, "b", "c"))
+ .build();
+
+
+ CursorBuildSpec expected = CursorBuildSpec.builder()
+ .setPhysicalColumns(Set.of("a",
"b", "c"))
+
.setVirtualColumns(VirtualColumns.create(identifier))
+ .build();
+
+ CursorBuildSpec transformed = UnnestCursorFactory.transformCursorBuildSpec(
+ buildSpec,
+ identifier,
+ null
+ );
+
+ Assertions.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void testTransformCursorFilter()
+ {
+ final String unnestColumn = "unnest";
+ ExpressionVirtualColumn identifier = new ExpressionVirtualColumn(
+ unnestColumn,
+ "\"a\"",
+ ColumnType.STRING_ARRAY,
+ ExprMacroTable.nil()
+ );
+ Filter unnestEquals = new EqualityFilter(unnestColumn, ColumnType.STRING,
"value", null);
+ Filter bRange = new RangeFilter("b", ColumnType.LONG, 1L, 10L, false,
false, null);
+ CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(unnestColumn, "b", "c"))
+
.setGroupingColumns(List.of(unnestColumn, "b", "c"))
+ .setFilter(bRange)
+ .build();
+
+ UnnestCursorFactory.UnnestFilterSplit split =
UnnestCursorFactory.computeBaseAndPostUnnestFilters(
+ buildSpec.getFilter(),
+ unnestEquals,
+ buildSpec.getVirtualColumns(),
+ identifier,
+ "a",
+ ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING_ARRAY)
+ );
+
+ // expect unnest filter not pushed down since is array column
+ CursorBuildSpec expected = CursorBuildSpec.builder()
+ .setPhysicalColumns(Set.of("a",
"b", "c"))
+
.setVirtualColumns(VirtualColumns.create(identifier))
+ .setFilter(bRange)
+ .build();
+
+ CursorBuildSpec transformed = UnnestCursorFactory.transformCursorBuildSpec(
+ buildSpec,
+ identifier,
+ split.getBaseTableFilter()
+ );
+
+ Assertions.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void testTransformCursorFilterMvd()
+ {
+ final String unnestColumn = "unnest";
+ ExpressionVirtualColumn identifier = new ExpressionVirtualColumn(
+ unnestColumn,
+ "\"a\"",
+ ColumnType.STRING_ARRAY,
+ ExprMacroTable.nil()
+ );
+ Filter unnestEquals = new EqualityFilter(unnestColumn, ColumnType.STRING,
"value", null);
+ Filter bRange = new RangeFilter("b", ColumnType.LONG, 1L, 10L, false,
false, null);
+ CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(unnestColumn, "b", "c"))
+
.setGroupingColumns(List.of(unnestColumn, "b", "c"))
+ .setFilter(bRange)
+ .build();
+
+ UnnestCursorFactory.UnnestFilterSplit split =
UnnestCursorFactory.computeBaseAndPostUnnestFilters(
+ buildSpec.getFilter(),
+ unnestEquals,
+ buildSpec.getVirtualColumns(),
+ identifier,
+ "a",
+
ColumnCapabilitiesImpl.createDefault().setHasMultipleValues(true).setType(ColumnType.STRING)
+ );
+
+ // since unnest column is mvd, expect pushdown filter as on base table
+ CursorBuildSpec expected = CursorBuildSpec.builder()
+ .setPhysicalColumns(Set.of("a",
"b", "c"))
+
.setVirtualColumns(VirtualColumns.create(identifier))
+ .setFilter(
+ new AndFilter(
+ List.of(
+ bRange,
+ new
EqualityFilter("a", ColumnType.STRING, "value", null)
+ )
+ )
+ )
+ .build();
+
+ CursorBuildSpec transformed = UnnestCursorFactory.transformCursorBuildSpec(
+ buildSpec,
+ identifier,
+ split.getBaseTableFilter()
+ );
+
+ Assertions.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void testTransformCursorArray()
+ {
+ final String unnestColumn = "unnest";
+ ExpressionVirtualColumn array = new ExpressionVirtualColumn(
+ unnestColumn,
+ "array(x, y, z)",
+ ColumnType.DOUBLE_ARRAY,
+ ExprMacroTable.nil()
+ );
+ CursorBuildSpec buildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of(unnestColumn))
+
.setGroupingColumns(List.of(unnestColumn))
+ .build();
+
+
+ CursorBuildSpec expected = CursorBuildSpec.builder()
+ .setPhysicalColumns(Set.of("x",
"y", "z"))
+
.setVirtualColumns(VirtualColumns.create(array))
+ .build();
+
+ CursorBuildSpec transformed = UnnestCursorFactory.transformCursorBuildSpec(
+ buildSpec,
+ array,
+ null
+ );
+
+ Assertions.assertEquals(expected, transformed);
+ }
+
public void testComputeBaseAndPostUnnestFilters(
Filter testQueryFilter,
String expectedBasePushDown,
@@ -858,16 +1045,18 @@ public class UnnestCursorFactoryTest extends
InitializedNullHandlingTest
)
{
final String inputColumn =
cursorFactory.getUnnestInputIfDirectAccess(cursorFactory.getUnnestColumn());
+ Assert.assertNotNull(inputColumn);
final VirtualColumn vc = cursorFactory.getUnnestColumn();
- Pair<Filter, Filter> filterPair =
cursorFactory.computeBaseAndPostUnnestFilters(
+ UnnestCursorFactory.UnnestFilterSplit filterSplit =
UnnestCursorFactory.computeBaseAndPostUnnestFilters(
testQueryFilter,
null,
VirtualColumns.EMPTY,
+ cursorFactory.getUnnestColumn(),
inputColumn,
vc.capabilities(cursorFactory, inputColumn)
);
- Filter actualPushDownFilter = filterPair.lhs;
- Filter actualPostUnnestFilter = filterPair.rhs;
+ Filter actualPushDownFilter = filterSplit.getBaseTableFilter();
+ Filter actualPostUnnestFilter = filterSplit.getPostUnnestFilter();
Assert.assertEquals(
"Expects only top level child of And Filter to push down to base",
expectedBasePushDown,
diff --git
a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
index dc86fb8ab2f..b8922a9a3f1 100644
---
a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
@@ -56,6 +56,7 @@ class ProjectionsTest
12345
);
CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of("c"))
.setPreferredOrdering(List.of())
.setAggregators(
List.of(
@@ -99,6 +100,7 @@ class ProjectionsTest
12345
);
CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of("c"))
.setPreferredOrdering(List.of())
.setAggregators(
List.of(
@@ -115,6 +117,7 @@ class ProjectionsTest
)
);
CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of("b", "c"))
.setPreferredOrdering(List.of())
.setFilter(
new
EqualityFilter(
@@ -166,6 +169,7 @@ class ProjectionsTest
);
CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder()
.setPreferredOrdering(List.of())
+
.setPhysicalColumns(Set.of("a", "b", "c"))
.setGroupingColumns(List.of("a", "b"))
.setAggregators(
List.of(
@@ -182,6 +186,7 @@ class ProjectionsTest
)
);
CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder()
+
.setPhysicalColumns(Set.of("a", "b", "c"))
.setGroupingColumns(List.of("a", "b"))
.setPreferredOrdering(List.of())
.setFilter(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
