This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new af3c64f7eb7 Make sorted ORDER BY late materialization automatic
(#20003)
af3c64f7eb7 is described below
commit af3c64f7eb7d85488900818ccf7b5c6909a266fc
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Oct 22 16:47:51 2026 -0700
Make sorted ORDER BY late materialization automatic (#20003)
---
.../common/utils/config/QueryOptionsUtils.java | 8 --
.../query/LinearSelectionOrderByOperator.java | 154 +++++++--------------
.../SelectionPartiallyOrderedByDescOperation.java | 16 +--
.../SelectionPartiallyOrderedByLinearOperator.java | 10 +-
.../apache/pinot/core/plan/SelectionPlanNode.java | 57 ++------
...nnerSegmentSelectionSingleValueQueriesTest.java | 113 +++++----------
.../queries/NullHandlingEnabledQueriesTest.java | 71 +++++++---
.../queries/TextMatchTransformFunctionTest.java | 12 +-
.../apache/pinot/spi/utils/CommonConstants.java | 10 --
9 files changed, 155 insertions(+), 296 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
index 491417d7315..936c968aaf0 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
@@ -964,14 +964,6 @@ public class QueryOptionsUtils {
return queryOptions.getOrDefault(QueryOptionKey.WORKLOAD_NAME,
CommonConstants.Accounting.DEFAULT_WORKLOAD_NAME);
}
- public static boolean isLateMaterializeSortedOrderByEnabled(Map<String,
String> queryOptions) {
- String value =
queryOptions.get(QueryOptionKey.LATE_MATERIALIZE_SORTED_ORDER_BY);
- if (value == null) {
- return QueryOptionKey.DEFAULT_LATE_MATERIALIZE_SORTED_ORDER_BY;
- }
- return Boolean.parseBoolean(value);
- }
-
public static boolean isReverseOrderAllowed(Map<String, String>
queryOptions) {
String value = queryOptions.get(QueryOptionKey.ALLOW_REVERSE_ORDER);
if (value == null) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/LinearSelectionOrderByOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/LinearSelectionOrderByOperator.java
index b5a16551b25..4e87193899e 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/LinearSelectionOrderByOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/LinearSelectionOrderByOperator.java
@@ -38,6 +38,7 @@ import javax.annotation.Nullable;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.OrderByExpressionContext;
import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.common.RowBasedBlockValueFetcher;
import org.apache.pinot.core.operator.BaseOperator;
@@ -67,11 +68,11 @@ import org.roaringbitmap.RoaringBitmap;
/// or: SELECT ... FROM Table WHERE predicates ORDER BY sorted_column,
not_sorted LIMIT 10 OFFSET 5
/// but not SELECT ... FROM Table WHERE predicates ORDER BY not_sorted,
sorted_column LIMIT 10 OFFSET 5
///
-/// Operators that derives from this class are going to have an almost linear
cost instead of the usual NlogN when
-/// actual sorting must be done, where N is the number of rows in the segment.
There is a degraded scenario when the
-/// cost is actually NlogL (where L is the limit of the query) when all the
first L rows have the exact same value for
-/// the prefix of the sorted columns. Even in that case, L should be quite
smaller than N, so this implementation is
-/// algorithmically better than the normal solution.
+/// Only order-by expressions are fetched during selection. Any remaining
output expressions are fetched for the
+/// surviving rows after selection, as in [SelectionOrderByOperator].
+///
+/// These operators reuse the segment's sorted prefix. Rows tied on that
prefix are ordered by the remaining
+/// expressions.
public abstract class LinearSelectionOrderByOperator extends
BaseOperator<SelectionResultsBlock> {
protected final IndexSegment _indexSegment;
protected final QueryContext _queryContext;
@@ -82,72 +83,49 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
protected final List<ExpressionContext> _toSort;
protected final BaseProjectOperator<?> _projectOperator;
- protected final List<OrderByExpressionContext> _orderByExpressions;
protected final ColumnContext[] _columnContexts;
- protected final int _numRowsToKeep;
protected final Comparator<Object[]> _comparator;
protected final Supplier<ListBuilder> _listBuilderSupplier;
protected final int _numOrderByExpressions;
- /// When true the project operator only produces the order-by expressions,
and the remaining output expressions are
- /// fetched for the rows that survive the limit. Rows carry their doc id at
index [#_numOrderByExpressions] until
- /// that fetch overwrites it.
- protected final boolean _lateMaterialize;
private long _numEntriesScannedPostFilterLate;
/// @param expressions Order-by expressions must be at the head of
the list.
/// @param numSortedExpressions Number of expressions in the order-by
expressions that are sorted.
public LinearSelectionOrderByOperator(IndexSegment indexSegment,
QueryContext queryContext,
List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions) {
- this(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions, false);
- }
-
- /// @param lateMaterialize whether `projectOperator` produces only the
order-by expressions, leaving the remaining
- /// output expressions to be fetched once the limit
has been applied.
- public LinearSelectionOrderByOperator(IndexSegment indexSegment,
QueryContext queryContext,
- List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions,
- boolean lateMaterialize) {
_indexSegment = indexSegment;
_queryContext = queryContext;
_nullHandlingEnabled = queryContext.isNullHandlingEnabled();
_expressions = expressions;
_projectOperator = projectOperator;
- _orderByExpressions = queryContext.getOrderByExpressions();
- assert _orderByExpressions != null;
- int numOrderByExpressions = _orderByExpressions.size();
-
- _numOrderByExpressions = numOrderByExpressions;
- _lateMaterialize = lateMaterialize;
- // Rows carry the doc id at index numOrderByExpressions, so there must be
at least one expression after the
- // order-by ones for that slot to exist (and something worth deferring).
- Preconditions.checkArgument(!lateMaterialize || expressions.size() >
numOrderByExpressions,
- "Late materialization requires output expressions beyond the %s
order-by ones, got %s",
- numOrderByExpressions, expressions.size());
+ List<OrderByExpressionContext> orderByExpressions =
queryContext.getOrderByExpressions();
+ assert orderByExpressions != null;
+ _numOrderByExpressions = orderByExpressions.size();
_alreadySorted = expressions.subList(0, numSortedExpressions);
- _toSort = expressions.subList(numSortedExpressions, numOrderByExpressions);
+ _toSort = expressions.subList(numSortedExpressions,
_numOrderByExpressions);
- // Only the projected expressions can be resolved against the project
operator. When materialising late it holds
- // the order-by expressions alone, which is also all the comparator needs.
- _columnContexts = new ColumnContext[lateMaterialize ?
numOrderByExpressions : _expressions.size()];
+ // Only order-by expressions are projected during selection; the remaining
expressions are fetched afterwards.
+ _columnContexts = new ColumnContext[_numOrderByExpressions];
for (int i = 0; i < _columnContexts.length; i++) {
ExpressionContext expression = _expressions.get(i);
_columnContexts[i] = _projectOperator.getResultColumnContext(expression);
}
- _numRowsToKeep = queryContext.getOffset() + queryContext.getLimit();
- _comparator = OrderByComparatorFactory.getComparator(_orderByExpressions,
_columnContexts, _nullHandlingEnabled);
+ int numRowsToKeep = queryContext.getOffset() + queryContext.getLimit();
+ _comparator = OrderByComparatorFactory.getComparator(orderByExpressions,
_columnContexts, _nullHandlingEnabled);
if (_toSort.isEmpty()) {
- _listBuilderSupplier = () -> new
TotallySortedListBuilder(_numRowsToKeep);
+ _listBuilderSupplier = () -> new TotallySortedListBuilder(numRowsToKeep);
} else {
Comparator<Object[]> sortedComparator =
- OrderByComparatorFactory.getComparator(_orderByExpressions,
_columnContexts, _nullHandlingEnabled, 0,
+ OrderByComparatorFactory.getComparator(orderByExpressions,
_columnContexts, _nullHandlingEnabled, 0,
numSortedExpressions);
Comparator<Object[]> unsortedComparator =
- OrderByComparatorFactory.getComparator(_orderByExpressions,
_columnContexts, _nullHandlingEnabled,
- numSortedExpressions, numOrderByExpressions);
- _listBuilderSupplier = () -> new
PartiallySortedListBuilder(_numRowsToKeep, sortedComparator,
unsortedComparator);
+ OrderByComparatorFactory.getComparator(orderByExpressions,
_columnContexts, _nullHandlingEnabled,
+ numSortedExpressions, _numOrderByExpressions);
+ _listBuilderSupplier = () -> new
PartiallySortedListBuilder(numRowsToKeep, sortedComparator, unsortedComparator);
}
}
@@ -167,16 +145,8 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
numTotalDocs);
}
- /// Number of expressions the project operator actually produces: all of
them, or just the order-by prefix when
- /// materialising late.
- protected int getNumProjectedExpressions() {
- return _columnContexts.length;
- }
-
protected IntFunction<Object[]> fetchBlock(ValueBlock valueBlock,
BlockValSet[] blockValSets) {
- int numProjected = getNumProjectedExpressions();
-
- for (int i = 0; i < numProjected; i++) {
+ for (int i = 0; i < _numOrderByExpressions; i++) {
ExpressionContext expression = _expressions.get(i);
blockValSets[i] = valueBlock.getBlockValueSet(expression);
}
@@ -184,21 +154,21 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
RoaringBitmap[] nullBitmaps;
if (_nullHandlingEnabled) {
- nullBitmaps = new RoaringBitmap[numProjected];
- for (int i = 0; i < numProjected; i++) {
+ nullBitmaps = new RoaringBitmap[_numOrderByExpressions];
+ for (int i = 0; i < _numOrderByExpressions; i++) {
nullBitmaps[i] = blockValSets[i].getNullBitmap();
}
} else {
nullBitmaps = null;
}
- if (!_lateMaterialize) {
+ if (_expressions.size() == _numOrderByExpressions) {
if (nullBitmaps == null) {
return blockValueFetcher::getRow;
}
return (rowIdInBlock) -> {
Object[] row = blockValueFetcher.getRow(rowIdInBlock);
- applyNulls(row, nullBitmaps, rowIdInBlock);
+ applyNulls(row, nullBitmaps, rowIdInBlock, 0);
return row;
};
}
@@ -211,17 +181,17 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
Object[] row = new Object[numExpressions];
blockValueFetcher.getRow(rowIdInBlock, row, 0);
if (nullBitmaps != null) {
- applyNulls(row, nullBitmaps, rowIdInBlock);
+ applyNulls(row, nullBitmaps, rowIdInBlock, 0);
}
row[_numOrderByExpressions] = docIds[rowIdInBlock];
return row;
};
}
- private static void applyNulls(Object[] row, RoaringBitmap[] nullBitmaps,
int rowIdInBlock) {
+ private static void applyNulls(Object[] row, RoaringBitmap[] nullBitmaps,
int rowIdInBlock, int columnOffset) {
for (int colId = 0; colId < nullBitmaps.length; colId++) {
if (nullBitmaps[colId] != null &&
nullBitmaps[colId].contains(rowIdInBlock)) {
- row[colId] = null;
+ row[columnOffset + colId] = null;
}
}
}
@@ -257,7 +227,7 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
return CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL,
getUpperCaseExplainName());
}
- // remember this ist he single-stage explain string
+ // Used by the single-stage explain string.
@Override
public String toExplainString() {
StringBuilder sb = new StringBuilder(getUpperCaseExplainName());
@@ -267,12 +237,9 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
sb.append(", unsortedList: ");
concatList(sb, _toSort);
- if (_lateMaterialize) {
- sb.append(", lateMaterialized: true");
- }
sb.append(", rest: ");
- concatList(sb, _expressions.subList(_alreadySorted.size() +
_toSort.size(), _expressions.size()));
+ concatList(sb, _expressions.subList(_numOrderByExpressions,
_expressions.size()));
sb.append(')');
return sb.toString();
@@ -281,15 +248,12 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
@Override
protected void explainAttributes(ExplainAttributeBuilder attributeBuilder) {
super.explainAttributes(attributeBuilder);
- List<ExpressionContext> rest = _expressions.subList(_alreadySorted.size()
+ _toSort.size(), _expressions.size());
+ List<ExpressionContext> rest =
_expressions.subList(_numOrderByExpressions, _expressions.size());
List<String> sortedList =
_alreadySorted.stream().map(ExpressionContext::toString).collect(Collectors.toList());
List<String> toSort =
_toSort.stream().map(ExpressionContext::toString).collect(Collectors.toList());
List<String> restStr =
rest.stream().map(ExpressionContext::toString).collect(Collectors.toList());
- if (_lateMaterialize) {
- attributeBuilder.putString("lateMaterialized", "true");
- }
attributeBuilder.putStringList("sortedList", sortedList)
.putStringList("unsortedList", toSort)
.putStringList("rest", restStr);
@@ -310,8 +274,8 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
@Override
protected SelectionResultsBlock getNextBlock() {
List<Object[]> rows = fetch(_listBuilderSupplier);
- if (!_lateMaterialize) {
- return new SelectionResultsBlock(createDataSchema(), rows, _comparator,
_queryContext);
+ if (_expressions.size() == _numOrderByExpressions) {
+ return new SelectionResultsBlock(createDataSchema(null), rows,
_comparator, _queryContext);
}
return fetchRemainingExpressions(rows);
}
@@ -321,8 +285,7 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
/// The rows are visited in doc-id order because that is the order
[BitmapDocIdSetOperator#ascending] returns, while
/// `rows` keeps the result order. Both views hold the same row arrays, so
filling one fills the other.
private SelectionResultsBlock fetchRemainingExpressions(List<Object[]> rows)
{
- int numExpressions = _expressions.size();
- List<ExpressionContext> nonOrderByExpressions =
_expressions.subList(_numOrderByExpressions, numExpressions);
+ List<ExpressionContext> nonOrderByExpressions =
_expressions.subList(_numOrderByExpressions, _expressions.size());
int numNonOrderByExpressions = nonOrderByExpressions.size();
int numRows = rows.size();
@@ -332,14 +295,10 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
for (Object[] row : rowsByDocId) {
docIds.add((int) row[_numOrderByExpressions]);
}
- // rowsByDocId is filled by walking the bitmap in ascending order, so the
two must line up one for one.
- // They do only because surviving rows have distinct doc ids. If that ever
stopped holding, the bitmap would
- // dedupe and the trailing rows would keep their unfilled values instead
of failing -- wrong data returned,
- // not an error. Checked rather than asserted because this build never
runs with -ea, and the cost is one
- // cardinality read per query against a projection pass over every
surviving row.
- Preconditions.checkState(docIds.getCardinality() == numRows,
- "Expected one distinct doc id per surviving row, got %s doc ids for %s
rows", docIds.getCardinality(),
- numRows);
+ // Surviving doc ids must be unique so the bitmap and rowsByDocId stay
aligned.
+ int numDocIds = docIds.getCardinality();
+ Preconditions.checkState(numDocIds == numRows,
+ "Expected one distinct doc id per surviving row, got %s doc ids for %s
rows", numDocIds, numRows);
Set<String> columns = new HashSet<>();
for (ExpressionContext expression : nonOrderByExpressions) {
@@ -347,13 +306,9 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
}
Map<String, DataSource> dataSourceMap = new HashMap<>();
for (String column : columns) {
- // Same resolution the eager path uses: ProjectPlanNode resolves every
selected column through this exact
- // call, including the ones fetched here. The schema is only
dereferenced for a column the segment does not
- // physically have, and it is set for every server query in
ServerQueryExecutorV1Impl.
dataSourceMap.put(column, _indexSegment.getDataSource(column,
_queryContext.getSchema()));
}
- // Distinct physical columns. A column that is both an order-by column and
referenced by a non-order-by
- // expression is read again here and counted again, which slightly
over-reports entries scanned.
+ // The scan count includes columns reread from the order-by pass.
int numColumns = columns.size();
BitmapDocIdSetOperator docIdSetOperator =
BitmapDocIdSetOperator.ascending(docIds, numRows);
@@ -380,12 +335,7 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
nullBitmaps[i] = blockValSets[i].getNullBitmap();
}
for (int i = 0; i < numDocsFetched; i++) {
- Object[] row = rowsByDocId[rowBaseId + i];
- for (int colId = 0; colId < numNonOrderByExpressions; colId++) {
- if (nullBitmaps[colId] != null &&
nullBitmaps[colId].contains(i)) {
- row[_numOrderByExpressions + colId] = null;
- }
- }
+ applyNulls(rowsByDocId[rowBaseId + i], nullBitmaps, i,
_numOrderByExpressions);
}
}
long entriesScanned = (long) numDocsFetched * numColumns;
@@ -396,35 +346,25 @@ public abstract class LinearSelectionOrderByOperator
extends BaseOperator<Select
}
rowBaseId += numDocsFetched;
}
- return new SelectionResultsBlock(createDataSchema(transformOperator,
nonOrderByExpressions), rows, _comparator,
- _queryContext);
+ return new SelectionResultsBlock(createDataSchema(transformOperator),
rows, _comparator, _queryContext);
}
}
- /// Data schema covering both cases. Types for the expressions the scan
projected come from the project
- /// operator's column contexts; types for any expressions fetched late come
from the transform operator built
- /// for that fetch. `lateExpressions` is empty when nothing was materialised
late, which leaves the first
- /// branch covering every expression.
- private DataSchema createDataSchema(@Nullable TransformOperator
lateTransformOperator,
- List<ExpressionContext> lateExpressions) {
+ /// Resolves order-by types from the initial projection and remaining types
from the late transform operator.
+ private DataSchema createDataSchema(@Nullable TransformOperator
lateTransformOperator) {
int numExpressions = _expressions.size();
- int numProjected = numExpressions - lateExpressions.size();
String[] columnNames = new String[numExpressions];
- DataSchema.ColumnDataType[] columnDataTypes = new
DataSchema.ColumnDataType[numExpressions];
+ ColumnDataType[] columnDataTypes = new ColumnDataType[numExpressions];
for (int i = 0; i < numExpressions; i++) {
columnNames[i] = _expressions.get(i).toString();
- ColumnContext columnContext = i < numProjected ? _columnContexts[i]
- : lateTransformOperator.getResultColumnContext(lateExpressions.get(i
- numProjected));
- columnDataTypes[i] =
- DataSchema.ColumnDataType.fromDataType(columnContext.getDataType(),
columnContext.isSingleValue());
+ ColumnContext columnContext = i < _numOrderByExpressions
+ ? _columnContexts[i]
+ : lateTransformOperator.getResultColumnContext(_expressions.get(i));
+ columnDataTypes[i] =
ColumnDataType.fromDataType(columnContext.getDataType(),
columnContext.isSingleValue());
}
return new DataSchema(columnNames, columnDataTypes);
}
- protected DataSchema createDataSchema() {
- return createDataSchema(null, List.of());
- }
-
/// A private class used to build a partially sorted list by adding
partially sorted data.
///
/// Specifically, this class has been designed to receive successive calls
to [#add(Object[])] follow by a
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByDescOperation.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByDescOperation.java
index 396bd01ab02..90ae5fc5cf3 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByDescOperation.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByDescOperation.java
@@ -44,13 +44,7 @@ public class SelectionPartiallyOrderedByDescOperation
extends LinearSelectionOrd
public SelectionPartiallyOrderedByDescOperation(IndexSegment indexSegment,
QueryContext queryContext,
List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions) {
- this(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions, false);
- }
-
- public SelectionPartiallyOrderedByDescOperation(IndexSegment indexSegment,
QueryContext queryContext,
- List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions,
- boolean lateMaterialize) {
- super(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions, lateMaterialize);
+ super(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions);
assert queryContext.getOrderByExpressions() != null;
Preconditions.checkArgument(!queryContext.getOrderByExpressions().stream()
.filter(expr -> expr.getExpression().getType() ==
ExpressionContext.Type.IDENTIFIER)
@@ -63,12 +57,10 @@ public class SelectionPartiallyOrderedByDescOperation
extends LinearSelectionOrd
@Override
protected List<Object[]> fetch(Supplier<ListBuilder> listBuilderSupplier) {
- // Ideally we would use a descending cursor, but we don't actually have
them
- // Alternatively, we could store all blocks in a list and iterate them in
reverse order, but ProjectionBlocks share
- // the same DataBlockCache, so they may be ephemeral and being overridden
by the next block.
- // The only alternative we have right now is to retrieve the last LIMIT
elements from each block
+ // This path handles projections that cannot iterate in descending order.
Keep the best rows from each block
+ // because the blocks share a mutable DataBlockCache and cannot be
buffered for reverse iteration.
- BlockValSet[] blockValSets = new BlockValSet[getNumProjectedExpressions()];
+ BlockValSet[] blockValSets = new BlockValSet[_numOrderByExpressions];
List<Object[]> localBestRows = new ArrayList<>();
ValueBlock valueBlock;
while ((valueBlock = _projectOperator.nextBlock()) != null) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByLinearOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByLinearOperator.java
index 33a43a144c8..845d65e0181 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByLinearOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionPartiallyOrderedByLinearOperator.java
@@ -46,13 +46,7 @@ public class SelectionPartiallyOrderedByLinearOperator
extends LinearSelectionOr
public SelectionPartiallyOrderedByLinearOperator(IndexSegment indexSegment,
QueryContext queryContext,
List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions) {
- this(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions, false);
- }
-
- public SelectionPartiallyOrderedByLinearOperator(IndexSegment indexSegment,
QueryContext queryContext,
- List<ExpressionContext> expressions, BaseProjectOperator<?>
projectOperator, int numSortedExpressions,
- boolean lateMaterialize) {
- super(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions, lateMaterialize);
+ super(indexSegment, queryContext, expressions, projectOperator,
numSortedExpressions);
boolean firstColIsAsc = queryContext.getOrderByExpressions().stream()
.filter(expr -> expr.getExpression().getType() ==
ExpressionContext.Type.IDENTIFIER)
.findFirst()
@@ -67,7 +61,7 @@ public class SelectionPartiallyOrderedByLinearOperator
extends LinearSelectionOr
@Override
protected List<Object[]> fetch(Supplier<ListBuilder> listBuilderSupplier) {
- BlockValSet[] blockValSets = new BlockValSet[getNumProjectedExpressions()];
+ BlockValSet[] blockValSets = new BlockValSet[_numOrderByExpressions];
ListBuilder listBuilder = listBuilderSupplier.get();
ValueBlock valueBlock;
while ((valueBlock = _projectOperator.nextBlock()) != null) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java
index 04b11cec49b..3e366d54140 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/SelectionPlanNode.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.plan;
-import java.util.ArrayList;
import java.util.List;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.OrderByExpressionContext;
@@ -74,8 +73,8 @@ public class SelectionPlanNode implements PlanNode {
return new SelectionOnlyOperator(_indexSegment, _queryContext,
expressions, projectOperator);
}
int numOrderByExpressions = orderByExpressions.size();
- // Although it is a break of abstraction, some code, specially merging,
assumes that if there is an order by
- // expression the operator will return a block whose selection result is a
priority queue.
+ // Order-by expressions come first; any remaining output expressions are
fetched after selection.
+ List<ExpressionContext> expressionsToProject = expressions.subList(0,
numOrderByExpressions);
int sortedColumnsPrefixSize = getSortedColumnsPrefix(orderByExpressions,
_queryContext.isNullHandlingEnabled());
if (sortedColumnsPrefixSize > 0) {
int maxDocsPerCall = DocIdSetPlanNode.MAX_DOC_PER_CALL;
@@ -84,59 +83,24 @@ public class SelectionPlanNode implements PlanNode {
// or: SELECT ... FROM Table WHERE predicates ORDER BY sorted_column,
not_sorted LIMIT 10 OFFSET 5
// but not SELECT ... FROM Table WHERE predicates ORDER BY not_sorted,
sorted_column LIMIT 10 OFFSET 5
- if (sortedColumnsPrefixSize == orderByExpressions.size()) {
+ if (sortedColumnsPrefixSize == numOrderByExpressions) {
maxDocsPerCall = Math.min(limit + _queryContext.getOffset(),
DocIdSetPlanNode.MAX_DOC_PER_CALL);
}
- // Whether the scan can iterate in query order, and so stop once the
list builder is full. Ascending always
- // can; descending only when reverse iteration is permitted, because
DocIdSetPlanNode only produces an
- // ascending DocIdSetOperator. getSortedByProject below performs the
actual switch and decides for real; this
- // is a conservative prediction, and being wrong only costs the
optimization, never correctness.
- boolean asc = orderByExpressions.get(0).isAsc();
- boolean canIterateInQueryOrder =
- asc ||
QueryOptionsUtils.isReverseOrderAllowed(_queryContext.getQueryOptions());
-
- // Projecting only the order-by expressions and fetching the rest for
the rows that survive the limit pays off
- // only when the scan cannot stop at the limit. The linear operator
stops as soon as the list builder is full,
- // which with a fully sorted order-by prefix is after limit+offset
documents, so there is nothing to save
- // there. The descending operator has to read the whole match set
because there is no descending cursor.
- boolean scanBoundedByLimit = canIterateInQueryOrder &&
sortedColumnsPrefixSize == orderByExpressions.size();
- boolean lateMaterialize = expressions.size() > numOrderByExpressions &&
!scanBoundedByLimit
- &&
QueryOptionsUtils.isLateMaterializeSortedOrderByEnabled(_queryContext.getQueryOptions());
-
- // Build the project operator once. Constructing one is not free -- it
resolves a DataSource and a
- // ColumnContext per selected column, and some implementations start
prefetching those columns -- so a
- // build-and-discard probe would read exactly the columns this is trying
to avoid reading.
- List<ExpressionContext> expressionsToProject =
- lateMaterialize ? expressions.subList(0, numOrderByExpressions) :
expressions;
BaseProjectOperator<?> projectOperator =
getSortedByProject(expressionsToProject, maxDocsPerCall,
orderByExpressions);
- // Remember that we cannot use asc == projectOperator.isAscending()
because empty operators are considered
- // both ascending and descending
- DocIdOrderedOperator.DocIdOrder queryOrder =
DocIdOrderedOperator.DocIdOrder.fromAsc(asc);
+ // Empty operators support both directions, so check compatibility with
the query order.
+ DocIdOrderedOperator.DocIdOrder queryOrder =
+
DocIdOrderedOperator.DocIdOrder.fromAsc(orderByExpressions.get(0).isAsc());
if (projectOperator.isCompatibleWith(queryOrder)) {
return new SelectionPartiallyOrderedByLinearOperator(_indexSegment,
_queryContext, expressions, projectOperator,
- sortedColumnsPrefixSize, lateMaterialize);
+ sortedColumnsPrefixSize);
} else {
return new SelectionPartiallyOrderedByDescOperation(_indexSegment,
_queryContext, expressions, projectOperator,
- sortedColumnsPrefixSize, lateMaterialize);
+ sortedColumnsPrefixSize);
}
}
- if (numOrderByExpressions == expressions.size()) {
- // All output expressions are ordered
- // ie: SELECT not_sorted1, not_sorted2 FROM Table WHERE ... ORDER BY
not_sorted1, not_sorted2 LIMIT 10 OFFSET 5
- BaseProjectOperator<?> projectOperator =
- new ProjectPlanNode(_segmentContext, _queryContext, expressions,
DocIdSetPlanNode.MAX_DOC_PER_CALL).run();
- return new SelectionOrderByOperator(_indexSegment, _queryContext,
expressions, projectOperator);
- }
- // Not all output expressions are ordered, only fetch the order-by
expressions and docId to avoid the
- // unnecessary data fetch
- // ie: SELECT ... FROM Table WHERE ... ORDER BY not_sorted1, not_sorted2
LIMIT 10
- List<ExpressionContext> expressionsToTransform = new
ArrayList<>(numOrderByExpressions);
- for (OrderByExpressionContext orderByExpression : orderByExpressions) {
- expressionsToTransform.add(orderByExpression.getExpression());
- }
- BaseProjectOperator<?> projectOperator = new
ProjectPlanNode(_segmentContext, _queryContext, expressionsToTransform,
+ BaseProjectOperator<?> projectOperator = new
ProjectPlanNode(_segmentContext, _queryContext, expressionsToProject,
DocIdSetPlanNode.MAX_DOC_PER_CALL).run();
return new SelectionOrderByOperator(_indexSegment, _queryContext,
expressions, projectOperator);
}
@@ -153,8 +117,7 @@ public class SelectionPlanNode implements PlanNode {
try {
return projectOperator.withOrder(DocIdOrderedOperator.DocIdOrder.DESC);
} catch (IllegalArgumentException | UnsupportedOperationException e) {
- // This happens when the operator cannot provide the required order
between blocks
- // Fallback to SelectionOrderByOperator
+ // Keep the ascending project operator for
SelectionPartiallyOrderedByDescOperation.
return projectOperator;
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/InnerSegmentSelectionSingleValueQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/InnerSegmentSelectionSingleValueQueriesTest.java
index 1a35f440ef9..b333f786e9d 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/InnerSegmentSelectionSingleValueQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/InnerSegmentSelectionSingleValueQueriesTest.java
@@ -290,7 +290,7 @@ public class InnerSegmentSelectionSingleValueQueriesTest
extends BaseSingleValue
public void testSelectionOrderBySingleSortedColumnDescDidOrder() {
String orderBy = " ORDER BY column5 DESC";
BaseOperator<SelectionResultsBlock> selectionOrderByOperator = getOperator(
- "SET allowReverseOrder = false; SET lateMaterializeSortedOrderBy =
true; " + SELECTION_QUERY + orderBy);
+ "SET allowReverseOrder = false; " + SELECTION_QUERY + orderBy);
assertTrue(selectionOrderByOperator instanceof
SelectionPartiallyOrderedByDescOperation, "Expected: "
+ SelectionPartiallyOrderedByLinearOperator.class + ", found: " +
selectionOrderByOperator.getClass());
SelectionResultsBlock resultsBlock = selectionOrderByOperator.nextBlock();
@@ -312,17 +312,6 @@ public class InnerSegmentSelectionSingleValueQueriesTest
extends BaseSingleValue
assertEquals(lastRow[0], "gFuH");
}
- /// The optimization is opt-in, so the same query without the option must
still read every selected column
- /// for the whole match set. Pins the default; delete or invert this when
the default flips.
- @Test
- public void testSelectionOrderBySingleSortedColumnDescIsEagerByDefault() {
- BaseOperator<SelectionResultsBlock> operator =
- getOperator("SET allowReverseOrder = false; " + SELECTION_QUERY + "
ORDER BY column5 DESC");
- operator.nextBlock();
- // 30000 docs * 3 selected columns, as before late materialisation existed.
-
assertEquals(operator.getExecutionStatistics().getNumEntriesScannedPostFilter(),
90000L);
- }
-
@Test
public void testSelectionOrderBySingleSortedColumnDescReverseOrder() {
String orderBy = " ORDER BY column5 DESC";
@@ -738,79 +727,45 @@ public class InnerSegmentSelectionSingleValueQueriesTest
extends BaseSingleValue
assertEquals(brokerResponse.getRealtimeThreadMemAllocatedBytes(), 0);
}
- /// Runs `query` with late materialisation on and off and asserts the
answers are identical. column5 is the sorted
- /// column in this fixture.
- ///
- /// @param expectFewerEntries whether late materialisation should read fewer
entries post-filter. It should not when
- /// the scan already stops at the limit, which is
the case for the linear operator with a
- /// fully sorted order-by prefix.
- private void assertSameAnswerWithAndWithoutLateMaterialization(String query,
boolean expectFewerEntries) {
- BaseOperator<SelectionResultsBlock> lateOperator = getOperator("SET
lateMaterializeSortedOrderBy = true; " + query);
- SelectionResultsBlock lateBlock = lateOperator.nextBlock();
- ExecutionStatistics lateStats = lateOperator.getExecutionStatistics();
-
- BaseOperator<SelectionResultsBlock> eagerOperator =
- getOperator("SET lateMaterializeSortedOrderBy = false; " + query);
- SelectionResultsBlock eagerBlock = eagerOperator.nextBlock();
- ExecutionStatistics eagerStats = eagerOperator.getExecutionStatistics();
-
- DataSchema lateSchema = lateBlock.getDataSchema();
- DataSchema eagerSchema = eagerBlock.getDataSchema();
- assertEquals(lateSchema.getColumnNames(), eagerSchema.getColumnNames());
- assertEquals(lateSchema.getColumnDataTypes(),
eagerSchema.getColumnDataTypes());
-
- List<Object[]> lateRows = lateBlock.getRows();
- List<Object[]> eagerRows = eagerBlock.getRows();
- assertEquals(lateRows.size(), eagerRows.size());
- for (int i = 0; i < lateRows.size(); i++) {
- assertEquals(lateRows.get(i), eagerRows.get(i), "row " + i + " differs
for query: " + query);
- }
- assertEquals(lateStats.getNumDocsScanned(),
eagerStats.getNumDocsScanned());
-
- long late = lateStats.getNumEntriesScannedPostFilter();
- long eager = eagerStats.getNumEntriesScannedPostFilter();
- if (expectFewerEntries) {
- assertTrue(late < eager, "expected fewer entries post-filter, but read "
+ late + " vs " + eager
- + " for query: " + query);
- } else {
- assertEquals(late, eager, "expected the same entries post-filter for
query: " + query);
- }
- }
-
- /// The linear operator stops at the limit, so there is nothing to defer and
the plan keeps projecting everything.
- @Test
- public void testLateMaterializationSkippedWhenScanStopsAtLimit() {
- assertSameAnswerWithAndWithoutLateMaterialization(SELECTION_QUERY + "
ORDER BY column5", false);
- assertSameAnswerWithAndWithoutLateMaterialization(
- "SET allowReverseOrder = true; " + SELECTION_QUERY + " ORDER BY
column5 DESC", false);
- }
-
- /// Descending without a reverse cursor reads the whole match set, so
deferring the other columns is a real saving.
- @Test
- public void testLateMaterializationSortedColumnDesc() {
- assertSameAnswerWithAndWithoutLateMaterialization(
- "SET allowReverseOrder = false; " + SELECTION_QUERY + " ORDER BY
column5 DESC", true);
- }
-
- @Test
- public void testLateMaterializationWithOffsetAndLimit() {
- assertSameAnswerWithAndWithoutLateMaterialization(
- "SET allowReverseOrder = false; " + SELECTION_QUERY + " ORDER BY
column5 DESC LIMIT 5 OFFSET 3", true);
- }
-
- /// An unsorted tail in the order-by removes the limit bound on the scan, so
the linear operator benefits too. The
- /// unsorted order-by column is still projected during the scan; only the
remaining output columns are deferred.
@Test
public void testLateMaterializationWithUnsortedOrderBySuffix() {
- assertSameAnswerWithAndWithoutLateMaterialization(SELECTION_QUERY + "
ORDER BY column5, column6", true);
+ BaseOperator<SelectionResultsBlock> operator = getOperator(
+ "SELECT column5, column1, ADD(column1, 1) FROM testTable ORDER BY
column5, column6, column1");
+ assertTrue(operator instanceof SelectionPartiallyOrderedByLinearOperator);
+ SelectionResultsBlock block = operator.nextBlock();
+ assertEquals(block.getDataSchema().getColumnNames(),
+ new String[]{"column5", "column6", "column1", "add(column1,'1')"});
+ assertEquals(block.getDataSchema().getColumnDataTypes(),
+ new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT,
ColumnDataType.INT, ColumnDataType.DOUBLE});
+ List<Object[]> rows = block.getRows();
+ assertEquals(rows.size(), 10);
+ for (Object[] row : rows) {
+ assertEquals(row[3], ((Integer) row[2]).doubleValue() + 1);
+ }
+ assertEquals(rows.get(9), new Object[]{"gFuH", 6043515, 10542595,
10542596.0});
+ assertEquals(operator.getExecutionStatistics().getNumDocsScanned(),
30000L);
+ // The transform rereads column1 for the selected rows after scanning the
three order-by columns.
+
assertEquals(operator.getExecutionStatistics().getNumEntriesScannedPostFilter(),
90010L);
}
- /// The late fetch builds its own TransformOperator, so a non-order-by
expression that is a function must resolve,
- /// both for its values and for its data type in the schema.
@Test
public void testLateMaterializationWithTransformProjection() {
- assertSameAnswerWithAndWithoutLateMaterialization(
+ BaseOperator<SelectionResultsBlock> operator = getOperator(
"SET allowReverseOrder = false; SELECT column5, ADD(column1, 1),
column11 FROM testTable "
- + "ORDER BY column5 DESC", true);
+ + "ORDER BY column5 DESC, column1, column11 LIMIT 5 OFFSET 3");
+ assertTrue(operator instanceof SelectionPartiallyOrderedByDescOperation);
+ SelectionResultsBlock block = operator.nextBlock();
+ assertEquals(block.getDataSchema().getColumnNames(),
+ new String[]{"column5", "column1", "column11", "add(column1,'1')"});
+ assertEquals(block.getDataSchema().getColumnDataTypes(),
+ new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT,
ColumnDataType.STRING, ColumnDataType.DOUBLE});
+ List<Object[]> rows = block.getRows();
+ // Segment operators retain limit + offset rows; the broker applies the
offset.
+ assertEquals(rows.size(), 8);
+ for (Object[] row : rows) {
+ assertEquals(row[3], ((Integer) row[1]).doubleValue() + 1);
+ }
+ assertEquals(operator.getExecutionStatistics().getNumDocsScanned(),
30000L);
+
assertEquals(operator.getExecutionStatistics().getNumEntriesScannedPostFilter(),
90008L);
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
index 61faccf6d37..c4a15643494 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
@@ -27,6 +27,10 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.response.broker.ResultTable;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.core.operator.BaseOperator;
+import org.apache.pinot.core.operator.blocks.results.SelectionResultsBlock;
+import
org.apache.pinot.core.operator.query.SelectionPartiallyOrderedByDescOperation;
import org.apache.pinot.core.plan.DocIdSetPlanNode;
import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.indexsegment.mutable.MutableSegmentImpl;
@@ -2103,11 +2107,13 @@ public class NullHandlingEnabledQueriesTest extends
BaseQueriesTest {
_indexSegment.destroy();
}
- /// Late materialisation fetches the non-order-by columns in a second pass,
so their null bitmaps come from a
- /// different block than the order-by column's. An off-by-one there would
silently drop or invent nulls, and the
- /// answer must not depend on whether the optimization ran.
- @Test
- public void testLateMaterializationPreservesNullsInDeferredColumns()
+ @DataProvider
+ public Object[][] sortedOrderByDirections() {
+ return new Object[][]{{true, false}, {false, false}, {false, true}};
+ }
+
+ @Test(dataProvider = "sortedOrderByDirections")
+ public void testLateMaterializationPreservesNullsInDeferredColumns(boolean
ascending, boolean allowReverseOrder)
throws Exception {
initializeRows();
insertRowWithTwoColumns(1, null);
@@ -2118,22 +2124,51 @@ public class NullHandlingEnabledQueriesTest extends
BaseQueriesTest {
.addSingleValueDimension(COLUMN1, DataType.INT)
.addSingleValueDimension(COLUMN2, DataType.INT)
.build();
- // column1 is sorted and has no nulls, so the sorted-prefix path is used;
column2 is the deferred column
setUpSegments(TABLE_CONFIG_WITH_SORTED_COLUMN, schema);
- String query =
- String.format("SELECT %s, %s FROM testTable ORDER BY %s DESC LIMIT 8",
COLUMN1, COLUMN2, COLUMN1);
- List<Object[]> lateRows = getRows("SET lateMaterializeSortedOrderBy =
true; " + query);
- List<Object[]> eagerRows = getRows("SET lateMaterializeSortedOrderBy =
false; " + query);
+ String direction = ascending ? "ASC" : "DESC";
+ String query = String.format(
+ "SET allowReverseOrder = %s; SELECT %s, %s FROM testTable ORDER BY %s
%s LIMIT 8 OFFSET 4",
+ allowReverseOrder, COLUMN1, COLUMN2, COLUMN1, direction);
+ List<Object[]> rows = getRows(query);
+ assertEquals(rows.size(), 8);
+ for (int i = 0; i < rows.size(); i++) {
+ int value = ascending ? 2 + i / NUM_OF_SEGMENT_COPIES : 3 - i /
NUM_OF_SEGMENT_COPIES;
+ assertEquals(rows.get(i), new Object[]{value, value == 2 ? 20 : null});
+ }
+ }
+
+ @Test
+ public void testLateMaterializationAcrossBlocksWithSparseDocIds()
+ throws Exception {
+ initializeRows();
+ for (int i = 0; i < 22010; i++) {
+ insertRowWithTwoColumns(i, i % 3 == 0 ? null : i * 10);
+ }
+ Schema schema = schemaBuilder()
+ .addSingleValueDimension(COLUMN1, DataType.INT)
+ .addSingleValueDimension(COLUMN2, DataType.INT)
+ .build();
+ setUpSegments(TABLE_CONFIG_WITH_SORTED_COLUMN, schema);
- assertEquals(lateRows.size(), eagerRows.size());
- for (int i = 0; i < lateRows.size(); i++) {
- assertEquals(lateRows.get(i), eagerRows.get(i), "row " + i + " differs");
+ BaseOperator<SelectionResultsBlock> operator = getOperator(
+ "SET enableNullHandling = true; SET allowReverseOrder = false; "
+ + "SELECT column1, column2, ADD(column2, 1) FROM testTable WHERE
MOD(column1, 2) = 0 "
+ + "ORDER BY column1 DESC LIMIT 10001 OFFSET 3");
+ assertTrue(operator instanceof SelectionPartiallyOrderedByDescOperation);
+ SelectionResultsBlock block = operator.nextBlock();
+ assertEquals(block.getDataSchema().getColumnNames(), new
String[]{"column1", "column2", "add(column2,'1')"});
+ assertEquals(block.getDataSchema().getColumnDataTypes(),
+ new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT,
ColumnDataType.DOUBLE});
+ List<Object[]> rows = block.getRows();
+ // Both the initial scan and the deferred fetch span the 10000-document
block boundary.
+ assertEquals(rows.size(), 10004);
+ for (int i = 0; i < rows.size(); i++) {
+ int value = 22008 - 2 * i;
+ assertEquals(rows.get(i),
+ new Object[]{value, value % 3 == 0 ? null : value * 10, value % 3 ==
0 ? null : value * 10.0 + 1});
}
- // The top rows are column1=4 (column2=40) across the segment copies, then
column1=3, whose column2 is null
- assertEquals(lateRows.get(0)[0], 4);
- assertEquals(lateRows.get(0)[1], 40);
- assertEquals(lateRows.get(lateRows.size() - 1)[0], 3);
- assertNull(lateRows.get(lateRows.size() - 1)[1], "the deferred column must
keep its null");
+ assertEquals(operator.getExecutionStatistics().getNumDocsScanned(),
11005L);
+
assertEquals(operator.getExecutionStatistics().getNumEntriesScannedPostFilter(),
11005L + 10004L);
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/TextMatchTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/TextMatchTransformFunctionTest.java
index 18c59ef2254..c237cafced8 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/TextMatchTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/TextMatchTransformFunctionTest.java
@@ -351,14 +351,12 @@ public class TextMatchTransformFunctionTest {
+ "SELECT_PARTIAL_ORDER_BY_LINEAR("
+ "sortedList: (startTime), unsortedList: (), rest:
(case(is_null(agent),"
+
"'N/A',and(text_match(part,'_zz_'),is_not_null(part)),agent,''))) | 3 | 2\n"
- +
"TRANSFORM(case(is_null(agent),'N/A',and(text_match(part,'_zz_'),is_not_null(part)),agent,''),
"
- + "startTime) | 4 | 3\n"
- + "PROJECT(agent, startTime, part) | 5 | 4\n"
- + "DOC_ID_SET | 6 | 5\n"
- + "FILTER_AND | 7 | 6\n"
- +
"FILTER_INVERTED_INDEX(indexLookUp:inverted_index,operator:EQ,predicate:customerId
= 'XYZ') | 8 | 7\n"
+ + "PROJECT(startTime) | 4 | 3\n"
+ + "DOC_ID_SET | 5 | 4\n"
+ + "FILTER_AND | 6 | 5\n"
+ +
"FILTER_INVERTED_INDEX(indexLookUp:inverted_index,operator:EQ,predicate:customerId
= 'XYZ') | 7 | 6\n"
+
"FILTER_RANGE_INDEX(indexLookUp:range_index,operator:RANGE,predicate:startTime
BETWEEN '0' AND "
- + "'1000000') | 9 | 7"
+ + "'1000000') | 8 | 6"
).whenQuery(query)
.thenResultTextIs("val[STRING]\n"
+ "N/A\n"
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 6e935bb3cec..363490d95b6 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1058,16 +1058,6 @@ public class CommonConstants {
public static final String ALLOW_REVERSE_ORDER = "allowReverseOrder";
public static final boolean DEFAULT_ALLOW_REVERSE_ORDER = false;
- /// Selection with an order-by prefix on a sorted column: project only
the order-by expressions during the
- /// scan and fetch the remaining output expressions for the rows that
survive the limit. Enabling it
- /// avoids reading every selected column for the whole match set.
- ///
- /// Off by default while the optimization bakes, following
`allowReverseOrder` and
- /// `usePhysicalOptimizer`. It also changes the observable
`numEntriesScannedPostFilter` on the queries it
- /// applies to, so turning it on is a deliberate choice. Note the win
shrinks as `limit + offset`
- /// approaches the match-set size, where the second pass reads what
the scan would have read anyway.
- public static final String LATE_MATERIALIZE_SORTED_ORDER_BY =
"lateMaterializeSortedOrderBy";
- public static final boolean DEFAULT_LATE_MATERIALIZE_SORTED_ORDER_BY =
false;
public static final String MULTI_STAGE_LEAF_LIMIT =
"multiStageLeafLimit";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
