xiangfu0 commented on code in PR #19311: URL: https://github.com/apache/pinot/pull/19311#discussion_r4486997817
########## pinot-query-planner/src/main/java/org/apache/pinot/query/validate/MatchRecognizeValidator.java: ########## @@ -0,0 +1,682 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.validate; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.function.UnaryOperator; +import javax.annotation.Nullable; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlUnresolvedFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/// Pinot specific validation and rewriting of SQL:2016 `MATCH_RECOGNIZE` (row pattern recognition) clauses. +/// +/// This visitor runs on the **raw parsed** [SqlNode] tree, before Calcite's own validation and before +/// [org.apache.calcite.sql2rel.SqlToRelConverter] runs. Running this early is deliberate and load bearing: +/// +/// 1. **The `AFTER MATCH` default must be corrected at the `SqlNode` level.** When the `AFTER MATCH` clause is +/// omitted, the parser leaves [SqlMatchRecognize#getAfter()] as `null`, and `SqlToRelConverter` then silently +/// substitutes [SqlMatchRecognize.AfterOption#SKIP_TO_NEXT_ROW]. SQL:2016 (and Trino, Snowflake and Oracle) all +/// default to `SKIP PAST LAST ROW`. The two differ in whether matches may overlap, so a query ported from +/// another engine would silently return different rows. Conversion erases the omitted-vs-explicit distinction, +/// so the fix has to happen here. See [#applyStandardAfterMatchDefault]. +/// 2. **Unsupported constructs must be rejected before conversion.** Several of them (`ORDER BY` / `PARTITION BY` +/// on arbitrary expressions in particular) blow up inside `SqlToRelConverter` with an `AssertionError` or +/// `ClassCastException` rather than a usable message. +/// 3. **`RUNNING` in `DEFINE` must be normalized before Calcite validation.** SQL:2016 and Trino allow it (and it is +/// the default processing mode for a definition), but Calcite 1.42 rejects both `RUNNING` and `FINAL` there. +/// [#validateDefinitions] removes the redundant `RUNNING` wrapper and rejects `FINAL`, whose future-row +/// semantics cannot be evaluated while a match is being built. +/// 4. **Unquoted pattern variables must be case-insensitive.** Pinot deliberately preserves the spelling of +/// unquoted identifiers in its parser, while Calcite stores pattern variables in exact-case string maps. Without +/// an early rewrite, `PATTERN (A) DEFINE a AS a.price > 0` either creates two symbols or fails to bind. See +/// [#normalizePatternVariableNames]. +/// 5. **Pattern variables and the row source alias need separate internal names.** SQL gives those names separate +/// namespaces inside MATCH_RECOGNIZE, but Calcite reuses the row source alias as the alpha of the universal row +/// pattern variable. [#isolateRowSourceAlias] gives a colliding input an internal alias before Calcite validation, +/// so explicit pattern references and unqualified universal references remain distinguishable. +/// 6. **`PREV` in `DEFINE` must be anchored before Calcite validation.** Calcite rewrites `PREV(x, n)` into the +/// logical `LAST(x, n)` unless `x` is qualified by the variable being defined, which silently changes the rows an +/// unqualified or cross-variable `PREV` reads. See [#anchorAtLastRow]. +/// +/// Everything this class rejects is deferred, not permanently unsupported; each message names the construct so the +/// user can rewrite the query. +/// +/// This class is stateless apart from the tree it is walking and is **not** thread-safe; create a new instance per +/// query, exactly like [RowExpressionValidationVisitor]. +public class MatchRecognizeValidator extends SqlBasicVisitor<Void> { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchRecognizeValidator.class); + + private static final String PERMUTE = "PERMUTE"; + private static final String PERMUTE_NOT_SUPPORTED = + "PERMUTE is not supported yet in the MATCH_RECOGNIZE PATTERN clause. Expand the permutation into an explicit " + + "alternation, e.g. PATTERN ((A B) | (B A))."; + + /// Whether the node currently being visited sits inside a MEASURES or DEFINE list. `RUNNING` wrappers in DEFINE + /// have already been removed and `FINAL` rejected by [#validateDefinitions] before this traversal. + private boolean _inMeasureOrDefine; + private boolean _matchRecognizeFound; + + /// Whether the visited SQL tree contains at least one MATCH_RECOGNIZE clause. + public boolean hasMatchRecognize() { + return _matchRecognizeFound; + } + + @Override + public Void visit(SqlCall call) { + if (call instanceof SqlMatchRecognize) { + visitMatchRecognize((SqlMatchRecognize) call); + return null; + } + SqlKind kind = call.getKind(); + if ((kind == SqlKind.RUNNING || kind == SqlKind.FINAL) && !_inMeasureOrDefine) { + // The RUNNING / FINAL prefix operators are registered for MATCH_RECOGNIZE, but unlike PREV / NEXT / FIRST / + // LAST / CLASSIFIER / MATCH_NUMBER, Calcite does not confine them to a MatchRecognizeScope. Reject them here + // so they cannot leak into an ordinary projection. + String legalClause = kind == SqlKind.FINAL ? "MEASURES" : "MEASURES or DEFINE"; + throw unsupported(kind + " can only be used in the " + legalClause + " clause of a MATCH_RECOGNIZE query: '" + + call + "'."); + } + return super.visit(call); + } + + private void visitMatchRecognize(SqlMatchRecognize match) { + _matchRecognizeFound = true; + validateAndRewrite(match); + // Only MEASURES and DEFINE may contain RUNNING / FINAL. Every other operand has already been checked by + // validateAndRewrite(), so the only one left to descend into is the table reference, which may itself hold a + // nested MATCH_RECOGNIZE. + visitIn(match.getMeasureList(), true); + visitIn(match.getPatternDefList(), true); + visitIn(match.getTableRef(), false); + } + + private void visitIn(SqlNode node, boolean inMeasureOrDefine) { + boolean saved = _inMeasureOrDefine; + _inMeasureOrDefine = inMeasureOrDefine; + try { + node.accept(this); + } finally { + _inMeasureOrDefine = saved; + } + } + + private void validateAndRewrite(SqlMatchRecognize match) { + normalizePatternVariableNames(match); + rejectUnsupportedClauses(match); + validatePartitionBy(match.getPartitionList()); + validateOrderBy(match.getOrderList()); + Set<String> definedVariables = definedVariables(match.getPatternDefList()); + validatePattern(match.getPattern(), definedVariables); + validateDefinitions(match.getPatternDefList()); + // Deliberately last: a query that also uses a deferred construct should be told about that construct first. + validateMeasures(match); + isolateRowSourceAlias(match, definedVariables); + applyStandardAfterMatchDefault(match); + logUndefinedPatternVariables(match, definedVariables); + } + + /// Canonicalizes every unquoted pattern variable to upper case while retaining quoted identifiers verbatim. + /// + /// Pinot configures its parser with `Casing.UNCHANGED`, but Calcite's MATCH_RECOGNIZE implementation keys PATTERN, + /// DEFINE, pattern-field references, and AFTER MATCH targets by exact [String] equality. SQL identifiers do not + /// have those semantics: unquoted names are case-insensitive, while quoted names are case-sensitive. Rewriting the + /// raw tree gives every downstream Calcite map one stable spelling and preserves each component's quoted parser + /// position so `"a"` remains distinct from unquoted `a` (canonical `A`). + private void normalizePatternVariableNames(SqlMatchRecognize match) { + match.setOperand(SqlMatchRecognize.OPERAND_PATTERN, normalizePattern(match.getPattern())); + + SqlNodeList definitions = match.getPatternDefList(); + for (SqlNode definition : definitions) { + SqlNode condition = patternDefinitionCondition(definition); + SqlIdentifier variable = patternDefinitionVariable(definition); + if (condition != null) { + ((SqlCall) definition).setOperand(0, normalizePatternReferences(condition)); + } + if (variable != null && variable.isSimple()) { + ((SqlCall) definition).setOperand(1, normalizeIdentifierComponent(variable, 0)); + } + } + + SqlNodeList measures = match.getMeasureList(); + for (int i = 0; i < measures.size(); i++) { + SqlNode measure = measures.get(i); + SqlNode expression = asOperand(measure, 0); + if (expression != null) { + ((SqlCall) measure).setOperand(0, normalizePatternReferences(expression)); + } else { + measures.set(i, normalizePatternReferences(measure)); + } + } + + SqlNode after = match.getAfter(); + if (after != null) { + match.setOperand(SqlMatchRecognize.OPERAND_AFTER, normalizePattern(after)); + } + } + + /// Rewrites the identifiers in the PATTERN grammar, where every identifier is a pattern variable. + private SqlNode normalizePattern(SqlNode node) { + if (node instanceof SqlIdentifier) { + SqlIdentifier identifier = (SqlIdentifier) node; + return identifier.isSimple() ? normalizeIdentifierComponent(identifier, 0) : identifier; + } + if (!(node instanceof SqlCall)) { + return node; + } + SqlCall call = (SqlCall) node; + List<SqlNode> operands = call.getOperandList(); + for (int i = 0; i < operands.size(); i++) { + SqlNode operand = operands.get(i); + if (operand != null) { + call.setOperand(i, normalizePattern(operand)); + } + } + return call; + } + + /// Rewrites only the alpha of a qualified pattern-field reference. A simple identifier is an input column, not a + /// pattern variable, so changing it here would also change column-name resolution. + private SqlNode normalizePatternReferences(SqlNode node) { + if (node instanceof SqlIdentifier) { + SqlIdentifier identifier = (SqlIdentifier) node; + return identifier.names.size() > 1 ? normalizeIdentifierComponent(identifier, 0) : identifier; + } + if (node instanceof SqlNodeList) { + return rewriteElements((SqlNodeList) node, this::normalizePatternReferences); + } + if (!(node instanceof SqlCall)) { + return node; + } + SqlCall call = (SqlCall) node; + List<SqlNode> operands = call.getOperandList(); + for (int i = 0; i < operands.size(); i++) { + SqlNode operand = operands.get(i); + if (operand != null) { + call.setOperand(i, normalizePatternReferences(operand)); + } + } + return call; + } + + private static SqlIdentifier normalizeIdentifierComponent(SqlIdentifier identifier, int component) { + // Calcite's single-name SqlIdentifier constructor stores quoting only on the node position and leaves + // componentPositions null; compound identifiers retain per-component positions. Consult both representations. + boolean quoted = identifier.isComponentQuoted(component) + || (identifier.isSimple() && identifier.getParserPosition().isQuoted()); + if (quoted) { + return identifier; + } + return identifier.setName(component, canonicalIdentifierComponent(identifier, component)); + } + + private static String canonicalIdentifierComponent(SqlIdentifier identifier, int component) { + boolean quoted = identifier.isComponentQuoted(component) + || (identifier.isSimple() && identifier.getParserPosition().isQuoted()); + String name = identifier.names.get(component); + return quoted ? name : name.toUpperCase(Locale.ROOT); + } + + /// Rewrites an omitted `AFTER MATCH` clause into an explicit `AFTER MATCH SKIP PAST LAST ROW`. + /// + /// Calcite's `SqlToRelConverter` defaults a `null` `AFTER` operand to `SKIP TO NEXT ROW`, which produces + /// overlapping matches. SQL:2016 mandates `SKIP PAST LAST ROW` (non-overlapping matches), and that is what + /// Trino, Snowflake and Oracle do. An explicitly written `AFTER MATCH SKIP TO NEXT ROW` is left untouched: the + /// parser only leaves the operand `null` when the clause is absent from the query text, so the omitted and the + /// explicit case are perfectly distinguishable here. + private void applyStandardAfterMatchDefault(SqlMatchRecognize match) { + if (match.getAfter() != null) { + return; + } + match.setOperand(SqlMatchRecognize.OPERAND_AFTER, + SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW.symbol(match.getParserPosition())); + } + + private void rejectUnsupportedClauses(SqlMatchRecognize match) { + SqlLiteral rowsPerMatch = match.getRowsPerMatch(); + if (rowsPerMatch != null + && rowsPerMatch.getValue() == SqlMatchRecognize.RowsPerMatchOption.ALL_ROWS) { + throw unsupported("ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the " + + "default) and expose the per-match values you need through the MEASURES clause."); + } + if (!isEmpty(match.getSubsetList())) { + throw unsupported("SUBSET is not supported yet in MATCH_RECOGNIZE. Remove the SUBSET clause and reference the " + + "individual pattern variables directly."); + } + if (match.getInterval() != null) { + throw unsupported("WITHIN is not supported yet in MATCH_RECOGNIZE. Remove the WITHIN clause and bound the " + + "match with a predicate in the DEFINE clause instead."); + } + } + + /// Rejects the two `MEASURES` shapes whose Calcite row type is not `partition columns ++ measures`, which is what + /// `MatchNode` and `MatchOperator` assume. Both plan without complaint today and only fail on the server, with a + /// message describing an internal invariant rather than the user's mistake. + /// + /// 1. **No `MEASURES` clause at all.** `SqlValidatorImpl.validateMatchRecognize` ends with + /// `if (measureList.size() == 0) ns.setType(<the table reference row type>)`, i.e. the whole *input* row type, + /// which is not the `ONE ROW PER MATCH` shape at all. `MEASURES` is optional in SQL:2016 and in Trino, so this + /// is ordinary valid SQL; we do not own the Calcite code, so rejecting is the only sound option until + /// measures-less `MATCH_RECOGNIZE` is implemented properly. + /// 2. **An alias that collides with a `PARTITION BY` column or with an earlier measure alias.** Calcite adds each + /// measure to the row type only `if (!typeBuilder.nameExists(name))`, so a colliding alias is dropped + /// altogether and the query plans with a column missing. + /// + /// The name comparison is case-sensitive on purpose, because `RelDataTypeFactory.Builder#nameExists` is: + /// `PARTITION BY col1 ... MEASURES LAST(x) AS COL1` really does produce two distinct columns and must keep working. + private void validateMeasures(SqlMatchRecognize match) { + SqlNodeList measureList = match.getMeasureList(); + if (isEmpty(measureList)) { + throw unsupported("MATCH_RECOGNIZE requires a MEASURES clause. ONE ROW PER MATCH emits the PARTITION BY " + + "columns plus the MEASURES, so add at least one measure, e.g. MEASURES MATCH_NUMBER() AS mno."); + } + Set<String> names = new LinkedHashSet<>(); + for (SqlNode partitionKey : match.getPartitionList()) { + // validatePartitionBy() already established that every partition key is a plain identifier. + names.add(lastName((SqlIdentifier) partitionKey)); + } + for (SqlNode measure : measureList) { + SqlNode alias = asOperand(measure, 1); + if (alias instanceof SqlIdentifier && ((SqlIdentifier) alias).isSimple() + && !names.add(((SqlIdentifier) alias).getSimple())) { + throw unsupported("MATCH_RECOGNIZE measure alias '" + ((SqlIdentifier) alias).getSimple() + "' collides with " + + "a PARTITION BY column or an earlier measure alias. Calcite drops the duplicate from the output row " + + "type, which would silently lose a column; rename the measure."); + } + } + } + + /// Gives the row source an internal alias when its SQL name equals a pattern variable. + /// + /// SQL places the input relation and pattern variables in separate namespaces: with input alias `A` and pattern + /// variable `A`, a qualifier in PARTITION BY / ORDER BY names the input, a qualifier in MEASURES / DEFINE names the + /// pattern variable, and an unqualified measure or definition column uses the universal row pattern variable. + /// + /// Calcite does not retain that distinction: it qualifies an unqualified column in MEASURES / DEFINE with the row + /// source alias and later represents both forms as `RexPatternFieldRef(alpha, column)`. Re-aliasing the input before + /// Calcite sees the tree gives universal references an alpha that cannot equal a pattern variable. Qualified input + /// references in PARTITION BY / ORDER BY are rewritten with it; pattern references deliberately are not. + private void isolateRowSourceAlias(SqlMatchRecognize match, Set<String> definedVariables) { + SqlNode tableRef = match.getTableRef(); + SqlIdentifier aliasIdentifier = rowSourceAliasIdentifier(tableRef); + if (aliasIdentifier == null) { + // A join or a sub-query with no explicit alias; Calcite does not qualify anything with a row source alias. + return; Review Comment: Fixed: every unaliased derived input (sub-query, UNION, VALUES, LATERAL and so on) now gets the internal alias, because the `EXPR$n` name Calcite generates depends on the rest of the statement, and hinted tables get an explicit one. Your example now returns 20 instead of 10 in the new `match_recognize_row_source_alias` case in MatchRecognize.json. MatchRecognizeValidatorTest covers the alias each input shape gets, and MatchNodeConverterTest covers the binding. _🤖 Addressed by [Claude Code](https://claude.com/claude-code)_ ########## pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MatchOperator.java: ########## @@ -0,0 +1,423 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.common.datatable.StatMap; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.runtime.blocks.MseBlock; +import org.apache.pinot.query.runtime.blocks.RowHeapDataBlock; +import org.apache.pinot.query.runtime.operator.match.MatchExpression; +import org.apache.pinot.query.runtime.operator.match.MatchLimits; +import org.apache.pinot.query.runtime.operator.match.MatchTape; +import org.apache.pinot.query.runtime.operator.match.PartitionMatcher; +import org.apache.pinot.query.runtime.operator.match.PatternNfa; +import org.apache.pinot.query.runtime.operator.match.PatternToNfaCompiler; +import org.apache.pinot.query.runtime.operator.utils.TypeUtils; +import org.apache.pinot.query.runtime.plan.OpChainExecutionContext; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/// Evaluates SQL:2016 `MATCH_RECOGNIZE` (row pattern recognition) with `ONE ROW PER MATCH`. +/// +/// ## What it does per partition +/// +/// For every `PARTITION BY` partition, in `ORDER BY` order, it walks a scan position from the first row to +/// the last. At each position it asks [PartitionMatcher] for the preferred match starting exactly there. On a +/// match it emits one row - the partition key columns followed by the `MEASURES` - and then moves the scan +/// position according to the `AFTER MATCH SKIP` mode. On no match it moves one row forward. +/// +/// ## What it expects from the plan +/// +/// Like [WindowAggregateOperator], this operator does not sort. `PinotMatchExchangeNodeInsertRule` puts a +/// sort exchange underneath that hash distributes on the partition keys and sorts the receiver side on +/// `(partitionKeys..., orderKeys...)`, so rows arrive grouped by partition and ordered within a partition. The +/// operator therefore buffers one partition at a time and releases it at each boundary, and never reads +/// [MatchNode#getCollations()]: the ordering has already been established below it. +/// +/// The grouping half of that assumption is verified rather than trusted. Partition keys are compared directly on the +/// incoming rows and must be monotonically increasing, so a key that reappears after its partition was closed fails +/// without retaining every previously seen key. The ordering half is not re-checked per row, because an exchange that +/// grouped correctly but sorted incorrectly is not a failure mode the exchange can produce - losing the sort loses the +/// grouping too, which the partition-key order check already catches. +/// +/// ## Guardrails throw, they never truncate +/// +/// [QueryOptionKey#MAX_ROWS_IN_MATCH_PARTITION] bounds the rows buffered for a partition and +/// [QueryOptionKey#MAX_STEPS_PER_MATCH_ATTEMPT] bounds the backtracking of one match attempt. Both raise an error, +/// because a truncated pattern result is a wrong result that nothing in the response would flag. +/// +/// ## Not supported yet +/// +/// `ALL ROWS PER MATCH` is rejected here as well as during planning: this operator emits exactly one row per +/// match, so accepting it would silently return the wrong shape of result. +public class MatchOperator extends MultiStageOperator { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchOperator.class); + private static final String EXPLAIN_NAME = "MATCH_RECOGNIZE"; + static final int MAX_OUTPUT_ROWS_PER_BLOCK = 1024; + + private final MultiStageOperator _input; + private final DataSchema _resultSchema; + private final ColumnDataType[] _resultStoredTypes; + private final int[] _partitionKeys; + private final int[] _partitionComparisonKeys; + private final List<PatternSymbol> _patternSymbols; + private final MatchExpression[] _measures; + private final PartitionMatcher _matcher; + private final MatchNode.AfterMatchSkipMode _skipMode; + private final int _skipToSymbolOrdinal; + private final int _maxRowsInMatchPartition; + private final StatMap<StatKey> _statMap = new StatMap<>(StatKey.class); + + private List<Object[]> _partitionRows = new ArrayList<>(); + @Nullable + private List<Object[]> _inputRows; + private int _inputRowIndex; + private boolean _matchingPartition; + private int _scanStart; + private long _matchNumber; + @Nullable + private MseBlock.Eos _inputEos; + private int _numRows; + + public MatchOperator(OpChainExecutionContext context, MultiStageOperator input, DataSchema inputSchema, + MatchNode node) { + super(context); + if (node.getRowsPerMatchMode() != MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the default) and " + + "expose the per-match values you need through the MEASURES clause."); + } + _input = input; + _resultSchema = node.getDataSchema(); + _resultStoredTypes = _resultSchema.getStoredColumnDataTypes(); + List<Integer> partitionKeys = node.getPartitionKeys(); + _partitionKeys = new int[partitionKeys.size()]; + for (int i = 0; i < _partitionKeys.length; i++) { + _partitionKeys[i] = partitionKeys.get(i); + ColumnDataType partitionKeyType = inputSchema.getColumnDataType(_partitionKeys[i]); + if (partitionKeyType.isArray()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY requires single-value columns, but column '" + + inputSchema.getColumnName(_partitionKeys[i]) + "' has type " + partitionKeyType); + } + } + // Calcite exposes the exchange distribution as an ImmutableBitSet, so the rule below this operator sorts + // partition fields by input ordinal. Preserve SQL declaration order in _partitionKeys for the output schema, but + // validate monotonic input against the order the exchange actually produces. + _partitionComparisonKeys = _partitionKeys.clone(); + Arrays.sort(_partitionComparisonKeys); + _patternSymbols = node.getPatternSymbols(); + List<MatchNode.Measure> measures = node.getMeasures(); + _measures = new MatchExpression[measures.size()]; + for (int i = 0; i < _measures.length; i++) { + _measures[i] = MatchExpression.compile(measures.get(i).getExpression(), inputSchema); + } + if (_resultSchema.size() != _partitionKeys.length + _measures.length) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE output schema " + _resultSchema + " does not match " + _partitionKeys.length + + " partition key(s) plus " + _measures.length + " measure(s)"); + } + _skipMode = node.getAfterMatchSkipMode(); + _skipToSymbolOrdinal = node.getAfterMatchSkipToSymbolOrdinal(); + + PatternNfa nfa = PatternToNfaCompiler.compile(node.getPattern()); + _maxRowsInMatchPartition = + MatchLimits.getMaxRowsInMatchPartition(context.getOpChainMetadata(), node.getNodeHint()); + long maxStepsPerMatchAttempt = + MatchLimits.getMaxStepsPerMatchAttempt(context.getOpChainMetadata(), node.getNodeHint()); + _matcher = new PartitionMatcher(nfa, _patternSymbols, inputSchema, maxStepsPerMatchAttempt, + this::checkTerminationAndSampleUsage); + } + + @Override + protected Logger logger() { + return LOGGER; + } + + @Override + public List<MultiStageOperator> getChildOperators() { + return List.of(_input); + } + + @Override + public Type getOperatorType() { + return Type.MATCH; + } + + @Override + public String toExplainString() { + return EXPLAIN_NAME; + } + + @Override + public void registerExecution(long time, int numRows, long memoryUsedBytes, long gcTimeMs) { + _statMap.merge(StatKey.EXECUTION_TIME_MS, time); + _statMap.merge(StatKey.EMITTED_ROWS, numRows); + _statMap.merge(StatKey.ALLOCATED_MEMORY_BYTES, memoryUsedBytes); + _statMap.merge(StatKey.GC_TIME_MS, gcTimeMs); + } + + @Override + public StatMap<StatKey> copyStatMaps() { + return new StatMap<>(_statMap); + } + + @Override + protected MseBlock getNextBlock() { + List<Object[]> outputRows = new ArrayList<>(MAX_OUTPUT_ROWS_PER_BLOCK); + while (outputRows.size() < MAX_OUTPUT_ROWS_PER_BLOCK) { + if (_matchingPartition) { + emitMatches(outputRows); + continue; Review Comment: Fixed: MATCH checks `_isEarlyTerminated` at the start of every call, drops its buffered state, drains the input and returns the input's own terminal block, so upstream errors and stats still come through. It also returns a block early once it holds a row and has spent 2^18 more matcher steps. `testPartialBlockLetsAnOuterLimitStopTheOperatorBeforeAnExpensiveSuffix` runs `PATTERN (A | B+ C)` under a real `LIMIT 1`, with a long non-matching run between the A match and the B-only suffix. Two gaps are left. The attempt in progress always finishes, so if the first `B+ C` attempt right after the row the LIMIT needs exceeds `maxStepsPerMatchAttempt` on its own, the query still fails; that takes a lowered budget, because the default covers one attempt over a full partition. And on the mailbox path MATCH, like filters and joins, never returns an empty block, so the block a sender computes before it learns the receiver terminated keeps matching until it finds a row or the input ends. I left a TODO for that one, since closing it needs a way to see early termination during a call, which is a separate MSE change. _🤖 Addressed by [Claude Code](https://claude.com/claude-code)_ ########## pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MatchOperator.java: ########## @@ -0,0 +1,423 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.common.datatable.StatMap; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.runtime.blocks.MseBlock; +import org.apache.pinot.query.runtime.blocks.RowHeapDataBlock; +import org.apache.pinot.query.runtime.operator.match.MatchExpression; +import org.apache.pinot.query.runtime.operator.match.MatchLimits; +import org.apache.pinot.query.runtime.operator.match.MatchTape; +import org.apache.pinot.query.runtime.operator.match.PartitionMatcher; +import org.apache.pinot.query.runtime.operator.match.PatternNfa; +import org.apache.pinot.query.runtime.operator.match.PatternToNfaCompiler; +import org.apache.pinot.query.runtime.operator.utils.TypeUtils; +import org.apache.pinot.query.runtime.plan.OpChainExecutionContext; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/// Evaluates SQL:2016 `MATCH_RECOGNIZE` (row pattern recognition) with `ONE ROW PER MATCH`. +/// +/// ## What it does per partition +/// +/// For every `PARTITION BY` partition, in `ORDER BY` order, it walks a scan position from the first row to +/// the last. At each position it asks [PartitionMatcher] for the preferred match starting exactly there. On a +/// match it emits one row - the partition key columns followed by the `MEASURES` - and then moves the scan +/// position according to the `AFTER MATCH SKIP` mode. On no match it moves one row forward. +/// +/// ## What it expects from the plan +/// +/// Like [WindowAggregateOperator], this operator does not sort. `PinotMatchExchangeNodeInsertRule` puts a +/// sort exchange underneath that hash distributes on the partition keys and sorts the receiver side on +/// `(partitionKeys..., orderKeys...)`, so rows arrive grouped by partition and ordered within a partition. The +/// operator therefore buffers one partition at a time and releases it at each boundary, and never reads +/// [MatchNode#getCollations()]: the ordering has already been established below it. +/// +/// The grouping half of that assumption is verified rather than trusted. Partition keys are compared directly on the +/// incoming rows and must be monotonically increasing, so a key that reappears after its partition was closed fails +/// without retaining every previously seen key. The ordering half is not re-checked per row, because an exchange that +/// grouped correctly but sorted incorrectly is not a failure mode the exchange can produce - losing the sort loses the +/// grouping too, which the partition-key order check already catches. +/// +/// ## Guardrails throw, they never truncate +/// +/// [QueryOptionKey#MAX_ROWS_IN_MATCH_PARTITION] bounds the rows buffered for a partition and +/// [QueryOptionKey#MAX_STEPS_PER_MATCH_ATTEMPT] bounds the backtracking of one match attempt. Both raise an error, +/// because a truncated pattern result is a wrong result that nothing in the response would flag. +/// +/// ## Not supported yet +/// +/// `ALL ROWS PER MATCH` is rejected here as well as during planning: this operator emits exactly one row per +/// match, so accepting it would silently return the wrong shape of result. +public class MatchOperator extends MultiStageOperator { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchOperator.class); + private static final String EXPLAIN_NAME = "MATCH_RECOGNIZE"; + static final int MAX_OUTPUT_ROWS_PER_BLOCK = 1024; + + private final MultiStageOperator _input; + private final DataSchema _resultSchema; + private final ColumnDataType[] _resultStoredTypes; + private final int[] _partitionKeys; + private final int[] _partitionComparisonKeys; + private final List<PatternSymbol> _patternSymbols; + private final MatchExpression[] _measures; + private final PartitionMatcher _matcher; + private final MatchNode.AfterMatchSkipMode _skipMode; + private final int _skipToSymbolOrdinal; + private final int _maxRowsInMatchPartition; + private final StatMap<StatKey> _statMap = new StatMap<>(StatKey.class); + + private List<Object[]> _partitionRows = new ArrayList<>(); + @Nullable + private List<Object[]> _inputRows; + private int _inputRowIndex; + private boolean _matchingPartition; + private int _scanStart; + private long _matchNumber; + @Nullable + private MseBlock.Eos _inputEos; + private int _numRows; + + public MatchOperator(OpChainExecutionContext context, MultiStageOperator input, DataSchema inputSchema, + MatchNode node) { + super(context); + if (node.getRowsPerMatchMode() != MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the default) and " + + "expose the per-match values you need through the MEASURES clause."); + } + _input = input; + _resultSchema = node.getDataSchema(); + _resultStoredTypes = _resultSchema.getStoredColumnDataTypes(); + List<Integer> partitionKeys = node.getPartitionKeys(); + _partitionKeys = new int[partitionKeys.size()]; + for (int i = 0; i < _partitionKeys.length; i++) { + _partitionKeys[i] = partitionKeys.get(i); + ColumnDataType partitionKeyType = inputSchema.getColumnDataType(_partitionKeys[i]); + if (partitionKeyType.isArray()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY requires single-value columns, but column '" + + inputSchema.getColumnName(_partitionKeys[i]) + "' has type " + partitionKeyType); + } + } + // Calcite exposes the exchange distribution as an ImmutableBitSet, so the rule below this operator sorts + // partition fields by input ordinal. Preserve SQL declaration order in _partitionKeys for the output schema, but + // validate monotonic input against the order the exchange actually produces. + _partitionComparisonKeys = _partitionKeys.clone(); + Arrays.sort(_partitionComparisonKeys); + _patternSymbols = node.getPatternSymbols(); + List<MatchNode.Measure> measures = node.getMeasures(); + _measures = new MatchExpression[measures.size()]; + for (int i = 0; i < _measures.length; i++) { + _measures[i] = MatchExpression.compile(measures.get(i).getExpression(), inputSchema); + } + if (_resultSchema.size() != _partitionKeys.length + _measures.length) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE output schema " + _resultSchema + " does not match " + _partitionKeys.length + + " partition key(s) plus " + _measures.length + " measure(s)"); + } + _skipMode = node.getAfterMatchSkipMode(); + _skipToSymbolOrdinal = node.getAfterMatchSkipToSymbolOrdinal(); + + PatternNfa nfa = PatternToNfaCompiler.compile(node.getPattern()); + _maxRowsInMatchPartition = + MatchLimits.getMaxRowsInMatchPartition(context.getOpChainMetadata(), node.getNodeHint()); + long maxStepsPerMatchAttempt = + MatchLimits.getMaxStepsPerMatchAttempt(context.getOpChainMetadata(), node.getNodeHint()); + _matcher = new PartitionMatcher(nfa, _patternSymbols, inputSchema, maxStepsPerMatchAttempt, + this::checkTerminationAndSampleUsage); + } + + @Override + protected Logger logger() { + return LOGGER; + } + + @Override + public List<MultiStageOperator> getChildOperators() { + return List.of(_input); + } + + @Override + public Type getOperatorType() { + return Type.MATCH; + } + + @Override + public String toExplainString() { + return EXPLAIN_NAME; + } + + @Override + public void registerExecution(long time, int numRows, long memoryUsedBytes, long gcTimeMs) { + _statMap.merge(StatKey.EXECUTION_TIME_MS, time); + _statMap.merge(StatKey.EMITTED_ROWS, numRows); + _statMap.merge(StatKey.ALLOCATED_MEMORY_BYTES, memoryUsedBytes); + _statMap.merge(StatKey.GC_TIME_MS, gcTimeMs); + } + + @Override + public StatMap<StatKey> copyStatMaps() { + return new StatMap<>(_statMap); + } + + @Override + protected MseBlock getNextBlock() { + List<Object[]> outputRows = new ArrayList<>(MAX_OUTPUT_ROWS_PER_BLOCK); + while (outputRows.size() < MAX_OUTPUT_ROWS_PER_BLOCK) { + if (_matchingPartition) { + emitMatches(outputRows); + continue; + } + + if (_inputRows != null) { + consumeInputRowsUntilPartitionBoundary(); + continue; + } + + if (_inputEos != null) { + if (outputRows.isEmpty()) { + return _inputEos; + } + break; + } + + MseBlock block = _input.nextBlock(); + if (block.isData()) { + _inputRows = ((MseBlock.Data) block).asRowHeap().getRows(); + _inputRowIndex = 0; + continue; + } + + _inputEos = (MseBlock.Eos) block; + if (_inputEos.isError()) { + return _inputEos; + } + if (!_partitionRows.isEmpty()) { + startMatchingPartition(); + } + } + return new RowHeapDataBlock(outputRows, _resultSchema); + } + + /// Consumes rows without allocating extracted keys. A boundary row stays in the input block until the preceding + /// partition has been fully matched and emitted. + private void consumeInputRowsUntilPartitionBoundary() { + assert _inputRows != null; + while (_inputRowIndex < _inputRows.size()) { + Object[] row = _inputRows.get(_inputRowIndex); + if (!_partitionRows.isEmpty()) { + int comparison = comparePartitionKeys(_partitionRows.get(0), row); + if (comparison > 0) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE input is not grouped by partition key in ascending order: partition " + + partitionKeyToString(row) + " appeared after " + partitionKeyToString(_partitionRows.get(0))); + } + if (comparison < 0) { + startMatchingPartition(); + return; + } + } + if (_partitionRows.size() >= _maxRowsInMatchPartition) { + throw QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.asException( + "MATCH_RECOGNIZE partition exceeds the maximum of " + _maxRowsInMatchPartition + + " rows. Partition on a column with more distinct values, or raise the '" + + QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION + "' query option."); + } + _partitionRows.add(row); + _inputRowIndex++; + _numRows++; + checkTerminationAndSampleUsagePeriodically(_numRows, EXPLAIN_NAME); + } + _inputRows = null; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private int comparePartitionKeys(Object[] leftRow, Object[] rightRow) { + for (int partitionKey : _partitionComparisonKeys) { + Object left = leftRow[partitionKey]; + Object right = rightRow[partitionKey]; + if (left == null) { + if (right == null) { + continue; + } + return 1; + } + if (right == null) { + return -1; + } + if (!(left instanceof Comparable)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY value is not sortable: " + left.getClass().getName()); + } + int comparison; + try { + comparison = ((Comparable) left).compareTo(right); + } catch (ClassCastException e) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY values have incompatible types: " + left.getClass().getName() + " and " + + right.getClass().getName()); + } + if (comparison != 0) { + return comparison; + } + } + return 0; + } + + private String partitionKeyToString(Object[] row) { + StringBuilder builder = new StringBuilder("["); + for (int i = 0; i < _partitionKeys.length; i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(row[_partitionKeys[i]]); + } + return builder.append(']').toString(); + } + + private void startMatchingPartition() { + _matchingPartition = true; + _scanStart = 0; + _matchNumber = 0; + } + + /// Resumes scanning the buffered partition until it is exhausted or the current output block is full. + private void emitMatches(List<Object[]> outputRows) { + List<Object[]> rows = _partitionRows; + int numPartitionRows = rows.size(); + MatchTape tape = _matcher.getTape(); + while (_scanStart < numPartitionRows && outputRows.size() < MAX_OUTPUT_ROWS_PER_BLOCK) { + checkTerminationAndSampleUsage(); + int endPos = _matcher.match(rows, _scanStart, _matchNumber + 1); + if (endPos == PartitionMatcher.NO_MATCH) { + _scanStart++; + continue; + } + _matchNumber++; + outputRows.add(buildOutputRow(rows, tape)); + _scanStart = nextScanStart(_scanStart, endPos, tape); + } + if (_scanStart >= numPartitionRows) { + _matcher.releasePartition(); + _partitionRows = new ArrayList<>(); + _matchingPartition = false; Review Comment: Fixed: MATCH implements `releaseBuffers()`/`hasBufferedState()` and drops the input block reference, the partition list, the tape and the matcher's backtracking arrays. OperatorBufferReleaseTest covers it on cancel and on the row-limit, step-budget and upstream-error paths, and PartitionMatcherStepBudgetTest checks that the final release drops grown backtracking arrays. _🤖 Addressed by [Claude Code](https://claude.com/claude-code)_ ########## pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTerm.java: ########## @@ -0,0 +1,394 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.runtime.operator.operands.TransformOperand; +import org.apache.pinot.query.runtime.operator.utils.TypeUtils; +import org.apache.pinot.spi.exception.QueryErrorCode; + + +/// A leaf of a MEASURES or DEFINE expression whose value depends on the match rather than on a single row: a row +/// pattern navigation, `CLASSIFIER()`, `MATCH_NUMBER()`, or a single variable aggregate. +/// +/// [MatchExpression] replaces every such leaf with a slot in a synthetic row, so that everything above the +/// leaves - comparisons, boolean connectives, arithmetic, scalar functions - is evaluated by Pinot's ordinary +/// [TransformOperand] machinery instead of a second expression interpreter. +/// +/// Implementations are stateless with respect to the match: all match state is passed in through the +/// [MatchTape], so one instance is reused across every match of every partition. +public interface MatchTerm { + + /// The type this term reports to the enclosing expression. Values returned by [#evaluate] are in the + /// corresponding [stored][ColumnDataType#getStoredType()] representation, exactly like the values of a real + /// input row. + ColumnDataType getResultType(); + + @Nullable + Object evaluate(MatchTape tape); + + /// A row pattern navigation: an optional logical step (`FIRST` / `LAST`) that designates a row of the + /// match, followed by an optional physical step (`PREV` / `NEXT`) that moves a fixed number of rows + /// relative to it, and finally a column read. + /// + /// Both steps are needed because SQL:2016 nests them, e.g. `PREV(LAST(A.price), 2)` designates the last row + /// mapped to `A` and then moves two rows back. The logical step is bounded by the match; the physical step is + /// bounded only by the partition, so it may legally read a row outside the match. Either step falling off its bound + /// yields `null`, as the standard requires. + final class Navigation implements MatchTerm { + private final int _symbolOrdinal; + private final boolean _fromEnd; + private final int _logicalOffset; + private final int _physicalDelta; + private final int _columnIndex; + private final ColumnDataType _resultType; + private final ColumnDataType _storedType; + private final boolean _requiresConversion; + + /// @param symbolOrdinal pattern variable to navigate, or + /// [org.apache.pinot.query.planner.logical.RexExpression.PatternFieldRef#UNIVERSAL_SYMBOL_ORDINAL] + /// for an unqualified column reference, which navigates the rows of the whole match + /// @param fromEnd `true` for `LAST`, `false` for `FIRST` + /// @param logicalOffset how many rows back from the end (or forward from the start) of the designated variable + /// @param physicalDelta rows to move in the partition afterwards; negative for `PREV`, positive for + /// `NEXT`, zero when there is no physical step + /// @param columnIndex column to read, as an index into the input row of the MATCH_RECOGNIZE node + /// @param sourceType type of that input column; values already use its stored representation + public Navigation(int symbolOrdinal, boolean fromEnd, int logicalOffset, int physicalDelta, int columnIndex, + ColumnDataType sourceType, ColumnDataType resultType) { + _symbolOrdinal = symbolOrdinal; + _fromEnd = fromEnd; + _logicalOffset = logicalOffset; + _physicalDelta = physicalDelta; + _columnIndex = columnIndex; + _resultType = resultType; + _storedType = resultType.getStoredType(); + _requiresConversion = sourceType.getStoredType() != _storedType; + } + + @Override + public ColumnDataType getResultType() { + return _resultType; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + int rowIndex = _fromEnd ? tape.lastRow(_symbolOrdinal, _logicalOffset) + : tape.firstRow(_symbolOrdinal, _logicalOffset); + if (rowIndex == MatchTape.NO_ROW) { + return null; + } + long physicalRowIndex = (long) rowIndex + _physicalDelta; + List<Object[]> rows = tape.getPartitionRows(); + if (physicalRowIndex < 0 || physicalRowIndex >= rows.size()) { + return null; + } + Object value = rows.get((int) physicalRowIndex)[_columnIndex]; + // The declared type of the navigation may be wider than the column's (e.g. BIG_DECIMAL for a DOUBLE column), + // and the enclosing operand compares against that declared type. + return value != null && _requiresConversion ? TypeUtils.convert(value, _storedType) : value; + } + } + + /// `CLASSIFIER()`: the name of the pattern variable the designated row is mapped to. A bare call designates the + /// last (current) row; FIRST / LAST may select another match row and PREV / NEXT may move relative to it. + final class Classifier implements MatchTerm { + public static final Classifier INSTANCE = new Classifier(true, 0, 0); + + private final boolean _fromEnd; + private final int _logicalOffset; + private final int _physicalDelta; + + public Classifier(boolean fromEnd, int logicalOffset, int physicalDelta) { + _fromEnd = fromEnd; + _logicalOffset = logicalOffset; + _physicalDelta = physicalDelta; + } + + @Override + public ColumnDataType getResultType() { + return ColumnDataType.STRING; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + int rowIndex = _fromEnd + ? tape.lastRow(RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL, _logicalOffset) + : tape.firstRow(RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL, _logicalOffset); + if (rowIndex == MatchTape.NO_ROW) { + return null; + } + long physicalRowIndex = (long) rowIndex + _physicalDelta; + if (physicalRowIndex < 0 || physicalRowIndex >= tape.getPartitionRows().size()) { + return null; + } + return tape.classifierAt((int) physicalRowIndex); + } + } + + /// `MATCH_NUMBER()`: the sequential number of the match within its partition, starting at 1. + final class MatchNumber implements MatchTerm { + public static final MatchNumber INSTANCE = new MatchNumber(); + + private MatchNumber() { + } + + @Override + public ColumnDataType getResultType() { + return ColumnDataType.LONG; + } + + @Override + public Object evaluate(MatchTape tape) { + return tape.getMatchNumber(); + } + } + + /// A single variable aggregate in MEASURES, e.g. `SUM(A.price)` or `COUNT(*)`: the aggregate of an + /// expression evaluated over every row of the match that is mapped to one pattern variable. + /// + /// The argument is evaluated by an ordinary [TransformOperand] against the raw input row, so any scalar + /// expression works, e.g. `SUM(A.price * A.quantity)`. Nulls are skipped, as SQL requires; an aggregate over + /// zero rows is `null` except for `COUNT`, which is `0`. + final class Aggregate implements MatchTerm { + private final Kind _kind; + private final int _symbolOrdinal; + @Nullable + private final TransformOperand _argument; + private final ColumnDataType _resultType; + private final ColumnDataType _storedType; + private final boolean _requiresResultConversion; + + /// @param argument the aggregated expression evaluated against an input row, or `null` for `COUNT(*)`, + /// which counts rows rather than values + public Aggregate(Kind kind, int symbolOrdinal, @Nullable TransformOperand argument, ColumnDataType resultType) { + _kind = kind; + _symbolOrdinal = symbolOrdinal; + _argument = argument; + _resultType = resultType; + _storedType = resultType.getStoredType(); + if (argument != null && argument.getResultType().isArray()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Multi-value operand type '" + argument.getResultType() + "' is not supported for " + kind + + " in a MATCH_RECOGNIZE MEASURES clause. Reduce the array to a scalar before aggregating it."); + } + if ((kind == Kind.SUM || kind == Kind.AVG) && !_storedType.isNumber()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE " + kind + " requires a numeric result type, got: " + resultType + "."); + } + _requiresResultConversion = argument != null && argument.getResultType().getStoredType() != _storedType; + } + + @Override + public ColumnDataType getResultType() { + return _resultType; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + int rowCount = tape.rowCount(_symbolOrdinal); + if (_kind == Kind.COUNT && _argument == null) { + return (long) rowCount; + } + List<Object[]> partitionRows = tape.getPartitionRows(); + switch (_kind) { + case COUNT: + return countNonNull(tape, rowCount, partitionRows); + case MIN: + case MAX: + return evaluateExtremum(tape, rowCount, partitionRows); + case SUM: + case AVG: + return evaluateSumOrAverage(tape, rowCount, partitionRows); + default: + throw new IllegalStateException("Unexpected MATCH_RECOGNIZE aggregate: " + _kind); + } + } + + private long countNonNull(MatchTape tape, int rowCount, List<Object[]> partitionRows) { + long count = 0; + for (int i = 0; i < rowCount; i++) { + Object value = _argument.apply(partitionRows.get(tape.rowAt(_symbolOrdinal, i))); + if (value != null) { + count++; + } + } + return count; + } + + @Nullable + private Object evaluateExtremum(MatchTape tape, int rowCount, List<Object[]> partitionRows) { + Object extremum = null; + for (int i = 0; i < rowCount; i++) { + Object value = _argument.apply(partitionRows.get(tape.rowAt(_symbolOrdinal, i))); + if (value == null) { + continue; + } + if (extremum == null || (_kind == Kind.MIN ? compare(value, extremum) < 0 : compare(value, extremum) > 0)) { + extremum = value; + } + } + if (extremum == null || !_requiresResultConversion) { + return extremum; + } + return TypeUtils.convert(extremum, _storedType); + } + + @Nullable + private Object evaluateSumOrAverage(MatchTape tape, int rowCount, List<Object[]> partitionRows) { + switch (_storedType) { + case INT: + case LONG: + return evaluateIntegralSumOrAverage(tape, rowCount, partitionRows); + case FLOAT: + case DOUBLE: + return evaluateFloatingPointSumOrAverage(tape, rowCount, partitionRows); + case BIG_DECIMAL: + return evaluateDecimalSumOrAverage(tape, rowCount, partitionRows); + default: + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE " + _kind + " requires a numeric result type, got: " + _resultType + "."); + } + } + + @Nullable + private Object evaluateIntegralSumOrAverage(MatchTape tape, int rowCount, List<Object[]> partitionRows) { + long sum = 0; + long count = 0; + for (int i = 0; i < rowCount; i++) { + Object value = _argument.apply(partitionRows.get(tape.rowAt(_symbolOrdinal, i))); + if (value != null) { + sum += numericValue(value).longValue(); + count++; + } + } + if (count == 0) { + return null; + } + long result = _kind == Kind.AVG ? sum / count : sum; + if (_storedType == ColumnDataType.INT) { + return (int) result; + } + return result; + } + + @Nullable + private Object evaluateFloatingPointSumOrAverage(MatchTape tape, int rowCount, List<Object[]> partitionRows) { + double sum = 0; + long count = 0; + for (int i = 0; i < rowCount; i++) { + Object value = _argument.apply(partitionRows.get(tape.rowAt(_symbolOrdinal, i))); + if (value != null) { + sum += numericValue(value).doubleValue(); + count++; + } Review Comment: Fixed: every SUM/AVG/COUNT/MIN/MAX scan in MEASURES now runs the operator's termination check every 8,192 visited rows. `testCancellationInterruptsAnAggregateMeasureScan` cancels after 100 argument evaluations and checks that the scan stops before visiting every row. _🤖 Addressed by [Claude Code](https://claude.com/claude-code)_ -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
