Jackie-Jiang commented on code in PR #20061: URL: https://github.com/apache/pinot/pull/20061#discussion_r4570692929
########## pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotArrayToMvCoalesceFilterRule.java: ########## @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.calcite.rel.rules; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilderFactory; + + +/// Rewrites a filter predicate that replaces a null `ARRAY_TO_MV(array)` with a default value, so that the leaf stage +/// can run it. +/// +/// The leaf stage runs `ARRAY_TO_MV(array) <op> value` as a multi-value predicate of the single-stage engine. It cannot +/// run `COALESCE(ARRAY_TO_MV(array), default)`, because the single-stage engine has no multi-value `COALESCE` or +/// `CASE`. This rule moves the null check out of the expression, with `A` for `ARRAY_TO_MV(array)`: +/// +/// | Before | After | +/// |---|---| +/// | `pred(COALESCE(A, d))` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND pred(d))` | +/// | `pred(CASE WHEN A IS NULL THEN d ELSE A END)` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND pred(d))` | +/// | `CASE WHEN A IS NULL THEN v ELSE e END` | `(A IS NULL AND v) OR (A IS NOT NULL AND e)` | +/// | `CASE WHEN A IS NOT NULL THEN e ELSE v END` | `(A IS NULL AND v) OR (A IS NOT NULL AND e)` | +/// +/// `pred` is a boolean call with `COALESCE(A, d)` (or the same `CASE`) as one operand, for example `=`, `<>`, `<` or +/// `LIKE`. Calcite can also move `pred` into the branches of the `CASE`, which gives the boolean `CASE` forms. When +/// `pred(d)` or `v` evaluates to a literal, the result is simpler: `A IS NULL OR pred(A)` for `true`, and +/// `A IS NOT NULL AND pred(A)` for `false`. +/// +/// The rewrite gives the same result for every row in three-valued logic. `pred(A)` keeps the multi-value meaning of +/// the single-stage engine: for example, `A = 'a'` matches when an element is `'a'`, and `A <> 'a'` matches when no +/// element is `'a'`. The `A IS NOT NULL` check is necessary because, without null handling, a null array is stored as +/// the default value, and `pred(A)` can match it. +/// +/// The rule does not change other uses of `ARRAY_TO_MV`, for example `UPPER(COALESCE(A, d)) = 'X'`. A predicate that +/// still has `ARRAY_TO_MV` in an intermediate stage fails the `ARRAY_TO_MV` validation, as before. +/// +/// The rule is stateless and thread-safe. +public class PinotArrayToMvCoalesceFilterRule extends RelOptRule { + public static final PinotArrayToMvCoalesceFilterRule INSTANCE = + new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, null); + + public static PinotArrayToMvCoalesceFilterRule instanceWithDescription(String description) { + return new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, description); + } + + private PinotArrayToMvCoalesceFilterRule(RelBuilderFactory factory, @Nullable String description) { + super(operand(LogicalFilter.class, any()), factory, description); + } + + @Override + public boolean matches(RelOptRuleCall call) { + LogicalFilter filter = call.rel(0); + return PinotProjectJoinTransposeRule.containsArrayToMv(filter.getCondition()); + } + + @Override + public void onMatch(RelOptRuleCall call) { + LogicalFilter filter = call.rel(0); + RexNode condition = filter.getCondition(); + RexNode newCondition = rewrite(condition, filter.getCluster().getRexBuilder()); + if (newCondition != condition) { + call.transformTo(filter.copy(filter.getTraitSet(), filter.getInput(), newCondition)); + } + } + + /// Returns `condition` with the predicates in the table of the class doc rewritten, or `condition` itself when no + /// predicate matches. + public static RexNode rewrite(RexNode condition, RexBuilder rexBuilder) { + RexNode newCondition = condition.accept(new Shuttle(rexBuilder)); + return newCondition == condition ? condition : RexUtil.flatten(rexBuilder, newCondition); + } + + private static class Shuttle extends RexShuttle { + private final RexBuilder _rexBuilder; + + Shuttle(RexBuilder rexBuilder) { + _rexBuilder = rexBuilder; + } + + @Override + public RexNode visitCall(RexCall call) { + RexCall visited = (RexCall) super.visitCall(call); + if (visited.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + return visited; + } + RexNode rewritten = visited.getKind() == SqlKind.CASE ? rewriteCase(visited) : rewritePredicate(visited); + return rewritten != null ? rewritten : visited; + } + + /// Rewrites `pred(COALESCE(A, d...))` and `pred(CASE WHEN A IS NULL THEN d ELSE A END)`, or returns `null` when + /// `call` does not have one of these forms. + @Nullable + private RexNode rewritePredicate(RexCall call) { + if (call.isA(SqlKind.AND) || call.isA(SqlKind.OR) || call.isA(SqlKind.NOT)) { + return null; + } + List<RexNode> operands = call.getOperands(); + int index = -1; + RexNode[] arrayToMvAndDefault = null; + for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + RexNode[] match = matchDefaultForNull(operand); + if (match != null) { + if (index >= 0) { + return null; + } + index = i; + arrayToMvAndDefault = match; + } else if (PinotProjectJoinTransposeRule.containsArrayToMv(operand)) { + return null; Review Comment: [P2] Preserve CAST-wrapped nullable-MV COALESCE filters A supported CAST around COALESCE still bypasses this rewrite. For a nullable INT MV column with null handling enabled, `SELECT COUNT(*) FROM tbl WHERE CAST(COALESCE(ARRAY_TO_MV(intMv), 0) AS VARCHAR) = '1'` previously worked; for `[1,2]`, NULL, and `[-1]`, it should return 1. The shuttle skips the non-boolean COALESCE and CAST calls, then this comparison sees CAST as its immediate operand and returns without rewriting. Leaf translation removes ARRAY_TO_MV, leaving COALESCE to read scalar values from an MV column, which fails. Before this PR, Calcite eliminated COALESCE because ARRAY_TO_MV was typed non-nullable, and the remaining CAST supports MV input. This also affects queries without GROUP BY and persists with `useMvAggregate=false`. Please retain the corrected nullability, extend the safe null-branch rewrite through supported CAST wrappers, and add a successful-result regression test. ########## pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java: ########## @@ -732,6 +733,59 @@ public static class Broker { public static final String CONFIG_OF_USE_PHYSICAL_OPTIMIZER = "pinot.broker.multistage.use.physical.optimizer"; public static final boolean DEFAULT_USE_PHYSICAL_OPTIMIZER = false; + /// How the multi-stage planner plans `GROUP BY ARRAY_TO_MV(col)` by default, as one of the [MvAggregateMode] + /// values: `auto` (the default), `true` or `false`. The value is case-insensitive. The MV aggregate groups each row + /// by every element of the multi-value key in every stage, as the single-stage engine does. + /// + /// A server that predates the MV aggregate cannot run it. The `auto` mode is safe during a rolling upgrade: it + /// sends the MV aggregate only for queries that fail without it. + /// + /// The value can also be set in the cluster config, under the same key or under the alias + /// `pinot.all.multistage.use.mv.aggregate`. The same key wins over the alias, and the cluster config wins over the + /// broker config. A change applies to the next query without a broker restart. When the keys are removed from the + /// cluster config, the broker config applies again. + /// + /// The query option [Request.QueryOptionKey#USE_MV_AGGREGATE] overrides this value for one query. + /// + /// These planner changes stay active regardless of this value: + /// - The stricter validation of `ARRAY_TO_MV` and its error messages. + /// - The leaf-stage check that rejects a plain `GROUP BY` on a multi-value column. + /// - The classification of the nodes above a lookup join or an `UNNEST` as intermediate stage nodes. + /// - A filter condition on an `ARRAY_TO_MV(expr)` group key stays above the aggregate. + public static final String CONFIG_OF_USE_MV_AGGREGATE = "pinot.broker.multistage.use.mv.aggregate"; + public static final String DEFAULT_USE_MV_AGGREGATE = "auto"; Review Comment: Suggestion: remove the MV-aggregate flag completely Could we remove the `useMvAggregate` query option and the broker/cluster configuration, and select the operator automatically? Preserve the legacy aggregate representation wherever grouping can execute inside the single-stage leaf engine, and introduce MvAggregateNode only where MV grouping needs intermediate-stage execution. Both leaf representations already lower through ServerPlanRequestVisitor.applyAggregate into the same SSE GROUP BY, so forcing the new node for these existing queries adds an old-server compatibility requirement without an expected execution benefit. I have not identified an existing supported query that current auto mode must move to the new node. The three modes, overrides, aliases, and dynamic configuration add substantial combinations to maintain. Automatic selection should follow the actual SSE boundary, including lookup joins and UNNEST, and keep the nullability, COALESCE, and HAVING correctness fixes. Please cover existing supported queries against the old runtime; newly supported query shapes can require upgraded servers. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
