gortiz commented on code in PR #20061: URL: https://github.com/apache/pinot/pull/20061#discussion_r4584533652
########## pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotArrayToMvCoalesceFilterRule.java: ########## @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.calcite.rel.rules; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilderFactory; + + +/// Rewrites a filter predicate that replaces a null `ARRAY_TO_MV(array)` with a default value, so that the leaf stage +/// can run it. +/// +/// The leaf stage runs `ARRAY_TO_MV(array) <op> value` as a multi-value predicate of the single-stage engine. It cannot +/// run `COALESCE(ARRAY_TO_MV(array), default)`, because the single-stage engine has no multi-value `COALESCE` or +/// `CASE`. This rule moves the null check out of the expression, with `A` for `ARRAY_TO_MV(array)`: +/// +/// | Before | After | +/// |---|---| +/// | `pred(COALESCE(A, d))` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND pred(d))` | +/// | `pred(CASE WHEN A IS NULL THEN d ELSE A END)` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND pred(d))` | +/// | `CASE WHEN A IS NULL THEN v ELSE e END` | `(A IS NULL AND v) OR (A IS NOT NULL AND e)` | +/// | `CASE WHEN A IS NOT NULL THEN e ELSE v END` | `(A IS NULL AND v) OR (A IS NOT NULL AND e)` | +/// +/// `pred` is a boolean call with `COALESCE(A, d)` (or the same `CASE`) as one operand, for example `=`, `<>`, `<` or +/// `LIKE`. Calcite can also move `pred` into the branches of the `CASE`, which gives the boolean `CASE` forms. When +/// `pred(d)` or `v` evaluates to a literal, the result is simpler: `A IS NULL OR pred(A)` for `true`, and +/// `A IS NOT NULL AND pred(A)` for `false`. +/// +/// The rewrite gives the same result for every row in three-valued logic. `pred(A)` keeps the multi-value meaning of +/// the single-stage engine: for example, `A = 'a'` matches when an element is `'a'`, and `A <> 'a'` matches when no +/// element is `'a'`. The `A IS NOT NULL` check is necessary because, without null handling, a null array is stored as +/// the default value, and `pred(A)` can match it. +/// +/// The rule does not change other uses of `ARRAY_TO_MV`, for example `UPPER(COALESCE(A, d)) = 'X'`. A predicate that +/// still has `ARRAY_TO_MV` in an intermediate stage fails the `ARRAY_TO_MV` validation, as before. +/// +/// The rule is stateless and thread-safe. +public class PinotArrayToMvCoalesceFilterRule extends RelOptRule { + public static final PinotArrayToMvCoalesceFilterRule INSTANCE = + new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, null); + + public static PinotArrayToMvCoalesceFilterRule instanceWithDescription(String description) { + return new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, description); + } + + private PinotArrayToMvCoalesceFilterRule(RelBuilderFactory factory, @Nullable String description) { + super(operand(LogicalFilter.class, any()), factory, description); + } + + @Override + public boolean matches(RelOptRuleCall call) { + LogicalFilter filter = call.rel(0); + return PinotProjectJoinTransposeRule.containsArrayToMv(filter.getCondition()); + } + + @Override + public void onMatch(RelOptRuleCall call) { + LogicalFilter filter = call.rel(0); + RexNode condition = filter.getCondition(); + RexNode newCondition = rewrite(condition, filter.getCluster().getRexBuilder()); + if (newCondition != condition) { + call.transformTo(filter.copy(filter.getTraitSet(), filter.getInput(), newCondition)); + } + } + + /// Returns `condition` with the predicates in the table of the class doc rewritten, or `condition` itself when no + /// predicate matches. + public static RexNode rewrite(RexNode condition, RexBuilder rexBuilder) { + RexNode newCondition = condition.accept(new Shuttle(rexBuilder)); + return newCondition == condition ? condition : RexUtil.flatten(rexBuilder, newCondition); + } + + private static class Shuttle extends RexShuttle { + private final RexBuilder _rexBuilder; + + Shuttle(RexBuilder rexBuilder) { + _rexBuilder = rexBuilder; + } + + @Override + public RexNode visitCall(RexCall call) { + RexCall visited = (RexCall) super.visitCall(call); + if (visited.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + return visited; + } + RexNode rewritten = visited.getKind() == SqlKind.CASE ? rewriteCase(visited) : rewritePredicate(visited); + return rewritten != null ? rewritten : visited; + } + + /// Rewrites `pred(COALESCE(A, d...))` and `pred(CASE WHEN A IS NULL THEN d ELSE A END)`, or returns `null` when + /// `call` does not have one of these forms. + @Nullable + private RexNode rewritePredicate(RexCall call) { + if (call.isA(SqlKind.AND) || call.isA(SqlKind.OR) || call.isA(SqlKind.NOT)) { + return null; + } + List<RexNode> operands = call.getOperands(); + int index = -1; + RexNode[] arrayToMvAndDefault = null; + for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + RexNode[] match = matchDefaultForNull(operand); + if (match != null) { + if (index >= 0) { + return null; + } + index = i; + arrayToMvAndDefault = match; + } else if (PinotProjectJoinTransposeRule.containsArrayToMv(operand)) { + return null; Review Comment: Thanks, fixed in 141f72fdb8, but in a different way. Instead of extending the filter rewrite to more shapes, I moved the nullable type to the place where it matters. `ARRAY_TO_MV` now has the type it has on master in the first plan. Only the second plan, the one with the MV aggregate, uses a nullable `ARRAY_TO_MV`. That plan starts again from a copy of the parsed query, which the planner takes before the first validation. Only a query that fails on master reaches the second plan, so the nullable type can't change a query that works today, whatever shape its filters have. `PinotArrayToMvCoalesceFilterRule` is gone. Your example returns 1 again, and it's in `MvAggregate.json`. The self-join from your first comment still gets the nullable key, so `CASE WHEN ... IS NULL` above the aggregate gives `none`, and `COALESCE` around the key fails as on master. The flip side is that the wrong results that master has on a scan stay as they are: `IS NOT NULL` folded to true, and `COALESCE(ARRAY_TO_MV(col), 'x') = 'x'` not matching a null row. They don't come from this PR, so I'd rather fix them separately. ########## pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java: ########## @@ -732,6 +733,59 @@ public static class Broker { public static final String CONFIG_OF_USE_PHYSICAL_OPTIMIZER = "pinot.broker.multistage.use.physical.optimizer"; public static final boolean DEFAULT_USE_PHYSICAL_OPTIMIZER = false; + /// How the multi-stage planner plans `GROUP BY ARRAY_TO_MV(col)` by default, as one of the [MvAggregateMode] + /// values: `auto` (the default), `true` or `false`. The value is case-insensitive. The MV aggregate groups each row + /// by every element of the multi-value key in every stage, as the single-stage engine does. + /// + /// A server that predates the MV aggregate cannot run it. The `auto` mode is safe during a rolling upgrade: it + /// sends the MV aggregate only for queries that fail without it. + /// + /// The value can also be set in the cluster config, under the same key or under the alias + /// `pinot.all.multistage.use.mv.aggregate`. The same key wins over the alias, and the cluster config wins over the + /// broker config. A change applies to the next query without a broker restart. When the keys are removed from the + /// cluster config, the broker config applies again. + /// + /// The query option [Request.QueryOptionKey#USE_MV_AGGREGATE] overrides this value for one query. + /// + /// These planner changes stay active regardless of this value: + /// - The stricter validation of `ARRAY_TO_MV` and its error messages. + /// - The leaf-stage check that rejects a plain `GROUP BY` on a multi-value column. + /// - The classification of the nodes above a lookup join or an `UNNEST` as intermediate stage nodes. + /// - A filter condition on an `ARRAY_TO_MV(expr)` group key stays above the aggregate. + public static final String CONFIG_OF_USE_MV_AGGREGATE = "pinot.broker.multistage.use.mv.aggregate"; + public static final String DEFAULT_USE_MV_AGGREGATE = "auto"; Review Comment: Agreed, done in 141f72fdb8. The query option, the broker and cluster configs and the `pinot.all` alias are gone. The planner always plans the query without the MV aggregate first, and plans it again with the MV aggregate only when the first plan fails the `ARRAY_TO_MV` validation. So the choice follows the real leaf boundary, lookup joins and `UNNEST` included, because the validation is what decides it. A leaf-only `GROUP BY ARRAY_TO_MV(col)` keeps the legacy aggregate, so old servers run every query that works today. The tests that forced the MV aggregate on a scan now use the hint `is_skip_leaf_stage_group_by`. With it, the first plan fails and the second plan puts the MV aggregate directly on the scan, which keeps the leaf lowering covered without a test-only switch. `MvAggregateFallbackTest` replaces `MvAggregateGateTest`. `MvAggregateFallbackTest` also checks that leaf-only queries (scan, filter, semi-join) keep the plain plan. For the old runtime, I added two leaf-only `GROUP BY ARRAY_TO_MV` queries to the multi-stage compatibility suite, so they run at every step of the rolling upgrade and rollback. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
