gortiz commented on code in PR #20061:
URL: https://github.com/apache/pinot/pull/20061#discussion_r4584533652


##########
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotArrayToMvCoalesceFilterRule.java:
##########
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.calcite.rel.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+
+/// Rewrites a filter predicate that replaces a null `ARRAY_TO_MV(array)` with 
a default value, so that the leaf stage
+/// can run it.
+///
+/// The leaf stage runs `ARRAY_TO_MV(array) <op> value` as a multi-value 
predicate of the single-stage engine. It cannot
+/// run `COALESCE(ARRAY_TO_MV(array), default)`, because the single-stage 
engine has no multi-value `COALESCE` or
+/// `CASE`. This rule moves the null check out of the expression, with `A` for 
`ARRAY_TO_MV(array)`:
+///
+/// | Before | After |
+/// |---|---|
+/// | `pred(COALESCE(A, d))` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND 
pred(d))` |
+/// | `pred(CASE WHEN A IS NULL THEN d ELSE A END)` | `(A IS NOT NULL AND 
pred(A)) OR (A IS NULL AND pred(d))` |
+/// | `CASE WHEN A IS NULL THEN v ELSE e END` | `(A IS NULL AND v) OR (A IS 
NOT NULL AND e)` |
+/// | `CASE WHEN A IS NOT NULL THEN e ELSE v END` | `(A IS NULL AND v) OR (A 
IS NOT NULL AND e)` |
+///
+/// `pred` is a boolean call with `COALESCE(A, d)` (or the same `CASE`) as one 
operand, for example `=`, `<>`, `<` or
+/// `LIKE`. Calcite can also move `pred` into the branches of the `CASE`, 
which gives the boolean `CASE` forms. When
+/// `pred(d)` or `v` evaluates to a literal, the result is simpler: `A IS NULL 
OR pred(A)` for `true`, and
+/// `A IS NOT NULL AND pred(A)` for `false`.
+///
+/// The rewrite gives the same result for every row in three-valued logic. 
`pred(A)` keeps the multi-value meaning of
+/// the single-stage engine: for example, `A = 'a'` matches when an element is 
`'a'`, and `A <> 'a'` matches when no
+/// element is `'a'`. The `A IS NOT NULL` check is necessary because, without 
null handling, a null array is stored as
+/// the default value, and `pred(A)` can match it.
+///
+/// The rule does not change other uses of `ARRAY_TO_MV`, for example 
`UPPER(COALESCE(A, d)) = 'X'`. A predicate that
+/// still has `ARRAY_TO_MV` in an intermediate stage fails the `ARRAY_TO_MV` 
validation, as before.
+///
+/// The rule is stateless and thread-safe.
+public class PinotArrayToMvCoalesceFilterRule extends RelOptRule {
+  public static final PinotArrayToMvCoalesceFilterRule INSTANCE =
+      new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, 
null);
+
+  public static PinotArrayToMvCoalesceFilterRule 
instanceWithDescription(String description) {
+    return new 
PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, description);
+  }
+
+  private PinotArrayToMvCoalesceFilterRule(RelBuilderFactory factory, 
@Nullable String description) {
+    super(operand(LogicalFilter.class, any()), factory, description);
+  }
+
+  @Override
+  public boolean matches(RelOptRuleCall call) {
+    LogicalFilter filter = call.rel(0);
+    return 
PinotProjectJoinTransposeRule.containsArrayToMv(filter.getCondition());
+  }
+
+  @Override
+  public void onMatch(RelOptRuleCall call) {
+    LogicalFilter filter = call.rel(0);
+    RexNode condition = filter.getCondition();
+    RexNode newCondition = rewrite(condition, 
filter.getCluster().getRexBuilder());
+    if (newCondition != condition) {
+      call.transformTo(filter.copy(filter.getTraitSet(), filter.getInput(), 
newCondition));
+    }
+  }
+
+  /// Returns `condition` with the predicates in the table of the class doc 
rewritten, or `condition` itself when no
+  /// predicate matches.
+  public static RexNode rewrite(RexNode condition, RexBuilder rexBuilder) {
+    RexNode newCondition = condition.accept(new Shuttle(rexBuilder));
+    return newCondition == condition ? condition : RexUtil.flatten(rexBuilder, 
newCondition);
+  }
+
+  private static class Shuttle extends RexShuttle {
+    private final RexBuilder _rexBuilder;
+
+    Shuttle(RexBuilder rexBuilder) {
+      _rexBuilder = rexBuilder;
+    }
+
+    @Override
+    public RexNode visitCall(RexCall call) {
+      RexCall visited = (RexCall) super.visitCall(call);
+      if (visited.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
+        return visited;
+      }
+      RexNode rewritten = visited.getKind() == SqlKind.CASE ? 
rewriteCase(visited) : rewritePredicate(visited);
+      return rewritten != null ? rewritten : visited;
+    }
+
+    /// Rewrites `pred(COALESCE(A, d...))` and `pred(CASE WHEN A IS NULL THEN 
d ELSE A END)`, or returns `null` when
+    /// `call` does not have one of these forms.
+    @Nullable
+    private RexNode rewritePredicate(RexCall call) {
+      if (call.isA(SqlKind.AND) || call.isA(SqlKind.OR) || 
call.isA(SqlKind.NOT)) {
+        return null;
+      }
+      List<RexNode> operands = call.getOperands();
+      int index = -1;
+      RexNode[] arrayToMvAndDefault = null;
+      for (int i = 0; i < operands.size(); i++) {
+        RexNode operand = operands.get(i);
+        RexNode[] match = matchDefaultForNull(operand);
+        if (match != null) {
+          if (index >= 0) {
+            return null;
+          }
+          index = i;
+          arrayToMvAndDefault = match;
+        } else if (PinotProjectJoinTransposeRule.containsArrayToMv(operand)) {
+          return null;

Review Comment:
   Thanks, fixed in 141f72fdb8, but in a different way. Instead of extending 
the filter rewrite to more shapes, I moved the nullable type to the place where 
it matters. `ARRAY_TO_MV` now has the type it has on master in the first plan. 
Only the second plan, the one with the MV aggregate, uses a nullable 
`ARRAY_TO_MV`. That plan starts again from a copy of the parsed query, which 
the planner takes before the first validation. Only a query that fails on 
master reaches the second plan, so the nullable type can't change a query that 
works today, whatever shape its filters have. 
`PinotArrayToMvCoalesceFilterRule` is gone.
   
   Your example returns 1 again, and it's in `MvAggregate.json`. The self-join 
from your first comment still gets the nullable key, so `CASE WHEN ... IS NULL` 
above the aggregate gives `none`, and `COALESCE` around the key fails as on 
master. The flip side is that the wrong results that master has on a scan stay 
as they are: `IS NOT NULL` folded to true, and `COALESCE(ARRAY_TO_MV(col), 'x') 
= 'x'` not matching a null row. They don't come from this PR, so I'd rather fix 
them separately.
   
   



##########
pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java:
##########
@@ -732,6 +733,59 @@ public static class Broker {
     public static final String CONFIG_OF_USE_PHYSICAL_OPTIMIZER = 
"pinot.broker.multistage.use.physical.optimizer";
     public static final boolean DEFAULT_USE_PHYSICAL_OPTIMIZER = false;
 
+    /// How the multi-stage planner plans `GROUP BY ARRAY_TO_MV(col)` by 
default, as one of the [MvAggregateMode]
+    /// values: `auto` (the default), `true` or `false`. The value is 
case-insensitive. The MV aggregate groups each row
+    /// by every element of the multi-value key in every stage, as the 
single-stage engine does.
+    ///
+    /// A server that predates the MV aggregate cannot run it. The `auto` mode 
is safe during a rolling upgrade: it
+    /// sends the MV aggregate only for queries that fail without it.
+    ///
+    /// The value can also be set in the cluster config, under the same key or 
under the alias
+    /// `pinot.all.multistage.use.mv.aggregate`. The same key wins over the 
alias, and the cluster config wins over the
+    /// broker config. A change applies to the next query without a broker 
restart. When the keys are removed from the
+    /// cluster config, the broker config applies again.
+    ///
+    /// The query option [Request.QueryOptionKey#USE_MV_AGGREGATE] overrides 
this value for one query.
+    ///
+    /// These planner changes stay active regardless of this value:
+    /// - The stricter validation of `ARRAY_TO_MV` and its error messages.
+    /// - The leaf-stage check that rejects a plain `GROUP BY` on a 
multi-value column.
+    /// - The classification of the nodes above a lookup join or an `UNNEST` 
as intermediate stage nodes.
+    /// - A filter condition on an `ARRAY_TO_MV(expr)` group key stays above 
the aggregate.
+    public static final String CONFIG_OF_USE_MV_AGGREGATE = 
"pinot.broker.multistage.use.mv.aggregate";
+    public static final String DEFAULT_USE_MV_AGGREGATE = "auto";

Review Comment:
   Agreed, done in 141f72fdb8. The query option, the broker and cluster configs 
and the `pinot.all` alias are gone. The planner always plans the query without 
the MV aggregate first, and plans it again with the MV aggregate only when the 
first plan fails the `ARRAY_TO_MV` validation. So the choice follows the real 
leaf boundary, lookup joins and `UNNEST` included, because the validation is 
what decides it. A leaf-only `GROUP BY ARRAY_TO_MV(col)` keeps the legacy 
aggregate, so old servers run every query that works today.
   
   The tests that forced the MV aggregate on a scan now use the hint 
`is_skip_leaf_stage_group_by`. With it, the first plan fails and the second 
plan puts the MV aggregate directly on the scan, which keeps the leaf lowering 
covered without a test-only switch. `MvAggregateFallbackTest` replaces 
`MvAggregateGateTest`. `MvAggregateFallbackTest` also checks that leaf-only 
queries (scan, filter, semi-join) keep the plain plan. For the old runtime, I 
added two leaf-only `GROUP BY ARRAY_TO_MV` queries to the multi-stage 
compatibility suite, so they run at every step of the rolling upgrade and 
rollback.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to