Jackie-Jiang commented on code in PR #20061:
URL: https://github.com/apache/pinot/pull/20061#discussion_r4570692929


##########
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotArrayToMvCoalesceFilterRule.java:
##########
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.calcite.rel.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+
+/// Rewrites a filter predicate that replaces a null `ARRAY_TO_MV(array)` with 
a default value, so that the leaf stage
+/// can run it.
+///
+/// The leaf stage runs `ARRAY_TO_MV(array) <op> value` as a multi-value 
predicate of the single-stage engine. It cannot
+/// run `COALESCE(ARRAY_TO_MV(array), default)`, because the single-stage 
engine has no multi-value `COALESCE` or
+/// `CASE`. This rule moves the null check out of the expression, with `A` for 
`ARRAY_TO_MV(array)`:
+///
+/// | Before | After |
+/// |---|---|
+/// | `pred(COALESCE(A, d))` | `(A IS NOT NULL AND pred(A)) OR (A IS NULL AND 
pred(d))` |
+/// | `pred(CASE WHEN A IS NULL THEN d ELSE A END)` | `(A IS NOT NULL AND 
pred(A)) OR (A IS NULL AND pred(d))` |
+/// | `CASE WHEN A IS NULL THEN v ELSE e END` | `(A IS NULL AND v) OR (A IS 
NOT NULL AND e)` |
+/// | `CASE WHEN A IS NOT NULL THEN e ELSE v END` | `(A IS NULL AND v) OR (A 
IS NOT NULL AND e)` |
+///
+/// `pred` is a boolean call with `COALESCE(A, d)` (or the same `CASE`) as one 
operand, for example `=`, `<>`, `<` or
+/// `LIKE`. Calcite can also move `pred` into the branches of the `CASE`, 
which gives the boolean `CASE` forms. When
+/// `pred(d)` or `v` evaluates to a literal, the result is simpler: `A IS NULL 
OR pred(A)` for `true`, and
+/// `A IS NOT NULL AND pred(A)` for `false`.
+///
+/// The rewrite gives the same result for every row in three-valued logic. 
`pred(A)` keeps the multi-value meaning of
+/// the single-stage engine: for example, `A = 'a'` matches when an element is 
`'a'`, and `A <> 'a'` matches when no
+/// element is `'a'`. The `A IS NOT NULL` check is necessary because, without 
null handling, a null array is stored as
+/// the default value, and `pred(A)` can match it.
+///
+/// The rule does not change other uses of `ARRAY_TO_MV`, for example 
`UPPER(COALESCE(A, d)) = 'X'`. A predicate that
+/// still has `ARRAY_TO_MV` in an intermediate stage fails the `ARRAY_TO_MV` 
validation, as before.
+///
+/// The rule is stateless and thread-safe.
+public class PinotArrayToMvCoalesceFilterRule extends RelOptRule {
+  public static final PinotArrayToMvCoalesceFilterRule INSTANCE =
+      new PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, 
null);
+
+  public static PinotArrayToMvCoalesceFilterRule 
instanceWithDescription(String description) {
+    return new 
PinotArrayToMvCoalesceFilterRule(PinotRuleUtils.PINOT_REL_FACTORY, description);
+  }
+
+  private PinotArrayToMvCoalesceFilterRule(RelBuilderFactory factory, 
@Nullable String description) {
+    super(operand(LogicalFilter.class, any()), factory, description);
+  }
+
+  @Override
+  public boolean matches(RelOptRuleCall call) {
+    LogicalFilter filter = call.rel(0);
+    return 
PinotProjectJoinTransposeRule.containsArrayToMv(filter.getCondition());
+  }
+
+  @Override
+  public void onMatch(RelOptRuleCall call) {
+    LogicalFilter filter = call.rel(0);
+    RexNode condition = filter.getCondition();
+    RexNode newCondition = rewrite(condition, 
filter.getCluster().getRexBuilder());
+    if (newCondition != condition) {
+      call.transformTo(filter.copy(filter.getTraitSet(), filter.getInput(), 
newCondition));
+    }
+  }
+
+  /// Returns `condition` with the predicates in the table of the class doc 
rewritten, or `condition` itself when no
+  /// predicate matches.
+  public static RexNode rewrite(RexNode condition, RexBuilder rexBuilder) {
+    RexNode newCondition = condition.accept(new Shuttle(rexBuilder));
+    return newCondition == condition ? condition : RexUtil.flatten(rexBuilder, 
newCondition);
+  }
+
+  private static class Shuttle extends RexShuttle {
+    private final RexBuilder _rexBuilder;
+
+    Shuttle(RexBuilder rexBuilder) {
+      _rexBuilder = rexBuilder;
+    }
+
+    @Override
+    public RexNode visitCall(RexCall call) {
+      RexCall visited = (RexCall) super.visitCall(call);
+      if (visited.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
+        return visited;
+      }
+      RexNode rewritten = visited.getKind() == SqlKind.CASE ? 
rewriteCase(visited) : rewritePredicate(visited);
+      return rewritten != null ? rewritten : visited;
+    }
+
+    /// Rewrites `pred(COALESCE(A, d...))` and `pred(CASE WHEN A IS NULL THEN 
d ELSE A END)`, or returns `null` when
+    /// `call` does not have one of these forms.
+    @Nullable
+    private RexNode rewritePredicate(RexCall call) {
+      if (call.isA(SqlKind.AND) || call.isA(SqlKind.OR) || 
call.isA(SqlKind.NOT)) {
+        return null;
+      }
+      List<RexNode> operands = call.getOperands();
+      int index = -1;
+      RexNode[] arrayToMvAndDefault = null;
+      for (int i = 0; i < operands.size(); i++) {
+        RexNode operand = operands.get(i);
+        RexNode[] match = matchDefaultForNull(operand);
+        if (match != null) {
+          if (index >= 0) {
+            return null;
+          }
+          index = i;
+          arrayToMvAndDefault = match;
+        } else if (PinotProjectJoinTransposeRule.containsArrayToMv(operand)) {
+          return null;

Review Comment:
   [P2] Preserve CAST-wrapped nullable-MV COALESCE filters
   
   A supported CAST around COALESCE still bypasses this rewrite. For a nullable 
INT MV column with null handling enabled, `SELECT COUNT(*) FROM tbl WHERE 
CAST(COALESCE(ARRAY_TO_MV(intMv), 0) AS VARCHAR) = '1'` previously worked; for 
`[1,2]`, NULL, and `[-1]`, it should return 1. The shuttle skips the 
non-boolean COALESCE and CAST calls, then this comparison sees CAST as its 
immediate operand and returns without rewriting. Leaf translation removes 
ARRAY_TO_MV, leaving COALESCE to read scalar values from an MV column, which 
fails. Before this PR, Calcite eliminated COALESCE because ARRAY_TO_MV was 
typed non-nullable, and the remaining CAST supports MV input. This also affects 
queries without GROUP BY and persists with `useMvAggregate=false`. Please 
retain the corrected nullability, extend the safe null-branch rewrite through 
supported CAST wrappers, and add a successful-result regression test.



##########
pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java:
##########
@@ -732,6 +733,59 @@ public static class Broker {
     public static final String CONFIG_OF_USE_PHYSICAL_OPTIMIZER = 
"pinot.broker.multistage.use.physical.optimizer";
     public static final boolean DEFAULT_USE_PHYSICAL_OPTIMIZER = false;
 
+    /// How the multi-stage planner plans `GROUP BY ARRAY_TO_MV(col)` by 
default, as one of the [MvAggregateMode]
+    /// values: `auto` (the default), `true` or `false`. The value is 
case-insensitive. The MV aggregate groups each row
+    /// by every element of the multi-value key in every stage, as the 
single-stage engine does.
+    ///
+    /// A server that predates the MV aggregate cannot run it. The `auto` mode 
is safe during a rolling upgrade: it
+    /// sends the MV aggregate only for queries that fail without it.
+    ///
+    /// The value can also be set in the cluster config, under the same key or 
under the alias
+    /// `pinot.all.multistage.use.mv.aggregate`. The same key wins over the 
alias, and the cluster config wins over the
+    /// broker config. A change applies to the next query without a broker 
restart. When the keys are removed from the
+    /// cluster config, the broker config applies again.
+    ///
+    /// The query option [Request.QueryOptionKey#USE_MV_AGGREGATE] overrides 
this value for one query.
+    ///
+    /// These planner changes stay active regardless of this value:
+    /// - The stricter validation of `ARRAY_TO_MV` and its error messages.
+    /// - The leaf-stage check that rejects a plain `GROUP BY` on a 
multi-value column.
+    /// - The classification of the nodes above a lookup join or an `UNNEST` 
as intermediate stage nodes.
+    /// - A filter condition on an `ARRAY_TO_MV(expr)` group key stays above 
the aggregate.
+    public static final String CONFIG_OF_USE_MV_AGGREGATE = 
"pinot.broker.multistage.use.mv.aggregate";
+    public static final String DEFAULT_USE_MV_AGGREGATE = "auto";

Review Comment:
   Suggestion: remove the MV-aggregate flag completely
   
   Could we remove the `useMvAggregate` query option and the broker/cluster 
configuration, and select the operator automatically? Preserve the legacy 
aggregate representation wherever grouping can execute inside the single-stage 
leaf engine, and introduce MvAggregateNode only where MV grouping needs 
intermediate-stage execution. Both leaf representations already lower through 
ServerPlanRequestVisitor.applyAggregate into the same SSE GROUP BY, so forcing 
the new node for these existing queries adds an old-server compatibility 
requirement without an expected execution benefit.
   
   I have not identified an existing supported query that current auto mode 
must move to the new node. The three modes, overrides, aliases, and dynamic 
configuration add substantial combinations to maintain. Automatic selection 
should follow the actual SSE boundary, including lookup joins and UNNEST, and 
keep the nullability, COALESCE, and HAVING correctness fixes. Please cover 
existing supported queries against the old runtime; newly supported query 
shapes can require upgraded servers.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to