github-actions[bot] commented on code in PR #63974:
URL: https://github.com/apache/doris/pull/63974#discussion_r3596373388
##########
fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java:
##########
@@ -254,153 +263,272 @@ private static void walkPlan(Plan plan,
Aggregate<?> agg = (Aggregate<?>) plan;
// GROUP BY keys
for (Expression expr : agg.getGroupByExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
// Columns consumed by aggregate functions (e.g. k2 in SUM(k2))
for (NamedExpression expr : agg.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ }
+ // HAVING: link each non-scan-backed aggregate output to its own
inputs, however
+ // computed those already are (e.g. HAVING SUM(x) where x is
itself computed).
+ for (NamedExpression ne : agg.getOutputExpressions()) {
+ if (exprIdToScan.containsKey(ne.getExprId()) || ne instanceof
Slot) {
+ // Bare pass-through output (e.g. two-phase DISTINCT's
merge phase reusing the
+ // local phase's slot) — already linked there;
getInputSlots() on a bare Slot
+ // includes itself, which would self-loop here.
+ continue;
+ }
+ Set<Slot> inputSlots = ne.getInputSlots();
+ if (!inputSlots.isEmpty()) {
+ derivedSlotInputs.put(ne.getExprId(), inputSlots);
+ }
}
}
if (plan instanceof AbstractPhysicalSort) {
for (OrderKey orderKey : ((AbstractPhysicalSort<?>)
plan).getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalPartitionTopN does not extend AbstractPhysicalSort but also
has ORDER BY and
// partition keys (used for row_number() / rank() per partition).
if (plan instanceof PhysicalPartitionTopN) {
PhysicalPartitionTopN<?> ptn = (PhysicalPartitionTopN<?>) plan;
for (Expression partKey : ptn.getPartitionKeys()) {
- recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (OrderKey orderKey : ptn.getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalRepeat handles ROLLUP/CUBE: group sets are like GROUP BY
keys.
if (plan instanceof PhysicalRepeat) {
PhysicalRepeat<?> repeat = (PhysicalRepeat<?>) plan;
for (List<Expression> groupSet : repeat.getGroupingSets()) {
for (Expression expr : groupSet) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
}
for (NamedExpression expr : repeat.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
Review Comment:
**[P1] Preserve provenance for `PhysicalRepeat` outputs**
The existing `analyze_repeat` query produces this shape:
```text
Filter(GROUPING_PREFIX_a#G = 1)
Aggregate(groupBy=[GROUPING_PREFIX_a#G], ...)
PhysicalRepeat(outputs=[..., Alias(Grouping(a#A), GROUPING_PREFIX_a#G)])
OlapScan(a#A)
```
This call records `a#A` as a `queryHit`, but it never stores `G -> A` in
`derivedSlotInputs`. The aggregate passes `G` through, so the parent HAVING
filter resolves `G` to nothing and misses `a.filterHit`. Please add
output-to-input provenance for non-scan-backed repeat expressions (with the
same bare-slot/self-edge guard used for aggregate/project outputs) and cover
`HAVING GROUPING(a) = 1` or `GROUPING_ID(...) > 0`; the current repeat test
stubs an empty output-expression list and cannot exercise this path.
##########
fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java:
##########
@@ -254,153 +263,272 @@ private static void walkPlan(Plan plan,
Aggregate<?> agg = (Aggregate<?>) plan;
// GROUP BY keys
for (Expression expr : agg.getGroupByExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
// Columns consumed by aggregate functions (e.g. k2 in SUM(k2))
for (NamedExpression expr : agg.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ }
+ // HAVING: link each non-scan-backed aggregate output to its own
inputs, however
+ // computed those already are (e.g. HAVING SUM(x) where x is
itself computed).
+ for (NamedExpression ne : agg.getOutputExpressions()) {
+ if (exprIdToScan.containsKey(ne.getExprId()) || ne instanceof
Slot) {
+ // Bare pass-through output (e.g. two-phase DISTINCT's
merge phase reusing the
+ // local phase's slot) — already linked there;
getInputSlots() on a bare Slot
+ // includes itself, which would self-loop here.
+ continue;
+ }
+ Set<Slot> inputSlots = ne.getInputSlots();
+ if (!inputSlots.isEmpty()) {
+ derivedSlotInputs.put(ne.getExprId(), inputSlots);
+ }
}
}
if (plan instanceof AbstractPhysicalSort) {
for (OrderKey orderKey : ((AbstractPhysicalSort<?>)
plan).getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalPartitionTopN does not extend AbstractPhysicalSort but also
has ORDER BY and
// partition keys (used for row_number() / rank() per partition).
if (plan instanceof PhysicalPartitionTopN) {
PhysicalPartitionTopN<?> ptn = (PhysicalPartitionTopN<?>) plan;
for (Expression partKey : ptn.getPartitionKeys()) {
- recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (OrderKey orderKey : ptn.getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalRepeat handles ROLLUP/CUBE: group sets are like GROUP BY
keys.
if (plan instanceof PhysicalRepeat) {
PhysicalRepeat<?> repeat = (PhysicalRepeat<?>) plan;
for (List<Expression> groupSet : repeat.getGroupingSets()) {
for (Expression expr : groupSet) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
}
for (NamedExpression expr : repeat.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
}
if (plan instanceof PhysicalWindow) {
WindowFrameGroup wfg = ((PhysicalWindow<?>)
plan).getWindowFrameGroup();
Set<Expression> partitionKeys = wfg.getPartitionKeys();
for (Expression expr : partitionKeys) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (OrderExpression orderExpr : wfg.getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
- // queryHit for the window function value columns (e.g. k2 in
SUM(k2) OVER (...)).
+ // queryHit for window value columns (e.g. k2 in SUM(k2) OVER
(...)), and link the
+ // alias to those inputs for a QUALIFY-style filter above;
positional functions
+ // like ROW_NUMBER have no column arguments, so nothing gets
linked for those.
for (NamedExpression windowAlias : wfg.getGroups()) {
Expression windowExpr = windowAlias.child(0);
if (windowExpr instanceof WindowExpression) {
- recordInputSlotsAsQueryHit(
- ((WindowExpression) windowExpr).getFunction(),
- exprIdToScan, exprIdToColName, deltas);
+ Expression function = ((WindowExpression)
windowExpr).getFunction();
+ recordInputSlotsAsQueryHit(function, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ if (!exprIdToScan.containsKey(windowAlias.getExprId())) {
+ Set<Slot> inputSlots = function.getInputSlots();
Review Comment:
**[P1] Include window partition/order inputs in alias provenance**
```text
Filter(rk#R > 1)
PhysicalWindow(Alias(row_number() OVER (ORDER BY year#Y), rk#R))
OlapScan(year#Y)
```
`QUALIFY` is rewritten to this alias filter, and the PartitionTopN
optimization retains the filter above the window.
`RowNumber`/`Rank`/`DenseRank` have no function arguments, so
`function.getInputSlots()` is empty and no `R -> Y` edge is stored; the filter
therefore records no `filterHit` for `year`. A value window has the same
partial problem: `SUM(k2) OVER (PARTITION BY k0 ORDER BY k1)` links only `k2`,
omitting `k0/k1` even though those inputs determine the filtered result.
`WindowExpression` already contains the function, partition keys, and order
keys, so please link the alias through its complete input set and add a
zero-argument `row_number` QUALIFY/subquery-filter case.
##########
fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java:
##########
@@ -254,153 +263,272 @@ private static void walkPlan(Plan plan,
Aggregate<?> agg = (Aggregate<?>) plan;
// GROUP BY keys
for (Expression expr : agg.getGroupByExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
// Columns consumed by aggregate functions (e.g. k2 in SUM(k2))
for (NamedExpression expr : agg.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ }
+ // HAVING: link each non-scan-backed aggregate output to its own
inputs, however
+ // computed those already are (e.g. HAVING SUM(x) where x is
itself computed).
+ for (NamedExpression ne : agg.getOutputExpressions()) {
+ if (exprIdToScan.containsKey(ne.getExprId()) || ne instanceof
Slot) {
+ // Bare pass-through output (e.g. two-phase DISTINCT's
merge phase reusing the
+ // local phase's slot) — already linked there;
getInputSlots() on a bare Slot
+ // includes itself, which would self-loop here.
+ continue;
+ }
+ Set<Slot> inputSlots = ne.getInputSlots();
+ if (!inputSlots.isEmpty()) {
+ derivedSlotInputs.put(ne.getExprId(), inputSlots);
+ }
}
}
if (plan instanceof AbstractPhysicalSort) {
for (OrderKey orderKey : ((AbstractPhysicalSort<?>)
plan).getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalPartitionTopN does not extend AbstractPhysicalSort but also
has ORDER BY and
// partition keys (used for row_number() / rank() per partition).
if (plan instanceof PhysicalPartitionTopN) {
PhysicalPartitionTopN<?> ptn = (PhysicalPartitionTopN<?>) plan;
for (Expression partKey : ptn.getPartitionKeys()) {
- recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(partKey, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (OrderKey orderKey : ptn.getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
}
// PhysicalRepeat handles ROLLUP/CUBE: group sets are like GROUP BY
keys.
if (plan instanceof PhysicalRepeat) {
PhysicalRepeat<?> repeat = (PhysicalRepeat<?>) plan;
for (List<Expression> groupSet : repeat.getGroupingSets()) {
for (Expression expr : groupSet) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
}
for (NamedExpression expr : repeat.getOutputExpressions()) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
}
if (plan instanceof PhysicalWindow) {
WindowFrameGroup wfg = ((PhysicalWindow<?>)
plan).getWindowFrameGroup();
Set<Expression> partitionKeys = wfg.getPartitionKeys();
for (Expression expr : partitionKeys) {
- recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(expr, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (OrderExpression orderExpr : wfg.getOrderKeys()) {
- recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan,
exprIdToColName, deltas,
+ derivedSlotInputs);
}
- // queryHit for the window function value columns (e.g. k2 in
SUM(k2) OVER (...)).
+ // queryHit for window value columns (e.g. k2 in SUM(k2) OVER
(...)), and link the
+ // alias to those inputs for a QUALIFY-style filter above;
positional functions
+ // like ROW_NUMBER have no column arguments, so nothing gets
linked for those.
for (NamedExpression windowAlias : wfg.getGroups()) {
Expression windowExpr = windowAlias.child(0);
if (windowExpr instanceof WindowExpression) {
- recordInputSlotsAsQueryHit(
- ((WindowExpression) windowExpr).getFunction(),
- exprIdToScan, exprIdToColName, deltas);
+ Expression function = ((WindowExpression)
windowExpr).getFunction();
+ recordInputSlotsAsQueryHit(function, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ if (!exprIdToScan.containsKey(windowAlias.getExprId())) {
+ Set<Slot> inputSlots = function.getInputSlots();
+ if (!inputSlots.isEmpty()) {
+ derivedSlotInputs.put(windowAlias.getExprId(),
inputSlots);
+ }
+ }
}
}
}
- // filterHit for JOIN ON conditions (hash equality and non-equality
predicates).
+ // filterHit for all JOIN ON conditions; mark conjuncts are a separate
field not included
+ // in hashJoinConjuncts or otherJoinConjuncts (IN/EXISTS subquery
correlation columns).
if (plan instanceof AbstractPhysicalJoin) {
AbstractPhysicalJoin<?, ?> join = (AbstractPhysicalJoin<?, ?>)
plan;
for (Expression conjunct : join.getHashJoinConjuncts()) {
- recordInputSlotsAsFilterHit(conjunct, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsFilterHit(conjunct, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
}
for (Expression conjunct : join.getOtherJoinConjuncts()) {
- recordInputSlotsAsFilterHit(conjunct, exprIdToScan,
exprIdToColName, deltas);
+ recordInputSlotsAsFilterHit(conjunct, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ }
+ for (Expression conjunct : join.getMarkJoinConjuncts()) {
+ recordInputSlotsAsFilterHit(conjunct, exprIdToScan,
exprIdToColName, deltas, derivedSlotInputs);
+ }
+ }
+ // UNION / INTERSECT / EXCEPT: queryHit per branch, plus link the set
op's own output
+ // slots to those branch slots so a filter kept above the set op (e.g.
a volatile
+ // predicate PushDownFilterThroughSetOperation can't push down) still
resolves.
+ if (plan instanceof PhysicalSetOperation) {
+ PhysicalSetOperation setOp = (PhysicalSetOperation) plan;
+ recordSetOpChildrenOutputs(setOp.getOutput(),
setOp.getRegularChildrenOutputs(),
+ exprIdToScan, exprIdToColName, deltas, derivedSlotInputs);
+ }
+ // PhysicalRecursiveUnion extends PhysicalBinary (not
PhysicalSetOperation); handle its
+ // getRegularChildrenOutputs() explicitly. Recursive-case
(WorkTableReference) slots skipped.
Review Comment:
**[P1] Preserve recursive work-table provenance before recording usages**
A real recursive plan has this reduced shape:
```text
PhysicalRecursiveUnion(out curr_id#U, total_score#V)
├─ Project(id, budget) -> Filter(parent_id IS NULL) -> OlapScan
└─ Project(t.id, c.total_score + t.budget)
└─ Filter(c.total_score#W2 < 50000)
└─ Join(t.parent_id = c.curr_id#W1)
├─ OlapScan t
└─ PhysicalWorkTableReference(curr_id#W1, total_score#W2)
```
`walkPlan` processes the right-child filter/join before reaching this block.
`W1/W2` are fresh work-table ExprIds with no scan or derived edge, so both
usages resolve to nothing. This block later links the recursive-union output to
its regular child outputs, but it never links work-table slots back to the
union output, and in any case that edge arrives too late to record the
already-visited predicates. The result is missing `filterHit` for the
originating `id`/`budget` columns. Please build provenance before recording
usages (including positional work-table-to-union edges) and add a real
recursive query-stats regression; the current mock explicitly expects the
recursive branch to be skipped.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
