This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 8c3a7c9fb3f branch-4.2: [fix](nested column pruning) Resolve captured
lambda access paths by ExprId #67047 (#70068)
8c3a7c9fb3f is described below
commit 8c3a7c9fb3f0b582be40deac69c751e52b4c4354
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:19:23 2026 +0800
branch-4.2: [fix](nested column pruning) Resolve captured lambda access
paths by ExprId #67047 (#70068)
picked from master #67047
Co-authored-by: linrrarity <[email protected]>
---
.../rewrite/AccessPathExpressionCollector.java | 36 ++++++---
.../rules/rewrite/PruneNestedColumnTest.java | 36 ++++++++-
.../nested_lambda_outer_argument_pruning.out | 12 +++
.../nested_lambda_outer_argument_pruning.groovy | 85 ++++++++++++++++++++++
4 files changed, 156 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index d86286acc35..01eeb8a6927 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -90,7 +90,7 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
private boolean bottomPredicate;
private boolean skipMetaPath;
private Multimap<Integer, CollectAccessPathResult> slotToAccessPaths;
- private Stack<Map<String, Expression>> nameToLambdaArguments = new
Stack<>();
+ private Stack<Map<ExprId, Expression>> exprIdToLambdaArguments = new
Stack<>();
public AccessPathExpressionCollector(
StatementContext statementContext, Multimap<Integer,
CollectAccessPathResult> slotToAccessPaths,
@@ -312,15 +312,17 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
@Override
public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot,
CollectorContext context) {
- if (nameToLambdaArguments.isEmpty()) {
+ if (exprIdToLambdaArguments.isEmpty()) {
return null;
}
context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_ALL);
- Expression argument =
nameToLambdaArguments.peek().get(arrayItemSlot.getName());
- if (argument == null) {
- return null;
+ for (int i = exprIdToLambdaArguments.size() - 1; i >= 0; i--) {
+ Expression argument =
exprIdToLambdaArguments.get(i).get(arrayItemSlot.getExprId());
+ if (argument != null) {
+ return continueCollectAccessPath(argument, context);
+ }
}
- return continueCollectAccessPath(argument, context);
+ return null;
}
@Override
@@ -376,7 +378,11 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
continueCollectAccessPath(first, context);
for (int i = 1; i < arguments.size(); i++) {
- visit(arguments.get(i), context);
+ // Dispatch every index/key expression with a fresh context.
For example, in
+ // element_at(a, i), generic visit(i, context) treats a leaf
lambda slot as having
+ // no children and skips visitArrayItemSlot, so the bound
index array loses its payload.
+ arguments.get(i).accept(this,
+ new CollectorContext(context.statementContext,
context.bottomFilter));
}
return null;
} else if (first.getDataType().isVariantType() && arguments.size() >= 2
@@ -555,7 +561,13 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
Expression argument = arraySort.getArgument(0);
if ((argument instanceof Lambda)) {
- return collectArrayPathInLambda((Lambda) argument, context);
+ Lambda lambda = (Lambda) argument;
+ // A comparator may inspect only array lengths, while array_sort
still returns every element.
+ // Propagate the result access path through the source expression
to preserve its payload.
+ CollectorContext resultContext = copyContext(context);
+ collectArrayPathInLambda(lambda, context);
+ return continueCollectAccessPath(
+ lambda.getLambdaArgument(0).getArrayExpression(),
resultContext);
}
return visit(arraySort, context);
}
@@ -771,10 +783,10 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
private Void collectArrayPathInLambda(Lambda lambda, CollectorContext
context) {
List<Expression> arguments = lambda.getArguments();
- Map<String, Expression> nameToArray = Maps.newLinkedHashMap();
+ Map<ExprId, Expression> exprIdToArray = Maps.newLinkedHashMap();
for (Expression argument : arguments) {
if (argument instanceof ArrayItemReference) {
- nameToArray.put(((ArrayItemReference) argument).getName(),
argument.child(0));
+ exprIdToArray.put(((ArrayItemReference) argument).getExprId(),
argument.child(0));
}
}
@@ -783,11 +795,11 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
context.accessPathBuilder.removePrefix();
}
- nameToLambdaArguments.push(nameToArray);
+ exprIdToLambdaArguments.push(exprIdToArray);
try {
continueCollectAccessPath(arguments.get(0), context);
} finally {
- nameToLambdaArguments.pop();
+ exprIdToLambdaArguments.pop();
}
// After visiting the lambda body, for any bound array whose lambda
variable
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index fb0f89738eb..18ec4114f71 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -137,7 +137,8 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
createTable("create table nested_array_tbl(\n"
+ " id int,\n"
- + " a array<array<int>>\n"
+ + " a array<array<int>>,\n"
+ + " indexes array<bigint>\n"
+ ") properties ('replication_num'='1')");
createTable("create table map_array_tbl(\n"
@@ -254,6 +255,32 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
ImmutableList.of(path("a", "*")));
}
+ @Test
+ public void testComparatorArraySortKeepsPayloadPath() throws Exception {
+ assertColumn("select array_sort((x, y) -> if(cardinality(x) <
cardinality(y), -1, "
+ + "if(cardinality(x) = cardinality(y), 0, 1)), a) from
nested_array_tbl",
+ "array<array<int>>",
+ ImmutableList.of(path("a")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testElementAtLambdaIndexKeepsPayloadPath() throws Exception {
+ assertColumns("select array_map((a, i) -> element_at(a, i), a,
indexes) "
+ + "from nested_array_tbl where indexes is not null",
+ ImmutableList.of(
+ Triple.of(
+ "array<array<int>>",
+ ImmutableList.of(path("a", "*", "*")),
+ ImmutableList.of()),
+ Triple.of(
+ "array<bigint>",
+ // The branch keeps element payload and typed
NULL metadata separately.
+ ImmutableList.of(path("indexes", "*"),
metaPath("indexes", "NULL")),
+ ImmutableList.of(metaPath("indexes", "NULL")))
+ ));
+ }
+
@Test
public void testCardinalityMapElementKeepsValueOffsetPath() throws
Exception {
assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl",
@@ -705,6 +732,13 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
ImmutableList.of(path("s", "data", "*", "VALUES", "a"),
path("s", "data", "*", "VALUES", "b")),
ImmutableList.of()
);
+
+ assertColumn("select array_map(m -> array_map(x ->
element_at(map_values(m)[0], 'a'), [1]), "
+ + "element_at(s, 'data')) from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
}
@Test
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.out
b/regression-test/data/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.out
new file mode 100644
index 00000000000..9706b9378ca
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !capture_parent --
+1 [[11]]
+2 [[21]]
+3 []
+4 \N
+
+-- !capture_sibling_lambdas --
+1 [[11, 101]]
+2 [[21, 201]]
+3 []
+4 \N
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.groovy
new file mode 100644
index 00000000000..6d700234b99
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/nested_lambda_outer_argument_pruning.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nested_lambda_outer_argument_pruning") {
+ sql """ DROP TABLE IF EXISTS nested_lambda_outer_argument_pruning_tbl """
+ sql """
+ CREATE TABLE nested_lambda_outer_argument_pruning_tbl (
+ id INT,
+ maps ARRAY<MAP<INT, STRUCT<a: INT, b: INT>>> NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO nested_lambda_outer_argument_pruning_tbl VALUES
+ (1, array(map(1, named_struct('a', 10, 'b', 100)))),
+ (2, array(map(2, named_struct('a', 20, 'b', 200)))),
+ (3, array()),
+ (4, NULL)
+ """
+
+ // The inner lambda captures m from the immediately enclosing lambda.
+ explain {
+ sql """
+ SELECT array_map(
+ m -> array_map(x -> element_at(map_values(m)[1], 'a') +
x, [1]),
+ maps)
+ FROM nested_lambda_outer_argument_pruning_tbl
+ """
+ contains "nested columns"
+ contains "maps.*.VALUES.a"
+ notContains "maps.*.VALUES.b"
+ }
+
+ order_qt_capture_parent """
+ SELECT id,
+ array_map(
+ m -> array_map(x -> element_at(map_values(m)[1], 'a') + x,
[1]),
+ maps)
+ FROM nested_lambda_outer_argument_pruning_tbl
+ ORDER BY id
+ """
+
+ // Two sibling lambdas both capture m. This also verifies that entering and
+ // leaving one inner lambda does not leak or discard the outer scope.
+ explain {
+ sql """
+ SELECT array_map(
+ m -> array_concat(
+ array_map(x -> element_at(map_values(m)[1], 'a') +
x, [1]),
+ array_map(y -> element_at(map_values(m)[1], 'b') +
y, [1])),
+ maps)
+ FROM nested_lambda_outer_argument_pruning_tbl
+ """
+ contains "nested columns"
+ contains "maps.*.VALUES.a"
+ contains "maps.*.VALUES.b"
+ }
+
+ order_qt_capture_sibling_lambdas """
+ SELECT id,
+ array_map(
+ m -> array_concat(
+ array_map(x -> element_at(map_values(m)[1], 'a') + x,
[1]),
+ array_map(y -> element_at(map_values(m)[1], 'b') + y,
[1])),
+ maps)
+ FROM nested_lambda_outer_argument_pruning_tbl
+ ORDER BY id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
