This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 1e9486cec3e branch-4.2: [fix](meta path) Keep whole-value TRY_CAST
semantics under nested column pruning #67718 (#70051)
1e9486cec3e is described below
commit 1e9486cec3e52dc857bce7c0a4824d5abd7a9ff7
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:12:41 2026 +0800
branch-4.2: [fix](meta path) Keep whole-value TRY_CAST semantics under
nested column pruning #67718 (#70051)
picked from master #67718
Co-authored-by: minghong <[email protected]>
---
.../rewrite/AccessPathExpressionCollector.java | 16 +++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 10 +-
.../rules/rewrite/PruneNestedColumnTest.java | 64 ++++++++++++
.../column_pruning/try_cast_nested_pruning.out | 25 +++++
.../column_pruning/try_cast_nested_pruning.groovy | 107 +++++++++++++++++++++
5 files changed, 221 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index a814617fbec..fcd84cf9d83 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.TryCast;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
@@ -337,6 +338,21 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
);
}
+ @Override
+ public Void visitTryCast(TryCast tryCast, CollectorContext context) {
+ // TRY_CAST semantics cover the WHOLE value: for a composite type any
element
+ // conversion failure makes the entire cast NULL. Narrowing the
read/type down to
+ // only the fields an outer expression accesses would drop the
conversion attempts
+ // of the other fields and silently change the result (e.g.
element_at(try_cast(s as
+ // struct<a:int,b:int>), 'a') must still yield NULL when only field b
is unparsable,
+ // even though field a alone converts fine). Plain Cast over nested
types is pruned
+ // field-by-field on purpose, a TryCast never is: read the whole child
value and keep
+ // the cast identity and target type intact.
+ return tryCast.child(0).accept(this,
+ new CollectorContext(context.statementContext,
context.bottomFilter)
+ );
+ }
+
// array element at
@Override
public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index e4ca2a09600..016092ee770 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -620,7 +620,15 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
newType = prunedTree.pruneCastType(originTree, castTree);
}
- return new Cast(newChild, newType);
+ // Rebuild through withChildren/withTargetType so the concrete cast
kind survives. A
+ // TRY_CAST that reaches here has kept its whole immediate child value
(see
+ // AccessPathExpressionCollector.visitTryCast, which never translates
an access path
+ // through the cast), but that child can still be rebuilt: for
+ // element_at(try_cast(element_at(wrapper, 'f') as struct<...>), 'g')
the collector
+ // records [wrapper, f], so pruning may drop a sibling of wrapper and
replace the
+ // inner element_at. Constructing a Cast here would silently downgrade
the whole-value
+ // TRY_CAST into a strict CAST, turning its NULL result into a cast
error.
+ return
cast.withChildren(ImmutableList.of(newChild)).withTargetType(newType);
}
private List<TColumnAccessPath> replaceIcebergAccessPathToId(
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 070bc4fcddc..1afe9326e94 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -27,6 +27,7 @@ import
org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.Coll
import
org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -163,6 +164,17 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
+ " s struct<`outer`: struct<`a`: string, `inner_f`:
string>>\n"
+ ") properties ('replication_num'='1')");
+ // Struct holding a nested struct plus a sibling field. The sibling
makes the column
+ // prunable, while element_at(wrapper, 'payload') gives the TRY_CAST a
derived child that
+ // is rebuilt (not just re-typed) when the sibling is pruned away.
+ createTable("create table try_cast_nested_tbl(\n"
+ + " id int,\n"
+ + " wrapper struct<\n"
+ + " payload: struct<good: string, bad: string>,\n"
+ + " unused: int\n"
+ + " >\n"
+ + ") properties ('replication_num'='1')");
+
// Tables for outer-join nullability test: verifying that synthetic
nullability
// from outer join does NOT cause META NULL paths on physically NOT
NULL columns.
createTable("create table driving_tbl(\n"
@@ -637,6 +649,38 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
);
}
+ @Test
+ public void testTryCastNotPrunedThroughFieldAccess() throws Exception {
+ // TRY_CAST of a composite type keeps whole-value semantics: the
conversion fails
+ // (and TRY_CAST returns NULL) if ANY field conversion fails. An outer
field access
+ // element_at(try_cast(s as struct<k:int,...>), 'k') must therefore
not narrow the
+ // underlying read to the accessed field only — the whole struct is
read and the
+ // cast target/identity are unchanged.
+ assertColumn(
+ "select element_at(try_cast(s as
struct<k:int,l:array<map<int,struct<a:int,b:double>>>>), 'k')"
+ + " from tbl",
+
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testTryCastKeptWhenDerivedChildIsRebuilt() throws Exception {
+ // Unlike testTryCastNotPrunedThroughFieldAccess, the TRY_CAST child
here is not the slot
+ // itself but a derived element_at. The whole-value context opened by
visitTryCast records
+ // [wrapper, payload], which only pins the payload value: the sibling
field wrapper.unused
+ // is still pruned away, the wrapper slot type changes, and
SlotTypeReplacer therefore
+ // rebuilds the inner element_at. Rebuilding the cast over that
changed child must not
+ // downgrade the whole-value TRY_CAST into a strict CAST.
+ String sql = "select element_at(try_cast(element_at(wrapper,
'payload') as struct<good:int,bad:int>),"
+ + " 'good') from try_cast_nested_tbl";
+ assertColumn(sql,
+ "struct<payload:struct<good:text,bad:text>>",
+ ImmutableList.of(path("wrapper", "payload")),
+ ImmutableList.of());
+ assertTryCastPreserved(sql, "STRUCT<good:INT,bad:INT>");
+ }
+
@Test
public void testPruneArrayLambda() throws Exception {
// map_values(element_at(s, 'data').*)[0].a
@@ -1537,6 +1581,26 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
assertColumns(sql, expectType == null ? null :
ImmutableList.of(Triple.of(expectType, expectAllAccessPaths,
expectPredicateAccessPaths)));
}
+ /** Assert the physical plan contains a TRY_CAST of the given target type
and no plain CAST of it. */
+ private void assertTryCastPreserved(String sql, String expectTargetType)
throws Exception {
+ PhysicalPlan physicalPlan = collectComplexSlots(sql).first;
+ List<String> casts = new ArrayList<>();
+ physicalPlan.foreachUp(plan -> {
+ for (Expression expression : ((PhysicalPlan)
plan).getExpressions()) {
+ expression.foreach(e -> {
+ if (e instanceof Cast) {
+ Cast cast = (Cast) e;
+ casts.add(cast.getClass().getSimpleName() + " -> " +
cast.getDataType().toSql());
+ }
+ });
+ }
+ });
+ Assertions.assertTrue(casts.contains("TryCast -> " + expectTargetType),
+ "expected TRY_CAST -> " + expectTargetType + " in the physical
plan, but found " + casts);
+ Assertions.assertFalse(casts.contains("Cast -> " + expectTargetType),
+ "TRY_CAST was downgraded to a plain CAST: " + casts);
+ }
+
private void assertAllAccessPathsContain(String sql,
List<TColumnAccessPath> expectContainAllAccessPaths,
List<TColumnAccessPath> expectNotContainAllAccessPaths) throws
Exception {
Pair<PhysicalPlan, List<SlotDescriptor>> result =
collectComplexSlots(sql);
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
b/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
new file mode 100644
index 00000000000..c7f5ea4e11f
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !direct_slot_try_cast --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !derived_child_try_cast --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !direct_slot_try_cast_pruning_off --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !derived_child_try_cast_pruning_off --
+1 10
+2 \N
+3 \N
+4 \N
+
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
new file mode 100644
index 00000000000..883b780d34c
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression tests for TRY_CAST under nested column pruning.
+//
+// TRY_CAST of a composite type keeps whole-value semantics: the conversion
fails (and the
+// result is NULL) if ANY field conversion fails. Nested column pruning may
narrow which
+// columns a scan reads, but it must never narrow the value a TRY_CAST
converts, and it must
+// never downgrade a TRY_CAST into a strict CAST while rebuilding the
expressions whose slots
+// were narrowed.
+//
+// Three shapes are covered:
+// 1. the TRY_CAST child is the slot itself: the whole struct stays read
([s]);
+// 2. the TRY_CAST child is a derived struct access: the container's sibling
field is still
+// pruned ([wrapper.payload]), so the derived element_at is rebuilt and
the TRY_CAST must
+// survive that rebuild. Without the fix the rebuilt expression becomes a
plain CAST and
+// raises "parse number fail" under enable_strict_cast instead of
returning NULL;
+// 3. plain CAST over a nested type: still pruned field by field ([s.a]),
unchanged.
+
+suite("try_cast_nested_pruning") {
+ sql "set enable_prune_nested_column = true"
+ sql "set enable_strict_cast = true"
+ sql "DROP TABLE IF EXISTS try_cast_pruning_tbl"
+ sql """
+ CREATE TABLE try_cast_pruning_tbl (
+ id INT,
+ s STRUCT<a: VARCHAR(10), b: VARCHAR(10)>,
+ wrapper STRUCT<payload: STRUCT<good: VARCHAR(10), bad:
VARCHAR(10)>, unused: INT>
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ sql """
+ INSERT INTO try_cast_pruning_tbl VALUES
+ (1, named_struct('a', '10', 'b', '20'),
+ named_struct('payload', named_struct('good', '10', 'bad',
'20'), 'unused', 1)),
+ (2, named_struct('a', '10', 'b', 'bad'),
+ named_struct('payload', named_struct('good', '10', 'bad',
'bad'), 'unused', 2)),
+ (3, NULL, NULL),
+ (4, named_struct('a', 'bad', 'b', '20'),
+ named_struct('payload', named_struct('good', 'bad', 'bad',
'20'), 'unused', 4))
+ """
+
+ // ── 1. TRY_CAST over the slot itself
────────────────────────────────────────
+ // The whole struct is read: narrowing to s.a would drop the failing b
conversion and
+ // silently turn the NULL of row 2 into 10.
+ explain {
+ sql "select element_at(try_cast(s as struct<a:int,b:int>), 'a') from
try_cast_pruning_tbl"
+ contains "all access paths: [s]"
+ }
+
+ order_qt_direct_slot_try_cast """
+ SELECT id, element_at(TRY_CAST(s AS STRUCT<a:INT,b:INT>), 'a') AS a_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+
+ // ── 2. TRY_CAST over a derived struct access
────────────────────────────────
+ // The whole-value context only pins payload, so the sibling field
wrapper.unused is still
+ // pruned away and the inner element_at is rebuilt on the narrowed wrapper
slot. The cast
+ // must keep both its TRY_CAST identity and its full target type across
that rebuild.
+ explain {
+ sql """
+ select element_at(try_cast(element_at(wrapper, 'payload') as
struct<good:int,bad:int>), 'good')
+ from try_cast_pruning_tbl
+ """
+ contains "all access paths: [wrapper.payload]"
+ }
+
+ order_qt_derived_child_try_cast """
+ SELECT id, element_at(TRY_CAST(element_at(wrapper, 'payload') AS
STRUCT<good:INT,bad:INT>), 'good') AS good_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+
+ // ── 3. Plain CAST over a nested type is still pruned field by field
─────────
+ explain {
+ sql "select element_at(cast(s as struct<a:int,b:int>), 'a') from
try_cast_pruning_tbl"
+ contains "all access paths: [s.a]"
+ }
+
+ // ── 4. Toggling pruning must not change any result
──────────────────────────
+ sql "set enable_prune_nested_column = false"
+ order_qt_direct_slot_try_cast_pruning_off """
+ SELECT id, element_at(TRY_CAST(s AS STRUCT<a:INT,b:INT>), 'a') AS a_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+ order_qt_derived_child_try_cast_pruning_off """
+ SELECT id, element_at(TRY_CAST(element_at(wrapper, 'payload') AS
STRUCT<good:INT,bad:INT>), 'good') AS good_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+ sql "set enable_prune_nested_column = true"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
