This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new bb038a6739 fix: ScalarSubqueryToJoin fails when CTE inside correlated
scalar subquery (#24363)
bb038a6739 is described below
commit bb038a6739091567fe32c196d8c568a98e4f42e2
Author: Ruchir Tripathi <[email protected]>
AuthorDate: Mon Aug 17 06:55:18 2026 +0000
fix: ScalarSubqueryToJoin fails when CTE inside correlated scalar subquery
(#24363)
## Which issue does this PR close?
- Closes #24139
## Rationale for this change
The `scalar_subquery_to_join` optimizer rule was failing with a `Schema
error: No field named __always_true` when a CTE was used inside a
correlated scalar subquery.
This occurred because the `SubqueryAlias` logical plan node wasn't
refreshing its schema after the `__always_true` column was injected by
the correlated expression
pull-up logic (`PullUpCorrelatedExpr`). Because the schema became stale,
the parent `Projection` node would attempt to reference the newly
injected `__always_true`
column and fail with a schema error.
## What changes are included in this PR?
- Updated `PullUpCorrelatedExpr::f_up` logic for
`LogicalPlan::SubqueryAlias` in
`datafusion/optimizer/src/decorrelate.rs`.
- We now check if the `SubqueryAlias`'s input schema has a different
number of fields than its own recorded schema. If so, we recreate the
`SubqueryAlias` using
`LogicalPlanBuilder` so that its schema correctly reflects any newly
added columns (such as `__always_true`).
## Are these changes tested?
Yes, they are covered by the existing sqllogictests and optimizer
integration tests. Queries with CTEs inside correlated scalar subqueries
that previously crashed
the optimizer will now correctly optimize and execute.
## Are there any user-facing changes?
No, this is strictly an internal optimizer bug fix.
---
datafusion/optimizer/src/decorrelate.rs | 22 +++++++++++++++++---
datafusion/sqllogictest/test_files/subquery.slt | 27 +++++++++++++++++++++++++
2 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/datafusion/optimizer/src/decorrelate.rs
b/datafusion/optimizer/src/decorrelate.rs
index 9490af0e59..aab58c5519 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -350,15 +350,31 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
new_correlated_cols
.insert(Column::new(Some(alias.alias.clone()),
col.name.clone()));
}
+
+ let new_plan = if alias.input.schema().fields().len()
+ != alias.schema.fields().len()
+ {
+ LogicalPlanBuilder::from((*alias.input).clone())
+ .alias(alias.alias.clone())?
+ .build()?
+ } else {
+ plan.clone()
+ };
+
self.correlated_subquery_cols_map
- .insert(plan.clone(), new_correlated_cols);
+ .insert(new_plan.clone(), new_correlated_cols);
if let Some(input_map) =
self.collected_count_expr_map.get(alias.input.deref())
{
self.collected_count_expr_map
- .insert(plan.clone(), input_map.clone());
+ .insert(new_plan.clone(), input_map.clone());
+ }
+
+ if new_plan != plan {
+ Ok(Transformed::yes(new_plan))
+ } else {
+ Ok(Transformed::no(plan))
}
- Ok(Transformed::no(plan))
}
LogicalPlan::Limit(limit) => {
let input_expr_map = self
diff --git a/datafusion/sqllogictest/test_files/subquery.slt
b/datafusion/sqllogictest/test_files/subquery.slt
index dcca13c416..594964778b 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2657,3 +2657,30 @@ DROP TABLE nia_right_no_null;
statement ok
reset datafusion.optimizer.prefer_hash_join;
+
+# Regression test for #24139: CTE inside a correlated scalar subquery causing
a schema error
+statement ok
+CREATE TABLE metrics (host VARCHAR, bytes_sent DOUBLE, ts BIGINT) AS VALUES
+ ('a', 100.0, 1),
+ ('b', 200.0, 2),
+ ('a', 300.0, 3),
+ ('b', 400.0, 4);
+
+query TR
+SELECT m.host, m.bytes_sent
+FROM metrics m
+WHERE m.bytes_sent > (
+ WITH avg_per_host AS (
+ SELECT avg(bytes_sent) AS avg_bytes
+ FROM metrics
+ WHERE host = m.host
+ )
+ SELECT avg_bytes FROM avg_per_host
+)
+ORDER BY m.host;
+----
+a 300
+b 400
+
+statement ok
+DROP TABLE metrics;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
