This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 0eb25a7e18b2 [SPARK-59983][SQL] Fix `StackOverflowError` in `toString`
of V2 expressions with unknown names
0eb25a7e18b2 is described below
commit 0eb25a7e18b2bbbe8d58cb5d96d64df67a8ef40c
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Oct 5 17:30:35 2026 -0700
[SPARK-59983][SQL] Fix `StackOverflowError` in `toString` of V2 expressions
with unknown names
### What changes were proposed in this pull request?
This PR aims to fix `ToStringSQLBuilder` to render a
`GeneralScalarExpression` with an unknown name as a function call, e.g.,
`DATE_TRUNC('MONTH', ts)`.
### Why are the changes needed?
Currently, `toString` and `describe` of a V2 expression with an unknown
name fail with `StackOverflowError`, because
`V2ExpressionSQLBuilder.visitUnexpectedExpr` builds its error message with
`toString` of the same expression.
```scala
import org.apache.spark.sql.connector.expressions.{Expression, Expressions,
GeneralScalarExpression}
val e = new GeneralScalarExpression("DATE_TRUNC",
Array[Expression](Expressions.literal("MONTH"), Expressions.column("ts")))
e.toString // java.lang.StackOverflowError
```
This is a regression since Apache Spark
[v3.4.0](https://github.com/apache/spark/releases/tag/v3.4.0) (2023-04-07) by
[SPARK-39139](https://issues.apache.org/jira/browse/SPARK-39139).
### Does this PR introduce _any_ user-facing change?
Yes. `toString` and `describe` return a string like `DATE_TRUNC('MONTH',
ts)` instead of throwing `StackOverflowError`. Known names are rendered as
before, and the SQL builders for pushdown still reject such expressions with
`UNEXPECTED_V2_EXPRESSION`.
### How was this patch tested?
Pass the CIs with the newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59241 from dongjoon-hyun/SPARK-59983.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 5926a10389e8eace9f655dc138f4bb30254f7bed)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../internal/connector/ToStringSQLBuilder.scala | 10 +++++++
.../expressions/V2ExpressionUtilsSuite.scala | 32 +++++++++++++++++++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
index 9df132418724..2d6c2256b7fa 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.internal.connector
+import org.apache.spark.sql.connector.expressions.Expression
+import org.apache.spark.sql.connector.expressions.GeneralScalarExpression
import org.apache.spark.sql.connector.expressions.GetArrayItem
import org.apache.spark.sql.connector.expressions.VariantGet
import org.apache.spark.sql.connector.util.V2ExpressionSQLBuilder
@@ -50,4 +52,12 @@ class ToStringSQLBuilder extends V2ExpressionSQLBuilder with
Serializable {
val tz = Option(variantGet.timeZoneId()).map(z => s", tz=$z").getOrElse("")
s"$funcName($col, '$path', $typ$tz)"
}
+
+ // `super.visitUnexpectedExpr` builds its error message from
`expr.toString`, which calls back
+ // into `ToStringSQLBuilder` for a `GeneralScalarExpression`, so an unknown
name would recurse
+ // until StackOverflowError. Render it as a function call instead.
+ override protected def visitUnexpectedExpr(expr: Expression): String = expr
match {
+ case e: GeneralScalarExpression => visitSQLFunction(e.name(), e.children())
+ case _ => super.visitUnexpectedExpr(expr)
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
index 899ddf5e9d3b..ffc1d5c48c96 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
@@ -17,11 +17,14 @@
package org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
import org.apache.spark.sql.connector.expressions._
+import org.apache.spark.sql.connector.expressions.filter.{Predicate =>
V2Predicate}
+import org.apache.spark.sql.connector.util.V2ExpressionSQLBuilder
import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.unsafe.types.UTF8String
class V2ExpressionUtilsSuite extends SparkFunSuite {
@@ -38,6 +41,33 @@ class V2ExpressionUtilsSuite extends SparkFunSuite {
assert(exc.message.contains("v2Fun(a) ASC NULLS FIRST is not currently
supported"))
}
+ test("SPARK-59983: V2 expression with an unknown name should be rendered as
a function call") {
+ val dateTrunc = new GeneralScalarExpression("DATE_TRUNC",
+ Array(LiteralValue(UTF8String.fromString("MONTH"), StringType),
FieldReference("a")))
+ assert(dateTrunc.toString === "DATE_TRUNC('MONTH', a)")
+ assert(dateTrunc.describe() === "DATE_TRUNC('MONTH', a)")
+ assert(new GeneralScalarExpression("ABS", Array(dateTrunc)).toString ===
+ "ABS(DATE_TRUNC('MONTH', a))")
+ val like = new V2Predicate("LIKE",
+ Array(FieldReference("a"), LiteralValue(UTF8String.fromString("x%"),
StringType)))
+ assert(like.toString === "LIKE(a, 'x%')")
+ // The SQL builder for pushdown still rejects an unknown name.
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ new V2ExpressionSQLBuilder().build(dateTrunc)
+ },
+ condition = "UNEXPECTED_V2_EXPRESSION",
+ parameters = Map("expr" -> "DATE_TRUNC('MONTH', a)"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ V2ExpressionUtils.toCatalystOrdering(
+ Array(SortValue(dateTrunc, SortDirection.ASCENDING,
NullOrdering.NULLS_FIRST)),
+ LocalRelation.apply(AttributeReference("a", StringType)()))
+ },
+ condition = "_LEGACY_ERROR_TEMP_3054",
+ parameters = Map("expr" -> "DATE_TRUNC('MONTH', a)"))
+ }
+
test("SPARK-59721: resolveRefOpt returns None for an unresolvable
reference") {
val plan = LocalRelation(AttributeReference("a", StringType)())
assert(V2ExpressionUtils.resolveRefOpt(FieldReference("a"),
plan).isDefined)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
