This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new cd3abf2ae027 [SPARK-59983][SQL] Fix `StackOverflowError` in `toString`
of V2 expressions with unknown names
cd3abf2ae027 is described below
commit cd3abf2ae027a001e09f4ff6106ddeeb1c8b0d59
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Oct 5 20:29:01 2026 -0700
[SPARK-59983][SQL] Fix `StackOverflowError` in `toString` of V2 expressions
with unknown names
This PR aims to fix `ToStringSQLBuilder` to render a
`GeneralScalarExpression` with an unknown name as a function call, e.g.,
`DATE_TRUNC('MONTH', ts)`.
Currently, `toString` and `describe` of a V2 expression with an unknown
name fail with `StackOverflowError`, because
`V2ExpressionSQLBuilder.visitUnexpectedExpr` builds its error message with
`toString` of the same expression.
```scala
import org.apache.spark.sql.connector.expressions.{Expression, Expressions,
GeneralScalarExpression}
val e = new GeneralScalarExpression("DATE_TRUNC",
Array[Expression](Expressions.literal("MONTH"), Expressions.column("ts")))
e.toString // java.lang.StackOverflowError
```
This is a regression since Apache Spark
[v3.4.0](https://github.com/apache/spark/releases/tag/v3.4.0) (2023-04-07) by
[SPARK-39139](https://issues.apache.org/jira/browse/SPARK-39139).
Yes. `toString` and `describe` return a string like `DATE_TRUNC('MONTH',
ts)` instead of throwing `StackOverflowError`. Known names are rendered as
before, and the SQL builders for pushdown still reject such expressions with
`UNEXPECTED_V2_EXPRESSION`.
Pass the CIs with the newly added test case.
Generated-by: Claude Opus 5.5
Closes #59241 from dongjoon-hyun/SPARK-59983.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 5926a10389e8eace9f655dc138f4bb30254f7bed)
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit f7b621b0447170691a3c079f99b58ceb1e50e11a)
(cherry picked from commit 6b99745a0a5e8a4c93ae541f889ab7f50d78027e)
(cherry picked from commit a26757ad997167ee8066952f69b9c3b3edf4ffc3)
---
.../internal/connector/ToStringSQLBuilder.scala | 11 ++++++++
.../expressions/V2ExpressionUtilsSuite.scala | 32 +++++++++++++++++++++-
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
index 118c1af97745..c21e4ca09158 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ToStringSQLBuilder.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.internal.connector
+import org.apache.spark.sql.connector.expressions.Expression
+import org.apache.spark.sql.connector.expressions.GeneralScalarExpression
import org.apache.spark.sql.connector.util.V2ExpressionSQLBuilder
/**
@@ -35,4 +37,13 @@ class ToStringSQLBuilder extends V2ExpressionSQLBuilder with
Serializable {
val distinct = if (isDistinct) "DISTINCT " else ""
s"""$funcName($distinct${inputs.mkString(", ")})"""
}
+
+ // `super.visitUnexpectedExpr` builds its error message from
`expr.toString`, which calls back
+ // into `ToStringSQLBuilder` for a `GeneralScalarExpression`, so an unknown
name would recurse
+ // until StackOverflowError. Render it as a function call instead.
+ override protected def visitUnexpectedExpr(expr: Expression): String = expr
match {
+ case e: GeneralScalarExpression =>
+ visitSQLFunction(e.name(), expressionsToStringArray(e.children()))
+ case _ => super.visitUnexpectedExpr(expr)
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
index d1c23d68555a..5245ad81841f 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtilsSuite.scala
@@ -17,11 +17,14 @@
package org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
import org.apache.spark.sql.connector.expressions._
+import org.apache.spark.sql.connector.expressions.filter.{Predicate =>
V2Predicate}
+import org.apache.spark.sql.connector.util.V2ExpressionSQLBuilder
import org.apache.spark.sql.types.StringType
+import org.apache.spark.unsafe.types.UTF8String
class V2ExpressionUtilsSuite extends SparkFunSuite {
@@ -37,4 +40,31 @@ class V2ExpressionUtilsSuite extends SparkFunSuite {
}
assert(exc.message.contains("v2Fun(a) ASC NULLS FIRST is not currently
supported"))
}
+
+ test("SPARK-59983: V2 expression with an unknown name should be rendered as
a function call") {
+ val dateTrunc = new GeneralScalarExpression("DATE_TRUNC",
+ Array(LiteralValue(UTF8String.fromString("MONTH"), StringType),
FieldReference("a")))
+ assert(dateTrunc.toString === "DATE_TRUNC('MONTH', a)")
+ assert(dateTrunc.describe() === "DATE_TRUNC('MONTH', a)")
+ assert(new GeneralScalarExpression("ABS", Array(dateTrunc)).toString ===
+ "ABS(DATE_TRUNC('MONTH', a))")
+ val like = new V2Predicate("LIKE",
+ Array(FieldReference("a"), LiteralValue(UTF8String.fromString("x%"),
StringType)))
+ assert(like.toString === "LIKE(a, 'x%')")
+ // The SQL builder for pushdown still rejects an unknown name.
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ new V2ExpressionSQLBuilder().build(dateTrunc)
+ },
+ condition = "_LEGACY_ERROR_TEMP_3207",
+ parameters = Map("expr" -> "DATE_TRUNC('MONTH', a)"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ V2ExpressionUtils.toCatalystOrdering(
+ Array(SortValue(dateTrunc, SortDirection.ASCENDING,
NullOrdering.NULLS_FIRST)),
+ LocalRelation.apply(AttributeReference("a", StringType)()))
+ },
+ condition = "_LEGACY_ERROR_TEMP_3054",
+ parameters = Map("expr" -> "DATE_TRUNC('MONTH', a)"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
