This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new f090eaca12c5 [SPARK-60762][SQL] Pass ScalaUDF error message parameters 
to generated code as references
f090eaca12c5 is described below

commit f090eaca12c5e8d57ffe3cb565becce4589a4a1e
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Oct 31 01:32:43 2026 -0700

    [SPARK-60762][SQL] Pass ScalaUDF error message parameters to generated code 
as references
    
    ### What changes were proposed in this pull request?
    
    This PR aims to pass `functionName`, `inputTypesString`, and `outputType` 
of `ScalaUDF` to the generated code via `ctx.addReferenceObj` instead of 
embedding them into Java string literals. The interpreted path is unchanged.
    
    ### Why are the changes needed?
    
    These values hold text from the query, such as the UDF name and the field 
names of struct types. The generated code puts them into Java string literals 
without escaping, so a double quote or an invalid escape sequence makes the 
generated class fail to compile.
    
    ```scala
    spark.udf.register("rowSize", (r: Row) => r.size)
    sql("""SELECT rowSize(named_struct('fieldName"b', id)) AS r FROM 
range(2)""").collect()
    ```
    
    This has been the case since SPARK-33541 (Apache Spark 3.2.0). Before that, 
the error message was passed via `ctx.addReferenceObj("errMsg", ...)`.
    
    References are preferred to escaping because the text stays out of the 
generated source:
    - A very long name does not hit the constant string limit of `javac`.
    - Queries that differ only in these names can share a cached generated 
class.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, for UDFs whose name or input and output types contain these characters.
    
    - Previously, the generated code failed to compile. Queries still returned 
correct results through the fallback, but compile errors were logged and 
whole-stage codegen was disabled for the plan. After this PR, the generated 
code compiles.
    - Previously, a valid Java escape sequence such as `\t` in a name compiled, 
but the `FAILED_EXECUTE_UDF` error showed the unescaped text, unlike the 
interpreted path. After this PR, both paths report the same text.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test cases.
    
    - `ScalaUDFSuite`: A UDF whose name, input type, and output type contain 
`"` and `\q`. The generated and interpreted paths must report the same 
`FAILED_EXECUTE_UDF` parameters, and none of the three values may appear in the 
generated code. The test fails if any one of the three values is put back into 
a string literal.
    - `UDFSuite`: The SQL query above must return the expected rows, and the 
field name must not appear in the whole-stage generated code. The test fails 
without the source change.
    
    The following suites also pass locally.
    
    ```bash
    build/sbt 'catalyst/testOnly *ScalaUDFSuite *CodeGenerationSuite 
*CodeCompilerSuite'
    ```
    
    ```bash
    build/sbt 'sql/testOnly org.apache.spark.sql.UDFSuite 
*WholeStageCodegenSuite'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #59969 from dongjoon-hyun/SPARK-60762.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../spark/sql/catalyst/expressions/ScalaUDF.scala    |  7 ++++++-
 .../sql/catalyst/expressions/ScalaUDFSuite.scala     | 20 +++++++++++++++++++-
 .../test/scala/org/apache/spark/sql/UDFSuite.scala   | 12 +++++++++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index f9a62309463a..6571a00d0b51 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -1163,6 +1163,11 @@ case class ScalaUDF(
     } else {
       s"$resultTerm = ($boxedType)$resultConverter.apply($getFuncResult)"
     }
+    // The names in these strings come from the query, so they stay out of the 
source.
+    val functionNameTerm = ctx.addReferenceObj("functionName", functionName)
+    val inputTypesTerm = ctx.addReferenceObj("inputTypes", inputTypesString)
+    val outputTypeTerm = ctx.addReferenceObj("outputType", outputType)
+
     // `failedExecuteUserDefinedFunctionError` returns a checked 
SparkException; throwing
     // it directly compiles under Janino but not under the JDK compiler 
("unreported
     // exception"). Re-throw via Platform.throwException (no declared 
`throws`), the same
@@ -1175,7 +1180,7 @@ case class ScalaUDF(
          |} catch (Throwable e) {
          |  org.apache.spark.unsafe.Platform.throwException(
          |    QueryExecutionErrors.failedExecuteUserDefinedFunctionError(
-         |      "$functionName", "$inputTypesString", "$outputType", e));
+         |      $functionNameTerm, $inputTypesTerm, $outputTypeTerm, e));
          |}
        """.stripMargin
 
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
index 2b0e6b6afac5..178eb2ffc5ef 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
@@ -22,11 +22,12 @@ import java.util.Locale
 import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.expressions.variant.{ParseJson, 
VariantGet}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType, 
VariantType}
+import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType, 
StructType, VariantType}
 import org.apache.spark.unsafe.types.VariantVal
 
 class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -121,4 +122,21 @@ class ScalaUDFSuite extends SparkFunSuite with 
ExpressionEvalHelper {
       checkEvaluation(udf, null)
     }
   }
+
+  test("SPARK-60762: The text of a UDF's error is not in the generated code") {
+    val udf = ScalaUDF(
+      (_: Any) => throw new IllegalStateException("harmless"),
+      new StructType().add("outputName\"\\q", IntegerType),
+      Literal.create(InternalRow(1), new StructType().add("fieldName\"\\q", 
IntegerType)) :: Nil,
+      None :: Nil,
+      udfName = Some("udfName\"\\q"))
+    val generated = intercept[SparkException] {
+      checkEvaluationWithUnsafeProjection(udf, null)
+    }
+    assert(generated.getCondition == "FAILED_EXECUTE_UDF")
+    val interpreted = intercept[SparkException](udf.eval())
+    assert(generated.getMessageParameters == interpreted.getMessageParameters)
+    val code = udf.genCode(new CodegenContext).code.toString
+    assert(!code.contains("fieldName") && !code.contains("udfName") && 
!code.contains("outputName"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index 21384d7c4b91..e7e8564568f6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.{Literal, 
ScalaUDF}
 import org.apache.spark.sql.catalyst.expressions.Cast._
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
+import org.apache.spark.sql.execution.{QueryExecution, SimpleMode, 
WholeStageCodegenExec}
 import org.apache.spark.sql.execution.aggregate.{ScalaAggregator, ScalaUDAF}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
 import 
org.apache.spark.sql.execution.command.{CreateDataSourceTableAsSelectCommand, 
ExplainCommand}
@@ -1282,4 +1282,14 @@ class UDFSuite extends SharedSparkSession {
     val result = input.select(kryoBufUDAF($"value").as("sum"))
     checkAnswer(result, Row(6L) :: Nil)
   }
+
+  test("SPARK-60762: The types of a UDF are not in the generated code") {
+    spark.udf.register("rowSize", (r: Row) => r.size)
+    val df = sql("""SELECT rowSize(named_struct('fieldName"b', id)) AS r FROM 
range(2)""")
+    val code = df.queryExecution.executedPlan.collect {
+      case w: WholeStageCodegenExec => w.doCodeGen()._2.body
+    }
+    assert(code.nonEmpty && code.forall(!_.contains("fieldName")))
+    checkAnswer(df, Row(1) :: Row(1) :: Nil)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to