This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 4ff8e564f004 [SPARK-60760][SQL] Use constant names for lambda variable 
references in higher-order function codegen
4ff8e564f004 is described below

commit 4ff8e564f004143c1682143de0c346466486da9c
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Oct 31 01:29:17 2026 -0700

    [SPARK-60760][SQL] Use constant names for lambda variable references in 
higher-order function codegen
    
    ### What changes were proposed in this pull request?
    
    This PR aims to use constant names when the codegen of higher-order 
functions registers lambda variables via `CodegenContext.addReferenceObj`, 
instead of the lambda variable names of the query.
    
    | Lambda variable | Before | After |
    |---|---|---|
    | Element | `elementVar.name` | `"lambdaElement"` |
    | Index | `indexVar.name` | `"lambdaIndex"` |
    | Accumulator for merge | `accForMergeVar.name` | `"lambdaAccForMerge"` |
    | Accumulator for finish | `accForFinishVar.name` | `"lambdaAccForFinish"` |
    
    In addition, this PR documents in the Scaladoc of `addReferenceObj` that 
`objName` must be a constant.
    
    ### Why are the changes needed?
    
    `addReferenceObj` writes `objName` into a comment of the generated Java 
source, like `((Class) references[0] /* objName */)`. Since SPARK-37019 added 
the codegen of array higher-order functions, this comment has held the lambda 
variable names of the query. This causes two issues.
    
    1. A valid lambda variable name can break the generated source. For 
example, the generated class fails to compile when the name contains `*/`, as 
in the following query on a view `t` with an array column `a`. Spark falls back 
and returns correct results, but the plan loses whole-stage codegen and errors 
are logged for each query.
    
       ```sql
       SELECT transform(a, (e, `lambdaName*/b`) -> e + `lambdaName*/b`) AS r 
FROM t
       ```
    
    2. The generated source depends on the name, so a compiled class cannot be 
reused across names. The DataFrame API creates a fresh name such as `x_1` from 
a global counter on each call, so running the same `transform` query again 
compiles its 2 classes again.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, this is a bug fix for Apache Spark 
[4.3.0](https://github.com/apache/spark/releases/tag/v4.3.0) (2026-10-27).
    
    - Queries with such lambda variable names keep whole-stage codegen and no 
longer log compilation errors. Query results do not change.
    - `EXPLAIN CODEGEN` shows the constant names instead of the lambda variable 
names in the comments of the generated source.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test cases. They fail without the main 
code change.
    
    - `HigherOrderFunctionsSuite`
      - `SPARK-60760: The generated code does not hold the names of lambda 
variables`
    - `WholeStageCodegenSuite`
      - `SPARK-60760: Lambda variable names are not in the generated code`
      - `SPARK-60760: A new lambda variable name does not compile a generated 
class again`
    
    ```
    build/sbt 'catalyst/testOnly *HigherOrderFunctionsSuite -- -z "SPARK-60760"'
    build/sbt 'sql/testOnly 
org.apache.spark.sql.execution.WholeStageCodegenSuite -- -z "SPARK-60760"'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #59968 from dongjoon-hyun/SPARK-60760.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../expressions/codegen/CodeGenerator.scala        |  3 ++
 .../expressions/higherOrderFunctions.scala         | 10 +++----
 .../expressions/HigherOrderFunctionsSuite.scala    | 21 +++++++++++++-
 .../sql/execution/WholeStageCodegenSuite.scala     | 32 ++++++++++++++++++++++
 4 files changed, 59 insertions(+), 7 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 8198c0a464f0..ff3cfbce3007 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -272,6 +272,9 @@ class CodegenContext extends Logging {
    *
    * This does not to store the object into field but refer it from the 
references field at the
    * time of use because number of fields in class is limited so we should 
reduce it.
+   *
+   * `objName` labels the reference in a comment of the generated source, so 
it must be a
+   * constant, never text that comes from a query.
    */
   def addReferenceObj(objName: String, obj: Any, className: String = null): 
String = {
     val idx = references.length
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 53b0ae38b08c..1c094f4f03d8 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -309,7 +309,7 @@ trait HigherOrderFunction extends Expression with 
ExpectsInputTypes {
       elementVar: NamedLambdaVariable,
       index: String): String = {
     val elementType = elementVar.dataType
-    val elementAtomic = ctx.addReferenceObj(elementVar.name, elementVar.value)
+    val elementAtomic = ctx.addReferenceObj("lambdaElement", elementVar.value)
     val extractElement = CodeGenerator.getValue(arrayName, elementType, index)
     val atomicAssign = assignAtomic(elementAtomic, elementCode.value,
       elementCode.isNull, elementVar.nullable)
@@ -333,7 +333,7 @@ trait HigherOrderFunction extends Expression with 
ExpectsInputTypes {
       indexCode: ExprCode,
       indexVar: NamedLambdaVariable,
       index: String): String = {
-    val indexAtomic = ctx.addReferenceObj(indexVar.name, indexVar.value)
+    val indexAtomic = ctx.addReferenceObj("lambdaIndex", indexVar.value)
     s"""
       ${indexCode.value} = $index;
       ${assignAtomic(indexAtomic, indexCode.value)}
@@ -1343,10 +1343,8 @@ case class ArrayAggregate(
         val finishCode = finishForCodegen.genCode(ctx)
 
         val elementAssignment = assignArrayElement(ctx, arg, elementCode, 
elementVar, i)
-        val mergeAtomic = ctx.addReferenceObj(accForMergeVar.name,
-          accForMergeVar.value)
-        val finishAtomic = ctx.addReferenceObj(accForFinishVar.name,
-          accForFinishVar.value)
+        val mergeAtomic = ctx.addReferenceObj("lambdaAccForMerge", 
accForMergeVar.value)
+        val finishAtomic = ctx.addReferenceObj("lambdaAccForFinish", 
accForFinishVar.value)
 
         val mergeJavaType = CodeGenerator.javaType(accForMergeVar.dataType)
         val finishJavaType = CodeGenerator.javaType(accForFinishVar.dataType)
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index 3c1f67109571..ebf1e5fd78f2 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
 import org.apache.spark.sql.catalyst.expressions.Cast._
-import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, 
CodegenFallback}
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -1012,6 +1012,25 @@ class HigherOrderFunctionsSuite extends SparkFunSuite 
with ExpressionEvalHelper
     assert(copy.asInstanceOf[NamedLambdaVariable].value ne lv.value,
       "fresh copy should have an independent AtomicReference value")
   }
+
+  test("SPARK-60760: The generated code does not hold the names of lambda 
variables") {
+    val name = "lambdaName*/b"
+    def v(): NamedLambdaVariable = NamedLambdaVariable(name, IntegerType, 
nullable = false)
+    val array = Literal.create(Seq(1, 2), ArrayType(IntegerType, containsNull 
= false))
+    val (e1, i1, e2, i2, e3, e4, e5, acc1, acc2) = (v(), v(), v(), v(), v(), 
v(), v(), v(), v())
+    Seq[(Expression, Any)](
+      ArrayTransform(array, LambdaFunction(e1 + i1, Seq(e1, i1))) -> Seq(1, 3),
+      ArrayFilter(array, LambdaFunction(e2 > i2, Seq(e2, i2))) -> Seq(1, 2),
+      ArrayExists(array, LambdaFunction(e3 > 1, Seq(e3))) -> true,
+      ArrayForAll(array, LambdaFunction(e4 > 0, Seq(e4))) -> true,
+      ArrayAggregate(array, Literal(0), LambdaFunction(acc1 + e5, Seq(acc1, 
e5)),
+        LambdaFunction(acc2, Seq(acc2))) -> 3
+    ).foreach { case (expr, expected) =>
+      checkEvaluation(expr, expected)
+      val code = expr.genCode(new CodegenContext).code.toString
+      assert(!code.contains("lambdaName"))
+    }
+  }
 }
 
 case class CodegenFallbackExpr(child: Expression) extends UnaryExpression with 
CodegenFallback {
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index 15396c3c661c..d8af608a1641 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -3217,4 +3217,36 @@ class WholeStageCodegenSuite extends SharedSparkSession
       }
     }
   }
+
+  test("SPARK-60760: Lambda variable names are not in the generated code") {
+    val name = "lambdaName*/b"
+    withTempView("t") {
+      spark.range(3).selectExpr("array(id, id + 1) AS 
a").createOrReplaceTempView("t")
+      Seq(
+        s"transform(a, (e, `$name`) -> e + `$name`)" ->
+          Seq(Row(Seq(0, 2)), Row(Seq(1, 3)), Row(Seq(2, 4))),
+        s"filter(a, `$name` -> `$name` > 0)" ->
+          Seq(Row(Seq(1)), Row(Seq(1, 2)), Row(Seq(2, 3))),
+        s"aggregate(a, 0L, (`$name`, e) -> `$name` + e, acc -> acc)" ->
+          Seq(Row(1), Row(3), Row(5)),
+        s"aggregate(a, 0L, (acc, e) -> acc + e, `$name` -> `$name`)" ->
+          Seq(Row(1), Row(3), Row(5))
+      ).foreach { case (expr, expected) =>
+        val df = sql(s"SELECT $expr AS r FROM t")
+        genCode(df).foreach(code => assert(!code.body.contains("lambdaName")))
+        checkAnswer(df, expected)
+      }
+    }
+  }
+
+  test("SPARK-60760: A new lambda variable name does not compile a generated 
class again") {
+    withTempView("t") {
+      spark.range(3).selectExpr("array(id, id + 1) AS 
a").createOrReplaceTempView("t")
+      val query = () => spark.table("t").select(transform($"a", x => x + 
98765))
+      query().collect()
+      val compiles = CodegenMetrics.METRIC_COMPILATION_TIME.getCount
+      query().collect()
+      assert(CodegenMetrics.METRIC_COMPILATION_TIME.getCount == compiles)
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to