This is an automated email from the ASF dual-hosted git repository.

LuciferYang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new a4c18a8ddf40 [SPARK-58437][SQL] Fix invalid Java constructs in 
generated code
a4c18a8ddf40 is described below

commit a4c18a8ddf40f8dda7b676487ab17b12206ec501
Author: Marcus Lin <[email protected]>
AuthorDate: Sat Aug 1 13:59:52 2026 +0800

    [SPARK-58437][SQL] Fix invalid Java constructs in generated code
    
    ### What changes were proposed in this pull request?
    
    This PR fixes several generated-code patterns that Janino accepts but javac 
rejects:
    
    - Avoids declaring the generated `sequence` array local as `final` before 
assigning to it.
    - Emits primitive `ArrayBuilder` inner class references with 
source-compatible names such as `ArrayBuilder.ofInt`.
    - Uses `Iterator<InternalRow>` for `SortExec`'s generated sorted iterator 
state, matching the sorter return type.
    - Creates mutable state arrays for generic Java types with raw array 
creation while preserving the generic field type.
    
    It also adds focused regression coverage for these generated-code shapes.
    
    ### Why are the changes needed?
    
    Spark currently relies on Janino for whole-stage codegen, and Janino 
tolerates some constructs that are not valid Java source for javac. These 
changes remove invalid Java source patterns as preparatory work for 
javac-compatible generated-code paths.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added regression tests and ran:
    
    ```
    build/sbt 'catalyst/testOnly 
org.apache.spark.sql.catalyst.expressions.CodeGenerationSuite -- -z 
"SPARK-58437"' 'sql/testOnly 
org.apache.spark.sql.execution.WholeStageCodegenSuite -- -z "SPARK-58437"'
    ```
    
    Also ran:
    
    ```
    git diff --check
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (OpenAI GPT-5) was used for code assistance and review 
support; the patch was guided by the contributor.
    
    Closes #57656 from marcuslin123/SPARK-58437-javac-codegen.
    
    Authored-by: Marcus Lin <[email protected]>
    Signed-off-by: yangjie01 <[email protected]>
---
 .../expressions/codegen/CodeGenerator.scala        | 19 ++++++++--
 .../expressions/collectionOperations.scala         | 10 +++---
 .../catalyst/expressions/CodeGenerationSuite.scala | 42 ++++++++++++++++++++++
 .../org/apache/spark/sql/execution/SortExec.scala  |  2 +-
 .../sql/execution/WholeStageCodegenSuite.scala     |  6 ++++
 5 files changed, 70 insertions(+), 9 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index def7ec39b571..4c86dbd77580 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -303,7 +303,8 @@ class CodegenContext extends Logging {
    *                 e.g. InternalRow, UnsafeRow, UnsafeArrayData, etc. Other 
types will have to
    *                 specify the fully-qualified Java type name. See the code 
in doCompile() for
    *                 the list of default imports available.
-   *                 Also, generic type arguments are accepted but ignored.
+   *                 Also, generic type arguments are kept in field 
declarations, but stripped
+   *                 from array creation expressions because Java forbids 
generic array creation.
    * @param variableName Name of the field.
    * @param initFunc Function includes statement(s) to put into the init() 
method to initialize
    *                 this field. The argument is the name of the mutable state 
variable.
@@ -402,6 +403,18 @@ class CodegenContext extends Logging {
   }
 
   def declareMutableStates(): String = {
+    def rawJavaType(javaType: String): String = {
+      val builder = new StringBuilder
+      var depth = 0
+      javaType.foreach {
+        case '<' => depth += 1
+        case '>' => depth -= 1
+        case c if depth == 0 => builder.append(c)
+        case _ =>
+      }
+      builder.toString
+    }
+
     // It's possible that we add same mutable state twice, e.g. the 
`mergeExpressions` in
     // `TypedAggregateExpression`, we should call `distinct` here to remove 
the duplicated ones.
     val inlinedStates = inlinedMutableStates.distinct.map { case (javaType, 
variableName) =>
@@ -419,10 +432,10 @@ class CodegenContext extends Logging {
         if (javaType.contains("[]")) {
           // initializer had an one-dimensional array variable
           val baseType = javaType.substring(0, javaType.length - 2)
-          s"private $javaType[] $arrayName = new $baseType[$length][];"
+          s"private $javaType[] $arrayName = new 
${rawJavaType(baseType)}[$length][];"
         } else {
           // initializer had a scalar variable
-          s"private $javaType[] $arrayName = new $javaType[$length];"
+          s"private $javaType[] $arrayName = new 
${rawJavaType(javaType)}[$length];"
         }
       }
     }
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 48096408c403..bae2e0f6b383 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -3535,7 +3535,7 @@ case class Sequence(
       val arr = ctx.freshName("arr")
       val arrElemType = CodeGenerator.javaType(dataType.elementType)
       s"""
-         |final $arrElemType[] $arr = null;
+         |$arrElemType[] $arr = null;
          |${impl.genCode(ctx, startGen.value, stopGen.value, stepGen.value, 
arr, arrElemType)}
          |${ev.value} = UnsafeArrayData.fromPrimitiveArray($arr);
        """.stripMargin
@@ -4473,7 +4473,7 @@ case class ArrayDistinct(child: Expression)
         val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
         val hashSet = ctx.freshName("hashSet")
         val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
-        val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+        val arrayBuilderClass = s"$arrayBuilder.of$ptName"
 
         // Only need to track null element index when array's element is 
nullable.
         val declareNullTrackVariables = if (resultArrayElementNullable) {
@@ -4674,7 +4674,7 @@ case class ArrayUnion(left: Expression, right: 
Expression) extends ArrayBinaryLi
         val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
         val hashSet = ctx.freshName("hashSet")
         val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
-        val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+        val arrayBuilderClass = s"$arrayBuilder.of$ptName"
 
         val body =
           s"""
@@ -4899,7 +4899,7 @@ case class ArrayIntersect(left: Expression, right: 
Expression) extends ArrayBina
         val hashSet = ctx.freshName("hashSet")
         val hashSetResult = ctx.freshName("hashSetResult")
         val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
-        val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+        val arrayBuilderClass = s"$arrayBuilder.of$ptName"
 
         val withArray2NaNCheckCodeGenerator =
           (array: String, index: String) =>
@@ -5124,7 +5124,7 @@ case class ArrayExcept(left: Expression, right: 
Expression) extends ArrayBinaryL
         val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
         val hashSet = ctx.freshName("hashSet")
         val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
-        val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+        val arrayBuilderClass = s"$arrayBuilder.of$ptName"
 
         val withArray2NaNCheckCodeGenerator =
           (array: String, index: String) =>
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index 7ce14bcedf4b..cf8009ae4c12 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -442,6 +442,48 @@ class CodeGenerationSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     assert(ctx2.mutableStateInitCode.size == 
CodeGenerator.MUTABLESTATEARRAY_SIZE_LIMIT + 10)
   }
 
+  test("SPARK-58437: declare generic mutable state arrays with raw array 
creation") {
+    val ctx = new CodegenContext
+    val samplerType =
+      "org.apache.spark.util.random.BernoulliCellSampler" +
+        "<org.apache.spark.sql.catalyst.expressions.UnsafeRow>"
+    for (_ <- 1 to CodeGenerator.OUTER_CLASS_VARIABLES_THRESHOLD + 1) {
+      ctx.addMutableState(samplerType, "sampler")
+    }
+
+    val states = ctx.declareMutableStates()
+    assert(states.contains(s"private $samplerType[]"))
+    assert(!states.contains(s"new $samplerType["))
+    assert(states.contains("new 
org.apache.spark.util.random.BernoulliCellSampler["))
+  }
+
+  test("SPARK-58437: generate javac-compatible source for collection 
expressions") {
+    val intArray =
+      BoundReference(0, ArrayType(IntegerType, containsNull = false), nullable 
= false)
+    val intArray2 =
+      BoundReference(1, ArrayType(IntegerType, containsNull = false), nullable 
= false)
+    val collectionExprs = Seq(
+      ArrayDistinct(intArray),
+      ArrayUnion(intArray, intArray2),
+      ArrayIntersect(intArray, intArray2),
+      ArrayExcept(intArray, intArray2))
+
+    collectionExprs.foreach { expr =>
+      val ctx = new CodegenContext
+      val exprCode = expr.genCode(ctx).code.toString
+      val code = exprCode + "\n" + ctx.declareAddedFunctions()
+      assert(!code.contains("ArrayBuilder$ofInt"))
+      assert(code.contains("ArrayBuilder.ofInt"))
+    }
+
+    val sequence = new Sequence(
+      BoundReference(0, IntegerType, nullable = false),
+      BoundReference(1, IntegerType, nullable = false),
+      BoundReference(2, IntegerType, nullable = false))
+    val sequenceCode = sequence.genCode(new CodegenContext).code.toString
+    assert(!sequenceCode.contains("final int[]"))
+  }
+
   test("SPARK-22750: addImmutableStateIfNotExists") {
     val ctx = new CodegenContext
     val mutableState1 = "field1"
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
index 74afa396264e..d0f7a6bc3291 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
@@ -146,7 +146,7 @@ case class SortExec(
       v => s"$v = $thisPlan.createSorter();", forceInline = true)
     val metrics = ctx.addMutableState(classOf[TaskMetrics].getName, "metrics",
       v => s"$v = org.apache.spark.TaskContext.get().taskMetrics();", 
forceInline = true)
-    val sortedIterator = 
ctx.addMutableState("scala.collection.Iterator<UnsafeRow>", "sortedIter",
+    val sortedIterator = 
ctx.addMutableState("scala.collection.Iterator<InternalRow>", "sortedIter",
       forceInline = true)
 
     val addToSorter = ctx.freshName("addToSorter")
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index 2e73417dd9ed..3f0c0ab97c40 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -753,6 +753,12 @@ class WholeStageCodegenSuite extends SharedSparkSession
     assert(df.collect() === Array(Row(1), Row(2), Row(3)))
   }
 
+  test("SPARK-58437: SortExec generates assignable iterator type") {
+    val code = genCode(spark.range(3, 0, 
-1).toDF().sort(col("id"))).map(_.body).mkString("\n")
+    assert(!code.contains("scala.collection.Iterator<UnsafeRow>"))
+    assert(code.contains("scala.collection.Iterator<InternalRow>"))
+  }
+
   test("MapElements should be included in WholeStageCodegen") {
     import testImplicits._
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to