This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a4c18a8ddf40 [SPARK-58437][SQL] Fix invalid Java constructs in
generated code
a4c18a8ddf40 is described below
commit a4c18a8ddf40f8dda7b676487ab17b12206ec501
Author: Marcus Lin <[email protected]>
AuthorDate: Sat Aug 1 13:59:52 2026 +0800
[SPARK-58437][SQL] Fix invalid Java constructs in generated code
### What changes were proposed in this pull request?
This PR fixes several generated-code patterns that Janino accepts but javac
rejects:
- Avoids declaring the generated `sequence` array local as `final` before
assigning to it.
- Emits primitive `ArrayBuilder` inner class references with
source-compatible names such as `ArrayBuilder.ofInt`.
- Uses `Iterator<InternalRow>` for `SortExec`'s generated sorted iterator
state, matching the sorter return type.
- Creates mutable state arrays for generic Java types with raw array
creation while preserving the generic field type.
It also adds focused regression coverage for these generated-code shapes.
### Why are the changes needed?
Spark currently relies on Janino for whole-stage codegen, and Janino
tolerates some constructs that are not valid Java source for javac. These
changes remove invalid Java source patterns as preparatory work for
javac-compatible generated-code paths.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added regression tests and ran:
```
build/sbt 'catalyst/testOnly
org.apache.spark.sql.catalyst.expressions.CodeGenerationSuite -- -z
"SPARK-58437"' 'sql/testOnly
org.apache.spark.sql.execution.WholeStageCodegenSuite -- -z "SPARK-58437"'
```
Also ran:
```
git diff --check
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (OpenAI GPT-5) was used for code assistance and review
support; the patch was guided by the contributor.
Closes #57656 from marcuslin123/SPARK-58437-javac-codegen.
Authored-by: Marcus Lin <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
.../expressions/codegen/CodeGenerator.scala | 19 ++++++++--
.../expressions/collectionOperations.scala | 10 +++---
.../catalyst/expressions/CodeGenerationSuite.scala | 42 ++++++++++++++++++++++
.../org/apache/spark/sql/execution/SortExec.scala | 2 +-
.../sql/execution/WholeStageCodegenSuite.scala | 6 ++++
5 files changed, 70 insertions(+), 9 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index def7ec39b571..4c86dbd77580 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -303,7 +303,8 @@ class CodegenContext extends Logging {
* e.g. InternalRow, UnsafeRow, UnsafeArrayData, etc. Other
types will have to
* specify the fully-qualified Java type name. See the code
in doCompile() for
* the list of default imports available.
- * Also, generic type arguments are accepted but ignored.
+ * Also, generic type arguments are kept in field
declarations, but stripped
+ * from array creation expressions because Java forbids
generic array creation.
* @param variableName Name of the field.
* @param initFunc Function includes statement(s) to put into the init()
method to initialize
* this field. The argument is the name of the mutable state
variable.
@@ -402,6 +403,18 @@ class CodegenContext extends Logging {
}
def declareMutableStates(): String = {
+ def rawJavaType(javaType: String): String = {
+ val builder = new StringBuilder
+ var depth = 0
+ javaType.foreach {
+ case '<' => depth += 1
+ case '>' => depth -= 1
+ case c if depth == 0 => builder.append(c)
+ case _ =>
+ }
+ builder.toString
+ }
+
// It's possible that we add same mutable state twice, e.g. the
`mergeExpressions` in
// `TypedAggregateExpression`, we should call `distinct` here to remove
the duplicated ones.
val inlinedStates = inlinedMutableStates.distinct.map { case (javaType,
variableName) =>
@@ -419,10 +432,10 @@ class CodegenContext extends Logging {
if (javaType.contains("[]")) {
// initializer had an one-dimensional array variable
val baseType = javaType.substring(0, javaType.length - 2)
- s"private $javaType[] $arrayName = new $baseType[$length][];"
+ s"private $javaType[] $arrayName = new
${rawJavaType(baseType)}[$length][];"
} else {
// initializer had a scalar variable
- s"private $javaType[] $arrayName = new $javaType[$length];"
+ s"private $javaType[] $arrayName = new
${rawJavaType(javaType)}[$length];"
}
}
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 48096408c403..bae2e0f6b383 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -3535,7 +3535,7 @@ case class Sequence(
val arr = ctx.freshName("arr")
val arrElemType = CodeGenerator.javaType(dataType.elementType)
s"""
- |final $arrElemType[] $arr = null;
+ |$arrElemType[] $arr = null;
|${impl.genCode(ctx, startGen.value, stopGen.value, stepGen.value,
arr, arrElemType)}
|${ev.value} = UnsafeArrayData.fromPrimitiveArray($arr);
""".stripMargin
@@ -4473,7 +4473,7 @@ case class ArrayDistinct(child: Expression)
val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
val hashSet = ctx.freshName("hashSet")
val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
- val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+ val arrayBuilderClass = s"$arrayBuilder.of$ptName"
// Only need to track null element index when array's element is
nullable.
val declareNullTrackVariables = if (resultArrayElementNullable) {
@@ -4674,7 +4674,7 @@ case class ArrayUnion(left: Expression, right:
Expression) extends ArrayBinaryLi
val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
val hashSet = ctx.freshName("hashSet")
val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
- val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+ val arrayBuilderClass = s"$arrayBuilder.of$ptName"
val body =
s"""
@@ -4899,7 +4899,7 @@ case class ArrayIntersect(left: Expression, right:
Expression) extends ArrayBina
val hashSet = ctx.freshName("hashSet")
val hashSetResult = ctx.freshName("hashSetResult")
val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
- val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+ val arrayBuilderClass = s"$arrayBuilder.of$ptName"
val withArray2NaNCheckCodeGenerator =
(array: String, index: String) =>
@@ -5124,7 +5124,7 @@ case class ArrayExcept(left: Expression, right:
Expression) extends ArrayBinaryL
val classTag = s"scala.reflect.ClassTag$$.MODULE$$.$hsTypeName()"
val hashSet = ctx.freshName("hashSet")
val arrayBuilder = classOf[mutable.ArrayBuilder[_]].getName
- val arrayBuilderClass = s"$arrayBuilder$$of$ptName"
+ val arrayBuilderClass = s"$arrayBuilder.of$ptName"
val withArray2NaNCheckCodeGenerator =
(array: String, index: String) =>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index 7ce14bcedf4b..cf8009ae4c12 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -442,6 +442,48 @@ class CodeGenerationSuite extends SparkFunSuite with
ExpressionEvalHelper {
assert(ctx2.mutableStateInitCode.size ==
CodeGenerator.MUTABLESTATEARRAY_SIZE_LIMIT + 10)
}
+ test("SPARK-58437: declare generic mutable state arrays with raw array
creation") {
+ val ctx = new CodegenContext
+ val samplerType =
+ "org.apache.spark.util.random.BernoulliCellSampler" +
+ "<org.apache.spark.sql.catalyst.expressions.UnsafeRow>"
+ for (_ <- 1 to CodeGenerator.OUTER_CLASS_VARIABLES_THRESHOLD + 1) {
+ ctx.addMutableState(samplerType, "sampler")
+ }
+
+ val states = ctx.declareMutableStates()
+ assert(states.contains(s"private $samplerType[]"))
+ assert(!states.contains(s"new $samplerType["))
+ assert(states.contains("new
org.apache.spark.util.random.BernoulliCellSampler["))
+ }
+
+ test("SPARK-58437: generate javac-compatible source for collection
expressions") {
+ val intArray =
+ BoundReference(0, ArrayType(IntegerType, containsNull = false), nullable
= false)
+ val intArray2 =
+ BoundReference(1, ArrayType(IntegerType, containsNull = false), nullable
= false)
+ val collectionExprs = Seq(
+ ArrayDistinct(intArray),
+ ArrayUnion(intArray, intArray2),
+ ArrayIntersect(intArray, intArray2),
+ ArrayExcept(intArray, intArray2))
+
+ collectionExprs.foreach { expr =>
+ val ctx = new CodegenContext
+ val exprCode = expr.genCode(ctx).code.toString
+ val code = exprCode + "\n" + ctx.declareAddedFunctions()
+ assert(!code.contains("ArrayBuilder$ofInt"))
+ assert(code.contains("ArrayBuilder.ofInt"))
+ }
+
+ val sequence = new Sequence(
+ BoundReference(0, IntegerType, nullable = false),
+ BoundReference(1, IntegerType, nullable = false),
+ BoundReference(2, IntegerType, nullable = false))
+ val sequenceCode = sequence.genCode(new CodegenContext).code.toString
+ assert(!sequenceCode.contains("final int[]"))
+ }
+
test("SPARK-22750: addImmutableStateIfNotExists") {
val ctx = new CodegenContext
val mutableState1 = "field1"
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
index 74afa396264e..d0f7a6bc3291 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
@@ -146,7 +146,7 @@ case class SortExec(
v => s"$v = $thisPlan.createSorter();", forceInline = true)
val metrics = ctx.addMutableState(classOf[TaskMetrics].getName, "metrics",
v => s"$v = org.apache.spark.TaskContext.get().taskMetrics();",
forceInline = true)
- val sortedIterator =
ctx.addMutableState("scala.collection.Iterator<UnsafeRow>", "sortedIter",
+ val sortedIterator =
ctx.addMutableState("scala.collection.Iterator<InternalRow>", "sortedIter",
forceInline = true)
val addToSorter = ctx.freshName("addToSorter")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index 2e73417dd9ed..3f0c0ab97c40 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -753,6 +753,12 @@ class WholeStageCodegenSuite extends SharedSparkSession
assert(df.collect() === Array(Row(1), Row(2), Row(3)))
}
+ test("SPARK-58437: SortExec generates assignable iterator type") {
+ val code = genCode(spark.range(3, 0,
-1).toDF().sort(col("id"))).map(_.body).mkString("\n")
+ assert(!code.contains("scala.collection.Iterator<UnsafeRow>"))
+ assert(code.contains("scala.collection.Iterator<InternalRow>"))
+ }
+
test("MapElements should be included in WholeStageCodegen") {
import testImplicits._
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]