This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 22b44016d8e1 [SPARK-60528][SQL] Fix `ExpressionEncoder` assertion
failure on nested empty structs
22b44016d8e1 is described below
commit 22b44016d8e183f6f267e58103aa35a72a65bacf
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Oct 25 15:55:10 2026 -0700
[SPARK-60528][SQL] Fix `ExpressionEncoder` assertion failure on nested
empty structs
### What changes were proposed in this pull request?
This PR aims to fix `SerializerBuildHelper.createSerializerForObject` to
check the input object for null even when it is not nullable if the struct is
empty, so that the serializer of an empty struct always refers to its input
object.
### Why are the changes needed?
An empty struct has no field serializers, so its serializer drops a
non-nullable input object. For example, `RowEncoder` serializes `s STRUCT<a:
STRUCT<> NOT NULL> NOT NULL` into `assertnotnull(named_struct(a,
assertnotnull(named_struct())))`, which does not refer to the input row. Since
`ExpressionEncoder` allows this only for a top-level empty struct
(SPARK-46539), such a nested empty struct fails with `AssertionError` at
`Dataset.collect` and `writeStream`.
```scala
spark.range(1).selectExpr("id AS i", "named_struct('a', struct()) AS
s").collect()
```
```
org.apache.spark.SparkException: [INTERNAL_ERROR] The "collect" action
failed. ...
Caused by: java.lang.AssertionError: assertion failed: each serializer
expression should contain at least one `BoundReference` or it should be an
empty struct. ...
```
### Does this PR introduce _any_ user-facing change?
Yes, compared to the released Spark versions.
- A nested empty struct can be collected and used in streaming queries.
- `RowEncoder` validates the value of an empty struct field like other
struct fields. A null value of a non-nullable empty struct field or a non-`Row`
value fails with `EXPRESSION_ENCODING_FAILED` instead of silently becoming an
empty struct.
### How was this patch tested?
Pass the CIs with the newly added test cases in `RowEncoderSuite` and
`DataFrameSuite`. All of them fail without this fix.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59771 from dongjoon-hyun/SPARK-60528.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 76af9a99e1c1326536b8073fcdb8f25e29572886)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/catalyst/SerializerBuildHelper.scala | 6 +++++-
.../sql/catalyst/encoders/RowEncoderSuite.scala | 21 +++++++++++++++++++++
.../scala/org/apache/spark/sql/DataFrameSuite.scala | 5 +++++
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
index 370749b1d9bf..036c126e466d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
@@ -293,7 +293,11 @@ object SerializerBuildHelper {
val nonNullOutput = CreateNamedStruct(fields.flatMap { case(fieldName,
fieldExpr) =>
argumentsForFieldSerializer(fieldName, fieldExpr)
})
- if (inputObject.nullable) {
+ // An empty struct has no field serializers that evaluate `inputObject`,
so check it for null
+ // even when it is not nullable. Otherwise, the serializer would not refer
to the input object,
+ // which `ExpressionEncoder` requires, and would skip its validation,
e.g., the not-null and
+ // external type checks of a `Row` field.
+ if (inputObject.nullable || fields.isEmpty) {
val nullOutput = expressions.Literal.create(null, nonNullOutput.dataType)
expressions.If(IsNull(inputObject), nullOutput, nonNullOutput)
} else {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index 6974caf14752..e3953d310bcd 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -254,6 +254,17 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
assert(encoder.serializer.head.nullable == false)
}
+ test("SPARK-60528: RowEncoder should support nested empty structs") {
+ val schema = new StructType().add(
+ "s",
+ new StructType().add("a", new StructType(), nullable = false),
+ nullable = false)
+ val encoder = ExpressionEncoder(schema).resolveAndBind()
+ assert(encoder.schema === schema)
+ val row = Row(Row(Row()))
+ assert(roundTrip(encoder, row) === row)
+ }
+
test("RowEncoder should support primitive arrays") {
val schema = new StructType()
.add("booleanPrimitiveArray", ArrayType(BooleanType, false))
@@ -339,6 +350,16 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
"for the type \"TIMESTAMP\""))
}
+ test("SPARK-60528: RowEncoder should validate the input of empty structs") {
+ val schema = new StructType().add("s", new StructType(), nullable = false)
+ val encoder = ExpressionEncoder(schema)
+ val e1 = intercept[RuntimeException](toRow(encoder, Row(null)))
+ assert(e1.getCause.getMessage.contains("The 0th field 's' of input row
cannot be null."))
+ val e2 = intercept[RuntimeException](toRow(encoder, Row("a")))
+ assert(e2.getCause.getMessage.contains("The external type java.lang.String
is not valid " +
+ "for the type \"STRUCT<>\""))
+ }
+
private def roundTripArray[T](dt: DataType, nullable: Boolean, data:
Array[T]): Unit = {
val schema = new StructType().add("a", ArrayType(dt, nullable))
test(s"RowEncoder should return mutable.ArraySeq with properly typed array
for $schema") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 05c2f37e22ec..b579924c67ad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2904,6 +2904,11 @@ class DataFrameSuite extends SharedSparkSession
assert(lateCalls.value === 10)
}
}
+
+ test("SPARK-60528: collect a nested empty struct") {
+ val df = spark.range(1).selectExpr("id AS i", "named_struct('a', struct())
AS s")
+ checkAnswer(df, Row(0L, Row(Row())))
+ }
}
case class GroupByKey(a: Int, b: Int)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]