This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 46a8f1f0df8 [SPARK-39818][SQL] Fix bug in ARRAY, STRUCT, MAP types
with DEFAULT values with NULL field(s)
46a8f1f0df8 is described below
commit 46a8f1f0df83deeda6309f8377988a66ba059c10
Author: Daniel Tenedorio <daniel.tenedo...@databricks.com>
AuthorDate: Tue Jul 19 18:53:23 2022 -0700
[SPARK-39818][SQL] Fix bug in ARRAY, STRUCT, MAP types with DEFAULT values
with NULL field(s)
### What changes were proposed in this pull request?
Fix bug in SQL string generation for literal values of ARRAY, STRUCT, MAP
types with DEFAULT values with NULL field(s). Specifically, prevent Scala
`MatchError`s from getting raised when attempting to call the `sql` method of
the `Literal` expression for such values when one of their fields is `null`.
### Why are the changes needed?
This fixes a bug by preventing exceptions from being inappropriately raised.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
This PR adds new unit test coverage.
Closes #37229 from dtenedor/fix-bug-sql.
Authored-by: Daniel Tenedorio <daniel.tenedo...@databricks.com>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
.../apache/spark/sql/catalyst/expressions/literals.scala | 14 +++++++-------
.../catalyst/analysis/ExpressionTypeCheckingSuite.scala | 15 +++++++++++++++
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 03678773ccc..a8c877a29de 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -494,20 +494,20 @@ case class Literal (value: Any, dataType: DataType)
extends LeafExpression {
case (row: GenericInternalRow, structType: StructType) =>
val structNames: Array[String] = structType.fields.map(_.name)
val structValues: Array[String] =
- row.values.zip(structType.fields.map(_.dataType)).map {
- case (value: Any, fieldType: DataType) =>
- Literal(value, fieldType).sql
+ row.values.zip(structType.fields.map(_.dataType)).map { kv =>
+ Literal(kv._1, kv._2).sql
}
val structFields: Array[String] =
- structNames.zip(structValues).map { kv => s"${kv._1}, ${kv._2}" }
+ structNames.zip(structValues).map {
+ kv => s"${kv._1}, ${kv._2}"
+ }
s"NAMED_STRUCT(${structFields.mkString(", ")})"
case (data: ArrayBasedMapData, mapType: MapType) =>
val keyData = data.keyArray.asInstanceOf[GenericArrayData]
val valueData = data.valueArray.asInstanceOf[GenericArrayData]
val keysAndValues: Array[String] =
- keyData.array.zip(valueData.array).map {
- case (key: Any, value: Any) =>
- s"${Literal(key, mapType.keyType).sql}, ${Literal(value,
mapType.valueType).sql}"
+ keyData.array.zip(valueData.array).map { kv =>
+ s"${Literal(kv._1, mapType.keyType).sql}, ${Literal(kv._2,
mapType.valueType).sql}"
}
s"MAP(${keysAndValues.mkString(", ")})"
case _ => value.toString
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 9225a7dcbe4..0969089bd92 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -240,4 +240,19 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite
with SQLHelper {
"does not support ordering")
}
}
+
+ test("check types for SQL string generation") {
+ assert(Literal.create(Array(1, 2, 3), ArrayType(IntegerType)).sql ==
+ "ARRAY(1, 2, 3)")
+ assert(Literal.create(Array(1, 2, null), ArrayType(IntegerType)).sql ==
+ "ARRAY(1, 2, CAST(NULL AS INT))")
+ assert(Literal.default(StructType(Seq(StructField("col",
StringType)))).sql ==
+ "NAMED_STRUCT(col, '')")
+ assert(Literal.default(StructType(Seq(StructField("col", NullType)))).sql
==
+ "NAMED_STRUCT(col, NULL)")
+ assert(Literal.create(Map(42L -> true), MapType(LongType,
BooleanType)).sql ==
+ "MAP(42L, true)")
+ assert(Literal.create(Map(42L -> null), MapType(LongType, NullType)).sql ==
+ "MAP(42L, NULL)")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
