cfmcgrady commented on a change in pull request #33212:
URL: https://github.com/apache/spark/pull/33212#discussion_r666705223
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
##########
@@ -2924,6 +2924,53 @@ abstract class JsonSuite
}
}
}
+
+ test("SPARK-35912: nullability with different parse mode -- struct") {
+ // JSON field is missing.
+ val input = """{"c1": 1}"""
+ // JSON filed is null.
+ val input2 =
+ """
+ |{
+ | "c1": 1,
+ | "c2": null
+ |}
+ |""".stripMargin
+
+ val load = (mode: String, schema: StructType, inputJson: String) => {
+ val json = spark.createDataset(
+ spark.sparkContext.parallelize(inputJson :: Nil))(Encoders.STRING)
+ spark.read
+ .option("mode", mode)
+ .schema(schema)
+ .json(json)
+ }
+ Seq(true, false).foreach { nullable =>
+ val schema = StructType(Seq(
+ StructField("c1", IntegerType, nullable = false),
+ StructField("c2", IntegerType, nullable = nullable)))
+
+ Seq(input, input2).foreach { jsonString =>
+ if (nullable) {
+ checkAnswer(load("DROPMALFORMED", schema, jsonString), Row(1, null)
:: Nil)
+ checkAnswer(load("FAILFAST", schema, jsonString), Row(1, null) ::
Nil)
+ checkAnswer(load("PERMISSIVE", schema, jsonString), Row(1, null) ::
Nil)
+ } else {
+ checkAnswer(load("DROPMALFORMED", schema, jsonString), Seq.empty)
+ val exceptionMsg1 = intercept[SparkException] {
+ load("FAILFAST", schema, jsonString).collect
+ }.getMessage
+ assert(exceptionMsg1.contains(
+ "the null value found when parsing non-nullable field c2."))
+ val exceptionMsg2 = intercept[SparkException] {
+ load("PERMISSIVE", schema, jsonString).collect
Review comment:
@cloud-fan Since the field is non-nullable, do we still return Row(1,
null) for PERMISSIVE mode? if yes, this may cause the cast struct problem as we
talked about before, the field is non-nullable but row.isNullAt(index) is true.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
