cloud-fan commented on a change in pull request #33212:
URL: https://github.com/apache/spark/pull/33212#discussion_r666905445
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
##########
@@ -2924,6 +2924,53 @@ abstract class JsonSuite
}
}
}
+
+ test("SPARK-35912: nullability with different parse mode -- struct") {
+ // JSON field is missing.
+ val input = """{"c1": 1}"""
+ // JSON filed is null.
+ val input2 =
+ """
+ |{
+ | "c1": 1,
+ | "c2": null
+ |}
+ |""".stripMargin
+
+ val load = (mode: String, schema: StructType, inputJson: String) => {
+ val json = spark.createDataset(
+ spark.sparkContext.parallelize(inputJson :: Nil))(Encoders.STRING)
+ spark.read
+ .option("mode", mode)
+ .schema(schema)
+ .json(json)
+ }
+ Seq(true, false).foreach { nullable =>
+ val schema = StructType(Seq(
+ StructField("c1", IntegerType, nullable = false),
+ StructField("c2", IntegerType, nullable = nullable)))
+
+ Seq(input, input2).foreach { jsonString =>
+ if (nullable) {
+ checkAnswer(load("DROPMALFORMED", schema, jsonString), Row(1, null)
:: Nil)
+ checkAnswer(load("FAILFAST", schema, jsonString), Row(1, null) ::
Nil)
+ checkAnswer(load("PERMISSIVE", schema, jsonString), Row(1, null) ::
Nil)
+ } else {
+ checkAnswer(load("DROPMALFORMED", schema, jsonString), Seq.empty)
+ val exceptionMsg1 = intercept[SparkException] {
+ load("FAILFAST", schema, jsonString).collect
+ }.getMessage
+ assert(exceptionMsg1.contains(
+ "the null value found when parsing non-nullable field c2."))
+ val exceptionMsg2 = intercept[SparkException] {
+ load("PERMISSIVE", schema, jsonString).collect
Review comment:
Yea, I think we can't apply permissive mode here. Failing looks fine to
me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
