cfmcgrady commented on a change in pull request #33212:
URL: https://github.com/apache/spark/pull/33212#discussion_r665002215
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
##########
@@ -2924,6 +2924,50 @@ abstract class JsonSuite
}
}
}
+
+ test("SPARK-35912: nullability with different parse mode -- struct") {
+ val input =
+ """
+ |{
+ | "c1": {
+ | "c2": 1
+ | }
+ |}
+ |""".stripMargin
+ val json = spark.createDataset(spark.sparkContext.parallelize(input ::
Nil))(Encoders.STRING)
+
+ val load = (mode: String, schema: StructType) => {
+ spark.read
+ .option("mode", mode)
+ .schema(schema)
+ .json(json)
+ }
+
+ Seq(true, false).foreach { nullable =>
+ val schema = StructType(Seq(
+ StructField("c1",
+ StructType(Seq(
+ StructField("c2", IntegerType, nullable = false),
+ StructField("not_exist_col", IntegerType, nullable = false)
+ )),
+ nullable = nullable))
+ )
+
+ checkAnswer(load("DROPMALFORMED", schema), Seq.empty)
+
+ val exception = intercept[SparkException] {
+ load("FAILFAST", schema).collect
+ }.getMessage
+ assert(exception.contains(
+ "the null value found when parsing non-nullable field not_exist_col."))
+
+ val e = intercept[SparkException] {
+ load("PERMISSIVE", schema).collect()
+ }
+ assert(e.getMessage.contains(
+ "the null value found when parsing non-nullable field not_exist_col."))
Review comment:
Do you mean we should return Row(null) when `c1.nullable = false` or
always return Row(null) for this case no matter what `c1.nullable` is true or
false?
The reason why I throw an exception here is that `not_exist_col.nullable =
false` but the value in the JSON string is missing. WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
