Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/19707#discussion_r150069049
--- Diff: sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
---
@@ -1408,6 +1409,23 @@ class DatasetSuite extends QueryTest with
SharedSQLContext {
checkDataset(ds, SpecialCharClass("1", "2"))
}
}
+
+ test("SPARK-22472: add null check for top-level primitive values") {
+ // If the primitive values are from Option, we need to do runtime null
check.
+ val ds = Seq(Some(1), None).toDS().as[Int]
+ intercept[NullPointerException](ds.collect())
+ val e = intercept[SparkException](ds.map(_ * 2).collect())
+ assert(e.getCause.isInstanceOf[NullPointerException])
+
+ withTempPath { path =>
+ Seq(new Integer(1),
null).toDF("i").write.parquet(path.getCanonicalPath)
--- End diff --
Thanks!
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
