Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/19492#discussion_r144757908
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala ---
@@ -170,6 +160,31 @@ class JsonFunctionsSuite extends QueryTest with
SharedSQLContext {
Row(Row(1, "haa")) :: Nil)
}
+ test("SPARK-22228: from_json should support also arrays of primitive
types") {
+ val dfInt = Seq("[1]", "[2, 3]").toDS()
+ checkAnswer(
+ dfInt.select(from_json($"value", ArrayType(IntegerType))),
+ Row(Seq(1)) :: Row(Seq(2, 3)) :: Nil)
+
+ val dfString = Seq("""["hello", "world", ""]""").toDS()
+ checkAnswer(
+ dfString.select(from_json($"value", ArrayType(StringType))),
+ Row(Seq("hello", "world", "")):: Nil)
+
+ val dfTimestamp = Seq("""["26/08/2015 18:00"]""").toDS()
+ val schema = ArrayType(TimestampType)
+ val options = Map("timestampFormat" -> "dd/MM/yyyy HH:mm")
+
+ checkAnswer(
+ dfTimestamp.select(from_json($"value", schema, options)),
+ Row(Seq(java.sql.Timestamp.valueOf("2015-08-26 18:00:00.0"))))
+
+ val dfEmpty = Seq("""[]""").toDS()
+ checkAnswer(
+ dfEmpty.select(from_json($"value", ArrayType(StringType))),
+ Row(Nil):: Nil)
--- End diff --
And also add a case that can fail the parsing like `Seq(""""[1]", "[2, 3]",
"[string]""").toDS()`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
