This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 298f5fad9cc [SPARK-39034][SQL][TESTS][DOCS] Add tests for options from
`to_json` and `from_json`
298f5fad9cc is described below
commit 298f5fad9cc0cb4b64ca2a4a9c5775b5f2532d42
Author: itholic <[email protected]>
AuthorDate: Sun May 1 08:55:09 2022 +0900
[SPARK-39034][SQL][TESTS][DOCS] Add tests for options from `to_json` and
`from_json`
### What changes were proposed in this pull request?
This PR proposes to fill up the missing test for options from JSON
read/write functions.
This PR also includes some typo fixing of docs.
### Why are the changes needed?
For better testability and correct the documents.
### Does this PR introduce _any_ user-facing change?
Some typo is fixed from documents.
Test-wise, it's dev-only.
### How was this patch tested?
Added some UTs for JSON read/write options.
Closes #36399 from itholic/SPARK-39034.
Authored-by: itholic <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
docs/sql-data-sources-json.md | 2 +-
.../org/apache/spark/sql/JsonFunctionsSuite.scala | 102 ++++++++++++++++++++-
2 files changed, 101 insertions(+), 3 deletions(-)
diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md
index d22ca1979a9..27d89875623 100644
--- a/docs/sql-data-sources-json.md
+++ b/docs/sql-data-sources-json.md
@@ -155,7 +155,7 @@ Data source options of JSON can be set via:
<td>read</td>
</tr>
<tr>
- <td><code>allowNumericLeadingZero</code></td>
+ <td><code>allowNumericLeadingZeros</code></td>
<td><code>false</code></td>
<td>Allows leading zeros in numbers (e.g. 00012).</td>
<td>read</td>
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index dd215de26e8..089c8a01e60 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -122,7 +122,7 @@ class JsonFunctionsSuite extends QueryTest with
SharedSparkSession {
Row(Row(1)) :: Nil)
}
- test("from_json with option") {
+ test("from_json with option (timestampFormat)") {
val df = Seq("""{"time": "26/08/2015 18:00"}""").toDS()
val schema = new StructType().add("time", TimestampType)
val options = Map("timestampFormat" -> "dd/MM/yyyy HH:mm")
@@ -132,6 +132,86 @@ class JsonFunctionsSuite extends QueryTest with
SharedSparkSession {
Row(Row(java.sql.Timestamp.valueOf("2015-08-26 18:00:00.0"))))
}
+ test("from_json with option (allowComments)") {
+ val df = Seq("""{"str": /* Hello */ "World"}""").toDS()
+ val schema = new StructType().add("str", StringType)
+ val options = Map("allowComments" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row("World")) :: Nil)
+ }
+
+ test("from_json with option (allowUnquotedFieldNames)") {
+ val df = Seq("""{str: "World"}""").toDS()
+ val schema = new StructType().add("str", StringType)
+ val options = Map("allowUnquotedFieldNames" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row("World")) :: Nil)
+ }
+
+ test("from_json with option (allowSingleQuotes)") {
+ val df = Seq("""{"str": 'World'}""").toDS()
+ val schema = new StructType().add("str", StringType)
+ val options = Map("allowSingleQuotes" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row("World")) :: Nil)
+ }
+
+ test("from_json with option (allowNumericLeadingZeros)") {
+ val df = Seq("""{"int": 0018}""").toDS()
+ val schema = new StructType().add("int", IntegerType)
+ val options = Map("allowNumericLeadingZeros" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row(18)) :: Nil)
+ }
+
+ test("from_json with option (allowBackslashEscapingAnyCharacter)") {
+ val df = Seq("""{"str": "\$10"}""").toDS()
+ val schema = new StructType().add("str", StringType)
+ val options = Map("allowBackslashEscapingAnyCharacter" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row("$10")) :: Nil)
+ }
+
+ test("from_json with option (dateFormat)") {
+ val df = Seq("""{"time": "26/08/2015"}""").toDS()
+ val schema = new StructType().add("time", DateType)
+ val options = Map("dateFormat" -> "dd/MM/yyyy")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row(java.sql.Date.valueOf("2015-08-26"))))
+ }
+
+ test("from_json with option (allowUnquotedControlChars)") {
+ val df = Seq("{\"str\": \"a\u0001b\"}").toDS()
+ val schema = new StructType().add("str", StringType)
+ val options = Map("allowUnquotedControlChars" -> "true")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row("a\u0001b")) :: Nil)
+ }
+
+ test("from_json with option (allowNonNumericNumbers)") {
+ val df = Seq("""{"int": +Infinity}""").toDS()
+ val schema = new StructType().add("int", FloatType)
+ val options = Map("allowNonNumericNumbers" -> "false")
+
+ checkAnswer(
+ df.select(from_json($"value", schema, options)),
+ Row(Row(null)) :: Nil)
+ }
+
test("from_json missing columns") {
val df = Seq("""{"a": 1}""").toDS()
val schema = new StructType().add("b", IntegerType)
@@ -215,7 +295,7 @@ class JsonFunctionsSuite extends QueryTest with
SharedSparkSession {
Row("""{"a":1}""") :: Nil)
}
- test("to_json with option") {
+ test("to_json with option (timestampFormat)") {
val df = Seq(Tuple1(Tuple1(java.sql.Timestamp.valueOf("2015-08-26
18:00:00.0")))).toDF("a")
val options = Map("timestampFormat" -> "dd/MM/yyyy HH:mm")
@@ -224,6 +304,24 @@ class JsonFunctionsSuite extends QueryTest with
SharedSparkSession {
Row("""{"_1":"26/08/2015 18:00"}""") :: Nil)
}
+ test("to_json with option (dateFormat)") {
+ val df = Seq(Tuple1(Tuple1(java.sql.Date.valueOf("2015-08-26")))).toDF("a")
+ val options = Map("dateFormat" -> "dd/MM/yyyy")
+
+ checkAnswer(
+ df.select(to_json($"a", options)),
+ Row("""{"_1":"26/08/2015"}""") :: Nil)
+ }
+
+ test("to_json with option (ignoreNullFields)") {
+ val df = Seq(Tuple1(Tuple1(null))).toDF("a")
+ val options = Map("ignoreNullFields" -> "true")
+
+ checkAnswer(
+ df.select(to_json($"a", options)),
+ Row("""{}""") :: Nil)
+ }
+
test("to_json - interval support") {
val baseDf = Seq(Tuple1(Tuple1("-3 month 7 hours"))).toDF("a")
val df =
baseDf.select(struct($"a._1".cast(CalendarIntervalType).as("a")).as("c"))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
