Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/22951#discussion_r231641960
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala ---
@@ -578,4 +581,20 @@ class JsonFunctionsSuite extends QueryTest with
SharedSQLContext {
"Acceptable modes are PERMISSIVE and FAILFAST."))
}
}
+
+ test("use locale while parsing timestamps") {
+ Seq("en-US", "ko-KR", "zh-CN", "ru-RU").foreach { langTag =>
+ val locale = Locale.forLanguageTag(langTag)
+ val ts = new SimpleDateFormat("dd/MM/yyyy HH:mm").parse("06/11/2018
18:00")
+ val timestampFormat = "dd MMM yyyy HH:mm"
+ val sdf = new SimpleDateFormat(timestampFormat, locale)
+ val input = Seq(s"""{"time": "${sdf.format(ts)}"}""").toDS()
+ val schema = new StructType().add("time", TimestampType)
+ val options = Map("timestampFormat" -> timestampFormat, "locale" ->
langTag)
+ val df = input.select(from_json($"value", schema, options))
--- End diff --
This one can be simplified like this, too.
```scala
- val schema = new StructType().add("time", TimestampType)
val options = Map("timestampFormat" -> timestampFormat, "locale" ->
langTag)
- val df = input.select(from_json($"value", schema, options))
+ val df = input.select(from_json($"value", "time timestamp", options))
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
