Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/22951#discussion_r231640870
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala ---
@@ -117,4 +120,20 @@ class CsvFunctionsSuite extends QueryTest with
SharedSQLContext {
"Acceptable modes are PERMISSIVE and FAILFAST."))
}
}
+
+ test("use locale while parsing timestamps") {
+ Seq("en-US", "ko-KR", "zh-CN", "ru-RU").foreach { langTag =>
+ val locale = Locale.forLanguageTag(langTag)
+ val ts = new SimpleDateFormat("dd/MM/yyyy HH:mm").parse("06/11/2018
18:00")
+ val timestampFormat = "dd MMM yyyy HH:mm"
+ val sdf = new SimpleDateFormat(timestampFormat, locale)
+ val input = Seq(s"""${sdf.format(ts)}""").toDS()
+ val schema = new StructType().add("time", TimestampType)
+ val options = Map("timestampFormat" -> timestampFormat, "locale" ->
langTag)
+ val df = input.select(from_csv($"value", schema, options))
--- End diff --
Can we simplify like this?
```scala
- val schema = new StructType().add("time", TimestampType)
val options = Map("timestampFormat" -> timestampFormat, "locale" ->
langTag)
- val df = input.select(from_csv($"value", schema, options))
+ val df = input.select(from_csv($"value", lit("time timestamp"),
options.asJava))
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
