Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/20937#discussion_r178427175
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/TextOptions.scala
---
@@ -41,13 +41,16 @@ private[text] class TextOptions(@transient private val
parameters: CaseInsensiti
*/
val wholeText = parameters.getOrElse(WHOLETEXT, "false").toBoolean
- private val lineSeparator: Option[String] =
parameters.get(LINE_SEPARATOR).map { sep =>
- require(sep.nonEmpty, s"'$LINE_SEPARATOR' cannot be an empty string.")
- sep
+ val charset: Option[String] = Some("UTF-8")
+
+ val lineSeparator: Option[Array[Byte]] = parameters.get("lineSep").map {
lineSep =>
+ require(lineSep.nonEmpty, s"'$LINE_SEPARATOR' cannot be an empty
string.")
+ lineSep.getBytes(charset.getOrElse(
--- End diff --
Shall we just `require` here too?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
