Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/20894#discussion_r178582307
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala
---
@@ -289,27 +295,19 @@ private[csv] object UnivocityParser {
*/
def parseIterator(
lines: Iterator[String],
- shouldDropHeader: Boolean,
parser: UnivocityParser,
schema: StructType): Iterator[InternalRow] = {
val options = parser.options
- val linesWithoutHeader = if (shouldDropHeader) {
- // Note that if there are only comments in the first block, the
header would probably
- // be not dropped.
- CSVUtils.dropHeaderLine(lines, options)
- } else {
- lines
- }
-
val filteredLines: Iterator[String] =
- CSVUtils.filterCommentAndEmpty(linesWithoutHeader, options)
+ CSVUtils.filterCommentAndEmpty(lines, options)
--- End diff --
Now, it fits a single line. Please combine line 302 and 303
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
