Github user MaxGekk commented on a diff in the pull request:
https://github.com/apache/spark/pull/23173#discussion_r238059569
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
---
@@ -1987,6 +1987,21 @@ class CSVSuite extends QueryTest with
SharedSQLContext with SQLTestUtils with Te
assert(errMsg2.contains("'lineSep' can contain only 1 character"))
}
+ test("SPARK-26208: write and read empty data to csv file with headers") {
+ withTempPath { path =>
+ val df1 = spark.range(10).repartition(2).filter(_ <
0).map(_.toString).toDF
+ // we have 2 partitions but they are both empty and will be filtered
out upon writing
+ // thanks to SPARK-23271 one new empty partition will be inserted
+ df1.write.format("csv").option("header",
true).save(path.getAbsolutePath)
+ val df2 = spark.read.format("csv").option("header",
true).option("inferSchema", false)
+ .load(path.getAbsolutePath)
+ assert(df1.rdd.getNumPartitions == 2)
+ assert(df2.rdd.getNumPartitions == 1)
--- End diff --
nit: I wouldn't check number of partition here since it is implementation
specific and doesn't matter for behavior checked in the test.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
