Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/19464#discussion_r144311831
--- Diff: core/src/test/scala/org/apache/spark/FileSuite.scala ---
@@ -510,4 +510,65 @@ class FileSuite extends SparkFunSuite with
LocalSparkContext {
}
}
+ test("allow user to filter out empty split (old Hadoop API)") {
+ val sf = new SparkConf()
+ sf.setAppName("test").setMaster("local").set(FILTER_OUT_EMPTY_SPLIT,
true)
+ sc = new SparkContext(sf)
+
+ // Ensure that if all of the splits are empty, we remove the splits
correctly
+ val emptyRDD = sc.parallelize(Array.empty[Tuple2[String, String]], 1)
+ emptyRDD.saveAsHadoopFile[TextOutputFormat[String,
String]](tempDir.getPath + "/output")
--- End diff --
don't hardcode the path separator, use `new File(tempDir, output)`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
