Github user imatiach-msft commented on a diff in the pull request:
https://github.com/apache/spark/pull/22356#discussion_r215730668
--- Diff: core/src/test/scala/org/apache/spark/FileSuite.scala ---
@@ -299,6 +301,25 @@ class FileSuite extends SparkFunSuite with
LocalSparkContext {
}
}
+ test("SPARK-22357 test binaryFiles minPartitions") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local")
+ .set("spark.files.openCostInBytes", "0")
+ .set("spark.default.parallelism", "1"))
+
+ val tempDir = Utils.createTempDir()
+ val tempDirPath = tempDir.getAbsolutePath
+
+ for (i <- 0 until 8) {
+ val tempFile = new File(tempDir, s"part-0000$i")
+ Files.write("someline1 in file1\nsomeline2 in file1\nsomeline3 in
file1", tempFile,
+ StandardCharsets.UTF_8)
+ }
+
+ assert(sc.binaryFiles(tempDirPath, minPartitions = 1).getNumPartitions
=== 1)
--- End diff --
nitpick: maybe put these three asserts in a loop
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
