dongjoon-hyun commented on a change in pull request #24865: [SPARK-27100][SQL]
Use `Array` instead of `Seq` in `FilePartition` to prevent `StackOverflowError `
URL: https://github.com/apache/spark/pull/24865#discussion_r296092353
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
##########
@@ -735,4 +745,93 @@ abstract class BucketedReadSuite extends QueryTest with
SQLTestUtils {
df1.groupBy("j").agg(max("k")))
}
}
+
+ // A test with a partition where the number of files in the partition is
+ // large. tests for the condition where the serialization of such a task
may result in a stack
+ // overflow if the files list is stored in a recursive data structure
+ test("SPARK-27100 stack overflow: read data with large partitions") {
+ // Need a large number of files in the partition for the overflow
+ val numFilesInPartition = 100000
+
+ def getInputRDD(isBucketed: Boolean): RDD[InternalRow] = {
+ val partitionValues = InternalRow.apply(Array("a"))
+
+ val schema = new StructType()
+ val fakeHadoopFsRelation =
+ HadoopFsRelation(null, schema, schema, null, new ParquetFileFormat,
null)(spark)
+ val optionalBucketSet = null
+ val files = (0 until numFilesInPartition).toStream.map { i =>
+ new FileStatus(10, false, 1, 512, 1000, new Path(s"file${i}_0.zzz"))
+ }
+ val partitionDirectory = PartitionDirectory(partitionValues, files)
+
+ val fileSource = FileSourceScanExec(fakeHadoopFsRelation,
+ null,
+ schema,
+ null,
+ Option(optionalBucketSet),
+ Seq.empty,
+ Option(new TableIdentifier("stackOverflow")))
+
+ val inputRDD = if (isBucketed) {
+ // Create a Bucketed RDD. This is a private method so we need to call
this indirectly.
+ val createBucketedReadRDD =
PrivateMethod[RDD[InternalRow]]('createBucketedReadRDD)
+ val bucketSpec = BucketSpec(1, Seq("a"), Seq("b"))
+
+ fileSource.invokePrivate(createBucketedReadRDD(bucketSpec,
+ (_: PartitionedFile) => Seq(InternalRow(1)).toIterator,
+ Array(partitionDirectory),
Review comment:
Ur, unfortunately, this test case seems to become unable to detect the
regression after changing from `Stream(partitionDirectory)` to
`Array(partitionDirectory)`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
