This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new c3d73f8180b7 [SPARK-57647][SQL][TEST] Make `FileSourceStrategySuite`
be independent from the default value of `maxPartitionBytes`
c3d73f8180b7 is described below
commit c3d73f8180b7f3b9f073882e04836e1f08542cc6
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Jun 23 22:46:26 2026 -0700
[SPARK-57647][SQL][TEST] Make `FileSourceStrategySuite` be independent from
the default value of `maxPartitionBytes`
### What changes were proposed in this pull request?
This PR aims to make `FileSourceStrategySuite` be independent from the
default value of `maxPartitionBytes`.
### Why are the changes needed?
The following `maxPartitionNum` test case fails like the following when the
default value of `maxPartitionBytes` is changed. If a test case assume some
other configurations, it should be set explicitly inside the test case.
```
[info] - SPARK-44021: Test spark.sql.files.maxPartitionNum works as
expected *** FAILED *** (33 milliseconds)
[info] 300000 did not equal 150000 (FileSourceStrategySuite.scala:613)
```
Since the current value of `maxPartitionBytes` is `128MB`, this PR
explicitly sets it.
https://github.com/apache/spark/blob/f2cbc7803c18648130db862cd85581a57c1dad98/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L2721-L2727
### Does this PR introduce _any_ user-facing change?
No, this is a test case fix.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56717 from dongjoon-hyun/SPARK-57647.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 0cc4d05aabb6aab612e947dec9dbeb7ecb2cef2a)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../datasources/FileSourceStrategySuite.scala | 36 ++++++++++++----------
1 file changed, 19 insertions(+), 17 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
index 21860122244a..195e7c7fce7f 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
@@ -605,25 +605,27 @@ class FileSourceStrategySuite extends SharedSparkSession {
}
test(s"SPARK-44021: Test ${SQLConf.FILES_MAX_PARTITION_NUM.key} works as
expected") {
- val files =
- Range(0, 300000).map(p => PartitionedFile(InternalRow.empty, sp(s"$p"),
0, 50000000))
- val maxPartitionBytes = conf.filesMaxPartitionBytes
- val defaultPartitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
- assert(defaultPartitions.size === 150000)
-
- withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "20000") {
- val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
- assert(partitions.size === 20000)
- }
+ withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "128MB") {
+ val files =
+ Range(0, 300000).map(p => PartitionedFile(InternalRow.empty,
sp(s"$p"), 0, 50000000))
+ val maxPartitionBytes = conf.filesMaxPartitionBytes
+ val defaultPartitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
+ assert(defaultPartitions.size === 150000)
+
+ withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "20000") {
+ val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
+ assert(partitions.size === 20000)
+ }
- withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "50000") {
- val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
- assert(partitions.size === 50000)
- }
+ withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "50000") {
+ val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
+ assert(partitions.size === 50000)
+ }
- withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "200000") {
- val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
- assert(partitions.size === defaultPartitions.size)
+ withSQLConf(SQLConf.FILES_MAX_PARTITION_NUM.key -> "200000") {
+ val partitions = FilePartition.getFilePartitions(spark, files,
maxPartitionBytes)
+ assert(partitions.size === defaultPartitions.size)
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
