This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 2da80d9997f0 [SPARK-57728][SQL][TEST] Make SQL tests independent of
the `ignoreDataLocality` default
2da80d9997f0 is described below
commit 2da80d9997f030f8b2db52c5006cfe6392ac070f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Jun 27 16:30:42 2026 -0700
[SPARK-57728][SQL][TEST] Make SQL tests independent of the
`ignoreDataLocality` default
### What changes were proposed in this pull request?
This PR pins `spark.sql.sources.ignoreDataLocality` to `false` in
- `FileIndexSuite`
- `FileSourceStrategySuite`
- `HadoopFsRelationTest`
### Why are the changes needed?
Pinning `spark.sql.sources.ignoreDataLocality` keeps the test independent
of the default value.
https://github.com/apache/spark/blob/225975d809a8a7c9f08a60e36576c391b82a2306/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L2435-L2445
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CI.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8
Closes #56835 from dongjoon-hyun/SPARK-57728.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a2724ac37d9e4c61eefe30fa707c62dcc256a3cc)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/execution/datasources/FileIndexSuite.scala | 11 ++++++++---
.../sql/execution/datasources/FileSourceStrategySuite.scala | 2 ++
.../org/apache/spark/sql/sources/HadoopFsRelationTest.scala | 4 +++-
3 files changed, 13 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index 6f72c31c6d4d..f6ccf1be62c5 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -209,6 +209,7 @@ class FileIndexSuite extends SharedSparkSession {
s"parDiscoveryThreshold=$parDiscoveryThreshold, sqlConf=$sqlConf,
options=$options"
) {
withSQLConf(
+ SQLConf.IGNORE_DATA_LOCALITY.key -> "false",
SQLConf.IGNORE_MISSING_FILES.key -> sqlConf,
SQLConf.PARALLEL_PARTITION_DISCOVERY_THRESHOLD.key ->
parDiscoveryThreshold.toString,
"fs.mockFs.impl" -> raceCondition.getName,
@@ -485,7 +486,9 @@ class FileIndexSuite extends SharedSparkSession {
test("SPARK-25062 - InMemoryFileIndex stores BlockLocation objects no matter
what subclass " +
"the FS returns") {
- withSQLConf("fs.file.impl" ->
classOf[SpecialBlockLocationFileSystem].getName) {
+ withSQLConf(
+ SQLConf.IGNORE_DATA_LOCALITY.key -> "false",
+ "fs.file.impl" -> classOf[SpecialBlockLocationFileSystem].getName) {
withTempDir { dir =>
val file = new File(dir, "text.txt")
stringToFile(file, "text")
@@ -544,8 +547,10 @@ class FileIndexSuite extends SharedSparkSession {
override def hasNext: Boolean = iter.hasNext
override def next(): LocatedFileStatus = iter.next()
})
- val fileIndex = new TestInMemoryFileIndex(spark, path)
- assert(fileIndex.leafFileStatuses.toSeq == statuses)
+ withSQLConf(SQLConf.IGNORE_DATA_LOCALITY.key -> "false") {
+ val fileIndex = new TestInMemoryFileIndex(spark, path)
+ assert(fileIndex.leafFileStatuses.toSeq == statuses)
+ }
}
test("SPARK-48649: Ignore invalid partitions") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
index 195e7c7fce7f..f38996adbd99 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
@@ -307,6 +307,7 @@ class FileSourceStrategySuite extends SharedSparkSession {
test("Locality support for FileScanRDD - one file per partition") {
withSQLConf(
SQLConf.FILES_MAX_PARTITION_BYTES.key -> "10",
+ SQLConf.IGNORE_DATA_LOCALITY.key -> "false",
"fs.file.impl" -> classOf[LocalityTestFileSystem].getName,
"fs.file.impl.disable.cache" -> "true") {
val table =
@@ -332,6 +333,7 @@ class FileSourceStrategySuite extends SharedSparkSession {
withSQLConf(
SQLConf.FILES_MAX_PARTITION_BYTES.key -> "10",
SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "0",
+ SQLConf.IGNORE_DATA_LOCALITY.key -> "false",
"fs.file.impl" -> classOf[LocalityTestFileSystem].getName,
"fs.file.impl.disable.cache" -> "true") {
val table =
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 42514e60a73c..92a33843a86a 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -855,7 +855,9 @@ abstract class HadoopFsRelationTest extends QueryTest with
TestHiveSingleton {
assert(preferredLocations.distinct.length == 2)
}
- withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> dataSourceName) {
+ withSQLConf(
+ SQLConf.USE_V1_SOURCE_LIST.key -> dataSourceName,
+ SQLConf.IGNORE_DATA_LOCALITY.key -> "false") {
checkLocality()
withSQLConf(SQLConf.PARALLEL_PARTITION_DISCOVERY_THRESHOLD.key -> "0")
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
