This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7ebd39500086 [SPARK-50729][SQL][TESTS] Remove `DebugFilesystem` from
`TPCDS(Collation)?QueryTestSuite`
7ebd39500086 is described below
commit 7ebd3950008694f5ed85420fbe620d3e2247bb90
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Jan 6 11:33:23 2025 +0800
[SPARK-50729][SQL][TESTS] Remove `DebugFilesystem` from
`TPCDS(Collation)?QueryTestSuite`
### What changes were proposed in this pull request?
This PR aims to remove `DebugFilesystem` from
`TPCDS(Collation)?QueryTestSuite` while keeping it in all other places.
### Why are the changes needed?
`TPCDS(Collation)?QueryTestSuite` extends `TPCDSBase` and `TPCHBase`
respectively and they are based on `TPCBase` which extends `SharedSparkSession`.
https://github.com/apache/spark/blob/91f3fdd25852b43095dd5273358fc394ffd11b66/sql/core/src/test/scala/org/apache/spark/sql/TPCBase.scala#L24
And, `SharedSparkSession` always uses `DebugFilesystem`.
https://github.com/apache/spark/blob/91f3fdd25852b43095dd5273358fc394ffd11b66/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala#L68
This PR aims to remove this resetting.
### Does this PR introduce _any_ user-facing change?
No, this is a test-only change.
### How was this patch tested?
Manual review.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49359 from dongjoon-hyun/SPARK-50729.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala | 1 +
sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 1 +
2 files changed, 2 insertions(+)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala
index 46a24acb475c..43e6111fc99c 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala
@@ -67,6 +67,7 @@ class TPCDSCollationQueryTestSuite extends QueryTest with
TPCDSBase with SQLQuer
// To make output results deterministic
override protected def sparkConf: SparkConf = super.sparkConf
.set(SQLConf.SHUFFLE_PARTITIONS.key, "1")
+ .remove("spark.hadoop.fs.file.impl")
protected override def createSparkSession: TestSparkSession = {
new TestSparkSession(new SparkContext("local[1]",
this.getClass.getSimpleName, sparkConf))
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
index bde615552987..c1246a167b8c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
@@ -62,6 +62,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase
with SQLQueryTestHelp
// To make output results deterministic
override protected def sparkConf: SparkConf = super.sparkConf
.set(SQLConf.SHUFFLE_PARTITIONS.key, "1")
+ .remove("spark.hadoop.fs.file.impl")
protected override def createSparkSession: TestSparkSession = {
new TestSparkSession(new SparkContext("local[1]",
this.getClass.getSimpleName, sparkConf))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
