This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 48d22e9f876 [SPARK-46643][SQL][TESTS] Fix ORC tests to be independent
from default compression
48d22e9f876 is described below
commit 48d22e9f876f070d35ff3dd011bfbd1b6bccb4ac
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Tue Jan 9 18:00:22 2024 -0800
[SPARK-46643][SQL][TESTS] Fix ORC tests to be independent from default
compression
### What changes were proposed in this pull request?
This PR aims to fix ORC tests to be independent from the change of default
ORC compression.
### Why are the changes needed?
Currently, a few test cases have implicit assumption.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44648 from dongjoon-hyun/SPARK-46643.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/sql/execution/datasources/orc/OrcQuerySuite.scala | 2 +-
.../spark/sql/execution/datasources/orc/OrcSourceSuite.scala | 5 +++--
.../apache/spark/sql/hive/orc/OrcHadoopFsRelationSuite.scala | 10 ----------
3 files changed, 4 insertions(+), 13 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index 7d666729bb4..3f3776bab8f 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -508,7 +508,7 @@ abstract class OrcQueryTest extends OrcTest {
conf.setBoolean("hive.io.file.read.all.columns", false)
val orcRecordReader = {
- val file = new
File(path).listFiles().find(_.getName.endsWith(".snappy.orc")).head
+ val file = new
File(path).listFiles().find(_.getName.endsWith(".orc")).head
val split = new FileSplit(new Path(file.toURI), 0, file.length,
Array.empty[String])
val attemptId = new TaskAttemptID(new TaskID(new JobID(),
TaskType.MAP, 0), 0)
val hadoopAttemptContext = new TaskAttemptContextImpl(conf, attemptId)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 1e98099361d..6166773fb09 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -332,8 +332,9 @@ abstract class OrcSuite
test("SPARK-21839: Add SQL config for ORC compression") {
val conf = spark.sessionState.conf
- // Test if the default of spark.sql.orc.compression.codec is snappy
- assert(new OrcOptions(Map.empty[String, String], conf).compressionCodec ==
SNAPPY.name())
+ // Test if the default of spark.sql.orc.compression.codec is used.
+ assert(new OrcOptions(Map.empty[String, String], conf).compressionCodec ==
+ SQLConf.ORC_COMPRESSION.defaultValueString.toUpperCase(Locale.ROOT))
// OrcOptions's parameters have a higher priority than SQL configuration.
// `compression` -> `orc.compression` -> `spark.sql.orc.compression.codec`
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcHadoopFsRelationSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcHadoopFsRelationSuite.scala
index aa2f110ceac..071035853b6 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcHadoopFsRelationSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcHadoopFsRelationSuite.scala
@@ -107,16 +107,6 @@ class OrcHadoopFsRelationSuite extends
HadoopFsRelationTest {
checkAnswer(df, copyDf)
}
}
-
- test("Default compression codec is snappy for ORC compression") {
- withTempPath { file =>
- spark.range(0, 10).write
- .orc(file.getCanonicalPath)
- val expectedCompressionKind =
- OrcFileOperator.getFileReader(file.getCanonicalPath).get.getCompression
- assert(OrcCompressionCodec.SNAPPY.name() ===
expectedCompressionKind.name())
- }
- }
}
class HiveOrcHadoopFsRelationSuite extends OrcHadoopFsRelationSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
