gatorsmile commented on a change in pull request #24043: [SPARK-11412][SQL]
Support merge schema for ORC
URL: https://github.com/apache/spark/pull/24043#discussion_r296469098
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
##########
@@ -332,6 +377,53 @@ abstract class OrcSuite extends OrcTest with
BeforeAndAfterAll {
assert(version === SPARK_VERSION_SHORT)
}
}
+
+ test("SPARK-11412 test orc merge schema option") {
+ val conf = spark.sessionState.conf
+ // Test if the default of spark.sql.orc.mergeSchema is false
+ assert(new OrcOptions(Map.empty[String, String], conf).mergeSchema ==
false)
+
+ // OrcOptions's parameters have a higher priority than SQL configuration.
+ // `mergeSchema` -> `spark.sql.orc.mergeSchema`
+ withSQLConf(SQLConf.ORC_SCHEMA_MERGING_ENABLED.key -> "true") {
+ val map1 = Map(OrcOptions.MERGE_SCHEMA -> "true")
+ val map2 = Map(OrcOptions.MERGE_SCHEMA -> "false")
+ assert(new OrcOptions(map1, conf).mergeSchema == true)
+ assert(new OrcOptions(map2, conf).mergeSchema == false)
+ }
+
+ withSQLConf(SQLConf.ORC_SCHEMA_MERGING_ENABLED.key -> "false") {
+ val map1 = Map(OrcOptions.MERGE_SCHEMA -> "true")
+ val map2 = Map(OrcOptions.MERGE_SCHEMA -> "false")
+ assert(new OrcOptions(map1, conf).mergeSchema == true)
+ assert(new OrcOptions(map2, conf).mergeSchema == false)
+ }
+ }
+
+ test("SPARK-11412 test enabling/disabling schema merging") {
Review comment:
Could you have a few negative test cases?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
