gaborgsomogyi commented on a change in pull request #25004: [SPARK-28205][SQL]
useV1SourceList configuration should be for all data sources
URL: https://github.com/apache/spark/pull/25004#discussion_r298602607
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/sources/v2/FileDataSourceV2FallBackSuite.scala
##########
@@ -170,4 +174,41 @@ class FileDataSourceV2FallBackSuite extends QueryTest
with SharedSQLContext {
}
}
}
+
+ test("Fallback Parquet V2 to V1") {
+ Seq("parquet", classOf[ParquetDataSourceV2].getCanonicalName).foreach {
format =>
+ withSQLConf(SQLConf.USE_V1_SOURCE_READER_LIST.key -> format,
+ SQLConf.USE_V1_SOURCE_WRITER_LIST.key -> format) {
+ val commands = ArrayBuffer.empty[(String, LogicalPlan)]
+ val exceptions = ArrayBuffer.empty[(String, Exception)]
+ val listener = new QueryExecutionListener {
+ override def onFailure(
+ funcName: String,
+ qe: QueryExecution,
+ exception: Exception): Unit = {
+ exceptions += funcName -> exception
+ }
+
+ override def onSuccess(funcName: String, qe: QueryExecution,
duration: Long): Unit = {
+ commands += funcName -> qe.logical
+ }
+ }
+ spark.listenerManager.register(listener)
+
+ withTempPath { path =>
+ val inputData = spark.range(10)
+ inputData.write.format(format).save(path.getCanonicalPath)
+ sparkContext.listenerBus.waitUntilEmpty(1000)
+ assert(commands.length == 1)
+ assert(commands.head._1 == "save")
+
assert(commands.head._2.isInstanceOf[InsertIntoHadoopFsRelationCommand])
+
assert(commands.head._2.asInstanceOf[InsertIntoHadoopFsRelationCommand]
+ .fileFormat.isInstanceOf[ParquetFileFormat])
+ val df = spark.read.format(format).load(path.getCanonicalPath)
+ checkAnswer(df, inputData.toDF())
+
assert(df.queryExecution.executedPlan.find(_.isInstanceOf[FileSourceScanExec]).isDefined)
Review comment:
Don't we need something like this?
```
spark.listenerManager.unregister(listener)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
