viirya commented on a change in pull request #23943: [SPARK-27034][SQL] Nested
schema pruning for ORC
URL: https://github.com/apache/spark/pull/23943#discussion_r262093206
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
##########
@@ -103,42 +43,13 @@ class ParquetSchemaPruningSuite
Nil)
}
- testSchemaPruning("select a single complex field array and its parent struct
array") {
- val query = sql("select friends.middle, friends from contacts where p=1")
- checkScan(query,
- "struct<friends:array<struct<first:string,middle:string,last:string>>>")
- checkAnswer(query.orderBy("id"),
- Row(Array("Z."), Array(Row("Susan", "Z.", "Smith"))) ::
- Row(Array.empty[String], Array.empty[Row]) ::
- Nil)
- }
-
- testSchemaPruning("select a single complex field from a map entry and its
parent map entry") {
- val query =
- sql("select relatives[\"brother\"].middle, relatives[\"brother\"] from
contacts where p=1")
- checkScan(query,
-
"struct<relatives:map<string,struct<first:string,middle:string,last:string>>>")
- checkAnswer(query.orderBy("id"),
- Row("Y.", Row("John", "Y.", "Doe")) ::
- Row(null, null) ::
- Nil)
- }
-
testSchemaPruning("select a single complex field and the partition column") {
val query = sql("select name.middle, p from contacts")
checkScan(query, "struct<name:struct<middle:string>>")
checkAnswer(query.orderBy("id"),
Row("X.", 1) :: Row("Y.", 1) :: Row(null, 2) :: Row(null, 2) :: Nil)
Review comment:
Those tests can't be shared with ORC, because they are depended on schema
merging.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
