Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/22357#discussion_r216694201
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
---
@@ -155,6 +163,60 @@ class ParquetSchemaPruningSuite
Row(null) :: Row(null) :: Nil)
}
+ testSchemaPruning("select a single complex field and in where clause") {
+ val query1 = sql("select name.first from contacts where name.first =
'Jane'")
+ checkScan(query1, "struct<name:struct<first:string>>")
+ checkAnswer(query1, Row("Jane") :: Nil)
+
+ val query2 = sql("select name.first, name.last from contacts where
name.first = 'Jane'")
+ checkScan(query2, "struct<name:struct<first:string,last:string>>")
+ checkAnswer(query2, Row("Jane", "Doe") :: Nil)
+
+ val query3 = sql("select name.first from contacts " +
+ "where employer.company.name = 'abc' and p = 1")
+ checkScan(query3, "struct<name:struct<first:string>," +
+ "employer:struct<company:struct<name:string>>>")
+ checkAnswer(query3, Row("Jane") :: Nil)
+
+ val query4 = sql("select name.first, employer.company.name from
contacts " +
+ "where employer.company is not null and p = 1")
+ checkScan(query4, "struct<name:struct<first:string>," +
+ "employer:struct<company:struct<name:string>>>")
+ checkAnswer(query4, Row("Jane", "abc") :: Nil)
+ }
+
+ testSchemaPruning("select nullable complex field and having is null
predicate") {
--- End diff --
Oops, yes, thanks.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
