Github user bersprockets commented on a diff in the pull request:
https://github.com/apache/spark/pull/21950#discussion_r217216975
--- Diff:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
---
@@ -91,4 +91,28 @@ class PruneFileSourcePartitionsSuite extends QueryTest
with SQLTestUtils with Te
assert(size2 < tableStats.get.sizeInBytes)
}
}
+
+ test("Test deserialization factor against partition") {
+ val factor = 10
+ withTable("tbl") {
+ spark.range(10).selectExpr("id", "id % 3 as
p").write.format("parquet")
+ .partitionBy("p").saveAsTable("tbl")
+ sql(s"ANALYZE TABLE tbl COMPUTE STATISTICS")
+
+ val df1 = sql("SELECT * FROM tbl WHERE p = 1")
+ val sizes1 = df1.queryExecution.optimizedPlan.collect {
+ case relation: LogicalRelation =>
relation.catalogTable.get.stats.get.sizeInBytes
+ }
+ assert(sizes1 != 0)
--- End diff --
Oops. Should be <code>assert(sizes1(0) != 0)</code>. I will fix.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
