dongjoon-hyun commented on a change in pull request #33975:
URL: https://github.com/apache/spark/pull/33975#discussion_r709790654
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
##########
@@ -468,30 +468,68 @@ abstract class DynamicPartitionPruningSuiteBase
Given("no stats and selective predicate with the size of dim too large")
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
- sql(
- """
- |SELECT f.date_id, f.product_id, f.units_sold, f.store_id
- |FROM fact_sk f WHERE store_id < 5
+ withTable("fact_aux") {
+ sql(
+ """
+ |SELECT f.date_id, f.product_id, f.units_sold, f.store_id
+ |FROM fact_sk f WHERE store_id < 5
+ """.stripMargin)
+ .write
+ .partitionBy("store_id")
+ .saveAsTable("fact_aux")
+
+ val df = sql(
+ """
+ |SELECT f.date_id, f.product_id, f.units_sold, f.store_id
+ |FROM fact_aux f JOIN dim_store s
+ |ON f.store_id = s.store_id WHERE s.country = 'US'
""".stripMargin)
- .write
- .partitionBy("store_id")
- .saveAsTable("fact_aux")
- val df = sql(
- """
- |SELECT f.date_id, f.product_id, f.units_sold, f.store_id
- |FROM fact_aux f JOIN dim_store s
- |ON f.store_id = s.store_id WHERE s.country = 'US'
+ checkPartitionPruningPredicate(df, false, false)
+
+ checkAnswer(df,
+ Row(1070, 2, 10, 4) ::
+ Row(1080, 3, 20, 4) ::
+ Row(1090, 3, 10, 4) ::
+ Row(1100, 3, 10, 4) :: Nil
+ )
+ }
+ }
+
+ Given("no stats and selective predicate with the size of dim too large
but cached")
+ withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+ SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
+ withTable("fact_aux") {
+ withTempView("cached_dim_store") {
+ sql(
+ """
+ |SELECT f.date_id, f.product_id, f.units_sold, f.store_id
+ |FROM fact_sk f WHERE store_id < 5
""".stripMargin)
Review comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
