viirya commented on a change in pull request #31300:
URL: https://github.com/apache/spark/pull/31300#discussion_r563037773
##########
File path: sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
##########
@@ -928,33 +928,61 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
}
}
- test("SPARK-24596 Non-cascading Cache Invalidation - drop temporary view") {
- withTempView("t1", "t2") {
- sql("CACHE TABLE t1 AS SELECT * FROM testData WHERE key > 1")
- sql("CACHE TABLE t2 as SELECT * FROM t1 WHERE value > 1")
+ test("SPARK-24596, SPARK-34052: cascading cache invalidation - drop
temporary view") {
+ Seq(true, false).foreach { storeAnalyzed =>
+ withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key ->
storeAnalyzed.toString) {
+ withTempView("t1", "t2") {
+ sql("CACHE TABLE t1 AS SELECT * FROM testData WHERE key > 1")
+ sql("CACHE TABLE t2 as SELECT * FROM t1 WHERE value > 1")
- assert(spark.catalog.isCached("t1"))
- assert(spark.catalog.isCached("t2"))
- sql("DROP VIEW t1")
- assert(spark.catalog.isCached("t2"))
+ assert(spark.catalog.isCached("t1"))
+ assert(spark.catalog.isCached("t2"))
+
+ val oldView = spark.table("t2")
+ sql("DROP VIEW t1")
+
+ // dropping a temp view trigger cache invalidation on dependents iff
the config is
+ // turned off
+ assert(storeAnalyzed ==
+ spark.sharedState.cacheManager.lookupCachedData(oldView).isDefined)
+ if (!storeAnalyzed) {
+ // t2 should become invalid after t1 is dropped
+ val e = intercept[AnalysisException](spark.catalog.isCached("t2"))
+ assert(e.message.contains(s"Table or view not found"))
+ }
+ }
+ }
}
}
- test("SPARK-24596 Non-cascading Cache Invalidation - drop persistent view") {
- withTable("t") {
- spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
- .write.format("json").saveAsTable("t")
- withView("t1") {
- withTempView("t2") {
- sql("CREATE VIEW t1 AS SELECT * FROM t WHERE key > 1")
+ test("SPARK-24596, SPARK-34052: cascading cache invalidation - drop
persistent view") {
+ Seq(true, false).foreach { storeAnalyzed =>
+ withSQLConf(SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW.key ->
storeAnalyzed.toString) {
+ withTable("t") {
+ spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
+ .write.format("json").saveAsTable("t")
+ withView("t1") {
+ withTempView("t2") {
+ sql("CREATE VIEW t1 AS SELECT * FROM t WHERE key > 1")
- sql("CACHE TABLE t1")
- sql("CACHE TABLE t2 AS SELECT * FROM t1 WHERE value > 1")
+ sql("CACHE TABLE t1")
+ sql("CACHE TABLE t2 AS SELECT * FROM t1 WHERE value > 1")
- assert(spark.catalog.isCached("t1"))
- assert(spark.catalog.isCached("t2"))
- sql("DROP VIEW t1")
- assert(!spark.catalog.isCached("t2"))
+ assert(spark.catalog.isCached("t1"))
+ assert(spark.catalog.isCached("t2"))
+
+ val oldView = spark.table("t2")
+ sql("DROP VIEW t1")
+
+ // dropping a permanent view always trigger cache invalidation
on dependents
+
assert(spark.sharedState.cacheManager.lookupCachedData(oldView).isEmpty)
+ if (!storeAnalyzed) {
+ // t2 should become invalid after t1 is dropped
+ val e =
intercept[AnalysisException](spark.catalog.isCached("t2"))
+ assert(e.message.contains(s"Table or view not found"))
+ }
Review comment:
Hm? Shouldn't we invalid t2 in any of cases if its source table t1 is
dropped?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
