This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d8336eb [SPARK-34076][SQL] SQLContext.dropTempTable fails if cache is
non-empty
d8336eb is described below
commit d8336ebf5e2ba507f76b06080c05859d3ca828f8
Author: Chao Sun <sunc...@apple.com>
AuthorDate: Wed Jan 13 13:22:21 2021 +0000
[SPARK-34076][SQL] SQLContext.dropTempTable fails if cache is non-empty
### What changes were proposed in this pull request?
This changes `CatalogImpl.dropTempView` and
`CatalogImpl.dropGlobalTempView` use analyzed logical plan instead of `viewDef`
which is unresolved.
### Why are the changes needed?
Currently, `CatalogImpl.dropTempView` is implemented as following:
```scala
override def dropTempView(viewName: String): Boolean = {
sparkSession.sessionState.catalog.getTempView(viewName).exists { viewDef
=>
sparkSession.sharedState.cacheManager.uncacheQuery(
sparkSession, viewDef, cascade = false)
sessionCatalog.dropTempView(viewName)
}
}
```
Here, the logical plan `viewDef` is not resolved, and when passing to
`uncacheQuery`, it could fail at `sameResult` call, where canonicalized plan is
compared. The error message looks like:
```
Invalid call to qualifier on unresolved object, tree: 'key
```
This can be reproduced via:
```scala
sql(s"CREATE TEMPORARY VIEW $v AS SELECT key FROM src LIMIT 10")
sql(s"CREATE TABLE $t AS SELECT * FROM src")
sql(s"CACHE TABLE $t")
dropTempTable(v)
```
### Does this PR introduce _any_ user-facing change?
The only user-facing change is that, previously `SQLContext.dropTempTable`
may fail in the above scenario but will work with this fix.
### How was this patch tested?
Added new unit tests.
Closes #31136 from sunchao/SPARK-34076.
Authored-by: Chao Sun <sunc...@apple.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
(cherry picked from commit 62d82b5b270c2eea27ba026e66b7d6598bbaa0a6)
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../apache/spark/sql/internal/CatalogImpl.scala | 18 ++++++++++++----
.../apache/spark/sql/hive/CachedTableSuite.scala | 24 ++++++++++++++++++++++
2 files changed, 38 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index d817fed..125a597 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -396,8 +396,13 @@ class CatalogImpl(sparkSession: SparkSession) extends
Catalog {
*/
override def dropTempView(viewName: String): Boolean = {
sparkSession.sessionState.catalog.getTempView(viewName).exists { viewDef =>
- sparkSession.sharedState.cacheManager.uncacheQuery(
- sparkSession, viewDef, cascade = false)
+ try {
+ val plan = sparkSession.sessionState.executePlan(viewDef)
+ sparkSession.sharedState.cacheManager.uncacheQuery(
+ sparkSession, plan.analyzed, cascade = false)
+ } catch {
+ case NonFatal(_) => // ignore
+ }
sessionCatalog.dropTempView(viewName)
}
}
@@ -412,8 +417,13 @@ class CatalogImpl(sparkSession: SparkSession) extends
Catalog {
*/
override def dropGlobalTempView(viewName: String): Boolean = {
sparkSession.sessionState.catalog.getGlobalTempView(viewName).exists {
viewDef =>
- sparkSession.sharedState.cacheManager.uncacheQuery(
- sparkSession, viewDef, cascade = false)
+ try {
+ val plan = sparkSession.sessionState.executePlan(viewDef)
+ sparkSession.sharedState.cacheManager.uncacheQuery(
+ sparkSession, plan.analyzed, cascade = false)
+ } catch {
+ case NonFatal(_) => // ignore
+ }
sessionCatalog.dropGlobalTempView(viewName)
}
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index b221a72..f6e5704 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -527,4 +527,28 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
with TestHiveSingleto
checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
}
}
+
+ test("SPARK-34076: should be able to drop temp view with cached tables") {
+ val t = "cachedTable"
+ val v = "tempView"
+ withTable(t) {
+ withTempView(v) {
+ sql(s"CREATE TEMPORARY VIEW $v AS SELECT key FROM src LIMIT 10")
+ sql(s"CREATE TABLE $t AS SELECT * FROM src")
+ sql(s"CACHE TABLE $t")
+ }
+ }
+ }
+
+ test("SPARK-34076: should be able to drop global temp view with cached
tables") {
+ val t = "cachedTable"
+ val v = "globalTempView"
+ withTable(t) {
+ withGlobalTempView(v) {
+ sql(s"CREATE GLOBAL TEMPORARY VIEW $v AS SELECT key FROM src LIMIT 10")
+ sql(s"CREATE TABLE $t AS SELECT * FROM src")
+ sql(s"CACHE TABLE $t")
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
