This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2e20297 [SPARK-33663][SQL] Uncaching should not be called on
non-existing temp views
2e20297 is described below
commit 2e202971183b4f1d54bc49294bb803dd8c30708f
Author: Terry Kim <[email protected]>
AuthorDate: Mon Dec 7 09:48:16 2020 +0900
[SPARK-33663][SQL] Uncaching should not be called on non-existing temp views
### What changes were proposed in this pull request?
This PR proposes to fix a misleading logs in the following scenario when
uncaching is called on non-existing views:
```
scala> sql("CREATE TABLE table USING parquet AS SELECT 2")
res0: org.apache.spark.sql.DataFrame = []
scala> val df = spark.table("table")
df: org.apache.spark.sql.DataFrame = [2: int]
scala> df.createOrReplaceTempView("t2")
20/12/04 10:16:24 WARN CommandUtils: Exception when attempting to uncache
$name
org.apache.spark.sql.AnalysisException: Table or view not found: t2;;
'UnresolvedRelation [t2], [], false
at
org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$1(CheckAnalysis.scala:113)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$1$adapted(CheckAnalysis.scala:93)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:183)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkAnalysis(CheckAnalysis.scala:93)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkAnalysis$(CheckAnalysis.scala:90)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:152)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$executeAndCheck$1(Analyzer.scala:172)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:214)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:169)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$analyzed$1(QueryExecution.scala:73)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:138)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:768)
at
org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:138)
at
org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:73)
at
org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:71)
at
org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:63)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$1(Dataset.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:768)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:88)
at org.apache.spark.sql.DataFrameReader.table(DataFrameReader.scala:889)
at org.apache.spark.sql.SparkSession.table(SparkSession.scala:589)
at
org.apache.spark.sql.internal.CatalogImpl.uncacheTable(CatalogImpl.scala:476)
at
org.apache.spark.sql.execution.command.CommandUtils$.uncacheTableOrView(CommandUtils.scala:392)
at
org.apache.spark.sql.execution.command.CreateViewCommand.run(views.scala:124)
```
Since `t2` does not exist yet, it shouldn't try to uncache.
### Why are the changes needed?
To fix misleading message.
### Does this PR introduce _any_ user-facing change?
Yes, the above message will not be displayed if the view doesn't exist yet.
### How was this patch tested?
Manually tested since this is a log message printed.
Closes #30608 from imback82/fix_cache_message.
Authored-by: Terry Kim <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
(cherry picked from commit 119539fd493af5ed0e37af79320787f145eaf3f1)
Signed-off-by: HyukjinKwon <[email protected]>
---
.../apache/spark/sql/execution/command/views.scala | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 4ad5edd..06b1e03 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -113,12 +113,12 @@ case class CreateViewCommand(
verifyTemporaryObjectsNotExists(catalog, isTemporary, name, child)
if (viewType == LocalTempView) {
- val samePlan = catalog.getTempView(name.table).exists {
- // Don't perform sameResult check for View logical plan, since it's
unresolved
- case _: View => false
- case other => other.sameResult(child)
+ val shouldUncache = replace && catalog.getTempView(name.table).exists {
+ // Uncache View logical plan without checking the same result check,
since it's unresolved.
+ case _: View => true
+ case other => !other.sameResult(child)
}
- if (replace && !samePlan) {
+ if (shouldUncache) {
logInfo(s"Try to uncache ${name.quotedString} before replacing.")
checkCyclicViewReference(analyzedPlan, Seq(name), name)
CommandUtils.uncacheTableOrView(sparkSession, name.quotedString)
@@ -141,12 +141,12 @@ case class CreateViewCommand(
} else if (viewType == GlobalTempView) {
val db =
sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
val viewIdent = TableIdentifier(name.table, Option(db))
- val samePlan = catalog.getGlobalTempView(name.table).exists {
- // Don't perform sameResult check for View logical plan, since it's
unresolved
- case _: View => false
- case other => other.sameResult(child)
+ val shouldUncache = replace &&
catalog.getGlobalTempView(name.table).exists {
+ // Uncache View logical plan without checking the same result check,
since it's unresolved.
+ case _: View => true
+ case other => !other.sameResult(child)
}
- if (replace && !samePlan) {
+ if (shouldUncache) {
logInfo(s"Try to uncache ${viewIdent.quotedString} before replacing.")
checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
CommandUtils.uncacheTableOrView(sparkSession, viewIdent.quotedString)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
