turboFei commented on a change in pull request #31431:
URL: https://github.com/apache/spark/pull/31431#discussion_r568502937
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
##########
@@ -1374,7 +1375,55 @@ case class RefreshTableCommand(tableIdent:
TableIdentifier)
override def run(sparkSession: SparkSession): Seq[Row] = {
// Refresh the given table's metadata. If this table is cached as an
InMemoryRelation,
// drop the original cached version and make the new version cached lazily.
- sparkSession.catalog.refreshTable(tableIdent.quotedString)
+ // If this table is a view, also refresh its underlying tables.
+ refreshAllUnderlyingTables(tableIdent, sparkSession)
Seq.empty[Row]
}
+
+ private def refreshAllUnderlyingTables(
+ tableIdent: TableIdentifier, sparkSession: SparkSession): Unit = {
+ val tableName = tableIdent.unquotedString
+ val catalog = sparkSession.sessionState.catalog
+ val table = catalog.getTempViewOrPermanentTableMetadata(tableIdent)
+ sparkSession.catalog.refreshTable(tableName)
+
+ if (table.tableType == CatalogTableType.VIEW) {
+ if (table.viewText.isDefined) {
+ val parser = sparkSession.sessionState.sqlParser
+ val unresolvedPlan = parser.parsePlan(table.viewText.get)
+ findOutUnderlyingTables(unresolvedPlan).foreach { t =>
+ refreshAllUnderlyingTables(t, sparkSession)
+ }
+ } else {
+ catalog.getTempView(tableName)
+ .orElse(catalog.getGlobalTempView(tableName))
+ .map(findOutUnderlyingTables).foreach { tables =>
Review comment:
In branch-2.3, the logical plan for temporary view is analyzed logical
plan.
In master branch, it is unresolved logical plan.
So I also enable refresh underlying tables for temporary view.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
