gengliangwang commented on code in PR #52764:
URL: https://github.com/apache/spark/pull/52764#discussion_r2543876147
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2TableRefreshUtil.scala:
##########
@@ -59,19 +61,35 @@ private[sql] object V2TableRefreshUtil extends
SQLConfHelper with Logging {
* Tables with time travel specifications are skipped as they reference a
specific point
* in time and don't have to be refreshed.
*
+ * @param spark the currently active Spark session
* @param plan the logical plan to refresh
* @param versionedOnly indicates whether to refresh only versioned tables
* @return plan with refreshed table metadata
*/
- def refresh(plan: LogicalPlan, versionedOnly: Boolean = false): LogicalPlan
= {
+ def refresh(
+ spark: SparkSession,
+ plan: LogicalPlan,
+ versionedOnly: Boolean = false): LogicalPlan = {
+ val sharedRelationCache = spark.sharedState.relationCache
val cache = mutable.HashMap.empty[(TableCatalog, Identifier), Table]
plan transform {
case r @ ExtractV2CatalogAndIdentifier(catalog, ident)
if (r.isVersioned || !versionedOnly) && r.timeTravelSpec.isEmpty =>
val currentTable = cache.getOrElseUpdate((catalog, ident), {
+ val nameParts = ident.toQualifiedNameParts(catalog)
val tableName = V2TableUtil.toQualifiedName(catalog, ident)
- logDebug(s"Refreshing table metadata for $tableName")
- catalog.loadTable(ident)
+ val cachedTable = sharedRelationCache.lookup(nameParts,
conf.resolver).collect {
+ case cached: DataSourceV2Relation
Review Comment:
There is similar code in
https://github.com/apache/spark/pull/52764/files#diff-cbcce0570cab073f758b082141fbc4efa3cdfc1da7395c63609c5f2fa5ad902eR138.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
