tedyu commented on code in PR #49029: URL: https://github.com/apache/spark/pull/49029#discussion_r1890543927
########## sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/MetadataResolver.scala: ########## @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis.resolver + +import java.util.ArrayDeque + +import org.apache.spark.sql.catalyst.analysis.{withPosition, RelationResolution, UnresolvedRelation} +import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression} +import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan} +import org.apache.spark.sql.connector.catalog.CatalogManager + +/** + * The [[MetadataResolver]] performs relation metadata resolution based on the unresolved plan + * at the start of the analysis phase. Usually it does RPC calls to some table catalog and to table + * metadata itself. + * + * [[RelationsWithResolvedMetadata]] is a map from relation ID to the relations with resolved + * metadata. It's produced by [[resolve]] and is used later in [[Resolver]] to replace + * [[UnresolvedRelation]]s. + * + * This object is one-shot per SQL query or DataFrame program resolution. + */ +class MetadataResolver( + override val catalogManager: CatalogManager, + override val relationResolution: RelationResolution, + override val extensions: Seq[ResolverExtension] = Seq.empty) + extends RelationMetadataProvider + with DelegatesResolutionToExtensions { + override val relationsWithResolvedMetadata = new RelationsWithResolvedMetadata + + /** + * Resolves the relation metadata for `unresolvedPlan`. Usually this involves several blocking + * calls for the [[UnresolvedRelation]]s present in that tree. During the `unresolvedPlan` + * traversal we fill [[relationsWithResolvedMetadata]] with resolved metadata by relation id. + * This map will be used to resolve the plan in single-pass by the [[Resolver]] using + * [[getRelationWithResolvedMetadata]]. If the generic metadata resolution using + * [[RelationResolution]] wasn't successful, we resort to using [[extensions]]. + * Otherwise, we fail with an exception. + */ + def resolve(unresolvedPlan: LogicalPlan): Unit = { + traverseLogicalPlanTree(unresolvedPlan) { unresolvedOperator => + unresolvedOperator match { + case unresolvedRelation: UnresolvedRelation => + val relationId = relationIdFromUnresolvedRelation(unresolvedRelation) + + if (!relationsWithResolvedMetadata.containsKey(relationId)) { + val relationWithResolvedMetadata = resolveRelation(unresolvedRelation).orElse { + // In case the generic metadata resolution returned `None`, we try to check if any + // of the [[extensions]] matches this `unresolvedRelation`, and resolve it using + // that extension. + tryDelegateResolutionToExtension(unresolvedRelation) + } + + relationWithResolvedMetadata match { + case Some(relationWithResolvedMetadata) => Review Comment: It is better to use different name for the variable, such as `relationCarryingResolvedMetadata`, so that the code is easier to read. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
