This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b7ac624d56c2 [SPARK-50174][SQL] Factor out `UnresolvedCatalogRelation`
resolution
b7ac624d56c2 is described below
commit b7ac624d56c283792beda8f630d415663999e28a
Author: Vladimir Golubev <[email protected]>
AuthorDate: Wed Oct 30 14:43:07 2024 +0100
[SPARK-50174][SQL] Factor out `UnresolvedCatalogRelation` resolution
### What changes were proposed in this pull request?
Factor out `UnresolvedCatalogRelation` resolution into a separate
`FindDataSourceTable` method to reuse it in the single-pass Analyzer.
### Why are the changes needed?
Context: https://issues.apache.org/jira/browse/SPARK-49834
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
copilot.vim.
Closes #48707 from vladimirg-db/vladimirg-db/master.
Authored-by: Vladimir Golubev <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../execution/datasources/DataSourceStrategy.scala | 26 ++++++++++++++--------
1 file changed, 17 insertions(+), 9 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index 4ea4ed289d41..cacb1ef928e8 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -298,15 +298,8 @@ class FindDataSourceTable(sparkSession: SparkSession)
extends Rule[LogicalPlan]
table.partitionColumnNames.map(name => name -> None).toMap,
Seq.empty, append.query, false, append.isByName)
- case UnresolvedCatalogRelation(tableMeta, options, false)
- if DDLUtils.isDatasourceTable(tableMeta) =>
- readDataSourceTable(tableMeta, options)
-
- case UnresolvedCatalogRelation(tableMeta, _, false) =>
- DDLUtils.readHiveTable(tableMeta)
-
- case UnresolvedCatalogRelation(tableMeta, extraOptions, true) =>
- getStreamingRelation(tableMeta, extraOptions)
+ case unresolvedCatalogRelation: UnresolvedCatalogRelation =>
+ resolveUnresolvedCatalogRelation(unresolvedCatalogRelation)
case s @ StreamingRelationV2(
_, _, table, extraOptions, _, _, _,
Some(UnresolvedCatalogRelation(tableMeta, _, true))) =>
@@ -320,6 +313,21 @@ class FindDataSourceTable(sparkSession: SparkSession)
extends Rule[LogicalPlan]
v1Relation
}
}
+
+ def resolveUnresolvedCatalogRelation(
+ unresolvedCatalogRelation: UnresolvedCatalogRelation): LogicalPlan = {
+ unresolvedCatalogRelation match {
+ case UnresolvedCatalogRelation(tableMeta, options, false)
+ if DDLUtils.isDatasourceTable(tableMeta) =>
+ readDataSourceTable(tableMeta, options)
+
+ case UnresolvedCatalogRelation(tableMeta, _, false) =>
+ DDLUtils.readHiveTable(tableMeta)
+
+ case UnresolvedCatalogRelation(tableMeta, extraOptions, true) =>
+ getStreamingRelation(tableMeta, extraOptions)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
