HyukjinKwon commented on a change in pull request #33922:
URL: https://github.com/apache/spark/pull/33922#discussion_r703114054
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Utils.scala
##########
@@ -83,4 +90,54 @@ private[sql] object DataSourceV2Utils extends Logging {
options.asCaseSensitiveMap())
}
}
+
+ def loadV2Source(sparkSession: SparkSession, provider: TableProvider,
+ userSpecifiedSchema: Option[StructType], extraOptions:
CaseInsensitiveMap[String],
+ source: String, paths: String*): Option[DataFrame] = {
+ val catalogManager = sparkSession.sessionState.catalogManager
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ source = provider, conf = sparkSession.sessionState.conf)
+
+ val optionsWithPath = getOptionsWithPaths(extraOptions, paths: _*)
+
+ val finalOptions =
sessionOptions.filterKeys(!optionsWithPath.contains(_)).toMap ++
+ optionsWithPath.originalMap
+ val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
+ val (table, catalog, ident) = provider match {
+ case _: SupportsCatalogOptions if userSpecifiedSchema.nonEmpty =>
+ throw new IllegalArgumentException(
+ s"$source does not support user specified schema. Please don't
specify the schema.")
+ case hasCatalog: SupportsCatalogOptions =>
+ val ident = hasCatalog.extractIdentifier(dsOptions)
+ val catalog = CatalogV2Util.getTableProviderCatalog(
+ hasCatalog,
+ catalogManager,
+ dsOptions)
+ (catalog.loadTable(ident), Some(catalog), Some(ident))
+ case _ =>
+ // TODO: Non-catalog paths for DSV2 are currently not well defined.
Review comment:
I know this comment was already existent before but wanted to make a
note. This isn't a good example of a comment. There's no JIRA. and we don't
know what's not well defined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
