xuanyuanking commented on a change in pull request #29712:
URL: https://github.com/apache/spark/pull/29712#discussion_r487239172
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
##########
@@ -190,4 +194,33 @@ object DataSourceUtils {
case LegacyBehaviorPolicy.LEGACY =>
RebaseDateTime.rebaseGregorianToJulianMicros
case LegacyBehaviorPolicy.CORRECTED => identity[Long]
}
+
+ def generateDatasourceOptions(
+ extraOptionsMap: CaseInsensitiveStringMap, table: CatalogTable):
Map[String, String] = {
+ val pathOption = table.storage.locationUri.map("path" ->
CatalogUtils.URIToString(_))
+ val options = table.storage.properties ++ pathOption
+ if (!SQLConf.get.getConf(SQLConf.LEGACY_EXTRA_OPTIONS_BEHAVIOR)) {
+ // Check the same key with different values
+ table.storage.properties.foreach { case (k, v) =>
+ if (extraOptionsMap.containsKey(k) && extraOptionsMap.get(k) != v) {
+ throw new AnalysisException(
+ s"Fail to create datasource for the table
${table.identifier.table} since the table " +
+ s"property has the duplicated key $k with input options. To fix
this, you can " +
+ "rollback to the legacy behavior of ignoring the input options
by setting the " +
Review comment:
Rephased in f3f8c8d.
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
##########
@@ -190,4 +194,33 @@ object DataSourceUtils {
case LegacyBehaviorPolicy.LEGACY =>
RebaseDateTime.rebaseGregorianToJulianMicros
case LegacyBehaviorPolicy.CORRECTED => identity[Long]
}
+
+ def generateDatasourceOptions(
+ extraOptionsMap: CaseInsensitiveStringMap, table: CatalogTable):
Map[String, String] = {
+ val pathOption = table.storage.locationUri.map("path" ->
CatalogUtils.URIToString(_))
+ val options = table.storage.properties ++ pathOption
+ if (!SQLConf.get.getConf(SQLConf.LEGACY_EXTRA_OPTIONS_BEHAVIOR)) {
+ // Check the same key with different values
+ table.storage.properties.foreach { case (k, v) =>
+ if (extraOptionsMap.containsKey(k) && extraOptionsMap.get(k) != v) {
+ throw new AnalysisException(
+ s"Fail to create datasource for the table
${table.identifier.table} since the table " +
+ s"property has the duplicated key $k with input options. To fix
this, you can " +
+ "rollback to the legacy behavior of ignoring the input options
by setting the " +
+ s"config ${SQLConf.LEGACY_EXTRA_OPTIONS_BEHAVIOR.key} to
`false`, or address the " +
+ s"conflicts of the same config.")
+ }
+ }
+ // To keep the original key from table properties, here we filter all
case insensitive
+ // duplicate keys out from extra options.
+ val caseInsensitiveDuplicateKeys =
Review comment:
Thanks, done in f3f8c8d.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
