Github user wangyum commented on a diff in the pull request:
https://github.com/apache/spark/pull/19231#discussion_r138946335
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
---
@@ -775,24 +775,23 @@ object JdbcUtils extends Logging {
tableSchema: StructType,
customSchema: String,
nameEquality: Resolver): StructType = {
- val userSchema = CatalystSqlParser.parseTableSchema(customSchema)
+ if (null != customSchema && customSchema.nonEmpty) {
+ val userSchema = CatalystSqlParser.parseTableSchema(customSchema)
- SchemaUtils.checkColumnNameDuplication(
- userSchema.map(_.name), "in the customSchema option value",
nameEquality)
-
- val colNames = tableSchema.fieldNames.mkString(",")
- val errorMsg = s"Please provide all the columns, all columns are:
$colNames"
- if (userSchema.size != tableSchema.size) {
- throw new AnalysisException(errorMsg)
- }
+ SchemaUtils.checkColumnNameDuplication(
+ userSchema.map(_.name), "in the customSchema option value",
nameEquality)
- // This is resolved by names, only check the column names.
- userSchema.fieldNames.foreach { col =>
- tableSchema.find(f => nameEquality(f.name, col)).getOrElse {
- throw new AnalysisException(errorMsg)
+ // This is resolved by names, use the custom filed dataType to
replace the default dateType.
+ val newSchema = tableSchema.map { col =>
+ userSchema.find(f => nameEquality(f.name, col.name)) match {
+ case Some(c) => col.copy(dataType = c.dataType, metadata =
Metadata.empty)
--- End diff --
Reset metadata to empty, otherwise it is not equal to the schema generated
by `CatalystSqlParser.parseTableSchema`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
