Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/21713#discussion_r200275396
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
---
@@ -166,6 +166,30 @@ object TypeCoercion {
case (l, r) => None
}
+ private def mergeComplexTypes(
+ t1: DataType,
+ t2: DataType,
+ mergeFunc: (DataType, DataType) => Option[DataType]):
Option[DataType] = (t1, t2) match {
+ case (ArrayType(et1, containsNull1), ArrayType(et2, containsNull2)) =>
+ mergeFunc(et1, et2).map(ArrayType(_, containsNull1 || containsNull2))
+ case (MapType(kt1, vt1, valueContainsNull1), MapType(kt2, vt2,
valueContainsNull2)) =>
+ mergeFunc(kt1, kt2).flatMap { kt =>
+ mergeFunc(vt1, vt2).map { vt =>
+ MapType(kt, vt, valueContainsNull1 || valueContainsNull2)
+ }
+ }
+ case (StructType(fields1), StructType(fields2)) if fields1.length ==
fields2.length =>
+ val resolver = SQLConf.get.resolver
+ fields1.zip(fields2).foldLeft(Option(new StructType())) {
+ case (Some(struct), (field1, field2)) if resolver(field1.name,
field2.name) =>
--- End diff --
In case the names between two fields are different,
`findTightestCommonType` uses the name of first field. This seems stricter?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
