xkrogen commented on a change in pull request #34038:
URL: https://github.com/apache/spark/pull/34038#discussion_r712557058
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
##########
@@ -401,16 +401,30 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog {
|the ${ordinalNumber(ti + 1)} table has
${child.output.length} columns
""".stripMargin.replace("\n", " ").trim())
}
+ val isUnion = operator.isInstanceOf[Union]
// Check if the data types match.
- dataTypes(child).zip(ref).zipWithIndex.foreach { case ((dt1,
dt2), ci) =>
- // SPARK-18058: we shall not care about the nullability of
columns
- if (TypeCoercion.findWiderTypeForTwo(dt1.asNullable,
dt2.asNullable).isEmpty) {
- failAnalysis(
- s"""
- |${operator.nodeName} can only be performed on tables
with the compatible
- |column types. ${dt1.catalogString} <>
${dt2.catalogString} at the
- |${ordinalNumber(ci)} column of the ${ordinalNumber(ti +
1)} table
- """.stripMargin.replace("\n", " ").trim())
+ if (!isUnion) {
Review comment:
How are top-level columns handled for other set operations? In general I
feel Spark SQL is built around column names by default, not positions, so I
would expect it to be by-name. I was surprised to realize recently that `union`
is by-position.
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
##########
@@ -401,16 +401,35 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog {
|the ${ordinalNumber(ti + 1)} table has
${child.output.length} columns
""".stripMargin.replace("\n", " ").trim())
}
+ val isUnion = operator.isInstanceOf[Union]
// Check if the data types match.
- dataTypes(child).zip(ref).zipWithIndex.foreach { case ((dt1,
dt2), ci) =>
- // SPARK-18058: we shall not care about the nullability of
columns
- if (TypeCoercion.findWiderTypeForTwo(dt1.asNullable,
dt2.asNullable).isEmpty) {
- failAnalysis(
- s"""
- |${operator.nodeName} can only be performed on tables
with the compatible
- |column types. ${dt1.catalogString} <>
${dt2.catalogString} at the
- |${ordinalNumber(ci)} column of the ${ordinalNumber(ti +
1)} table
- """.stripMargin.replace("\n", " ").trim())
+ if (!isUnion) {
+ dataTypes(child).zip(ref).zipWithIndex.foreach { case ((dt1,
dt2), ci) =>
+ // SPARK-18058: we shall not care about the nullability of
columns
+ if (TypeCoercion.findWiderTypeForTwo(dt1.asNullable,
dt2.asNullable).isEmpty) {
+ failAnalysis(
+ s"""
+ |${operator.nodeName} can only be performed on tables
with the compatible
+ |column types. ${dt1.catalogString} <>
${dt2.catalogString} at the
+ |${ordinalNumber(ci)} column of the
${ordinalNumber(ti + 1)} table
+ """.stripMargin.replace("\n", " ").trim())
+ }
+ }
+ } else {
+ // `TypeCoercion` takes care of type coercion already. If any
columns or nested
+ // columns are not compatible, we detect it here and throw
analysis exception.
+ val typeChecker = (dt1: DataType, dt2: DataType) => {
+ !TypeCoercion.findWiderTypeForTwo(dt1.asNullable,
dt2.asNullable).isEmpty
+ }
+ dataTypes(child).zip(ref).zipWithIndex.foreach { case ((dt1,
dt2), ci) =>
+ if (!DataType.equalsStructurally(dt1, dt2, true,
typeChecker)) {
+ failAnalysis(
+ s"""
+ |${operator.nodeName} can only be performed on tables
with the compatible
+ |column types. ${dt1.catalogString} <>
${dt2.catalogString} at the
+ |${ordinalNumber(ci)} column of the
${ordinalNumber(ti + 1)} table
+ """.stripMargin.replace("\n", " ").trim())
+ }
Review comment:
Maybe we can simplify like:
```scala
val dataTypesAreCompatibleFn = if (isUnion) {
// `TypeCoercion` takes care of type coercion already. If
any columns or nested
// columns are not compatible, we detect it here and throw
analysis exception.
val typeChecker = (dt1: DataType, dt2: DataType) => {
!TypeCoercion.findWiderTypeForTwo(dt1.asNullable,
dt2.asNullable).isEmpty
}
(dt1: DataType, dt2: DataType) =>
!DataType.equalsStructurally(dt1, dt2, true, typeChecker)
} else {
// SPARK-18058: we shall not care about the nullability of
columns
(dt1: DataType, dt2: DataType) =>
TypeCoercion.findWiderTypeForTwo(dt1.asNullable, dt2.asNullable).isEmpty
}
dataTypes(child).zip(ref).zipWithIndex.foreach { case ((dt1,
dt2), ci) =>
if (dataTypesAreCompatibleFn(dt1, dt2)) {
failAnalysis(
s"""
|${operator.nodeName} can only be performed on
tables with the compatible
|column types. ${dt1.catalogString} <>
${dt2.catalogString} at the
|${ordinalNumber(ci)} column of the
${ordinalNumber(ti + 1)} table
""".stripMargin.replace("\n", " ").trim())
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
