nikolamand-db commented on code in PR #46734:
URL: https://github.com/apache/spark/pull/46734#discussion_r1615599084
##########
sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala:
##########
@@ -408,6 +408,37 @@ object DataType {
}
}
+ /**
+ * Check if `from` is equal to `to` type except for collations and
nullability, which are
+ * both checked to be compatible so that data of type `from` can be
interpreted as of type `to`.
+ */
+ private[sql] def equalsIgnoreCompatibleCollationAndNullability(
+ from: DataType,
+ to: DataType): Boolean = {
+ (from, to) match {
+ case (_: StringType, _: StringType) => true
+
+ case (ArrayType(fromElement, fn), ArrayType(toElement, tn)) =>
+ (tn || !fn) &&
equalsIgnoreCompatibleCollationAndNullability(fromElement, toElement)
+
+ case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
+ (tn || !fn) &&
+ // Map keys cannot change collation.
+ equalsIgnoreCompatibleNullability(fromKey, toKey) &&
+ equalsIgnoreCompatibleCollationAndNullability(fromValue, toValue)
+
+ case (StructType(fromFields), StructType(toFields)) =>
+ fromFields.length == toFields.length &&
+ fromFields.zip(toFields).forall { case (fromField, toField) =>
Review Comment:
Please check
https://github.com/apache/spark/pull/46734#discussion_r1615598800.
##########
sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala:
##########
@@ -408,6 +408,37 @@ object DataType {
}
}
+ /**
+ * Check if `from` is equal to `to` type except for collations and
nullability, which are
+ * both checked to be compatible so that data of type `from` can be
interpreted as of type `to`.
+ */
+ private[sql] def equalsIgnoreCompatibleCollationAndNullability(
+ from: DataType,
+ to: DataType): Boolean = {
+ (from, to) match {
+ case (_: StringType, _: StringType) => true
+
+ case (ArrayType(fromElement, fn), ArrayType(toElement, tn)) =>
+ (tn || !fn) &&
equalsIgnoreCompatibleCollationAndNullability(fromElement, toElement)
+
+ case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
+ (tn || !fn) &&
+ // Map keys cannot change collation.
+ equalsIgnoreCompatibleNullability(fromKey, toKey) &&
+ equalsIgnoreCompatibleCollationAndNullability(fromValue, toValue)
+
+ case (StructType(fromFields), StructType(toFields)) =>
+ fromFields.length == toFields.length &&
Review Comment:
Please check
https://github.com/apache/spark/pull/46734#discussion_r1615598800.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
