maropu commented on a change in pull request #30260:
URL: https://github.com/apache/spark/pull/30260#discussion_r520241004
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
##########
@@ -1770,6 +1782,64 @@ case class AnsiCast(child: Expression, dataType:
DataType, timeZoneId: Option[St
copy(timeZoneId = Option(timeZoneId))
override protected val ansiEnabled: Boolean = true
+
+ override def canCast(from: DataType, to: DataType): Boolean =
AnsiCast.canCast(from, to)
+}
+
+object AnsiCast {
+ def canCast(from: DataType, to: DataType): Boolean = (from, to) match {
+ case (fromType, toType) if fromType == toType => true
+
+ case (NullType, _) => true
+
+ case (_: NumericType, _: NumericType) => true
+ case (StringType, _: NumericType) => true
+ case (BooleanType, _: NumericType) => true
Review comment:
(Just a suggestion) For readability, could you reorder these entries
according to `Cast.canCast` where possible? For example, the numeric entries
for `Cast` seems to be placed just before complex types:
https://github.com/apache/spark/blob/35ac314181129374b02f8f8c07341b43a734e1c7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala#L70-L74
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
##########
@@ -1396,14 +1337,125 @@ class CastSuite extends CastSuiteBase {
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
}
+
+ test("SPARK-22825 Cast array to string") {
Review comment:
(nit and this is not related to this PR though...) `SPARK-22825` ->
`SPARK-22825:`
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
##########
@@ -756,6 +756,47 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
}
}
+ test("SPARK-33354: Throw exceptions on inserting invalid cast with ANSI
casting policy") {
+ withSQLConf(
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.ANSI.toString) {
+ withTable("t") {
+ sql("create table t(i int, t timestamp) using parquet")
Review comment:
nit: please use uppercases for SQL keywords, e.g., `create` -> `CREATE`.
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
##########
@@ -1396,14 +1337,125 @@ class CastSuite extends CastSuiteBase {
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
}
+
+ test("SPARK-22825 Cast array to string") {
+ val ret1 = cast(Literal.create(Array(1, 2, 3, 4, 5)), StringType)
+ checkEvaluation(ret1, "[1, 2, 3, 4, 5]")
+ val ret2 = cast(Literal.create(Array("ab", "cde", "f")), StringType)
+ checkEvaluation(ret2, "[ab, cde, f]")
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret3 = cast(Literal.create(Array("ab", null, "c")), StringType)
+ checkEvaluation(ret3, s"[ab,${if (omitNull) "" else " null"}, c]")
+ }
+ }
+ val ret4 =
+ cast(Literal.create(Array("ab".getBytes, "cde".getBytes, "f".getBytes)),
StringType)
+ checkEvaluation(ret4, "[ab, cde, f]")
+ val ret5 = cast(
+ Literal.create(Array("2014-12-03", "2014-12-04",
"2014-12-06").map(Date.valueOf)),
+ StringType)
+ checkEvaluation(ret5, "[2014-12-03, 2014-12-04, 2014-12-06]")
+ val ret6 = cast(
+ Literal.create(Array("2014-12-03 13:01:00", "2014-12-04 15:05:00")
+ .map(Timestamp.valueOf)),
+ StringType)
+ checkEvaluation(ret6, "[2014-12-03 13:01:00, 2014-12-04 15:05:00]")
+ val ret7 = cast(Literal.create(Array(Array(1, 2, 3), Array(4, 5))),
StringType)
+ checkEvaluation(ret7, "[[1, 2, 3], [4, 5]]")
+ val ret8 = cast(
+ Literal.create(Array(Array(Array("a"), Array("b", "c")),
Array(Array("d")))),
+ StringType)
+ checkEvaluation(ret8, "[[[a], [b, c]], [[d]]]")
+ }
+
+ test("SPARK-33291: Cast array with null elements to string") {
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret1 = cast(Literal.create(Array(null, null)), StringType)
+ checkEvaluation(
+ ret1,
+ s"[${if (omitNull) "" else "null"},${if (omitNull) "" else "
null"}]")
+ }
+ }
+ }
+
+ test("SPARK-22973 Cast map to string") {
Review comment:
ditto
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
##########
@@ -1396,14 +1337,125 @@ class CastSuite extends CastSuiteBase {
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
}
+
+ test("SPARK-22825 Cast array to string") {
+ val ret1 = cast(Literal.create(Array(1, 2, 3, 4, 5)), StringType)
+ checkEvaluation(ret1, "[1, 2, 3, 4, 5]")
+ val ret2 = cast(Literal.create(Array("ab", "cde", "f")), StringType)
+ checkEvaluation(ret2, "[ab, cde, f]")
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret3 = cast(Literal.create(Array("ab", null, "c")), StringType)
+ checkEvaluation(ret3, s"[ab,${if (omitNull) "" else " null"}, c]")
+ }
+ }
+ val ret4 =
+ cast(Literal.create(Array("ab".getBytes, "cde".getBytes, "f".getBytes)),
StringType)
+ checkEvaluation(ret4, "[ab, cde, f]")
+ val ret5 = cast(
+ Literal.create(Array("2014-12-03", "2014-12-04",
"2014-12-06").map(Date.valueOf)),
+ StringType)
+ checkEvaluation(ret5, "[2014-12-03, 2014-12-04, 2014-12-06]")
+ val ret6 = cast(
+ Literal.create(Array("2014-12-03 13:01:00", "2014-12-04 15:05:00")
+ .map(Timestamp.valueOf)),
+ StringType)
+ checkEvaluation(ret6, "[2014-12-03 13:01:00, 2014-12-04 15:05:00]")
+ val ret7 = cast(Literal.create(Array(Array(1, 2, 3), Array(4, 5))),
StringType)
+ checkEvaluation(ret7, "[[1, 2, 3], [4, 5]]")
+ val ret8 = cast(
+ Literal.create(Array(Array(Array("a"), Array("b", "c")),
Array(Array("d")))),
+ StringType)
+ checkEvaluation(ret8, "[[[a], [b, c]], [[d]]]")
+ }
+
+ test("SPARK-33291: Cast array with null elements to string") {
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret1 = cast(Literal.create(Array(null, null)), StringType)
+ checkEvaluation(
+ ret1,
+ s"[${if (omitNull) "" else "null"},${if (omitNull) "" else "
null"}]")
+ }
+ }
+ }
+
+ test("SPARK-22973 Cast map to string") {
+ Seq(
+ false -> ("{", "}"),
+ true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
legacyCast.toString) {
+ val ret1 = cast(Literal.create(Map(1 -> "a", 2 -> "b", 3 -> "c")),
StringType)
+ checkEvaluation(ret1, s"${lb}1 -> a, 2 -> b, 3 -> c$rb")
+ val ret2 = cast(
+ Literal.create(Map("1" -> "a".getBytes, "2" -> null, "3" ->
"c".getBytes)),
+ StringType)
+ checkEvaluation(ret2, s"${lb}1 -> a, 2 ->${if (legacyCast) "" else "
null"}, 3 -> c$rb")
+ val ret3 = cast(
+ Literal.create(Map(
+ 1 -> Date.valueOf("2014-12-03"),
+ 2 -> Date.valueOf("2014-12-04"),
+ 3 -> Date.valueOf("2014-12-05"))),
+ StringType)
+ checkEvaluation(ret3, s"${lb}1 -> 2014-12-03, 2 -> 2014-12-04, 3 ->
2014-12-05$rb")
+ val ret4 = cast(
+ Literal.create(Map(
+ 1 -> Timestamp.valueOf("2014-12-03 13:01:00"),
+ 2 -> Timestamp.valueOf("2014-12-04 15:05:00"))),
+ StringType)
+ checkEvaluation(ret4, s"${lb}1 -> 2014-12-03 13:01:00, 2 -> 2014-12-04
15:05:00$rb")
+ val ret5 = cast(
+ Literal.create(Map(
+ 1 -> Array(1, 2, 3),
+ 2 -> Array(4, 5, 6))),
+ StringType)
+ checkEvaluation(ret5, s"${lb}1 -> [1, 2, 3], 2 -> [4, 5, 6]$rb")
+ }
+ }
+ }
+
+ test("SPARK-22981 Cast struct to string") {
Review comment:
ditto
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
