gengliangwang commented on a change in pull request #30260:
URL: https://github.com/apache/spark/pull/30260#discussion_r526112368
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
##########
@@ -1396,93 +1355,199 @@ class CastSuite extends CastSuiteBase {
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
}
+
+ test("SPARK-22825 Cast array to string") {
+ val ret1 = cast(Literal.create(Array(1, 2, 3, 4, 5)), StringType)
+ checkEvaluation(ret1, "[1, 2, 3, 4, 5]")
+ val ret2 = cast(Literal.create(Array("ab", "cde", "f")), StringType)
+ checkEvaluation(ret2, "[ab, cde, f]")
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret3 = cast(Literal.create(Array("ab", null, "c")), StringType)
+ checkEvaluation(ret3, s"[ab,${if (omitNull) "" else " null"}, c]")
+ }
+ }
+ val ret4 =
+ cast(Literal.create(Array("ab".getBytes, "cde".getBytes, "f".getBytes)),
StringType)
+ checkEvaluation(ret4, "[ab, cde, f]")
+ val ret5 = cast(
+ Literal.create(Array("2014-12-03", "2014-12-04",
"2014-12-06").map(Date.valueOf)),
+ StringType)
+ checkEvaluation(ret5, "[2014-12-03, 2014-12-04, 2014-12-06]")
+ val ret6 = cast(
+ Literal.create(Array("2014-12-03 13:01:00", "2014-12-04 15:05:00")
+ .map(Timestamp.valueOf)),
+ StringType)
+ checkEvaluation(ret6, "[2014-12-03 13:01:00, 2014-12-04 15:05:00]")
+ val ret7 = cast(Literal.create(Array(Array(1, 2, 3), Array(4, 5))),
StringType)
+ checkEvaluation(ret7, "[[1, 2, 3], [4, 5]]")
+ val ret8 = cast(
+ Literal.create(Array(Array(Array("a"), Array("b", "c")),
Array(Array("d")))),
+ StringType)
+ checkEvaluation(ret8, "[[[a], [b, c]], [[d]]]")
+ }
+
+ test("SPARK-33291: Cast array with null elements to string") {
+ Seq(false, true).foreach { omitNull =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
omitNull.toString) {
+ val ret1 = cast(Literal.create(Array(null, null)), StringType)
+ checkEvaluation(
+ ret1,
+ s"[${if (omitNull) "" else "null"},${if (omitNull) "" else "
null"}]")
+ }
+ }
+ }
+
+ test("SPARK-22973 Cast map to string") {
+ Seq(
+ false -> ("{", "}"),
+ true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
legacyCast.toString) {
+ val ret1 = cast(Literal.create(Map(1 -> "a", 2 -> "b", 3 -> "c")),
StringType)
+ checkEvaluation(ret1, s"${lb}1 -> a, 2 -> b, 3 -> c$rb")
+ val ret2 = cast(
+ Literal.create(Map("1" -> "a".getBytes, "2" -> null, "3" ->
"c".getBytes)),
+ StringType)
+ checkEvaluation(ret2, s"${lb}1 -> a, 2 ->${if (legacyCast) "" else "
null"}, 3 -> c$rb")
+ val ret3 = cast(
+ Literal.create(Map(
+ 1 -> Date.valueOf("2014-12-03"),
+ 2 -> Date.valueOf("2014-12-04"),
+ 3 -> Date.valueOf("2014-12-05"))),
+ StringType)
+ checkEvaluation(ret3, s"${lb}1 -> 2014-12-03, 2 -> 2014-12-04, 3 ->
2014-12-05$rb")
+ val ret4 = cast(
+ Literal.create(Map(
+ 1 -> Timestamp.valueOf("2014-12-03 13:01:00"),
+ 2 -> Timestamp.valueOf("2014-12-04 15:05:00"))),
+ StringType)
+ checkEvaluation(ret4, s"${lb}1 -> 2014-12-03 13:01:00, 2 -> 2014-12-04
15:05:00$rb")
+ val ret5 = cast(
+ Literal.create(Map(
+ 1 -> Array(1, 2, 3),
+ 2 -> Array(4, 5, 6))),
+ StringType)
+ checkEvaluation(ret5, s"${lb}1 -> [1, 2, 3], 2 -> [4, 5, 6]$rb")
+ }
+ }
+ }
+
+ test("SPARK-22981 Cast struct to string") {
+ Seq(
+ false -> ("{", "}"),
+ true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
legacyCast.toString) {
+ val ret1 = cast(Literal.create((1, "a", 0.1)), StringType)
+ checkEvaluation(ret1, s"${lb}1, a, 0.1$rb")
+ val ret2 = cast(Literal.create(Tuple3[Int, String, String](1, null,
"a")), StringType)
+ checkEvaluation(ret2, s"${lb}1,${if (legacyCast) "" else " null"},
a$rb")
+ val ret3 = cast(Literal.create(
+ (Date.valueOf("2014-12-03"), Timestamp.valueOf("2014-12-03
15:05:00"))), StringType)
+ checkEvaluation(ret3, s"${lb}2014-12-03, 2014-12-03 15:05:00$rb")
+ val ret4 = cast(Literal.create(((1, "a"), 5, 0.1)), StringType)
+ checkEvaluation(ret4, s"$lb${lb}1, a$rb, 5, 0.1$rb")
+ val ret5 = cast(Literal.create((Seq(1, 2, 3), "a", 0.1)), StringType)
+ checkEvaluation(ret5, s"$lb[1, 2, 3], a, 0.1$rb")
+ val ret6 = cast(Literal.create((1, Map(1 -> "a", 2 -> "b", 3 ->
"c"))), StringType)
+ checkEvaluation(ret6, s"${lb}1, ${lb}1 -> a, 2 -> b, 3 -> c$rb$rb")
+ }
+ }
+ }
+
+ test("SPARK-33291: Cast struct with null elements to string") {
+ Seq(
+ false -> ("{", "}"),
+ true -> ("[", "]")).foreach { case (legacyCast, (lb, rb)) =>
+ withSQLConf(SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING.key ->
legacyCast.toString) {
+ val ret1 = cast(Literal.create(Tuple2[String, String](null, null)),
StringType)
+ checkEvaluation(
+ ret1,
+ s"$lb${if (legacyCast) "" else "null"},${if (legacyCast) "" else "
null"}$rb")
+ }
+ }
+ }
+
+ test("data type casting II") {
+ checkEvaluation(
+ cast(cast(cast(cast(cast(cast("5", ByteType), TimestampType),
+ DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
+ 5.toShort)
+ checkEvaluation(
+ cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
+ DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
+ null)
+ checkEvaluation(cast(cast(cast(cast(cast(cast("5",
DecimalType.SYSTEM_DEFAULT),
+ ByteType), TimestampType), LongType), StringType), ShortType),
+ 5.toShort)
+ }
+
+ test("Cast from double II") {
+ checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType),
1.toDouble)
+ }
}
/**
- * Test suite for data type casting expression [[AnsiCast]].
+ * Test suite for data type casting expression [[Cast]] with ANSI mode
disabled.
*/
-class AnsiCastSuite extends CastSuiteBase {
- // It is not required to set SQLConf.ANSI_ENABLED as true for testing
numeric overflow.
- override protected def requiredAnsiEnabledForOverflowTestCases: Boolean =
false
+class CastSuiteWithAnsiModeOn extends AnsiCastSuiteBase {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true)
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED)
+ }
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): CastBase = {
v match {
- case lit: Expression => AnsiCast(lit, targetType, timeZoneId)
- case _ => AnsiCast(Literal(v), targetType, timeZoneId)
+ case lit: Expression => Cast(lit, targetType, timeZoneId)
+ case _ => Cast(Literal(v), targetType, timeZoneId)
}
}
+}
- test("cast from invalid string to numeric should throw
NumberFormatException") {
- // cast to IntegerType
- Seq(IntegerType, ShortType, ByteType, LongType).foreach { dataType =>
- val array = Literal.create(Seq("123", "true", "f", null),
- ArrayType(StringType, containsNull = true))
- checkExceptionInExpression[NumberFormatException](
- cast(array, ArrayType(dataType, containsNull = true)),
- "invalid input syntax for type numeric: true")
- checkExceptionInExpression[NumberFormatException](
- cast("string", dataType), "invalid input syntax for type numeric:
string")
- checkExceptionInExpression[NumberFormatException](
- cast("123-string", dataType), "invalid input syntax for type numeric:
123-string")
- checkExceptionInExpression[NumberFormatException](
- cast("2020-07-19", dataType), "invalid input syntax for type numeric:
2020-07-19")
- checkExceptionInExpression[NumberFormatException](
- cast("1.23", dataType), "invalid input syntax for type numeric: 1.23")
- }
+/**
+ * Test suite for data type casting expression [[AnsiCast]] with ANSI mode
enabled.
+ */
+class AnsiCastSuiteWithAnsiModeOn extends CastSuiteBase {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true)
Review comment:
For `SQL.get`, when there is no active SparkSession, it actually
creates a new SQLConf instance per thread:
```
/**
* Default config. Only used when there is no active SparkSession for the
thread.
* See [[get]] for more information.
*/
private lazy val fallbackConf = new ThreadLocal[SQLConf] {
override def initialValue: SQLConf = new SQLConf
}
```
Thus we don't need to worry about modifying the SQLConf will break the other
concurrent tests here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
