srielau commented on code in PR #59302:
URL: https://github.com/apache/spark/pull/59302#discussion_r4234768636
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala:
##########
@@ -42,8 +42,91 @@ class SparkScriptTransformationSuite extends
BaseScriptTransformationSuite with
)
}
- test("SPARK-59683: TRANSFORM view keeps bound CHAR/VARCHAR mode") {
+ // Nested complex TRANSFORM output is JSON without SerDe. Hive sessions
default to
+ // LazySimpleSerDe, so keep these no-SerDe JSON cases in the Spark suite.
+ test("SPARK-60090: TRANSFORM nested CHAR/VARCHAR assignment via Project") {
assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<CHAR(4)>)
+ |FROM VALUES ('["ab"]') t(value)
+ |""".stripMargin),
+ Row(Seq("ab ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<VARCHAR(4)>)
+ |FROM VALUES ('["xy"]') t(value)
+ |""".stripMargin),
+ Row(Seq("xy")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value STRUCT<value: CHAR(5)>)
+ |FROM VALUES ('{"value":"xy"}') t(value)
+ |""".stripMargin),
+ Row(Row("xy ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value MAP<STRING, CHAR(4)>)
+ |FROM VALUES ('{"k":"ab"}') t(value)
+ |""".stripMargin),
+ Row(Map("k" -> "ab ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value MAP<CHAR(4), INT>)
+ |FROM VALUES ('{"ab":1}') t(value)
+ |""".stripMargin),
+ Row(Map("ab " -> 1)))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<CHAR(4)>)
+ |FROM VALUES ('[null]') t(value)
+ |""".stripMargin),
+ Row(Seq(null)))
+ }
+ assert(uncaughtExceptionHandler.exception.isEmpty)
+ }
+
+ test("SPARK-60090: TRANSFORM nested CHAR/VARCHAR overflow via Project") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ Seq(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<CHAR(4)>)
+ |FROM VALUES ('["abcdef"]') t(value)
+ |""".stripMargin,
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value STRUCT<value: VARCHAR(4)>)
+ |FROM VALUES ('{"value":"abcdef"}') t(value)
+ |""".stripMargin,
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value MAP<STRING, CHAR(4)>)
+ |FROM VALUES ('{"k":"abcdef"}') t(value)
+ |""".stripMargin).foreach { query =>
+ checkExceedLimitLength(intercept[Exception](sql(query).collect()), "4")
+ }
+ }
+ }
+
+ test("SPARK-60090: TRANSFORM view keeps CHAR/VARCHAR Project assignment") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ // The Project with stringLengthCheck is baked into the parsed view plan,
so a later
+ // session conf change cannot drop pad / EXCEED_LIMIT_LENGTH.
Review Comment:
Reworded: the view SQL is re-parsed on each read; `standardSemantics` is
PERSISTED, so a view created with the flag on keeps the assignment Project even
if the caller session has it off.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
