srielau commented on code in PR #59188:
URL: https://github.com/apache/spark/pull/59188#discussion_r4186108571
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2444,42 +2444,47 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
test("SPARK-59274: from_json/csv/xml honor CHAR/VARCHAR under
standardSemantics") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
- def checkTextInput(query: String, inputType: DataType): DataFrame = {
- val df = sql(query)
- val inputTypes =
df.queryExecution.analyzed.expressions.flatMap(_.collect {
- case e: SupportTrimmedCharInput => e.child.dataType
- })
- assert(inputTypes === Seq(inputType), df.queryExecution.analyzed)
- df
- }
-
+ // CHAR padding is part of the STRING value after CAST. Parsers do not
rtrim it.
+ // from_json on padded CHAR must match the same document as STRING with
trailing spaces.
checkAnswer(
- checkTextInput(
- """SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')""",
- CharType(12)),
+ sql("""SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')"""),
Review Comment:
Agreed. The valid JSON comparison still holds with or without rtrim. Added a
PERMISSIVE corrupt-record case:
```sql
from_json(CAST('{"a":' AS CHAR(12)), 'a INT, _unparsed STRING',
map('columnNameOfCorruptRecord', '_unparsed'))
```
`_unparsed` is `{"a": ` (declared width 12). Restoring rtrim would
drop those trailing spaces.
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2444,42 +2444,47 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
test("SPARK-59274: from_json/csv/xml honor CHAR/VARCHAR under
standardSemantics") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
- def checkTextInput(query: String, inputType: DataType): DataFrame = {
- val df = sql(query)
- val inputTypes =
df.queryExecution.analyzed.expressions.flatMap(_.collect {
- case e: SupportTrimmedCharInput => e.child.dataType
- })
- assert(inputTypes === Seq(inputType), df.queryExecution.analyzed)
- df
- }
-
+ // CHAR padding is part of the STRING value after CAST. Parsers do not
rtrim it.
+ // from_json on padded CHAR must match the same document as STRING with
trailing spaces.
checkAnswer(
- checkTextInput(
- """SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')""",
- CharType(12)),
+ sql("""SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')"""),
+ sql("""SELECT from_json('{"a":1} ', 'a INT')"""))
+ // XML ignores trailing whitespace, so a padded CHAR document still
parses.
+ checkAnswer(
+ sql(
+ """SELECT from_xml(
Review Comment:
Same issue for XML. Kept the well-formed CHAR(30) parse and added a
malformed padded document:
```sql
from_xml(CAST('<ROW>' AS CHAR(10)), 'a INT, _unparsed STRING',
map('columnNameOfCorruptRecord', '_unparsed'))
```
`_unparsed` is `<ROW> `. Restoring rtrim would drop the pad.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
