srielau commented on code in PR #59188:
URL: https://github.com/apache/spark/pull/59188#discussion_r4157269142
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2444,42 +2444,23 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
test("SPARK-59274: from_json/csv/xml honor CHAR/VARCHAR under
standardSemantics") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
- def checkTextInput(query: String, inputType: DataType): DataFrame = {
- val df = sql(query)
- val inputTypes =
df.queryExecution.analyzed.expressions.flatMap(_.collect {
- case e: SupportTrimmedCharInput => e.child.dataType
- })
- assert(inputTypes === Seq(inputType), df.queryExecution.analyzed)
- df
- }
-
+ // CHAR padding is part of the STRING value after CAST. Parsers do not
rtrim it.
+ // from_json on padded CHAR must match the same document as STRING with
trailing spaces.
checkAnswer(
- checkTextInput(
- """SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')""",
- CharType(12)),
- Row(Row(1)))
+ sql("""SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')"""),
+ sql("""SELECT from_json('{"a":1} ', 'a INT')"""))
+ // With a space delimiter, CHAR padding shows up as empty CSV fields.
checkAnswer(
- checkTextInput(
- """SELECT from_csv(
+ sql(
+ """SELECT schema_of_csv(
Review Comment:
Restored the `from_csv(CAST('1' AS CHAR(3)), '_c0 INT', map('delimiter', '
', 'mode', 'FAILFAST'))` case and asserted
`MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION`. PERMISSIVE still returns the
first field (`named_struct('_c0', 1)`).
Also added an expression-level `CsvToStructs` check with a `CharType`
literal via `checkErrorInExpression`, so interpreted and codegen evaluation
both hit the untrimmed `child.genCode` path.
The PR description now calls out the FAILFAST `from_csv` behavior as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
