srielau commented on code in PR #59302:
URL: https://github.com/apache/spark/pull/59302#discussion_r4234763795
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala:
##########
@@ -1814,11 +1814,12 @@ class AstBuilder extends DataTypeAstBuilder
if (transformClause.setQuantifier != null) {
throw
QueryParsingErrors.transformNotSupportQuantifierError(transformClause.setQuantifier)
}
- // Create the attributes.
+ // Create the attributes. Script I/O is always unbounded STRING (Hive wire
format), with
+ // the declared CHAR/VARCHAR kept in metadata for optional assignment
below.
val (attributes, schemaLess) = if (transformClause.colTypeList != null) {
// Typed return columns.
val schema = createSchema(transformClause.colTypeList)
- val replacedSchema =
CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)
+ val replacedSchema =
CharVarcharUtils.replaceCharVarcharWithStringInSchemaAlways(schema)
Review Comment:
Went with gating the Project on `charVarcharFirstClassTypes` (standard or
preserve) while keeping the Always STRING rewrite for the exec. Preserve-only
now returns typed CHAR/VARCHAR with pad / `EXCEED_LIMIT_LENGTH`, which matches
first-class write-side assignment elsewhere and avoids STRING-plus-metadata.
Called out in the PR description.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala:
##########
@@ -1856,6 +1857,21 @@ class AstBuilder extends DataTypeAstBuilder
schemaLess
)
)
+ // Under standard semantics, assign declared CHAR/VARCHAR (pad / length
check) in a
+ // Project so the query result type matches AS (...). Flag off and
preserve-only keep
+ // the historical STRING result with no pad and no EXCEED_LIMIT_LENGTH.
+ if (conf.charVarcharStandardSemantics &&
+ attributes.exists(a =>
CharVarcharUtils.getRawType(a.metadata).isDefined)) {
Review Comment:
Done. The Project now zips the original `createSchema` fields with the
STRING attributes and calls `stringLengthCheck(a, field.dataType)` only for
CHAR/VARCHAR fields, so quoted names are not re-parsed from `catalogString`.
Added `AS (s STRUCT<\`a b\`: CHAR(4)>)`.
##########
sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala:
##########
@@ -70,6 +70,28 @@ trait SparkCharVarcharUtils {
case v: VarcharType if !SqlApiConf.get.charVarcharFirstClassTypes =>
v.toStringType
case _ => dt
}
+
+ /**
+ * Always replaces CharType/VarcharType with unbounded StringType, including
when first-class
+ * CHAR/VARCHAR types are enabled. Used for Hive TRANSFORM script I/O, which
is
+ * STRING-on-the-wire regardless of the declared AS types.
+ */
+ def replaceCharVarcharWithStringAlways(dt: DataType): DataType = dt match {
Review Comment:
`replaceCharVarcharWithString` now delegates to
`replaceCharVarcharWithStringAlways` when first-class types are off, Always is
`private[sql]`, and the two schema helpers share one private rewrite.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala:
##########
@@ -42,8 +42,91 @@ class SparkScriptTransformationSuite extends
BaseScriptTransformationSuite with
)
}
- test("SPARK-59683: TRANSFORM view keeps bound CHAR/VARCHAR mode") {
+ // Nested complex TRANSFORM output is JSON without SerDe. Hive sessions
default to
+ // LazySimpleSerDe, so keep these no-SerDe JSON cases in the Spark suite.
+ test("SPARK-60090: TRANSFORM nested CHAR/VARCHAR assignment via Project") {
assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<CHAR(4)>)
+ |FROM VALUES ('["ab"]') t(value)
+ |""".stripMargin),
+ Row(Seq("ab ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value ARRAY<VARCHAR(4)>)
+ |FROM VALUES ('["xy"]') t(value)
+ |""".stripMargin),
+ Row(Seq("xy")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value STRUCT<value: CHAR(5)>)
+ |FROM VALUES ('{"value":"xy"}') t(value)
+ |""".stripMargin),
+ Row(Row("xy ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value MAP<STRING, CHAR(4)>)
+ |FROM VALUES ('{"k":"ab"}') t(value)
+ |""".stripMargin),
+ Row(Map("k" -> "ab ")))
+ checkAnswer(
+ sql(
+ """
+ |SELECT TRANSFORM(value)
+ |USING 'cat' AS (value MAP<CHAR(4), INT>)
Review Comment:
Ported those. Nested cases now assert `query.schema`; added MAP CHAR key
`DUPLICATED_MAP_KEY`, STRING-keyed map-in-map, malformed JSON -> null, quoted
struct names, collated nested struct, and a view created at the default flag.
The default `false` is not stored on the view, so a later session with the flag
on re-parses and assigns; the create-true / read-false case is the PERSISTED
guarantee.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
