srielau commented on code in PR #59302:
URL: https://github.com/apache/spark/pull/59302#discussion_r4234766149
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -277,34 +247,6 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
data => IntervalUtils.microsToDuration(
IntervalUtils.castStringToDTInterval(UTF8String.fromString(data),
start, end)),
converter)
- case dt @ (_: ArrayType | _: MapType | _: StructType)
- if standardCharVarcharSemantics &&
CharVarcharUtils.hasCharVarchar(dt) =>
- val physicalType =
ScriptTransformationIOSchema.toUnboundedStringType(dt)
- // JSON object keys are strings. Restore non-string keys inside the
malformed
- // wrapper, then apply CHAR/VARCHAR (and CHAR/VARCHAR map-key dedup)
outside it.
- val jsonType =
ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
- val tz = Some(conf.sessionLocalTimeZone)
- val complexTypeFactory = JsonToStructs(
- jsonType,
- ioschema.outputSerdeProps.toMap,
- Literal(null),
- tz)
- val restoreJsonKeys = if (jsonType.sameType(physicalType)) {
- identity[Any] _
- } else {
- ScriptTransformationIOSchema.makeJsonToTargetRestorer(
- jsonType, physicalType, tz)
- }
- val toDeclaredType =
ScriptTransformationIOSchema.makeJsonToTargetRestorer(
- physicalType, dt, tz)
- val parser = wrapperConvertException(
- data => restoreJsonKeys(
- complexTypeFactory.nullSafeEval(UTF8String.fromString(data))),
- identity)
- (data: String) => {
- val parsed = parser(data)
- if (parsed == null) null else toDeclaredType(parsed)
- }
case _: ArrayType | _: MapType | _: StructType =>
Review Comment:
Leaving this out of scope, matching `MAP<INT, STRING>`. A parse-time error
would also reject empty/null `MAP<INT, STRING>` rows if we put it on
`outputFieldWriters`. Noted in the PR description.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -224,19 +201,12 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr
=>
val converter =
CatalystTypeConverters.createToCatalystConverter(attr.dataType)
attr.dataType match {
+ // CHAR/VARCHAR must not reach the script reader; AstBuilder rewrites
them to
+ // unbounded STRING and applies assignment in a Project when standard
semantics
+ // are enabled. Reject any that still arrive (e.g. hand-built plans).
Review Comment:
The no-SerDe reader now rejects `CharVarcharUtils.hasCharVarchar(dt)`,
including nested types. Hand-built `ARRAY<CHAR(4)>` is covered. The Hive SerDe
path still has no extra guard: SQL already rewrites to STRING, so leftover
nested CHAR there is only a hand-built plan.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala:
##########
@@ -1856,6 +1857,21 @@ class AstBuilder extends DataTypeAstBuilder
schemaLess
)
)
+ // Under standard semantics, assign declared CHAR/VARCHAR (pad / length
check) in a
+ // Project so the query result type matches AS (...). Flag off and
preserve-only keep
+ // the historical STRING result with no pad and no EXCEED_LIMIT_LENGTH.
+ if (conf.charVarcharStandardSemantics &&
+ attributes.exists(a =>
CharVarcharUtils.getRawType(a.metadata).isDefined)) {
+ Project(
Review Comment:
Intended, same as scan-side assignment. Added a `count(*)` test that
overflowing unreferenced CHAR does not raise, and mentioned it in the PR
description.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -317,7 +259,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
}
}
- // Match Hive `LazySimpleSerDe`: return null when a type cast fails.
+ // Keep consistent with Hive `LazySimpleSerde`, when there is a type case
error, return null
Review Comment:
Restored the SPARK-59683 wording: `Match Hive LazySimpleSerDe: return null
when a type cast fails.`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
