srielau commented on code in PR #58936:
URL: https://github.com/apache/spark/pull/58936#discussion_r4168977689
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -241,6 +276,29 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
data => IntervalUtils.microsToDuration(
IntervalUtils.castStringToDTInterval(UTF8String.fromString(data),
start, end)),
converter)
+ case dt @ (_: ArrayType | _: MapType | _: StructType)
+ if standardCharVarcharSemantics &&
CharVarcharUtils.hasCharVarchar(dt) =>
+ val physicalType =
ScriptTransformationIOSchema.toUnboundedStringType(dt)
+ // JSON object keys are strings. Cast them to the declared map key
type after parsing.
+ val jsonType =
ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
+ val complexTypeFactory = JsonToStructs(
+ jsonType,
+ ioschema.outputSerdeProps.toMap,
+ Literal(null),
+ Some(conf.sessionLocalTimeZone))
+ val parsedToPhysical = if (jsonType.sameType(physicalType)) {
+ identity[Any] _
+ } else {
+ val restoreMapKeys =
ScriptTransformationIOSchema.makeJsonMapKeyRestorer(
+ physicalType, Some(conf.sessionLocalTimeZone))
+ value: Any => restoreMapKeys(value)
+ }
+ val toScala =
CatalystTypeConverters.createToScalaConverter(physicalType)
+ val parser = wrapperConvertException(
+ data => parsedToPhysical(
+ complexTypeFactory.nullSafeEval(UTF8String.fromString(data))),
+ identity)
+ data => converter(toScala(parser(data)))
Review Comment:
Addressed. Nested conversion now stays in Catalyst: JSON parse and
non-string key restore remain inside the malformed-field wrapper, then
`makeJsonToTargetRestorer` applies CHAR/VARCHAR write-side checks and rebuilds
every map with a fresh `ArrayBasedMapBuilder`. Distinct JSON keys that pad to
the same CHAR/VARCHAR key now raise `DUPLICATED_MAP_KEY` under the default
EXCEPTION policy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
