srielau commented on code in PR #58545:
URL: https://github.com/apache/spark/pull/58545#discussion_r3942731118
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala:
##########
@@ -617,13 +618,15 @@ class JacksonParser(
*/
private def convertMap(
parser: JsonParser,
- fieldConverter: ValueConverter): MapData = {
+ fieldConverter: ValueConverter,
+ keyType: DataType): MapData = {
val keys = ArrayBuffer.empty[UTF8String]
val values = ArrayBuffer.empty[Any]
var badRecordException: Option[Throwable] = None
while (nextUntil(parser, JsonToken.END_OBJECT)) {
- keys += UTF8String.fromString(parser.currentName)
+ keys += CharVarcharUtils.applyTextParseSemantics(
Review Comment:
Fixed. `convertMap` now builds through `ArrayBasedMapBuilder` so
CHAR-normalized keys go through `spark.sql.mapKeyDedupPolicy` instead of
constructing `ArrayBasedMapData` with assumed-unique keys.
`DUPLICATED_MAP_KEY` is rethrown from the JSON parser and
`FailureSafeParser` so EXCEPTION vs LAST_WIN is independent of parse mode.
Regression:
```sql
SELECT from_json('{"a":1,"a ":2}', 'MAP<CHAR(2), INT>')
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -374,11 +374,15 @@ class StaxXmlParser(
*/
private def convertMap(
parser: XMLEventReader,
+ keyType: DataType,
valueType: DataType,
attributes: Array[Attribute]): MapData = {
val kvPairs = ArrayBuffer.empty[(UTF8String, Any)]
+ def mapKey(raw: String): UTF8String = {
+ CharVarcharUtils.applyTextParseSemantics(UTF8String.fromString(raw),
keyType)
Review Comment:
Fixed. XML `convertMap` now uses `ArrayBasedMapBuilder` instead of
`kvPairs.toMap`, so CHAR-normalized collisions honor
`spark.sql.mapKeyDedupPolicy` rather than silently dropping a key.
Regression (valueTag `'a '` collides with padded `<a>`):
```sql
SELECT from_xml(
'<ROW><m><a>1</a>9</m></ROW>',
'm MAP<CHAR(2), INT>',
map('valueTag', 'a ')).m
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
