Github user dbtsai commented on a diff in the pull request:
https://github.com/apache/spark/pull/21847#discussion_r206353416
--- Diff:
external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala ---
@@ -120,7 +133,7 @@ class AvroSerializer(rootCatalystType: DataType,
rootAvroType: Schema, nullable:
case MapType(kt, vt, valueContainsNull) if kt == StringType =>
val valueConverter = newConverter(
- vt, resolveNullableType(avroType.getValueType,
valueContainsNull))
+ vt, resolveUnionType(avroType.getValueType, vt,
valueContainsNull))
--- End diff --
ditto for MapType,
```scala
(getter, ordinal) =>
val mapData = getter.getMap(ordinal)
val length = mapData.numElements()
val result = new java.util.HashMap[String, Any](length)
val keyArray = mapData.keyArray()
val valueArray = mapData.valueArray()
var i = 0
while (i < length) {
val key = keyArray.getUTF8String(i).toString
if (valueContainsNull && valueArray.isNullAt(i)) {
result.put(key, null)
} else {
result.put(key, valueConverter(valueArray, i))
}
i += 1
}
result
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
