Github user maropu commented on a diff in the pull request:
https://github.com/apache/spark/pull/20980#discussion_r182933940
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
---
@@ -1197,8 +1197,64 @@ case class ExternalMapToCatalyst private(
override def dataType: MapType = MapType(
keyConverter.dataType, valueConverter.dataType, valueContainsNull =
valueConverter.nullable)
- override def eval(input: InternalRow): Any =
- throw new UnsupportedOperationException("Only code-generated
evaluation is supported")
+ private lazy val mapCatalystConverter: Any => (Array[Any], Array[Any]) =
child.dataType match {
+ case ObjectType(cls) if classOf[java.util.Map[_,
_]].isAssignableFrom(cls) =>
+ (input: Any) => {
+ val data = input.asInstanceOf[java.util.Map[Any, Any]]
+ val keys = new Array[Any](data.size)
+ val values = new Array[Any](data.size)
+ val iter = data.entrySet().iterator()
+ var i = 0
+ while (iter.hasNext) {
+ val entry = iter.next()
+ val (key, value) = (entry.getKey, entry.getValue)
+ keys(i) = if (key != null) {
+ keyConverter.eval(InternalRow.fromSeq(key :: Nil))
+ } else {
+ throw new RuntimeException("Cannot use null as map key!")
+ }
+ values(i) = if (value != null) {
+ valueConverter.eval(InternalRow.fromSeq(value :: Nil))
+ } else {
+ null
+ }
+ i += 1
+ }
+ (keys, values)
+ }
+
+ case ObjectType(cls) if classOf[scala.collection.Map[_,
_]].isAssignableFrom(cls) =>
+ (input: Any) => {
+ val data = input.asInstanceOf[scala.collection.Map[Any, Any]]
+ val keys = new Array[Any](data.size)
+ val values = new Array[Any](data.size)
+ var i = 0
+ for ((key, value) <- data) {
+ keys(i) = if (key != null) {
+ keyConverter.eval(InternalRow.fromSeq(key :: Nil))
+ } else {
+ throw new RuntimeException("Cannot use null as map key!")
+ }
+ values(i) = if (value != null) {
+ valueConverter.eval(InternalRow.fromSeq(value :: Nil))
+ } else {
+ null
+ }
+ i += 1
+ }
+ (keys, values)
+ }
+ }
+
+ override def eval(input: InternalRow): Any = {
+ val result = child.eval(input)
+ if (result != null) {
+ val (keys, values) = mapCatalystConverter(result)
+ new ArrayBasedMapData(ArrayData.toArrayData(keys),
ArrayData.toArrayData(values))
--- End diff --
ya, I also think you're right. I'll fix.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
