[incubator-paimon] branch master updated: [Bug] Fix spark array and map type error when write (#1932)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new fd3ae1ff9 [Bug] Fix spark array and map type error when write (#1932)
fd3ae1ff9 is described below

commit fd3ae1ff92935258ad0163ceb1a6ea782725b5a1
Author: leoyy0316 <105328124+leoyy0...@users.noreply.github.com>
AuthorDate: Wed Sep 6 15:18:00 2023 +0800

    [Bug] Fix spark array and map type error when write (#1932)
---
 .../java/org/apache/paimon/spark/SparkRow.java     | 23 ++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git 
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
 
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
index ecd1472ef..e4928974b 100644
--- 
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
+++ 
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
@@ -32,6 +32,8 @@ import org.apache.paimon.types.RowKind;
 import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.DateTimeUtils;
 
+import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
+
 import org.apache.spark.sql.Row;
 
 import java.io.Serializable;
@@ -44,6 +46,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import scala.collection.JavaConverters;
+import scala.collection.mutable.WrappedArray;
+
 /** A {@link InternalRow} wraps spark {@link Row}. */
 public class SparkRow implements InternalRow, Serializable {
 
@@ -286,12 +291,26 @@ public class SparkRow implements InternalRow, 
Serializable {
 
         @Override
         public InternalArray getArray(int i) {
-            return new PaimonArray(((ArrayType) elementType).getElementType(), 
getAs(i));
+            Object array = getAs(i);
+            if (array instanceof WrappedArray) {
+                List<Object> result = Lists.newArrayList();
+                ((WrappedArray) array).iterator().foreach(x -> result.add(x));
+                return new PaimonArray(((ArrayType) 
elementType).getElementType(), result);
+            }
+            return new PaimonArray(
+                    ((ArrayType) elementType).getElementType(), (List<Object>) 
array);
         }
 
         @Override
         public InternalMap getMap(int i) {
-            return toPaimonMap((MapType) elementType, getAs(i));
+            Object map = getAs(i);
+            if (map instanceof scala.collection.immutable.Map) {
+                return toPaimonMap(
+                        (MapType) elementType,
+                        JavaConverters.mapAsJavaMap(
+                                (scala.collection.immutable.Map<Object, 
Object>) map));
+            }
+            return toPaimonMap((MapType) elementType, (Map<Object, Object>) 
map);
         }
 
         @Override