(paimon) branch master updated: [parquet] Parquet write supports complex types in complex type field

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 4009edea7 [parquet] Parquet write supports complex types in complex 
type field
4009edea7 is described below

commit 4009edea7b333e240e8b9a5e6ffa9d7bd587b600
Author: Jingsong <jingsongl...@gmail.com>
AuthorDate: Tue Jun 11 15:37:53 2024 +0800

    [parquet] Parquet write supports complex types in complex type field
---
 .../parquet/writer/ParquetRowDataWriter.java       | 34 +++++++++++++++-------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git 
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java
 
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java
index 6312ed051..ae71b5dad 100644
--- 
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java
+++ 
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java
@@ -404,9 +404,16 @@ public class ParquetRowDataWriter {
 
         @Override
         public void write(InternalRow row, int ordinal) {
-            recordConsumer.startGroup();
+            writeMapData(row.getMap(ordinal));
+        }
 
-            InternalMap mapData = row.getMap(ordinal);
+        @Override
+        public void write(InternalArray arrayData, int ordinal) {
+            writeMapData(arrayData.getMap(ordinal));
+        }
+
+        private void writeMapData(InternalMap mapData) {
+            recordConsumer.startGroup();
 
             if (mapData != null && mapData.size() > 0) {
                 recordConsumer.startField(repeatedGroupName, 0);
@@ -435,9 +442,6 @@ public class ParquetRowDataWriter {
             }
             recordConsumer.endGroup();
         }
-
-        @Override
-        public void write(InternalArray arrayData, int ordinal) {}
     }
 
     /** It writes an array type field to parquet. */
@@ -461,8 +465,16 @@ public class ParquetRowDataWriter {
 
         @Override
         public void write(InternalRow row, int ordinal) {
+            writeArrayData(row.getArray(ordinal));
+        }
+
+        @Override
+        public void write(InternalArray arrayData, int ordinal) {
+            writeArrayData(arrayData.getArray(ordinal));
+        }
+
+        private void writeArrayData(InternalArray arrayData) {
             recordConsumer.startGroup();
-            InternalArray arrayData = row.getArray(ordinal);
             int listLength = arrayData.size();
 
             if (listLength > 0) {
@@ -481,9 +493,6 @@ public class ParquetRowDataWriter {
             }
             recordConsumer.endGroup();
         }
-
-        @Override
-        public void write(InternalArray arrayData, int ordinal) {}
     }
 
     /** It writes a row type field to parquet. */
@@ -522,7 +531,12 @@ public class ParquetRowDataWriter {
         }
 
         @Override
-        public void write(InternalArray arrayData, int ordinal) {}
+        public void write(InternalArray arrayData, int ordinal) {
+            recordConsumer.startGroup();
+            InternalRow rowData = arrayData.getRow(ordinal, 
fieldWriters.length);
+            write(rowData);
+            recordConsumer.endGroup();
+        }
     }
 
     private Binary timestampToInt96(Timestamp timestamp) {