This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 548169cbe8 [mosaic] Update mosaic dependency to 0.2.0 (#8264)
548169cbe8 is described below
commit 548169cbe884d4d15bf7b7a21ad95f90385f5790
Author: jianguotian <[email protected]>
AuthorDate: Wed Jun 17 20:10:52 2026 +0800
[mosaic] Update mosaic dependency to 0.2.0 (#8264)
Mosaic now supports ARRAY and MAP, starting from version 0.2.0 (released
on Maven Central). #8178 already let these types pass schema validation,
but kept the dependency on 0.1.0, so ARRAY/MAP columns still could not
actually be written or read.
This PR updates the mosaic dependency from 0.1.0 to 0.2.0, so ARRAY and
MAP columns can be written and read for real.
---
paimon-mosaic/pom.xml | 2 +-
.../format/mosaic/MosaicFormatReadWriteTest.java | 84 +++++++++++++++++++++-
2 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/paimon-mosaic/pom.xml b/paimon-mosaic/pom.xml
index a265af539b..39b11114bc 100644
--- a/paimon-mosaic/pom.xml
+++ b/paimon-mosaic/pom.xml
@@ -35,7 +35,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>mosaic</artifactId>
- <version>0.1.0</version>
+ <version>0.2.0</version>
</dependency>
<dependency>
diff --git
a/paimon-mosaic/src/test/java/org/apache/paimon/format/mosaic/MosaicFormatReadWriteTest.java
b/paimon-mosaic/src/test/java/org/apache/paimon/format/mosaic/MosaicFormatReadWriteTest.java
index 41f632b3ee..c286c74f5b 100644
---
a/paimon-mosaic/src/test/java/org/apache/paimon/format/mosaic/MosaicFormatReadWriteTest.java
+++
b/paimon-mosaic/src/test/java/org/apache/paimon/format/mosaic/MosaicFormatReadWriteTest.java
@@ -20,19 +20,30 @@ package org.apache.paimon.format.mosaic;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.format.FormatReadWriteTest;
+import org.apache.paimon.format.FormatReaderContext;
+import org.apache.paimon.format.FormatWriterFactory;
import org.apache.paimon.options.Options;
+import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -81,6 +92,8 @@ class MosaicFormatReadWriteTest extends FormatReadWriteTest {
.field("f_timestamp6", DataTypes.TIMESTAMP(6))
.field("f_decimal_5_2", DataTypes.DECIMAL(5, 2))
.field("f_decimal_20_0", DataTypes.DECIMAL(20, 0))
+ .field("f_array", DataTypes.ARRAY(DataTypes.INT()))
+ .field("f_map", DataTypes.MAP(DataTypes.STRING(),
DataTypes.INT()))
.build();
}
@@ -100,12 +113,21 @@ class MosaicFormatReadWriteTest extends
FormatReadWriteTest {
Timestamp.fromEpochMillis(1700000000000L),
Timestamp.fromMicros(1700000000000000L),
Decimal.fromBigDecimal(new BigDecimal("123.45"), 5, 2),
- Decimal.fromBigDecimal(new BigDecimal("12345678901234567890"),
20, 0));
+ Decimal.fromBigDecimal(new BigDecimal("12345678901234567890"),
20, 0),
+ new GenericArray(new int[] {10, 20, 30}),
+ new GenericMap(buildMap()));
+ }
+
+ private static Map<BinaryString, Integer> buildMap() {
+ Map<BinaryString, Integer> map = new HashMap<>();
+ map.put(BinaryString.fromString("k1"), 1);
+ map.put(BinaryString.fromString("k2"), 2);
+ return map;
}
@Override
protected void validateFullTypesResult(InternalRow actual, InternalRow
expected) {
- for (int i = 0; i < 14; i++) {
+ for (int i = 0; i < 16; i++) {
if (expected.isNullAt(i)) {
assertThat(actual.isNullAt(i)).isTrue();
}
@@ -124,6 +146,64 @@ class MosaicFormatReadWriteTest extends
FormatReadWriteTest {
assertThat(actual.getTimestamp(11,
6)).isEqualTo(expected.getTimestamp(11, 6));
assertThat(actual.getDecimal(12, 5,
2)).isEqualTo(expected.getDecimal(12, 5, 2));
assertThat(actual.getDecimal(13, 20,
0)).isEqualTo(expected.getDecimal(13, 20, 0));
+
assertThat(actual.getArray(14).toIntArray()).isEqualTo(expected.getArray(14).toIntArray());
+
assertThat(toJavaMap(actual.getMap(15))).isEqualTo(toJavaMap(expected.getMap(15)));
+ }
+
+ private static Map<String, Integer> toJavaMap(InternalMap m) {
+ Map<String, Integer> out = new HashMap<>();
+ for (int i = 0; i < m.size(); i++) {
+ out.put(m.keyArray().getString(i).toString(),
m.valueArray().getInt(i));
+ }
+ return out;
+ }
+
+ /** ARRAY/MAP with null fields, null elements and empty collections. */
+ @Test
+ void testArrayMapNullAndEmpty() throws Exception {
+ RowType rowType =
+ RowType.builder()
+ .field("f_arr", DataTypes.ARRAY(DataTypes.STRING()))
+ .field("f_map", DataTypes.MAP(DataTypes.INT(),
DataTypes.STRING()))
+ .build();
+
+ Map<Integer, BinaryString> m0 = new HashMap<>();
+ m0.put(1, BinaryString.fromString("a"));
+ InternalRow populated =
+ GenericRow.of(
+ new GenericArray(new BinaryString[]
{BinaryString.fromString("x"), null}),
+ new GenericMap(m0));
+ InternalRow nulls = GenericRow.of(null, null);
+ InternalRow empties =
+ GenericRow.of(
+ new GenericArray(new BinaryString[0]), new
GenericMap(new HashMap<>()));
+
+ FormatWriterFactory factory =
fileFormat().createWriterFactory(rowType);
+ write(factory, file, populated, nulls, empties);
+
+ // Materialize inside the callback: reader reuses native-backed
vectors per batch.
+ List<String> arrSummary = new ArrayList<>();
+ List<String> mapSummary = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ fileFormat()
+ .createReaderFactory(rowType, rowType, new
ArrayList<>())
+ .createReader(
+ new FormatReaderContext(fileIO, file,
fileIO.getFileSize(file)))) {
+ reader.forEachRemaining(
+ r -> {
+ arrSummary.add(
+ r.isNullAt(0)
+ ? "null"
+ : r.getArray(0).size()
+ + ":"
+ + (r.getArray(0).size() > 1
+ &&
r.getArray(0).isNullAt(1)));
+ mapSummary.add(r.isNullAt(1) ? "null" :
String.valueOf(r.getMap(1).size()));
+ });
+ }
+
+ assertThat(arrSummary).containsExactly("2:true", "null", "0:false");
+ assertThat(mapSummary).containsExactly("1", "null", "0");
}
private static boolean isNativeAvailable() {
