This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ca8c2ba6d5 [arrow] Use the nested field's own nullability for sub
writers (#8758)
ca8c2ba6d5 is described below
commit ca8c2ba6d57c600f1893e74f0f52b3a371a2bd9b
Author: Eunbin Son <[email protected]>
AuthorDate: Thu Jul 23 19:42:11 2026 +0900
[arrow] Use the nested field's own nullability for sub writers (#8758)
---
.../java/org/apache/paimon/arrow/ArrowUtils.java | 2 +-
.../writer/ArrowFieldWriterFactoryVisitor.java | 19 +++--
.../paimon/arrow/vector/ArrowFormatWriterTest.java | 86 ++++++++++++++++++++++
3 files changed, 101 insertions(+), 6 deletions(-)
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
index 83b6123219..404e6c009e 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
@@ -255,7 +255,7 @@ public class ArrowUtils {
fieldWriters[i] =
rowType.getTypeAt(i)
.accept(ArrowFieldWriterFactoryVisitor.INSTANCE)
- .create(vectors.get(i), rowType.isNullable());
+ .create(vectors.get(i),
rowType.getTypeAt(i).isNullable());
}
return fieldWriters;
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
index ccff6d6a24..419da16afd 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
@@ -163,7 +163,8 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
new ArrowFieldWriters.ArrayWriter(
fieldVector,
elementWriterFactory.create(
- ((ListVector) fieldVector).getDataVector(),
isNullable),
+ ((ListVector) fieldVector).getDataVector(),
+ arrayType.getElementType().isNullable()),
isNullable);
}
@@ -175,7 +176,8 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
fieldVector,
vectorType.getLength(),
elementWriterFactory.create(
- ((FixedSizeListVector)
fieldVector).getDataVector(), isNullable),
+ ((FixedSizeListVector)
fieldVector).getDataVector(),
+ vectorType.getElementType().isNullable()),
isNullable);
}
@@ -194,8 +196,13 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
List<FieldVector> keyValueVectors =
mapVector.getDataVector().getChildrenFromFields();
return new ArrowFieldWriters.MapWriter(
fieldVector,
- keyWriterFactory.create(keyValueVectors.get(0),
isNullable),
- valueWriterFactory.create(keyValueVectors.get(1),
isNullable),
+ // The Arrow map key is always declared NOT NULL by
ArrowUtils.toArrowField
+ // (per the Arrow spec), so the key writer must stay
non-nullable regardless of
+ // the declared key type's nullability. A null key then
fails loud instead of
+ // producing data that conflicts with the schema.
+ keyWriterFactory.create(keyValueVectors.get(0), false),
+ valueWriterFactory.create(
+ keyValueVectors.get(1),
mapType.getValueType().isNullable()),
isNullable);
};
}
@@ -207,7 +214,9 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
ArrowFieldWriter[] fieldWriters = new
ArrowFieldWriter[children.size()];
for (int i = 0; i < children.size(); i++) {
fieldWriters[i] =
-
rowType.getTypeAt(i).accept(this).create(children.get(i), isNullable);
+ rowType.getTypeAt(i)
+ .accept(this)
+ .create(children.get(i),
rowType.getTypeAt(i).isNullable());
}
return new ArrowFieldWriters.RowWriter(fieldVector, fieldWriters,
isNullable);
};
diff --git
a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
index c6942e1efe..40aa87dae9 100644
---
a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
+++
b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
@@ -29,6 +29,8 @@ import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.columnar.AllNullColumnVector;
@@ -74,6 +76,7 @@ import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link org.apache.paimon.arrow.vector.ArrowFormatWriter}. */
public class ArrowFormatWriterTest {
@@ -193,6 +196,89 @@ public class ArrowFormatWriterTest {
}
}
+ @Test
+ public void testWriteNullElementInNotNullArrayColumn() {
+ // the array column is NOT NULL, but its element type is nullable
+ RowType rowType =
RowType.of(DataTypes.ARRAY(DataTypes.INT()).notNull());
+ try (ArrowFormatWriter writer = new ArrowFormatWriter(rowType, 16,
true)) {
+ writer.write(GenericRow.of(new GenericArray(new Object[] {1, null,
3})));
+ writer.flush();
+
+ VectorSchemaRoot vectorSchemaRoot = writer.getVectorSchemaRoot();
+ ArrowBatchReader arrowBatchReader = new ArrowBatchReader(rowType,
true);
+ InternalRow row =
arrowBatchReader.readBatch(vectorSchemaRoot).iterator().next();
+
+ InternalArray array = row.getArray(0);
+ assertThat(array.size()).isEqualTo(3);
+ assertThat(array.isNullAt(0)).isFalse();
+ assertThat(array.getInt(0)).isEqualTo(1);
+ assertThat(array.isNullAt(1)).isTrue();
+ assertThat(array.isNullAt(2)).isFalse();
+ assertThat(array.getInt(2)).isEqualTo(3);
+ }
+ }
+
+ @Test
+ public void testWriteNullElementInNotNullElementArrayColumn() {
+ // the array column is nullable, but its element type is NOT NULL
+ RowType rowType =
RowType.of(DataTypes.ARRAY(DataTypes.INT().notNull()));
+ try (ArrowFormatWriter writer = new ArrowFormatWriter(rowType, 16,
true)) {
+ InternalRow row = GenericRow.of(new GenericArray(new Object[] {1,
null}));
+ assertThatThrownBy(() -> writer.write(row))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected not null but found null
value");
+ }
+ }
+
+ @Test
+ public void testWriteNullValueInNotNullMapColumn() {
+ // the map column is NOT NULL, but its value type is nullable
+ RowType rowType = RowType.of(DataTypes.MAP(DataTypes.INT(),
DataTypes.INT()).notNull());
+ try (ArrowFormatWriter writer = new ArrowFormatWriter(rowType, 16,
true)) {
+ Map<Integer, Integer> map = new HashMap<>();
+ map.put(1, null);
+ writer.write(GenericRow.of(new GenericMap(map)));
+ writer.flush();
+
+ VectorSchemaRoot vectorSchemaRoot = writer.getVectorSchemaRoot();
+ ArrowBatchReader arrowBatchReader = new ArrowBatchReader(rowType,
true);
+ InternalRow row =
arrowBatchReader.readBatch(vectorSchemaRoot).iterator().next();
+
+ InternalMap actualMap = row.getMap(0);
+ assertThat(actualMap.size()).isEqualTo(1);
+ assertThat(actualMap.keyArray().getInt(0)).isEqualTo(1);
+ assertThat(actualMap.valueArray().isNullAt(0)).isTrue();
+ }
+ }
+
+ @Test
+ public void testWriteNullKeyInMapColumnFailsLoud() {
+ // the map key type is nullable, but the Arrow map key is always NOT
NULL by spec, so a null
+ // key must fail loud rather than be written into a schema that
forbids it
+ RowType rowType = RowType.of(DataTypes.MAP(DataTypes.INT(),
DataTypes.INT()));
+ try (ArrowFormatWriter writer = new ArrowFormatWriter(rowType, 16,
true)) {
+ Map<Integer, Integer> map = new HashMap<>();
+ map.put(null, 1);
+ InternalRow row = GenericRow.of(new GenericMap(map));
+ assertThatThrownBy(() -> writer.write(row))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected not null but found null
value");
+ }
+ }
+
+ @Test
+ public void testWriteNullFieldInNotNullNestedRowField() {
+ // the row column is nullable, but its nested field is NOT NULL
+ RowType rowType =
+ RowType.of(DataTypes.ROW(DataTypes.FIELD(0, "a",
DataTypes.INT().notNull())));
+ try (ArrowFormatWriter writer = new ArrowFormatWriter(rowType, 16,
true)) {
+ InternalRow row = GenericRow.of(GenericRow.of((Object) null));
+ assertThatThrownBy(() -> writer.write(row))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected not null but found null
value");
+ }
+ }
+
@Test
public void testWriteVector() {
RowType rowType =
