(iceberg) branch main updated: Arrow: Cover null decimal values with defaults across batches (#18271)

[russellspitzer]

This is an automated email from the ASF dual-hosted git repository.

RussellSpitzer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 587f7984ff Arrow: Cover null decimal values with defaults across 
batches (#18271)
587f7984ff is described below

commit 587f7984ff811b1240b0ccf8e34e73e1a52f94d0
Author: Sho Tanaka <4620750+t...@users.noreply.github.com>
AuthorDate: Tue Sep 29 04:13:13 2026 +0900

    Arrow: Cover null decimal values with defaults across batches (#18271)
---
 .../vectorized/TestVectorizedDefaultValues.java    | 32 ++++++++++++++++------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git 
a/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedDefaultValues.java
 
b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedDefaultValues.java
index 48a9aa1b77..cbbce0d4fa 100644
--- 
a/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedDefaultValues.java
+++ 
b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedDefaultValues.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.io.File;
 import java.math.BigDecimal;
 import java.util.List;
+import java.util.stream.Stream;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DataFiles;
 import org.apache.iceberg.FileFormat;
@@ -41,16 +42,28 @@ import 
org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.iceberg.types.Types;
 import org.apache.parquet.hadoop.ParquetOutputFormat;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 /** Vectorized-read tests focused on Iceberg field defaults. */
 public class TestVectorizedDefaultValues {
 
+  private static final int NUM_ROWS = 7;
+
   @TempDir private File tempDir;
 
-  @Test
-  public void testDecimalWithDefaultValueNotDictionaryEncoded() throws 
Exception {
+  private static Stream<Arguments> readConfigurations() {
+    return Stream.of(2, NUM_ROWS, 1024)
+        .flatMap(
+            batchSize -> Stream.of(Arguments.of(batchSize, false), 
Arguments.of(batchSize, true)));
+  }
+
+  @ParameterizedTest(name = "batchSize={0}, reuseContainers={1}")
+  @MethodSource("readConfigurations")
+  void decimalWithDefaultsPreservesStoredValues(int batchSize, boolean 
reuseContainers)
+      throws Exception {
     Schema schema =
         new Schema(
             
Types.NestedField.required("id").withId(1).ofType(Types.LongType.get()).build(),
@@ -83,12 +96,15 @@ public class TestVectorizedDefaultValues {
 
     List<GenericRecord> records = Lists.newArrayList();
     GenericRecord template = GenericRecord.create(schema);
-    for (long i = 0; i < 5; i++) {
+    for (long i = 0; i < NUM_ROWS; i++) {
       GenericRecord rec = template.copy();
       rec.setField("id", i);
-      rec.setField("int_backed", new BigDecimal("12.34"));
-      rec.setField("long_backed", new BigDecimal("1234567890.12"));
-      rec.setField("fixed_backed", new BigDecimal("9876543210.99"));
+      BigDecimal multiplier = BigDecimal.valueOf(i + 1);
+      rec.setField("int_backed", i % 3 == 0 ? null : new 
BigDecimal("12.34").multiply(multiplier));
+      rec.setField(
+          "long_backed", i % 3 == 1 ? null : new 
BigDecimal("1234567890.12").multiply(multiplier));
+      rec.setField(
+          "fixed_backed", i % 3 == 2 ? null : new 
BigDecimal("9876543210.99").multiply(multiplier));
       records.add(rec);
     }
 
@@ -113,7 +129,7 @@ public class TestVectorizedDefaultValues {
 
     int rowsRead = 0;
     try (VectorizedTableScanIterable reader =
-        new VectorizedTableScanIterable(table.newScan(), 1024, false)) {
+        new VectorizedTableScanIterable(table.newScan(), batchSize, 
reuseContainers)) {
       for (ColumnarBatch batch : reader) {
         ColumnVector idColumn = batch.column(0);
         ColumnVector intBackedColumn = batch.column(1);