This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4118-0cc389bc5e1793ad3cf0f74bdfa7cfa199644ec0 in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit 208fd5147aae781aaeb68b204b47e9b6fe08fcc8 Author: Syed Ahnaf Raza <[email protected]> AuthorDate: Sun Oct 18 12:18:02 2026 +0000 Preserve null maps when projecting a map of structs (#4118) * Preserve null maps when projecting map of structs * Add test for projecting null map of structs * Build map entries with StructArray.from_arrays --- pyiceberg/io/pyarrow.py | 13 ++++++++++++- tests/io/test_pyarrow.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index e85973408..c55067d39 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2120,7 +2120,18 @@ class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None] ) if isinstance(value_result, pa.StructArray): # Arrow does not allow reordering of fields, therefore we have to copy the array :( - return pa.MapArray.from_arrays(map_array.offsets, key_result, value_result, arrow_field) + entries = pa.StructArray.from_arrays( + [key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field] + ) + # Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps + return pa.Array.from_buffers( + arrow_field, + len(map_array), + map_array.buffers()[:2], + map_array.null_count, + map_array.offset, + [entries], + ) else: return map_array.cast(arrow_field) else: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 6e66ff43e..956dc9e1d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3211,6 +3211,48 @@ def test__to_requested_schema_timestamps_without_downcast_raises_exception( assert "Unsupported schema projection from timestamp[ns] to timestamp[us]" in str(exc_info.value) +def test__to_requested_schema_null_map_of_structs() -> None: + requested_schema = Schema( + NestedField( + 1, + "col_map_with_struct", + MapType( + 11, + StringType(), + 12, + StructType(NestedField(121, "source", StringType(), required=False)), + value_required=False, + ), + required=False, + ), + NestedField(2, "col_map", MapType(21, StringType(), 22, IntegerType(), value_required=False), required=False), + ) + arrow_schema = pa.schema( + [ + pa.field("col_map_with_struct", pa.map_(pa.string(), pa.struct([pa.field("source", pa.string())]))), + pa.field("col_map", pa.map_(pa.string(), pa.int32())), + ] + ) + batch = pa.RecordBatch.from_arrays( + [ + pa.array( + [[("a", {"source": "x"})], [], None, [("b", {"source": "y"}), ("c", None)]], type=arrow_schema.field(0).type + ), + pa.array([[("a", 1)], [], None, [("b", 2), ("c", None)]], type=arrow_schema.field(1).type), + ], + schema=arrow_schema, + ) + + result = _to_requested_schema(requested_schema, requested_schema, batch) + assert result.column(0).to_pylist() == [[("a", {"source": "x"})], [], None, [("b", {"source": "y"}), ("c", None)]] + assert result.column(1).to_pylist() == [[("a", 1)], [], None, [("b", 2), ("c", None)]] + + # A sliced batch has to keep the nulls as well + result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3)) + assert result.column(0).to_pylist() == [[], None, [("b", {"source": "y"}), ("c", None)]] + assert result.column(1).to_pylist() == [[], None, [("b", 2), ("c", None)]] + + @pytest.mark.parametrize( "arrow_type,iceberg_type,expected_arrow_type", [
