This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4127-d37149c8b5d1b737da0eb9fdcb585bc3ef3bac69 in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit e3fbd9440ad0bb25c45848797ff9ec5ad7758624 Author: Yuang Gao <[email protected]> AuthorDate: Thu Oct 22 18:55:40 2026 +0000 fix: map write.parquet.page-row-limit to pyarrow's max_rows_per_page (#4127) --- pyiceberg/io/pyarrow.py | 13 +++++++++---- tests/integration/test_writes/test_writes.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7fd0c85b7..874646b71 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2986,7 +2986,7 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: if compression_codec == ICEBERG_UNCOMPRESSED_CODEC: compression_codec = PYARROW_UNCOMPRESSED_CODEC - return { + kwargs: dict[str, Any] = { "compression": compression_codec, "compression_level": compression_level, "data_page_size": property_as_int( @@ -2999,12 +2999,17 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: property_name=TableProperties.PARQUET_DICT_SIZE_BYTES, default=TableProperties.PARQUET_DICT_SIZE_BYTES_DEFAULT, ), - "write_batch_size": property_as_int( + } + # pyarrow added max_rows_per_page, the per-page row limit, in 23.0.0. + if tuple(map(int, pa.__version__.split(".")[:2])) >= (23, 0): + kwargs["max_rows_per_page"] = property_as_int( properties=table_properties, property_name=TableProperties.PARQUET_PAGE_ROW_LIMIT, default=TableProperties.PARQUET_PAGE_ROW_LIMIT_DEFAULT, - ), - } + ) + elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties: + warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2) + return kwargs def _dataframe_to_data_files( diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab..fe720440f 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -669,6 +669,7 @@ def test_write_parquet_compression_properties( [ ({"write.parquet.page-size-bytes": "42"}, {"data_page_size": 42}), ({"write.parquet.dict-size-bytes": "42"}, {"dictionary_pagesize_limit": 42}), + ({"write.parquet.page-row-limit": "42"}, {"max_rows_per_page": 42, "write_batch_size": None}), ], ) def test_write_parquet_other_properties(
