kevinjqliu commented on code in PR #4328:
URL: https://github.com/apache/iceberg-python/pull/4328#discussion_r4567881250
##########
pyiceberg/io/pyarrow.py:
##########
@@ -2766,13 +2775,21 @@ def data_file_statistics_from_parquet_metadata(
class ParquetFormatWriter(FileFormatWriter):
"""Writes Arrow tables to a Parquet file."""
- def __init__(self, output_file: OutputFile, file_schema: Schema,
properties: Properties) -> None:
+ def __init__(
+ self,
+ output_file: OutputFile,
+ file_schema: Schema,
+ properties: Properties,
+ key_metadata: StandardKeyMetadata | None = None,
+ ) -> None:
self._output_file = output_file
self._file_schema = file_schema
self._properties = properties
self._writer: pq.ParquetWriter | None = None
self._fos: OutputStream | None = None
self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties)
+ if key_metadata is not None:
+ self._parquet_writer_kwargs["encryption_properties"] =
_parquet_encryption_properties(key_metadata)
Review Comment:
https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html
##########
pyiceberg/io/pyarrow.py:
##########
@@ -2873,7 +2891,10 @@ def write_data_file(task: WriteTask) -> DataFile:
partition_key=task.partition_key,
)
fo = io.new_output(file_path)
- writer = format_model.create_writer(fo, file_schema,
table_metadata.properties)
+ if (key_metadata := _new_data_file_key_metadata(table_metadata)) is
None:
+ writer = format_model.create_writer(fo, file_schema,
table_metadata.properties)
+ else:
+ writer = format_model.create_writer(fo, file_schema,
table_metadata.properties, key_metadata=key_metadata)
Review Comment:
i think we can simplify this and just always pass `key_metadata`, even if
its none. and let the downstream figure it out
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]