emkornfield commented on code in PR #1851:
URL: https://github.com/apache/iceberg-rust/pull/1851#discussion_r2535639739
##########
crates/iceberg/src/spec/table_properties.rs:
##########
@@ -137,6 +162,21 @@ impl TableProperties {
pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str =
"write.target-file-size-bytes";
/// Default target file size
pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 *
1024 * 1024; // 512 MB
+
+ /// Compression codec for metadata files (JSON)
+ pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str =
"write.metadata.compression-codec";
+ /// Default metadata compression codec - uncompressed
+ pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str =
"uncompressed";
Review Comment:
done.
##########
crates/iceberg/src/spec/table_metadata.rs:
##########
@@ -461,9 +461,57 @@ impl TableMetadata {
file_io: &FileIO,
metadata_location: impl AsRef<str>,
) -> Result<()> {
+ use std::io::Write as _;
+
+ use flate2::write::GzEncoder;
+
+ let json_data = serde_json::to_vec(self)?;
+
+ // Check if compression is enabled via table properties
+ let codec = self
+ .properties
+
.get(crate::spec::table_properties::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC)
+ .map(|s| s.as_str())
+
.unwrap_or(crate::spec::table_properties::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT);
+
+ let (data_to_write, actual_location) = match codec {
+ "gzip" => {
+ let mut encoder = GzEncoder::new(Vec::new(),
flate2::Compression::default());
+ encoder.write_all(&json_data).map_err(|e| {
+ Error::new(
+ ErrorKind::DataInvalid,
+ "Failed to compress metadata with gzip",
+ )
+ .with_source(e)
+ })?;
+ let compressed_data = encoder.finish().map_err(|e| {
+ Error::new(ErrorKind::DataInvalid, "Failed to finish gzip
compression")
+ .with_source(e)
+ })?;
+
+ // Modify filename to add .gz before .metadata.json
+ let location = metadata_location.as_ref();
+ let new_location = if location.ends_with(".metadata.json") {
+ location.replace(".metadata.json", ".gz.metadata.json")
+ } else {
+ // If it doesn't end with expected pattern, just append .gz
+ format!("{}.gz", location)
+ };
+
+ (compressed_data, new_location)
+ }
+ "uncompressed" | "" => (json_data,
metadata_location.as_ref().to_string()),
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
