mbutrovich commented on code in PR #2916:
URL: https://github.com/apache/iceberg-rust/pull/2916#discussion_r4604420740
##########
crates/iceberg/src/spec/values/serde.rs:
##########
@@ -106,6 +109,12 @@ pub(crate) mod _serde {
Double(f64),
String(String),
Bytes(ByteBuf),
+ // Note with the use of `serialize_with` here:
+ // - it is to ensure we get an Avro Value::Fixed, and not
Value::Bytes.
+ // This is for UUID support. The spec requires that UUIDs _must_
use `Value::Fixed`.
+ // - `apache_avro::serde::fixed::serialize` only affects the
behaviour of
+ // `apache_avro::serde::to_value`. Internally it sets a thread
local flag for Avro.
Review Comment:
Could we correct the reason this comment gives? The spec doesn't say
anything about apache-avro's `Value` type. It defines the Avro schema for
`uuid` as a [16-byte `fixed` with `logicalType:
uuid`](https://github.com/apache/iceberg/blob/9203eaff9eb97fef9d1f5a3ef4460e8453adaed9/format/spec.md?plain=1#L1867).
`Value::Fixed` is needed because apache-avro 0.22 accepts `Value::Bytes` for a
plain `fixed` schema
([`types.rs#L527`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L527))
but only `Value::Uuid` or `Value::Fixed` for a fixed uuid schema
([`types.rs#L490`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L490),
[`types.rs#L547`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L547)).
The variant is also used for `fixed[N]` values now, so "This is for UUID
support" no longer covers it.
```suggestion
// apache-avro accepts only `Value::Uuid` or `Value::Fixed` for the
spec's `fixed`
// uuid schema, and a `ByteBuf` serializes to `Value::Bytes` by
default.
// `apache_avro::serde::fixed::serialize` makes
`apache_avro::to_value` produce
// `Value::Fixed`. Other serializers, such as serde_json, still see
plain bytes.
```
##########
crates/iceberg/src/spec/manifest/writer.rs:
##########
@@ -1058,4 +1071,184 @@ mod tests {
"content_size_in_bytes",
]);
}
+
+ #[tokio::test]
+ async fn test_manifest_v1_roundtrips_uuid_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::uuid(
+ uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v1,
+ Type::Primitive(PrimitiveType::Uuid),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
+
+ #[tokio::test]
+ async fn test_manifest_v2_roundtrips_uuid_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::uuid(
+ uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v2_data,
+ Type::Primitive(PrimitiveType::Uuid),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
+
+ #[tokio::test]
+ async fn test_manifest_v3_roundtrips_uuid_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::uuid(
+ uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v3_data,
+ Type::Primitive(PrimitiveType::Uuid),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
+
+ #[tokio::test]
+ async fn test_manifest_v1_roundtrips_16_byte_fixed_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::fixed(
+ 0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v1,
+ Type::Primitive(PrimitiveType::Fixed(16)),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
+
+ #[tokio::test]
+ async fn test_manifest_v2_roundtrips_16_byte_fixed_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::fixed(
+ 0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v2_data,
+ Type::Primitive(PrimitiveType::Fixed(16)),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
+
+ #[tokio::test]
+ async fn test_manifest_v3_roundtrips_16_byte_fixed_partition() {
+ let data_file = DataFileBuilder::default()
+ .content(DataContentType::Data)
+ .file_path("memory:///table/data/00000.parquet".to_string())
+ .file_format(DataFileFormat::Parquet)
+ .partition(Struct::from_iter([Some(Literal::fixed(
+ 0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+ ))]))
+ .record_count(123)
+ .file_size_in_bytes(456)
+ .build()
+ .unwrap();
+
+ let output = write_and_read_data_file(
+ ManifestWriterBuilder::build_v3_data,
+ Type::Primitive(PrimitiveType::Fixed(16)),
+ data_file.clone(),
+ )
+ .await;
+
+ assert_eq!(output, data_file);
+ }
Review Comment:
These six tests differ only in the writer builder and the partition type and
value. The v2 cases check the same partition round trip as
`test_write_manifest_with_repeated_uuid_partition_type` and
`test_write_manifest_with_repeated_fixed_partition_type` above. Could we fold
them into one test that loops over the three builders? That also gives us a
place to cover a null uuid partition value against the new `["null",
"uuid_fixed"]` union, which none of the tests write today.
```suggestion
#[tokio::test]
async fn
test_manifest_roundtrips_uuid_and_fixed_partitions_in_each_format_version() {
let uuid =
uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7);
let cases = [
(
Type::Primitive(PrimitiveType::Uuid),
Some(Literal::uuid(uuid)),
),
(Type::Primitive(PrimitiveType::Uuid), None),
(
Type::Primitive(PrimitiveType::Fixed(16)),
Some(Literal::fixed(uuid.into_bytes())),
),
];
let builds: [fn(ManifestWriterBuilder) -> ManifestWriter; 3] = [
ManifestWriterBuilder::build_v1,
ManifestWriterBuilder::build_v2_data,
ManifestWriterBuilder::build_v3_data,
];
for build in builds {
for (partition_field_type, value) in cases.clone() {
let data_file = DataFileBuilder::default()
.content(DataContentType::Data)
.file_path("memory:///table/data/00000.parquet".to_string())
.file_format(DataFileFormat::Parquet)
.partition(Struct::from_iter([value]))
.record_count(123)
.file_size_in_bytes(456)
.build()
.unwrap();
let output =
write_and_read_data_file(build, partition_field_type,
data_file.clone()).await;
assert_eq!(output, data_file);
}
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]