mbutrovich commented on code in PR #2916:
URL: https://github.com/apache/iceberg-rust/pull/2916#discussion_r4604420740


##########
crates/iceberg/src/spec/values/serde.rs:
##########
@@ -106,6 +109,12 @@ pub(crate) mod _serde {
         Double(f64),
         String(String),
         Bytes(ByteBuf),
+        // Note with the use of `serialize_with` here:
+        //  - it is to ensure we get an Avro Value::Fixed, and not 
Value::Bytes.
+        //    This is for UUID support. The spec requires that UUIDs _must_ 
use `Value::Fixed`.
+        //  - `apache_avro::serde::fixed::serialize` only affects the 
behaviour of
+        //    `apache_avro::serde::to_value`. Internally it sets a thread 
local flag for Avro.

Review Comment:
   Could we correct the reason this comment gives? The spec doesn't say 
anything about apache-avro's `Value` type. It defines the Avro schema for 
`uuid` as a [16-byte `fixed` with `logicalType: 
uuid`](https://github.com/apache/iceberg/blob/9203eaff9eb97fef9d1f5a3ef4460e8453adaed9/format/spec.md?plain=1#L1867).
 `Value::Fixed` is needed because apache-avro 0.22 accepts `Value::Bytes` for a 
plain `fixed` schema 
([`types.rs#L527`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L527))
 but only `Value::Uuid` or `Value::Fixed` for a fixed uuid schema 
([`types.rs#L490`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L490),
 
[`types.rs#L547`](https://github.com/apache/avro-rs/blob/ec5721cb0c80dcde56c1049a004f1d785abd88cf/avro/src/types.rs#L547)).
 The variant is also used for `fixed[N]` values now, so "This is for UUID 
support" no longer covers it.
   
   ```suggestion
           // apache-avro accepts only `Value::Uuid` or `Value::Fixed` for the 
spec's `fixed`
           // uuid schema, and a `ByteBuf` serializes to `Value::Bytes` by 
default.
           // `apache_avro::serde::fixed::serialize` makes 
`apache_avro::to_value` produce
           // `Value::Fixed`. Other serializers, such as serde_json, still see 
plain bytes.
   ```



##########
crates/iceberg/src/spec/manifest/writer.rs:
##########
@@ -1058,4 +1071,184 @@ mod tests {
             "content_size_in_bytes",
         ]);
     }
+
+    #[tokio::test]
+    async fn test_manifest_v1_roundtrips_uuid_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::uuid(
+                uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v1,
+            Type::Primitive(PrimitiveType::Uuid),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }
+
+    #[tokio::test]
+    async fn test_manifest_v2_roundtrips_uuid_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::uuid(
+                uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v2_data,
+            Type::Primitive(PrimitiveType::Uuid),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }
+
+    #[tokio::test]
+    async fn test_manifest_v3_roundtrips_uuid_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::uuid(
+                uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v3_data,
+            Type::Primitive(PrimitiveType::Uuid),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }
+
+    #[tokio::test]
+    async fn test_manifest_v1_roundtrips_16_byte_fixed_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::fixed(
+                0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v1,
+            Type::Primitive(PrimitiveType::Fixed(16)),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }
+
+    #[tokio::test]
+    async fn test_manifest_v2_roundtrips_16_byte_fixed_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::fixed(
+                0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v2_data,
+            Type::Primitive(PrimitiveType::Fixed(16)),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }
+
+    #[tokio::test]
+    async fn test_manifest_v3_roundtrips_16_byte_fixed_partition() {
+        let data_file = DataFileBuilder::default()
+            .content(DataContentType::Data)
+            .file_path("memory:///table/data/00000.parquet".to_string())
+            .file_format(DataFileFormat::Parquet)
+            .partition(Struct::from_iter([Some(Literal::fixed(
+                0xf79c3e09_677c_4bbd_a479_3f349cb785e7_u128.to_be_bytes(),
+            ))]))
+            .record_count(123)
+            .file_size_in_bytes(456)
+            .build()
+            .unwrap();
+
+        let output = write_and_read_data_file(
+            ManifestWriterBuilder::build_v3_data,
+            Type::Primitive(PrimitiveType::Fixed(16)),
+            data_file.clone(),
+        )
+        .await;
+
+        assert_eq!(output, data_file);
+    }

Review Comment:
   These six tests differ only in the writer builder and the partition type and 
value. The v2 cases check the same partition round trip as 
`test_write_manifest_with_repeated_uuid_partition_type` and 
`test_write_manifest_with_repeated_fixed_partition_type` above. Could we fold 
them into one test that loops over the three builders? That also gives us a 
place to cover a null uuid partition value against the new `["null", 
"uuid_fixed"]` union, which none of the tests write today.
   
   ```suggestion
       #[tokio::test]
       async fn 
test_manifest_roundtrips_uuid_and_fixed_partitions_in_each_format_version() {
           let uuid = 
uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7);
           let cases = [
               (
                   Type::Primitive(PrimitiveType::Uuid),
                   Some(Literal::uuid(uuid)),
               ),
               (Type::Primitive(PrimitiveType::Uuid), None),
               (
                   Type::Primitive(PrimitiveType::Fixed(16)),
                   Some(Literal::fixed(uuid.into_bytes())),
               ),
           ];
           let builds: [fn(ManifestWriterBuilder) -> ManifestWriter; 3] = [
               ManifestWriterBuilder::build_v1,
               ManifestWriterBuilder::build_v2_data,
               ManifestWriterBuilder::build_v3_data,
           ];
           for build in builds {
               for (partition_field_type, value) in cases.clone() {
                   let data_file = DataFileBuilder::default()
                       .content(DataContentType::Data)
                       
.file_path("memory:///table/data/00000.parquet".to_string())
                       .file_format(DataFileFormat::Parquet)
                       .partition(Struct::from_iter([value]))
                       .record_count(123)
                       .file_size_in_bytes(456)
                       .build()
                       .unwrap();
   
                   let output =
                       write_and_read_data_file(build, partition_field_type, 
data_file.clone()).await;
   
                   assert_eq!(output, data_file);
               }
           }
       }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to