c-thiel commented on code in PR #2880:
URL: https://github.com/apache/iceberg-rust/pull/2880#discussion_r4548272336


##########
crates/iceberg/src/arrow/reader/projection.rs:
##########
@@ -2482,4 +3209,611 @@ message schema {
             .unwrap();
         assert_eq!(result, vec![expected]);
     }
+
+    // ---------------------------------------------------------------------
+    // End-to-end variant read tests (synthetic Parquet).
+    //
+    // Unless a test says otherwise, the files carry a field id on the variant 
group, none on
+    // its leaves, and no Parquet VARIANT annotation. `value` is required, as 
in iceberg-java's
+    // unshredded layout.
+    // ---------------------------------------------------------------------
+
+    // Known variant bytes the reader must round-trip unchanged. `value` is 
distinct per row,
+    // so a dropped or reordered row is caught, not just the struct shape.
+    const VARIANT_METADATA: [&[u8]; 3] = [b"\x01", b"\x01", b"\x01"];
+    const VARIANT_VALUE: [&[u8]; 3] = [b"\x0a", b"\x0b", b"\x0c"];
+
+    /// The 3-row variant `StructArray` for 
[`VARIANT_METADATA`]/[`VARIANT_VALUE`].
+    fn variant_struct_array() -> ArrayRef {
+        use arrow_array::{BinaryArray, StructArray};
+        let metadata = Arc::new(BinaryArray::from(VARIANT_METADATA.to_vec())) 
as ArrayRef;
+        let value = Arc::new(BinaryArray::from(VARIANT_VALUE.to_vec())) as 
ArrayRef;
+        Arc::new(StructArray::new(
+            variant_arrow_fields(),
+            vec![metadata, value],
+            None,
+        )) as ArrayRef
+    }
+
+    /// Asserts `field` is a variant and `column` holds the bytes of 
[`variant_struct_array`]
+    /// in the table schema's Arrow type.
+    fn assert_variant(field: &Field, column: &ArrayRef) {
+        use arrow_array::StructArray;
+        let DataType::Struct(table_fields) =
+            
crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap()
+        else {
+            unreachable!()
+        };
+        let expected = StructArray::new(
+            table_fields,
+            variant_struct_array().as_struct().columns().to_vec(),
+            None,
+        );
+        assert_eq!(field.extension_type_name(), Some("arrow.parquet.variant"));
+        assert_eq!(column.as_struct(), &expected);
+    }
+
+    /// Asserts the top-level variant `v` of `batch` round-tripped.
+    fn assert_top_level_variant(batch: &RecordBatch) {
+        let schema = batch.schema();
+        assert_variant(
+            schema.field_with_name("v").unwrap(),
+            batch.column_by_name("v").unwrap(),
+        );
+    }
+
+    /// Asserts the `id` column holds exactly `[1, 2, 3]`.
+    fn assert_id_values(col: &ArrayRef) {
+        let ids = col.as_primitive::<arrow_array::types::Int32Type>();
+        assert_eq!(ids.values(), &[1, 2, 3]);
+    }
+
+    fn write_variant_parquet(
+        arrow_schema: Arc<ArrowSchema>,
+        columns: Vec<ArrayRef>,
+        dir: &str,
+    ) -> String {
+        let batch = RecordBatch::try_new(arrow_schema.clone(), 
columns).unwrap();
+        let path = format!("{dir}/variant.parquet");
+        let props = WriterProperties::builder()
+            .set_compression(Compression::SNAPPY)
+            .build();
+        let mut writer =
+            ArrowWriter::try_new(File::create(&path).unwrap(), arrow_schema, 
Some(props)).unwrap();
+        writer.write(&batch).unwrap();
+        writer.close().unwrap();
+        path
+    }
+
+    async fn try_read_variant(
+        schema: Arc<Schema>,
+        path: &str,
+        project_field_ids: Vec<i32>,
+        name_mapping: Option<Arc<NameMapping>>,
+    ) -> crate::Result<Vec<RecordBatch>> {
+        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), 
Runtime::current()).build();
+        let task = FileScanTask::builder()
+            .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len())
+            .with_start(0)
+            .with_length(0)
+            .with_data_file_path(path.to_string())
+            .with_data_file_format(DataFileFormat::Parquet)
+            .with_schema(schema)
+            .with_project_field_ids(project_field_ids)
+            .with_name_mapping(name_mapping)
+            .with_case_sensitive(false)
+            .build()?;
+        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as 
FileScanTaskStream;
+        reader.read(tasks)?.stream().try_collect().await
+    }
+
+    /// Reads `path`, checking that each batch's schema matches its columns, 
nested field
+    /// metadata included.
+    async fn read_variant(
+        schema: Arc<Schema>,
+        path: &str,
+        project_field_ids: Vec<i32>,
+    ) -> Vec<RecordBatch> {
+        let batches = try_read_variant(schema, path, project_field_ids, None)
+            .await
+            .unwrap();
+        for batch in &batches {
+            RecordBatch::try_new(batch.schema(), 
batch.columns().to_vec()).unwrap();
+        }
+        batches
+    }
+
+    /// `id` (int, id 1) + optional `v` (variant, id 2).
+    fn top_level_variant_schema() -> Arc<Schema> {
+        Arc::new(
+            Schema::builder()
+                .with_fields(vec![
+                    NestedField::required(1, "id", 
Type::Primitive(PrimitiveType::Int)).into(),
+                    NestedField::optional(2, "v", 
Type::Variant(VariantType)).into(),
+                ])
+                .build()
+                .unwrap(),
+        )
+    }
+
+    /// Writes [`top_level_variant_schema`] rows with `v` stored as 
`variant_field`.
+    fn write_top_level_variant_as(dir: &str, id_field: Field, variant_field: 
Field) -> String {
+        use arrow_array::{Int32Array, StructArray};
+        let DataType::Struct(storage) = variant_field.data_type() else {
+            unreachable!()
+        };
+        let variant = Arc::new(StructArray::new(
+            storage.clone(),
+            variant_struct_array().as_struct().columns().to_vec(),
+            None,
+        )) as ArrayRef;
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![id_field, 
variant_field]));
+        let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
+        write_variant_parquet(arrow_schema, vec![id, variant], dir)
+    }
+
+    fn write_top_level_variant(dir: &str) -> Arc<Schema> {
+        write_top_level_variant_as(
+            dir,
+            Field::new("id", DataType::Int32, 
false).with_metadata(field_id_meta(1)),
+            variant_arrow_field("v", 2),
+        );
+        top_level_variant_schema()
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_full_scan() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let schema = write_top_level_variant(dir);
+
+        let batches = read_variant(schema, &format!("{dir}/variant.parquet"), 
vec![1, 2]).await;
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].num_rows(), 3);
+        assert_id_values(batches[0].column_by_name("id").unwrap());
+        assert_top_level_variant(&batches[0]);
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_only() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let schema = write_top_level_variant(dir);
+
+        let batches = read_variant(schema, &format!("{dir}/variant.parquet"), 
vec![2]).await;
+        assert_eq!(batches[0].num_rows(), 3);
+        assert_top_level_variant(&batches[0]);
+        assert!(batches[0].column_by_name("id").is_none());
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_sibling_only_excludes_variant() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let schema = write_top_level_variant(dir);
+
+        let batches = read_variant(schema, &format!("{dir}/variant.parquet"), 
vec![1]).await;
+        assert_eq!(batches[0].num_rows(), 3);
+        assert_id_values(batches[0].column_by_name("id").unwrap());
+        assert!(batches[0].column_by_name("v").is_none());
+    }
+
+    /// Null variant rows stay null when the variant is rebuilt into the table 
type, here because
+    /// the file's `value` is required and the table type's is nullable.
+    #[tokio::test]
+    async fn test_read_variant_with_null_rows() {
+        use arrow_array::StructArray;
+        use arrow_buffer::NullBuffer;
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let with_null_row = |fields: Fields| {
+            Arc::new(StructArray::new(
+                fields,
+                variant_struct_array().as_struct().columns().to_vec(),
+                Some(NullBuffer::from(vec![true, false, true])),
+            )) as ArrayRef
+        };
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("v", DataType::Struct(variant_arrow_fields()), true)
+                .with_metadata(field_id_meta(2)),
+        ]));
+        let path = write_variant_parquet(
+            arrow_schema,
+            vec![with_null_row(variant_arrow_fields())],
+            dir,
+        );
+
+        let batches = read_variant(top_level_variant_schema(), &path, 
vec![2]).await;
+        let DataType::Struct(table_fields) =
+            
crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap()
+        else {
+            unreachable!()
+        };
+        let schema = batches[0].schema();
+        assert_eq!(
+            schema.field_with_name("v").unwrap().extension_type_name(),
+            Some("arrow.parquet.variant")
+        );
+        assert_eq!(
+            batches[0].column_by_name("v").unwrap(),
+            &with_null_row(table_fields)
+        );
+    }
+
+    /// A file written before `v` was added to the table reads `v` as null in 
every row.
+    #[tokio::test]
+    async fn test_read_variant_missing_from_file() {
+        use arrow_array::{Int32Array, new_null_array};
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, 
false).with_metadata(field_id_meta(1)),
+        ]));
+        let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
+        let path = write_variant_parquet(arrow_schema, vec![id], dir);
+        let table_type = 
crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap();
+
+        for project_field_ids in [vec![1, 2], vec![2]] {
+            let batches = read_variant(top_level_variant_schema(), &path, 
project_field_ids).await;
+            let schema = batches[0].schema();
+            assert_eq!(
+                schema.field_with_name("v").unwrap().extension_type_name(),
+                Some("arrow.parquet.variant")
+            );
+            assert_eq!(
+                batches[0].column_by_name("v").unwrap(),
+                &new_null_array(&table_type, 3)
+            );
+        }
+    }
+
+    /// The file stores the variant exactly as the table schema's Arrow type, 
so only the
+    /// extension has to be added.
+    #[tokio::test]
+    async fn test_read_variant_stored_as_table_type_gets_extension() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let storage = Fields::from(vec![
+            Field::new("metadata", DataType::Binary, false),
+            Field::new("value", DataType::Binary, true),
+        ]);
+        let path = write_top_level_variant_as(
+            dir,
+            Field::new("id", DataType::Int32, 
false).with_metadata(field_id_meta(1)),
+            Field::new("v", DataType::Struct(storage), 
true).with_metadata(field_id_meta(2)),
+        );
+
+        let batches = read_variant(top_level_variant_schema(), &path, vec![1, 
2]).await;
+        assert_id_values(batches[0].column_by_name("id").unwrap());
+        assert_top_level_variant(&batches[0]);
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_matches_storage_fields_by_name() {
+        use arrow_array::{LargeBinaryArray, StructArray};
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let storage = Fields::from(vec![
+            Field::new("value", DataType::LargeBinary, false),
+            Field::new("metadata", DataType::LargeBinary, false),
+        ]);
+        let variant = Arc::new(StructArray::new(
+            storage.clone(),
+            vec![
+                Arc::new(LargeBinaryArray::from(VARIANT_VALUE.to_vec())) as 
ArrayRef,
+                Arc::new(LargeBinaryArray::from(VARIANT_METADATA.to_vec())) as 
ArrayRef,
+            ],
+            None,
+        )) as ArrayRef;
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("v", DataType::Struct(storage), 
false).with_metadata(field_id_meta(2)),
+        ]));
+        let path = write_variant_parquet(arrow_schema, vec![variant], dir);
+
+        let batches = read_variant(top_level_variant_schema(), &path, 
vec![2]).await;
+        assert_top_level_variant(&batches[0]);
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_with_dictionary_storage() {
+        use arrow_array::BinaryArray;
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let dictionary =
+            DataType::Dictionary(Box::new(DataType::Int32), 
Box::new(DataType::Binary));
+        let storage = Fields::from(vec![
+            Field::new("metadata", dictionary.clone(), false),
+            Field::new("value", dictionary.clone(), false),
+        ]);
+        let encode = |bytes: [&[u8]; 3]| {
+            arrow_cast::cast(&BinaryArray::from(bytes.to_vec()), 
&dictionary).unwrap()
+        };
+        let variant = Arc::new(arrow_array::StructArray::new(
+            storage.clone(),
+            vec![encode(VARIANT_METADATA), encode(VARIANT_VALUE)],
+            None,
+        )) as ArrayRef;
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("v", DataType::Struct(storage), 
false).with_metadata(field_id_meta(2)),
+        ]));
+        let path = write_variant_parquet(arrow_schema, vec![variant], dir);
+
+        let batches = read_variant(top_level_variant_schema(), &path, 
vec![2]).await;
+        assert_top_level_variant(&batches[0]);
+    }
+
+    /// The variant group carries the Parquet VARIANT annotation, as 
iceberg-java writes it.
+    #[tokio::test]
+    async fn test_read_variant_with_parquet_annotation() {
+        use parquet::basic::{LogicalType, Repetition, Type as PhysicalType};
+        use parquet::data_type::{ByteArray, ByteArrayType, Int32Type};
+        use parquet::file::writer::SerializedFileWriter;
+        use parquet::schema::types::Type as ParquetType;
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let binary = |name: &str| {
+            Arc::new(
+                ParquetType::primitive_type_builder(name, 
PhysicalType::BYTE_ARRAY)
+                    .with_repetition(Repetition::REQUIRED)
+                    .build()
+                    .unwrap(),
+            )
+        };
+        let id = ParquetType::primitive_type_builder("id", PhysicalType::INT32)
+            .with_repetition(Repetition::REQUIRED)
+            .with_id(Some(1))
+            .build()
+            .unwrap();
+        let variant = ParquetType::group_type_builder("v")
+            .with_repetition(Repetition::OPTIONAL)
+            .with_logical_type(Some(LogicalType::variant(Some(1))))
+            .with_id(Some(2))
+            .with_fields(vec![binary("metadata"), binary("value")])
+            .build()
+            .unwrap();
+        let parquet_schema = ParquetType::group_type_builder("schema")
+            .with_fields(vec![Arc::new(id), Arc::new(variant)])
+            .build()
+            .unwrap();
+        let path = format!("{dir}/annotated.parquet");
+        let mut writer = SerializedFileWriter::new(
+            File::create(&path).unwrap(),
+            Arc::new(parquet_schema),
+            Arc::new(WriterProperties::builder().build()),
+        )
+        .unwrap();
+        let mut row_group = writer.next_row_group().unwrap();
+        let mut column = row_group.next_column().unwrap().unwrap();
+        column
+            .typed::<Int32Type>()
+            .write_batch(&[1, 2, 3], None, None)
+            .unwrap();
+        column.close().unwrap();
+        for bytes in [VARIANT_METADATA, VARIANT_VALUE] {
+            let values: Vec<ByteArray> =
+                bytes.iter().map(|b| ByteArray::from(b.to_vec())).collect();
+            let mut column = row_group.next_column().unwrap().unwrap();
+            column
+                .typed::<ByteArrayType>()
+                .write_batch(&values, Some(&[1, 1, 1]), None)
+                .unwrap();
+            column.close().unwrap();
+        }
+        row_group.close().unwrap();
+        writer.close().unwrap();
+
+        let batches = read_variant(top_level_variant_schema(), &path, vec![1, 
2]).await;
+        assert_id_values(batches[0].column_by_name("id").unwrap());
+        assert_top_level_variant(&batches[0]);
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_with_name_mapping() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let path = write_top_level_variant_as(
+            dir,
+            Field::new("id", DataType::Int32, false),
+            Field::new("v", DataType::Struct(variant_arrow_fields()), false),
+        );
+        let name_mapping = Arc::new(NameMapping::new(vec![
+            MappedField::new(Some(1), vec!["id".to_string()], vec![]),
+            MappedField::new(Some(2), vec!["v".to_string()], vec![]),
+        ]));
+
+        let batches = try_read_variant(
+            top_level_variant_schema(),
+            &path,
+            vec![1, 2],
+            Some(name_mapping),
+        )
+        .await
+        .unwrap();
+        assert_id_values(batches[0].column_by_name("id").unwrap());
+        assert_top_level_variant(&batches[0]);
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_without_field_ids_is_rejected() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let path = write_top_level_variant_as(
+            dir,
+            Field::new("id", DataType::Int32, false),
+            Field::new("v", DataType::Struct(variant_arrow_fields()), false),
+        );
+
+        let err = try_read_variant(top_level_variant_schema(), &path, vec![1, 
2], None)
+            .await
+            .unwrap_err();
+        assert_eq!(
+            (err.kind(), err.message()),
+            (
+                ErrorKind::FeatureUnsupported,
+                "Reading variant columns from Parquet files without field ids 
is not supported yet"
+            )
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_variant_nested_in_struct() {

Review Comment:
   Added test_read_variant_nested_in_null_struct_row in 133b42098. It makes the 
middle nested row null and covers both shapes: an optional payload, and a 
required payload inside the optional nested. Both read correctly. In the 
required shape payload has no null buffer of its own, and the test pins that. I 
also merged main to resolve the conflict with #3629.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to