laskoviymishka commented on code in PR #3683:
URL: https://github.com/apache/iceberg-rust/pull/3683#discussion_r4611065510


##########
crates/iceberg/src/arrow/record_batch_transformer.rs:
##########
@@ -494,19 +524,49 @@ impl PromotePlan {
                     *sorted,
                 )?))
             }
+            PromotePlan::Unshred {
+                target_type,
+                name,
+                field_id,
+            } => {
+                let (storage_fields, columns, nulls) = array
+                    .as_struct_opt()
+                    .ok_or_else(|| promote_err(array, "struct"))?
+                    .clone()
+                    .into_parts();
+                // `unshred_variant` skips null rows, so mark the rows under a 
null parent as
+                // null. The output keeps the nulls of `array`, because Arrow 
rejects a null in
+                // a required child of a valid row.
+                let skip = arrow_buffer::NullBuffer::union(parent_nulls, 
nulls.as_ref());
+                let storage = StructArray::try_new(storage_fields, columns, 
skip)?;
+                let unshredded = VariantArray::try_new(&storage)
+                    .and_then(|variant| unshred_variant(&variant))
+                    .map_err(|e| {
+                        invalid_data!(
+                            "Variant column `{name}` (field id {field_id}) has 
invalid shredded data: {e}"

Review Comment:
   Could we attach the arrow error with `.with_source(e)` instead of formatting 
it into the message? 
[`invalid_data!`](https://github.com/apache/iceberg-rust/blob/9964df486f7252747ded2bbb4f693ee71d214fab/crates/iceberg/src/error.rs#L500-L508)
 builds a plain `Error::new`, so the `ArrowError` from `VariantArray::try_new` 
/ `unshred_variant` survives only as text and callers can't reach it through 
`source()`.



##########
crates/iceberg/src/arrow/record_batch_transformer.rs:
##########
@@ -1917,6 +1977,42 @@ mod test {
         assert_eq!(promoted.as_struct(), &expected);
     }
 
+    /// Only an object can have both `value` and `typed_value`.
+    #[test]
+    fn promote_shredded_variant_rejects_invalid_row() {
+        use arrow_array::{BinaryArray, Int64Array};
+        let source = Arc::new(StructArray::new(
+            Fields::from(vec![
+                Field::new("metadata", DataType::Binary, false),
+                Field::new("value", DataType::Binary, true),
+                Field::new("typed_value", DataType::Int64, true),
+            ]),
+            vec![
+                Arc::new(BinaryArray::from(vec![b"\x01\x00\x00".as_ref()])) as 
ArrayRef,
+                Arc::new(BinaryArray::from(vec![Some(b"\x0c\x01".as_ref())])) 
as ArrayRef,
+                Arc::new(Int64Array::from(vec![Some(5)])) as ArrayRef,
+            ],
+            None,
+        )) as ArrayRef;
+
+        let err = PromotePlan::build_field(
+            &Field::new("v", source.data_type().clone(), true),
+            &variant_target(),
+            &empty_schema(),
+            &HashMap::new(),
+        )
+        .unwrap()
+        .apply(&source)
+        .unwrap_err();
+        assert_eq!(
+            (err.kind(), err.message()),
+            (
+                ErrorKind::DataInvalid,
+                "Variant column `v` (field id 2) has invalid shredded data: 
Invalid argument error: Invalid shredded variant: both value and typed_value 
are non-null"

Review Comment:
   Could this check only our part of the message? The tail is arrow-rs wording 
from 
[`unshred_variant.rs`](https://github.com/apache/arrow-rs/blob/b391e5af7dff9f6909af45c01f7dddb68ba08c0f/parquet-variant-compute/src/unshred_variant.rs#L436-L439),
 in a crate that [calls itself work in 
progress](https://github.com/apache/arrow-rs/blob/b391e5af7dff9f6909af45c01f7dddb68ba08c0f/parquet-variant-compute/src/lib.rs#L27-L29),
 so an upstream rewording would fail this test with no behavior change. With 
the source chained, the kind plus our prefix would be enough.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to