scovich commented on code in PR #8354:
URL: https://github.com/apache/arrow-rs/pull/8354#discussion_r2357571633
##########
parquet-variant-compute/src/variant_get.rs:
##########
@@ -1122,64 +1121,109 @@ mod test {
let result = variant_get(&array, options).unwrap();
// Should get StringArray
- let expected: ArrayRef =
Arc::new(StringArray::from(vec![Some("comedy"), Some("drama")]));
+ let expected: ArrayRef =
+ Arc::new(StringArray::from(vec![Some("comedy"), None,
Some("drama")]));
assert_eq!(&result, &expected);
}
/// Helper function to create a shredded variant array representing lists
///
/// This creates an array that represents:
/// Row 0: ["comedy", "drama"] ([0] is shredded, [1] is shredded -
perfectly shredded)
- /// Row 1: ["horror", null] ([0] is shredded, [1] is binary null -
partially shredded)
- /// Row 2: ["comedy", "drama", "romance"] (perfectly shredded)
+ /// Row 1: ["horror", 123] ([0] is shredded, [1] is int - partially
shredded)
///
/// The physical layout follows the shredding spec where:
/// - metadata: contains list metadata
/// - typed_value: StructArray with 0 index value
/// - value: contains fallback for
fn shredded_list_variant_array() -> ArrayRef {
- // Create the base metadata for lists
-
- // Could add this as an api for VariantList, like VariantList::from()
- fn build_list_metadata(vector: Vec<Variant>) -> (Vec<u8>, Vec<u8>) {
- let mut builder = parquet_variant::VariantBuilder::new();
- let mut list = builder.new_list();
- for value in vector {
- list.append_value(value);
- }
- list.finish();
- builder.finish()
- }
- let (metadata1, _) =
- build_list_metadata(vec![Variant::String("comedy"),
Variant::String("drama")]);
+ // Create metadata array
+ let metadata_array =
+
BinaryViewArray::from_iter_values(std::iter::repeat_n(EMPTY_VARIANT_METADATA_BYTES,
2));
- let (metadata2, _) =
build_list_metadata(vec![Variant::String("horror"), Variant::Null]);
+ // Building the typed_value ListArray
- let (metadata3, _) = build_list_metadata(vec![
- Variant::String("comedy"),
- Variant::String("drama"),
- Variant::String("romance"),
+ // Need a StructBuilder to create a ListBuilder
+ let fields = Fields::from(vec![
+ Field::new("value", DataType::Binary, true),
+ Field::new("typed_value", DataType::Utf8, true),
]);
+ let field_builders = vec![
+ make_builder(&DataType::Binary, 4),
+ make_builder(&DataType::Utf8, 4),
+ ];
+ let struct_builder = StructBuilder::new(fields, field_builders);
+
+ let mut builder = GenericListBuilder::<i32,
StructBuilder>::new(struct_builder);
+
+ // Row 0 index 0
+ builder
+ .values()
+ .field_builder::<BinaryBuilder>(0)
+ .unwrap()
+ .append_null();
+ builder
+ .values()
+ .field_builder::<StringBuilder>(1)
+ .unwrap()
+ .append_value("comedy");
+ builder.values().append(true);
+
+ // Row 0 index 1
+ builder
+ .values()
+ .field_builder::<BinaryBuilder>(0)
+ .unwrap()
+ .append_null();
+ builder
+ .values()
+ .field_builder::<StringBuilder>(1)
+ .unwrap()
+ .append_value("drama");
+ builder.values().append(true);
+
+ // Next row
+ builder.append(true);
+
+ // Row 1 index 0
+ builder
+ .values()
+ .field_builder::<BinaryBuilder>(0)
+ .unwrap()
+ .append_null();
+ builder
+ .values()
+ .field_builder::<StringBuilder>(1)
+ .unwrap()
+ .append_value("horror");
+ builder.values().append(true);
+
+ // Row 1 index 1
+ let mut variant_builder = VariantBuilder::new();
+ variant_builder.append_value(123i32); // <------ couldn't find the
right way to do it, used this as placeholder for binary
+ let (_, value) = variant_builder.finish();
+
+ builder
+ .values()
+ .field_builder::<BinaryBuilder>(0)
+ .unwrap()
+ .append_value(value);
+ builder
+ .values()
+ .field_builder::<StringBuilder>(1)
+ .unwrap()
+ .append_null();
+ builder.values().append(true);
+
+ // Next row
+ builder.append(true);
+
+ let typed_value_array = builder.finish();
- // Create metadata array
- let metadata_array =
- BinaryViewArray::from_iter_values(vec![metadata1, metadata2,
metadata3]);
-
- // Create the untyped value array
- let value_array =
BinaryViewArray::from(vec![Variant::Null.as_u8_slice()]);
- // Maybe I should try with an actual primitive array
- let typed_value_array = StringArray::from(vec![
- Some("comedy"),
- Some("drama"),
- Some("horror"),
- Some("comedy"),
- Some("drama"),
- Some("romance"),
- ]);
// Build the main VariantArray
let main_struct = crate::variant_array::StructArrayBuilder::new()
- .with_field("metadata", Arc::new(metadata_array))
- .with_field("value", Arc::new(value_array))
- .with_field("typed_value", Arc::new(typed_value_array))
+ .with_field("metadata", Arc::new(metadata_array), false)
+ // .with_field("value", Arc::new(value_array), true)
Review Comment:
There are two different `value` fields:
```
array_col: {
value: BINARY, -- for non-array variant objects
typed_value: {
elements: {
value: BINARY, -- for wrong-type array elements
typed_value: <ELEMENT TYPE>,
}
},
}
```
AFAIK, both those columns can be missing if every row contains a list with
all-correct element types.
Note that variant arrays cannot contain (SQL) NULL tho -- so the list
`[Some(1), None, Some(3)]` would produce a NULL `elements.typed_value` with
`Variant::Null` in `elements.value`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
