Rich-T-kid commented on code in PR #11071:
URL: https://github.com/apache/arrow-rs/pull/11071#discussion_r4473005831
##########
parquet-variant-compute/src/variant_array.rs:
##########
@@ -1564,6 +1573,204 @@ mod test {
}
}
+ #[test]
+ fn dictionary_inputs_preserve_nulls_and_slices() {
+ use arrow::array::{BooleanArray, DictionaryArray};
+ use arrow::compute::cast;
+
+ let values =
+ VariantArray::from_iter([Some(Variant::Int32(123)), None,
Some(Variant::Int32(-45))]);
+ let raw_values = BinaryArray::from(vec![
+ binary_array_value(values.value_column(), 0),
+ None,
+ binary_array_value(values.value_column(), 2),
+ ]);
+ let mut cases = Vec::new();
+ for data_type in [
+ DataType::Binary,
+ DataType::LargeBinary,
+ DataType::BinaryView,
+ ] {
+ cases.push(("value", cast(&raw_values, &data_type).unwrap()));
+ }
+ let typed_values: Vec<ArrayRef> = vec![
+ Arc::new(Int32Array::from(vec![Some(123), None, Some(-45)])),
+ Arc::new(StringArray::from(vec![Some("hello"), None,
Some("world")])),
+ Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)])),
+ Arc::new(
+ Decimal128Array::from(vec![Some(123), None, Some(-45)])
+ .with_precision_and_scale(9, 2)
+ .unwrap(),
+ ),
+ Arc::new(
+ FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+ [Some([1u8; 16]), None, Some([2u8; 16])].into_iter(),
+ 16,
+ )
+ .unwrap(),
+ ),
+ ];
+ cases.extend(
+ typed_values
+ .into_iter()
+ .map(|values| ("typed_value", values)),
+ );
+
+ for (name, values) in cases {
+ for key_type in [
+ DataType::Int8,
+ DataType::Int16,
+ DataType::Int32,
+ DataType::Int64,
+ DataType::UInt8,
+ DataType::UInt16,
+ DataType::UInt32,
+ DataType::UInt64,
+ ] {
+ // Slice starts after the first key, and includes both a null
dictionary value
+ // and a null key. The final non-null value is masked by the
parent.
+ let keys = Int8Array::from(vec![Some(2), Some(0), Some(1),
None, Some(2), Some(0)]);
+ let dictionary = DictionaryArray::<Int8Type>::new(keys,
values.clone());
+ let dictionary = cast(
+ &dictionary,
+ &DataType::Dictionary(Box::new(key_type),
Box::new(values.data_type().clone())),
+ )
+ .unwrap();
+ let metadata: ArrayRef =
+
Arc::new(BinaryViewArray::from(vec![EMPTY_VARIANT_METADATA_BYTES; 6]));
+ let metadata = cast(
+ &metadata,
+ &DataType::Dictionary(Box::new(DataType::Int8),
Box::new(DataType::BinaryView)),
+ )
+ .unwrap();
+ let metadata_type = metadata.data_type().clone();
+ let mut builder =
StructArrayBuilder::new().with_field("metadata", metadata, false);
Review Comment:
**Blocking:** Current replaced with , so a clean merge fails with . Please
rebase, use , and unwrap before slicing:\n\n
##########
parquet-variant-compute/src/variant_array.rs:
##########
@@ -787,6 +787,7 @@ impl ShreddedVariantFieldArray {
/// or be a list, large_list, list_view or struct
///
pub fn try_new(inner: &dyn Array) -> Result<Self> {
+ let inner = canonicalize_shredded_types(inner)?;
Review Comment:
**Should-fix:** Both constructors still describe only the canonical and
types. Please document that supported scalar arrays may be dictionary-encoded,
are decoded into canonical storage during construction, and may allocate. The
comment should mention too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
