AdamGS commented on code in PR #25122:
URL: https://github.com/apache/datafusion/pull/25122#discussion_r4006909637
##########
datafusion/functions/src/core/getfield.rs:
##########
@@ -182,6 +191,37 @@ fn process_map_with_nested_key(
Ok(ColumnarValue::Array(data))
}
+/// Apply a struct's nulls to one of its fields.
+fn apply_parent_nulls(col: &ArrayRef, parent_nulls: &NullBuffer) ->
Result<ArrayRef> {
Review Comment:
done, does make things simpler
##########
datafusion/functions/src/core/getfield.rs:
##########
@@ -733,6 +780,160 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_get_field_nested_struct_outer_nulls() -> Result<()> {
+ let inner_array = StructArray::new(
+ vec![Field::new("value", DataType::Int32, false)].into(),
+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+ None,
+ );
+
+ // Only the outer struct marks row 1 as null; its children are all
valid.
+ let outer_array = StructArray::new(
+ vec![Field::new("inner", inner_array.data_type().clone(),
false)].into(),
+ vec![Arc::new(inner_array)],
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+
+ let inner = extract_single_field(
+ ColumnarValue::Array(Arc::new(outer_array)),
+ ScalarValue::Utf8(Some("inner".to_string())),
+ )?;
+ let result =
+ extract_single_field(inner,
ScalarValue::Utf8(Some("value".to_string())))?
+ .into_array(3)?;
+
+ let expected = Int32Array::from(vec![Some(1), None, Some(3)]);
+ assert_eq!(result.as_ref(), &expected as &dyn Array);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_get_field_map_parent_nulls() -> Result<()> {
+ use arrow::array::{FixedSizeListArray, MapArray};
+ use arrow_buffer::OffsetBuffer;
+
+ let keys = Arc::new(Int32Array::from(vec![7; 3])) as ArrayRef;
+ let nested_keys = Arc::new(FixedSizeListArray::new(
+ Arc::new(Field::new("item", DataType::Int32, false)),
+ 1,
+ Arc::clone(&keys),
+ None,
+ )) as ArrayRef;
+
+ // Exercise both map lookup paths. The null map has a valid matching
entry.
+ for keys in [keys, nested_keys] {
+ let key = ScalarValue::try_from_array(keys.as_ref(), 0)?;
+ let entries = StructArray::new(
+ vec![
+ Field::new("key", keys.data_type().clone(), false),
+ Field::new("value", DataType::Int32, false),
+ ]
+ .into(),
+ vec![keys, Arc::new(Int32Array::from(vec![1, 2, 3]))],
+ None,
+ );
+ let map = MapArray::new(
+ Arc::new(Field::new("entries", entries.data_type().clone(),
false)),
+ OffsetBuffer::new(vec![0, 1, 2, 3].into()),
+ entries,
+ Some(NullBuffer::from(vec![true, false, true])),
+ false,
+ );
+ let result =
extract_single_field(ColumnarValue::Array(Arc::new(map)), key)?
+ .into_array(3)?;
+ let expected = Int32Array::from(vec![Some(1), None, Some(3)]);
+ assert_eq!(result.as_ref(), &expected as &dyn Array);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn test_get_field_null_typed_child() -> Result<()> {
+ use arrow::array::{DictionaryArray, NullArray, UInt32Array};
+ use arrow::datatypes::UInt32Type;
+
+ let values = Arc::new(StructArray::new(
+ vec![Field::new("value", DataType::Null, true)].into(),
+ vec![Arc::new(NullArray::new(2))],
+ Some(NullBuffer::from(vec![true, false])),
+ )) as ArrayRef;
+ let dictionary = Arc::new(DictionaryArray::<UInt32Type>::try_new(
+ UInt32Array::from(vec![0, 1]),
+ Arc::clone(&values),
+ )?) as ArrayRef;
+
+ for input in [values, dictionary] {
+ let result = extract_single_field(
+ ColumnarValue::Array(input),
+ ScalarValue::Utf8(Some("value".to_string())),
+ )?
+ .into_array(2)?;
+ assert_eq!(result.logical_null_count(), 2);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn test_get_field_union_parent_nulls() -> Result<()> {
+ use arrow::array::{DictionaryArray, StringArray, UInt32Array};
+ use arrow::datatypes::UInt32Type;
+
+ let fields = UnionFields::try_new(
+ [3, 7],
+ [
+ Field::new("int", DataType::Int32, true),
+ Field::new("string", DataType::Utf8, true),
+ ],
+ )?;
+ let ints = Int32Array::from(vec![Some(9), Some(1), Some(2), None,
Some(4)]);
+ let strings = StringArray::from(vec!["x"; 5]);
+
+ // Dense rows 1 and 2 share a value, but only row 2 has a null parent.
+ for offsets in [None, Some(vec![0, 1, 1, 3, 0].into())] {
+ let child = UnionArray::try_new(
+ fields.clone(),
+ vec![7, 3, 3, 3, 7].into(),
+ offsets,
+ vec![Arc::new(ints.clone()), Arc::new(strings.clone())],
+ )?;
+ let union_type = child.data_type().clone();
+ let parent = StructArray::new(
+ vec![Field::new("u", union_type.clone(), true)].into(),
+ vec![Arc::new(child)],
+ Some(NullBuffer::from(vec![true, true, false, true, true])),
+ );
+ let values = Arc::new(parent.slice(1, 4)) as ArrayRef;
Review Comment:
added that case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
