kosiew commented on code in PR #24590:
URL: https://github.com/apache/datafusion/pull/24590#discussion_r3844802130
##########
datafusion/common/src/nested_struct.rs:
##########
@@ -263,23 +276,127 @@ fn cast_list_view_column<O:
arrow::array::OffsetSizeTrait>(
cast_options: &CastOptions,
) -> Result<ArrayRef> {
let source_list = source_col.as_list_view::<O>();
+ let compacted_values = compact_list_view_values(source_list)?;
+ let (offsets, sizes, values) = match compacted_values.as_ref() {
+ Some((offsets, sizes, values)) => (offsets, sizes, values),
+ None => (
+ source_list.offsets(),
+ source_list.sizes(),
+ source_list.values(),
+ ),
+ };
- let cast_values = cast_column(
- source_list.values(),
- target_inner_field.data_type(),
- cast_options,
- )?;
+ let cast_values = cast_column(values, target_inner_field.data_type(),
cast_options)?;
let result = GenericListViewArray::<O>::try_new(
Arc::clone(target_inner_field),
- source_list.offsets().clone(),
- source_list.sizes().clone(),
+ offsets.clone(),
+ sizes.clone(),
cast_values,
source_list.nulls().cloned(),
)?;
Ok(Arc::new(result))
}
+fn compact_list_values<O: arrow::array::OffsetSizeTrait>(
+ list: &GenericListArray<O>,
+) -> Result<GenericListArray<O>> {
+ let indices = UInt64Array::from_iter_values(0..list.len() as u64);
+ Ok(take(list, &indices, None)?.as_list::<O>().clone())
+}
+
Review Comment:
Could we also add a regression test for an all-null `ListView` with invalid
backing values? That would exercise the empty `merged_ranges` compaction path,
which is a little different from the current null-plus-valid-row case, and
confirm that we don't try to cast any unreachable values when all parent rows
are null. This is non-blocking.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
