alamb commented on code in PR #5525:
URL: https://github.com/apache/arrow-rs/pull/5525#discussion_r1535861073
##########
arrow-ipc/src/writer.rs:
##########
@@ -547,6 +564,57 @@ impl IpcDataGenerator {
}
}
+fn set_variadic_buffer_counts(counts: &mut Vec<i64>, array: &dyn Array) {
+ match array.data_type() {
+ DataType::BinaryView | DataType::Utf8View => {
+ // The spec is not clear on whether the view/null buffer should be
included in the variadic buffer count.
+ // But from C++ impl
https://github.com/apache/arrow/blob/b448b33808f2dd42866195fa4bb44198e2fc26b9/cpp/src/arrow/ipc/writer.cc#L477
+ // we know they are not included.
+ counts.push(array.to_data().buffers().len() as i64 - 1);
+ }
+ DataType::Struct(_) => {
+ let array = array.as_any().downcast_ref::<StructArray>().unwrap();
+ for column in array.columns() {
+ set_variadic_buffer_counts(counts, column.as_ref());
+ }
+ }
+ DataType::LargeList(_) => {
+ let array =
array.as_any().downcast_ref::<LargeListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::List(_) => {
+ let array = array.as_any().downcast_ref::<ListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::FixedSizeList(_, _) => {
+ let array =
array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::Dictionary(kt, _) => {
+ macro_rules! set_subarray_counts {
+ ($array:expr, $counts:expr, $type:ty, $variant:ident) => {
+ if &DataType::$variant == kt.as_ref() {
+ let array = $array
+ .as_any()
+ .downcast_ref::<DictionaryArray<$type>>()
+ .unwrap();
+ set_variadic_buffer_counts($counts, array.values());
+ }
+ };
+ }
+ set_subarray_counts!(array, counts, Int8Type, Int8);
+ set_subarray_counts!(array, counts, Int16Type, Int16);
+ set_subarray_counts!(array, counts, Int32Type, Int32);
+ set_subarray_counts!(array, counts, Int64Type, Int64);
+ set_subarray_counts!(array, counts, UInt8Type, UInt8);
+ set_subarray_counts!(array, counts, UInt16Type, UInt16);
+ set_subarray_counts!(array, counts, UInt32Type, UInt32);
+ set_subarray_counts!(array, counts, UInt64Type, UInt64);
+ }
+ _ => {}
Review Comment:
The reason I suggested it was that we have had issues where default `_` /
`..` matching has introduced bugs (basically because a new case was added but
not handled by the code) in DataFusion -- e.g.
https://github.com/apache/arrow-datafusion/pull/9521
However, I think it a preference and would be find to leave the code in this
PR as is
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
