tustvold commented on code in PR #5525:
URL: https://github.com/apache/arrow-rs/pull/5525#discussion_r1538322420
##########
arrow-ipc/src/writer.rs:
##########
@@ -547,6 +564,57 @@ impl IpcDataGenerator {
}
}
+fn set_variadic_buffer_counts(counts: &mut Vec<i64>, array: &dyn Array) {
+ match array.data_type() {
+ DataType::BinaryView | DataType::Utf8View => {
+ // The spec is not clear on whether the view/null buffer should be
included in the variadic buffer count.
+ // But from C++ impl
https://github.com/apache/arrow/blob/b448b33808f2dd42866195fa4bb44198e2fc26b9/cpp/src/arrow/ipc/writer.cc#L477
+ // we know they are not included.
+ counts.push(array.to_data().buffers().len() as i64 - 1);
+ }
+ DataType::Struct(_) => {
+ let array = array.as_any().downcast_ref::<StructArray>().unwrap();
+ for column in array.columns() {
+ set_variadic_buffer_counts(counts, column.as_ref());
+ }
+ }
+ DataType::LargeList(_) => {
+ let array =
array.as_any().downcast_ref::<LargeListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::List(_) => {
+ let array = array.as_any().downcast_ref::<ListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::FixedSizeList(_, _) => {
+ let array =
array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ set_variadic_buffer_counts(counts, array.values());
+ }
+ DataType::Dictionary(kt, _) => {
+ macro_rules! set_subarray_counts {
+ ($array:expr, $counts:expr, $type:ty, $variant:ident) => {
+ if &DataType::$variant == kt.as_ref() {
+ let array = $array
+ .as_any()
+ .downcast_ref::<DictionaryArray<$type>>()
+ .unwrap();
+ set_variadic_buffer_counts($counts, array.values());
+ }
+ };
+ }
+ set_subarray_counts!(array, counts, Int8Type, Int8);
+ set_subarray_counts!(array, counts, Int16Type, Int16);
+ set_subarray_counts!(array, counts, Int32Type, Int32);
+ set_subarray_counts!(array, counts, Int64Type, Int64);
+ set_subarray_counts!(array, counts, UInt8Type, UInt8);
+ set_subarray_counts!(array, counts, UInt16Type, UInt16);
+ set_subarray_counts!(array, counts, UInt32Type, UInt32);
+ set_subarray_counts!(array, counts, UInt64Type, UInt64);
+ }
+ _ => {}
Review Comment:
Makes sense, https://github.com/apache/arrow-rs/pull/5525/files#r1538321869
suggests a way we could avoid needing this explicit matching at all
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
