alamb commented on code in PR #6998:
URL: https://github.com/apache/arrow-rs/pull/6998#discussion_r1927214791
##########
arrow-ipc/src/writer.rs:
##########
@@ -2433,6 +2430,126 @@ mod tests {
);
}
+ #[test]
+ fn test_large_slice_uint32() {
+ ensure_roundtrip(Arc::new(UInt32Array::from_iter((0..8000).map(|i| {
+ if i % 2 == 0 {
+ Some(i)
+ } else {
+ None
+ }
+ }))));
+ }
+
+ #[test]
+ fn test_large_slice_string() {
+ let strings: Vec<_> = (0..8000)
+ .map(|i| {
+ if i % 2 == 0 {
+ Some(format!("value{}", i))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ ensure_roundtrip(Arc::new(StringArray::from(strings)));
+ }
+
+ #[test]
+ fn test_large_slice_string_list() {
+ let mut ls = ListBuilder::new(StringBuilder::new());
+
+ let mut s = String::new();
+ for row_number in 0..8000 {
+ if row_number % 2 == 0 {
+ for list_element in 0..1000 {
+ s.clear();
+ use std::fmt::Write;
+ write!(&mut s,
"value{row_number}-{list_element}").unwrap();
+ ls.values().append_value(&s);
+ }
+ ls.append(true)
+ } else {
+ ls.append(false); // null
+ }
+ }
+
+ ensure_roundtrip(Arc::new(ls.finish()));
+ }
+
+ #[test]
+ fn test_large_slice_string_list_of_lists() {
+ // The reason for the special test is to verify reencode_offsets which
looks both at
+ // the starting offset and the data offset. So need a dataset where
the starting_offset
+ // is zero but the data offset is not.
Review Comment:
👍 Ah, Lists of Lists 🤯
I verified this fails without the tests in this PR like this:
```
the offset of the new Buffer cannot exceed the existing length: slice
offset=4 length=24004 selflen=24004
thread 'writer::tests::test_large_slice_string_list_of_lists' panicked at
arrow-buffer/src/buffer/immutable.rs:281:9:
the offset of the new Buffer cannot exceed the existing length: slice
offset=4 length=24004 selflen=24004
```
💯
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
