This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 4a1c655c75 fix(arrow-data): account for view payload buffers in slice
memory size (#10519)
4a1c655c75 is described below
commit 4a1c655c75d86ea5c9064e01985d2cf075150e9e
Author: wterrr <[email protected]>
AuthorDate: Tue Aug 4 06:38:58 2026 +0700
fix(arrow-data): account for view payload buffers in slice memory size
(#10519)
Closes #8230.
## Rationale
`ArrayData::get_slice_memory_size` accounts for the fixed-width view
buffer of `Utf8View` and `BinaryView` arrays, but previously omitted
their variadic payload buffers.
Slicing these arrays retains all payload buffers unchanged, so this
could substantially underestimate the memory retained by the slice.
## Changes
- Include the capacity of all trailing variadic payload buffers in
`get_slice_memory_size`
- Document the retained-buffer accounting behavior for variadic layouts
- Add regression coverage for both `Utf8View` and `BinaryView`
- Cover inline-only values, multiple payload buffers, and payload
capacity greater than logical length
## Testing
- `cargo test -p arrow-data --lib test_slice_memory_size`
- `cargo test -p arrow-data --all-features`
- `cargo clippy -p arrow-data --all-targets --all-features -- -D
warnings`
- `cargo fmt --all -- --check`
All affected crate tests, doc tests, formatting, and focused Clippy
checks pass.
## AI assistance
Claude was used to help investigate the issue, draft the implementation
and regression tests, and review the change. I reviewed the final change
and take responsibility for the submitted code.
---
arrow-data/src/data.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 61 insertions(+), 1 deletion(-)
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
index 2c9dfe6ddb..465c96352b 100644
--- a/arrow-data/src/data.rs
+++ b/arrow-data/src/data.rs
@@ -516,7 +516,9 @@ impl ArrayData {
///
/// This is approximately the number of bytes if a new
/// [`ArrayData`] was formed by creating new [`Buffer`]s with
- /// exactly the data needed.
+ /// exactly the data needed. For variadic layouts, this includes the full
+ /// capacity of every variadic buffer retained by a zero-copy slice,
without
+ /// inspecting which buffers or ranges are referenced by the slice.
///
/// For example, a [`DataType::Int64`] with `100` elements,
/// [`Self::get_slice_memory_size`] would return `100 * 8 = 800`. If
@@ -578,6 +580,13 @@ impl ArrayData {
}
}
+ if layout.variadic {
+ // Slicing view arrays retains all variadic data buffers unchanged.
+ for buffer in self.buffers.iter().skip(layout.buffers.len()) {
+ result += buffer.capacity();
+ }
+ }
+
if self.nulls().is_some() {
result += bit_util::ceil(self.len, 8);
}
@@ -2344,6 +2353,7 @@ pub(crate) fn get_fixed_size_binary_width(data_type:
&DataType) -> usize {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::ByteView;
use arrow_buffer::{OffsetBuffer, ScalarBuffer};
use arrow_schema::{Field, Fields};
@@ -2628,6 +2638,56 @@ mod tests {
assert!(!string_data_slice.ptr_eq(&string_data))
}
+ #[test]
+ fn test_slice_memory_size_view_payload_buffers() {
+ for data_type in [DataType::Utf8View, DataType::BinaryView] {
+ let inline_only = ArrayData::builder(data_type.clone())
+ .len(2)
+ .add_buffer(Buffer::from_vec(vec![0_u128; 2]))
+ .build()
+ .unwrap();
+ assert_eq!(
+ inline_only.get_slice_memory_size().unwrap(),
+ 2 * mem::size_of::<u128>()
+ );
+
+ let mut first_payload = Vec::with_capacity(32);
+ first_payload.extend_from_slice(b"first payload");
+ let first_view =
+ ByteView::new(first_payload.len().try_into().unwrap(),
&first_payload[..4])
+ .as_u128();
+ let first_payload = Buffer::from_vec(first_payload);
+ assert!(first_payload.capacity() > first_payload.len());
+ let first_payload_capacity = first_payload.capacity();
+
+ let mut second_payload = Vec::with_capacity(64);
+ second_payload.extend_from_slice(b"second payload");
+ let second_view = ByteView::new(
+ second_payload.len().try_into().unwrap(),
+ &second_payload[..4],
+ )
+ .with_buffer_index(1)
+ .as_u128();
+ let second_payload = Buffer::from_vec(second_payload);
+ assert!(second_payload.capacity() > second_payload.len());
+ let second_payload_capacity = second_payload.capacity();
+
+ let data = ArrayData::builder(data_type)
+ .len(3)
+ .add_buffer(Buffer::from_vec(vec![first_view, 0_u128,
second_view]))
+ .add_buffer(first_payload)
+ .add_buffer(second_payload)
+ .build()
+ .unwrap();
+ let sliced = data.slice(1, 1);
+
+ assert_eq!(
+ sliced.get_slice_memory_size().unwrap(),
+ mem::size_of::<u128>() + first_payload_capacity +
second_payload_capacity
+ );
+ }
+ }
+
#[test]
fn test_slice_memory_size_utf8_offset_buffer_len_plus_one() {
// 2-element array ["hello", "world"]: array len = 2, 10 bytes
