This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new eecf5f3870 fix(arrow-array): reset the lone offset of an empty FFI
byte array (#11661)
eecf5f3870 is described below
commit eecf5f3870d7e8e13cb5ba58ddf1c811defa631e
Author: Chang Yong Lik <[email protected]>
AuthorDate: Sun Oct 25 16:16:27 2026 +0000
fix(arrow-array): reset the lone offset of an empty FFI byte array (#11661)
# Which issue does this PR close?
Closes #11585.
# Rationale for this change
`StringArray::slice(n, 0)` keeps the offset it was sliced at as the
array's single offset. When such an array is exported and imported
again, `buffer_len` sizes the values buffer of an empty array at offset
0 as 0, because that lone offset can be any value from the producer (see
`test_empty_string_with_non_zero_offset`). The offset itself was kept,
so the imported `ArrayData` failed `validate_full` with `First offset 2
of Utf8 is larger than values length 0`.
# What changes are included in this PR?
In `ImportedArrowArray::consume`, an empty
`Utf8`/`Binary`/`LargeUtf8`/`LargeBinary` array at offset 0 gets a
zeroed offsets buffer, which matches the empty values buffer. Sizing the
values buffer from the lone offset instead would read past the buffer
for producers like the one in `test_empty_string_with_non_zero_offset`
(offset 123 over an empty buffer).
# Are these changes tested?
Yes. `test_empty_slice_of_bytes` round-trips `slice(1, 0)` of each of
the four types through `to_ffi`/`from_ffi`, then checks `validate_full`
and equality with the exported data. It fails without the fix with the
error above. `cargo test -p arrow-array --features ffi` and `--features
ffi,force_validate` pass, and `cargo clippy -p arrow-array --all-targets
--all-features -- -D warnings` is clean.
# Are there any user-facing changes?
No API changes. Empty sliced byte arrays imported through the C Data
Interface now pass validation.
AI use: Claude Code (AI) helped trace the import path and wrote the
first draft of the fix, the test and this description; I reviewed and
ran everything.
---
arrow-array/src/ffi.rs | 82 +++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 78 insertions(+), 4 deletions(-)
diff --git a/arrow-array/src/ffi.rs b/arrow-array/src/ffi.rs
index 1ee6ee1a9c..d96c940547 100644
--- a/arrow-array/src/ffi.rs
+++ b/arrow-array/src/ffi.rs
@@ -293,7 +293,19 @@ impl ImportedArrowArray<'_> {
};
let data_layout = layout(&self.data_type);
- let buffers = self.buffers(data_layout.can_contain_null_mask,
data_layout.variadic)?;
+ let mut buffers = self.buffers(data_layout.can_contain_null_mask,
data_layout.variadic)?;
+
+ // The values buffer of an empty array at offset 0 is imported as
empty (see
+ // `buffer_len`), so its lone offset, which can be any value, is reset
to 0.
+ if len == 0
+ && offset == 0
+ && matches!(
+ self.data_type,
+ DataType::Utf8 | DataType::Binary | DataType::LargeUtf8 |
DataType::LargeBinary
+ )
+ {
+ buffers[0] =
MutableBuffer::from_len_zeroed(buffers[0].len()).into();
+ }
let null_bit_buffer = if data_layout.can_contain_null_mask {
self.null_bit_buffer()
@@ -1402,10 +1414,11 @@ mod tests_from_ffi {
use crate::{
ArrayRef, GenericByteViewArray, ListArray,
array::{
- Array, BooleanArray, DictionaryArray, FixedSizeBinaryArray,
FixedSizeListArray,
- Int32Array, Int64Array, StringArray, StructArray, UInt32Array,
UInt64Array,
+ Array, BinaryArray, BooleanArray, DictionaryArray,
FixedSizeBinaryArray,
+ FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray,
LargeStringArray,
+ StringArray, StructArray, UInt32Array, UInt64Array,
},
- ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi},
+ ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi, to_ffi},
make_array,
};
@@ -1741,6 +1754,67 @@ mod tests_from_ffi {
Ok(())
}
+ /// An empty slice keeps the offset it was sliced at as its lone offset,
which
+ /// must not make the imported array invalid.
+ ///
+ /// <https://github.com/apache/arrow-rs/issues/11585>
+ #[test]
+ fn test_empty_slice_of_bytes() -> Result<()> {
+ let arrays: [ArrayRef; 4] = [
+ Arc::new(StringArray::from(vec!["ab", "c"])),
+ Arc::new(BinaryArray::from(vec![b"ab".as_ref(), b"c"])),
+ Arc::new(LargeStringArray::from(vec!["ab", "c"])),
+ Arc::new(LargeBinaryArray::from(vec![b"ab".as_ref(), b"c"])),
+ ];
+ for array in arrays {
+ let data = array.slice(1, 0).to_data();
+ let (array, schema) = to_ffi(&data)?;
+ let imported = unsafe { from_ffi(array, &schema) }?;
+
+ imported.validate_full()?;
+ assert_eq!(imported, data);
+ }
+
+ Ok(())
+ }
+
+ /// Unlike bytes, the values of lists and maps are a child array imported
with its
+ /// own length, so the lone offset of an empty slice stays within it.
+ #[test]
+ fn test_empty_slice_of_lists() -> Result<()> {
+ use crate::builder::{
+ Int32Builder, LargeListBuilder, ListBuilder, MapBuilder,
StringBuilder,
+ };
+
+ let mut list = ListBuilder::new(Int32Builder::new());
+ list.append_value([Some(1), Some(2)]);
+ list.append_value([Some(3)]);
+ let mut large_list = LargeListBuilder::new(Int32Builder::new());
+ large_list.append_value([Some(1), Some(2)]);
+ large_list.append_value([Some(3)]);
+ let mut map = MapBuilder::new(None, StringBuilder::new(),
Int32Builder::new());
+ for (k, v) in [("a", 1), ("b", 2)] {
+ map.keys().append_value(k);
+ map.values().append_value(v);
+ map.append(true).unwrap();
+ }
+ let arrays: [ArrayRef; 3] = [
+ Arc::new(list.finish()),
+ Arc::new(large_list.finish()),
+ Arc::new(map.finish()),
+ ];
+ for array in arrays {
+ let data = array.slice(1, 0).to_data();
+ let (array, schema) = to_ffi(&data)?;
+ let imported = unsafe { from_ffi(array, &schema) }?;
+
+ imported.validate_full()?;
+ assert_eq!(imported, data);
+ }
+
+ Ok(())
+ }
+
fn roundtrip_string_array(array: StringArray) -> StringArray {
let data = array.into_data();
