This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new e6dbdbdeed Support Utf8View and BinaryView in substring (#10672)
e6dbdbdeed is described below
commit e6dbdbdeedd99121fa7166f1e6775bf956b55836
Author: Stefan Wang <[email protected]>
AuthorDate: Fri Aug 14 00:11:42 2026 -0700
Support Utf8View and BinaryView in substring (#10672)
# Which issue does this PR close?
- Closes #10671.
# Rationale for this change
`substring()` takes `Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`,
`FixedSizeBinary` and `Dictionary`, but rejects the view types, so a
caller holding a `StringViewArray` has to cast to `Utf8` first and pay
for the copy. The other string kernels in this crate already accept
them: `length`, `bit_length`, `like` and `concat_elements` all have
`Utf8View` arms.
# What changes are included in this PR?
`Utf8View` and `BinaryView` arms. Views carry their own offsets rather
than sharing one value buffer, so the byte range is computed per element
and the result is built through the view builder. The existing
`Dictionary` arm recurses, so a dictionary of views now works too.
The utf-8 boundary check applies to `Utf8View` the same way it does to
`Utf8`. The offset quoted in that error is per element, because there is
no shared buffer to index into.
# Are these changes tested?
Yes. The view result is compared against the `Utf8` and `Binary` result
for the same inputs, so the two paths cannot drift: empty strings,
nulls, values longer than the twelve-byte inline limit, zero length,
lengths past the end, and starts past either end in both directions.
Plus the boundary error and a dictionary of views.
The cross-check earned its keep — it caught the error offsets differing
between the two paths, which is what prompted the note above.
<details><summary>Tests</summary>
```
$ cargo test -p arrow-string
186 passed; 0 failed
```
Removing the two new match arms fails exactly the four new tests, so
they cannot pass vacuously:
```
test substring::tests::string_view_matches_utf8 ... FAILED
test substring::tests::binary_view_matches_binary ... FAILED
test substring::tests::string_view_rejects_an_invalid_char_boundary ...
FAILED
test substring::tests::dictionary_of_string_view ... FAILED
```
</details>
---------
Signed-off-by: 1fanwang <[email protected]>
---
arrow-string/src/substring.rs | 197 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 195 insertions(+), 2 deletions(-)
diff --git a/arrow-string/src/substring.rs b/arrow-string/src/substring.rs
index c8762499ca..66aece9d05 100644
--- a/arrow-string/src/substring.rs
+++ b/arrow-string/src/substring.rs
@@ -17,9 +17,10 @@
//! Defines kernel to extract a substring of an Array
//! Supported array types:
-//! [GenericStringArray], [GenericBinaryArray], [FixedSizeBinaryArray],
[DictionaryArray]
+//! [GenericStringArray], [GenericBinaryArray], [GenericByteViewArray],
+//! [FixedSizeBinaryArray], [DictionaryArray]
-use arrow_array::builder::BufferBuilder;
+use arrow_array::builder::{BinaryViewBuilder, BufferBuilder,
StringViewBuilder};
use arrow_array::cast::AsArray;
use arrow_array::types::*;
use arrow_array::*;
@@ -102,6 +103,8 @@ pub fn substring(
start as i32,
length.map(|e| e as i32),
),
+ DataType::Utf8View => string_view_substring(array.as_string_view(),
start, length),
+ DataType::BinaryView => binary_view_substring(array.as_binary_view(),
start, length),
_ => Err(ArrowError::ComputeError(format!(
"substring does not support type {:?}",
array.data_type()
@@ -247,6 +250,72 @@ fn utf8_bounds(val: &str, start: i64, length:
Option<usize>) -> (usize, usize) {
(start_offset, end_offset)
}
+/// Byte range of one element, following the same rules as [`byte_substring`].
+fn view_substring_range(
+ original_length: usize,
+ start: i64,
+ substring_length: Option<u64>,
+) -> (usize, usize) {
+ let original_length = original_length as i64;
+ let new_start = match start.cmp(&0) {
+ Ordering::Greater => start.min(original_length),
+ Ordering::Equal => 0,
+ Ordering::Less => (original_length + start).max(0),
+ };
+ let new_end = match substring_length {
+ Some(length) => new_start.saturating_add(length as
i64).min(original_length),
+ None => original_length,
+ };
+ (new_start as usize, new_end as usize)
+}
+
+fn string_view_substring(
+ array: &StringViewArray,
+ start: i64,
+ length: Option<u64>,
+) -> Result<ArrayRef, ArrowError> {
+ let mut builder = StringViewBuilder::with_capacity(array.len());
+
+ for idx in 0..array.len() {
+ if array.is_null(idx) {
+ builder.append_null();
+ continue;
+ }
+ let value = array.value(idx);
+ let (new_start, new_end) = view_substring_range(value.len(), start,
length);
+ for offset in [new_start, new_end] {
+ if !value.is_char_boundary(offset) {
+ return Err(ArrowError::ComputeError(format!(
+ "The offset {offset} is at an invalid utf-8 boundary."
+ )));
+ }
+ }
+ builder.append_value(&value[new_start..new_end]);
+ }
+
+ Ok(Arc::new(builder.finish()))
+}
+
+fn binary_view_substring(
+ array: &BinaryViewArray,
+ start: i64,
+ length: Option<u64>,
+) -> Result<ArrayRef, ArrowError> {
+ let mut builder = BinaryViewBuilder::with_capacity(array.len());
+
+ for idx in 0..array.len() {
+ if array.is_null(idx) {
+ builder.append_null();
+ continue;
+ }
+ let value = array.value(idx);
+ let (new_start, new_end) = view_substring_range(value.len(), start,
length);
+ builder.append_value(&value[new_start..new_end]);
+ }
+
+ Ok(Arc::new(builder.finish()))
+}
+
fn byte_substring<T: ByteArrayType>(
array: &GenericByteArray<T>,
start: T::Offset,
@@ -1035,4 +1104,128 @@ mod tests {
let expected = BinaryArray::from(vec![Some(expected_bytes)]);
assert_eq!(expected, *actual);
}
+
+ #[test]
+ fn string_view_matches_utf8() {
+ let values = vec![
+ Some("hello world"),
+ Some(""),
+ None,
+ Some("a"),
+ Some("this one is definitely longer than twelve bytes"),
+ ];
+ let utf8 = StringArray::from(values.clone());
+ let view = StringViewArray::from(values);
+
+ for (start, length) in [
+ (0, None),
+ (0, Some(0)),
+ (0, Some(5)),
+ (0, Some(1000)),
+ (1, Some(3)),
+ (5, None),
+ (100, Some(2)),
+ (100, None),
+ (-3, None),
+ (-3, Some(2)),
+ (-100, Some(4)),
+ (-100, None),
+ ] {
+ let expected = substring(&utf8, start, length).unwrap();
+ let expected = expected.as_string::<i32>();
+ let actual = substring(&view, start, length).unwrap();
+ let actual = actual.as_string_view();
+ assert_eq!(
+ expected.iter().collect::<Vec<_>>(),
+ actual.iter().collect::<Vec<_>>(),
+ "start={start} length={length:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn binary_view_matches_binary() {
+ let values: Vec<Option<&[u8]>> = vec![
+ Some(b"hello world"),
+ Some(b""),
+ None,
+ Some(b"abc"),
+ Some(b"this one is definitely longer than twelve bytes"),
+ ];
+ let binary = BinaryArray::from(values.clone());
+ let view = BinaryViewArray::from(values);
+
+ for (start, length) in [
+ (0, None),
+ (0, Some(5)),
+ (2, Some(3)),
+ (-3, None),
+ (100, Some(2)),
+ ] {
+ let expected = substring(&binary, start, length).unwrap();
+ let expected = expected.as_binary::<i32>();
+ let actual = substring(&view, start, length).unwrap();
+ let actual = actual.as_binary_view();
+ assert_eq!(
+ expected.iter().collect::<Vec<_>>(),
+ actual.iter().collect::<Vec<_>>(),
+ "start={start} length={length:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn string_view_slices_on_a_multi_byte_boundary() {
+ // "é" and "ö" are two bytes, the CJK chars three, so 0/3/6 are the
offsets that land
+ // on a char boundary in both values.
+ let values = vec![Some("héllo wörld"), Some("日本語"), None];
+ let utf8 = StringArray::from(values.clone());
+ let view = StringViewArray::from(values);
+
+ for (start, length) in [
+ (0, Some(3)),
+ (3, Some(3)),
+ (0, Some(6)),
+ (3, None),
+ (-3, None),
+ (-6, None),
+ ] {
+ let expected = substring(&utf8, start, length).unwrap();
+ let expected = expected.as_string::<i32>();
+ let actual = substring(&view, start, length).unwrap();
+ let actual = actual.as_string_view();
+ assert_eq!(
+ expected.iter().collect::<Vec<_>>(),
+ actual.iter().collect::<Vec<_>>(),
+ "start={start} length={length:?}"
+ );
+ }
+
+ let actual = substring(&view, 0, Some(3)).unwrap();
+ let actual = actual.as_string_view();
+ assert_eq!(actual.value(0), "hé");
+ assert_eq!(actual.value(1), "日");
+ }
+
+ #[test]
+ fn string_view_rejects_an_invalid_char_boundary() {
+ let view = StringViewArray::from(vec![Some("E=mc²")]);
+ let err = substring(&view, 0, Some(5)).unwrap_err().to_string();
+ assert!(err.contains("invalid utf-8 boundary"), "{err}");
+ }
+
+ #[test]
+ fn dictionary_of_string_view() {
+ let view = StringViewArray::from(vec![Some("hello world"),
Some("bye")]);
+ let dict = DictionaryArray::new(Int32Array::from(vec![0, 1, 0]),
Arc::new(view));
+
+ let actual = substring(&dict, 0, Some(3)).unwrap();
+ let actual = actual.as_any_dictionary();
+
+ let values = actual.values().as_string_view();
+ assert_eq!(
+ values.iter().collect::<Vec<_>>(),
+ vec![Some("hel"), Some("bye")]
+ );
+ }
}
