neilconway commented on code in PR #26377:
URL: https://github.com/apache/datafusion/pull/26377#discussion_r4285909022
##########
datafusion/functions/src/strings.rs:
##########
@@ -829,38 +832,78 @@ impl StringViewArrayBuilder {
Self::make_long_view_checked(length, buffer_index, offset,
prefix_bytes)
}
- /// See [`BulkNullStringArrayBuilder::append_byte_map`].
+ /// Fallible variant of [`Self::append_byte_map`].
///
/// # Safety
///
/// The bytes produced by applying `map` to each byte of `src`, in order,
/// must form valid UTF-8.
///
- /// # Panics
+ /// # Errors
///
- /// Panics under the same conditions as [`Self::append_value`]: if
- /// `src.len()`, the in-progress buffer offset, or the number of completed
- /// buffers exceeds `i32::MAX`.
+ /// Returns an error under the same conditions as
+ /// [`Self::try_append_value`].
#[inline]
- pub unsafe fn append_byte_map<F: FnMut(u8) -> u8>(&mut self, src: &[u8],
mut map: F) {
- let length: u32 =
- i32::try_from(src.len()).expect("value length exceeds i32::MAX")
as u32;
- if length <= 12 {
- let mut bytes = [0u8; 12];
- for (d, &b) in bytes[..src.len()].iter_mut().zip(src) {
+ pub unsafe fn try_append_byte_map<F: FnMut(u8) -> u8>(
+ &mut self,
+ src: &[u8],
+ mut map: F,
+ ) -> Result<()> {
+ if src.len() <= MAX_INLINE_LEN {
+ // An inline view holds the length in its low 4 bytes, followed by
+ // the value zero-padded to 12 bytes. Writing the mapped bytes
+ // straight into it is faster than going through `inline_view`.
+ let mut view = (src.len() as u128).to_le_bytes();
+ for (d, &b) in view[4..].iter_mut().zip(src) {
*d = map(b);
}
- self.views.push(make_view(&bytes[..src.len()], 0, 0));
- return;
+ self.views.push(u128::from_le_bytes(view));
+ return Ok(());
}
- self.ensure_long_capacity(length);
+ let length: u32 = i32::try_from(src.len())
+ .map_err(|_| string_view_overflow_error("value length"))?
+ as u32;
+ self.try_ensure_long_capacity(length)?;
let cursor = self.in_progress.len();
- let offset: u32 = i32::try_from(cursor).expect("offset exceeds
i32::MAX") as u32;
+ let offset: u32 = i32::try_from(cursor)
+ .map_err(|_| string_view_overflow_error("offset"))?
+ as u32;
+ let buffer_index: u32 = i32::try_from(self.completed.len())
+ .map_err(|_| string_view_overflow_error("buffer count"))?
+ as u32;
self.in_progress.extend(src.iter().map(|&b| map(b)));
Review Comment:
The performance problem was observed with `MutableBuffer::extend()`
(separately, that could probably be optimized; looking into it now). The
Utf8View code path does `Vec::extend()`, which is fast. I double-checked this
and switching the vec path to copy-then-map is about 1.5x slower on ARM.
So I think we're okay as-is here.
##########
datafusion/functions/src/string/common.rs:
##########
@@ -390,38 +390,48 @@ fn unicode_case(s: &str, lower: bool) -> String {
}
}
-/// Like [`unicode_case`], but uppercases into `buf` so its allocation is
reused
-/// across rows. Lowercasing still allocates, see [`write_unicode_case`].
+/// Appends `s` to `builder` with its case converted.
+///
+/// An ASCII value is converted byte by byte straight into the builder. Other
+/// values go through `str::to_lowercase` / `str::to_uppercase`, which allocate
+/// a `String` but convert any leading run of ASCII bytes in bulk.
#[inline]
-fn unicode_case_into<'a>(s: &str, lower: bool, buf: &'a mut String) -> &'a str
{
- if lower {
- *buf = s.to_lowercase();
- } else {
- buf.clear();
- buf.extend(s.chars().flat_map(char::to_uppercase));
+fn append_case_converted<O: OffsetSizeTrait>(
+ builder: &mut GenericStringArrayBuilder<O>,
+ s: &str,
+ lower: bool,
+) -> Result<()> {
Review Comment:
Thanks, extended the comment to discuss this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
