alamb commented on code in PR #23223:
URL: https://github.com/apache/datafusion/pull/23223#discussion_r3535910574
##########
datafusion/functions/src/strings.rs:
##########
@@ -343,6 +343,12 @@ fn string_view_overflow_error(field: &str) ->
DataFusionError {
exec_datafusion_err!("byte array offset overflow: {field} exceeds
i32::MAX")
}
+fn try_string_view_part(field: &str, value: usize) -> Result<u32> {
+ // StringView stores these fields as u32, but Arrow limits lengths,
Review Comment:
I think it is unlikely buffer indices will ever overflow a i32 -- that would
mean there are 2 billion *buffers* (not values)
However, I see this is already checked below
##########
datafusion/functions/src/strings.rs:
##########
@@ -818,96 +765,117 @@ impl StringViewArrayBuilder {
.into()
}
- #[inline]
- fn make_long_view(&self, length: u32, offset: u32, prefix_bytes: &[u8]) ->
u128 {
- let buffer_index: u32 = i32::try_from(self.completed.len())
- .expect("buffer count exceeds i32::MAX")
- as u32;
- Self::make_long_view_checked(length, buffer_index, offset,
prefix_bytes)
- }
-
- /// See [`BulkNullStringArrayBuilder::append_byte_map`].
+ /// Fallibly append a row whose bytes are produced by mapping each byte of
+ /// `src` through `map`, in order.
///
/// # Safety
///
/// The bytes produced by applying `map` to each byte of `src`, in order,
/// must form valid UTF-8.
///
- /// # Panics
+ /// # Errors
///
- /// Panics under the same conditions as [`Self::append_value`]: if
- /// `src.len()`, the in-progress buffer offset, or the number of completed
- /// buffers exceeds `i32::MAX`.
+ /// Returns an error under the same conditions as
[`Self::try_append_value`].
#[inline]
- pub unsafe fn append_byte_map<F: FnMut(u8) -> u8>(&mut self, src: &[u8],
mut map: F) {
- let length: u32 =
- i32::try_from(src.len()).expect("value length exceeds i32::MAX")
as u32;
+ pub unsafe fn try_append_byte_map<F: FnMut(u8) -> u8>(
+ &mut self,
+ src: &[u8],
+ mut map: F,
+ ) -> Result<()> {
+ let length = try_string_view_part("value length", src.len())?;
if length <= 12 {
let mut bytes = [0u8; 12];
for (d, &b) in bytes[..src.len()].iter_mut().zip(src) {
*d = map(b);
}
self.views.push(make_view(&bytes[..src.len()], 0, 0));
- return;
+ return Ok(());
}
- self.ensure_long_capacity(length);
+ self.try_ensure_long_capacity(length)?;
let cursor = self.in_progress.len();
- let offset: u32 = i32::try_from(cursor).expect("offset exceeds
i32::MAX") as u32;
+ let offset = try_string_view_part("offset", cursor)?;
+ let buffer_index = try_string_view_part("buffer count",
self.completed.len())?;
self.in_progress.extend(src.iter().map(|&b| map(b)));
- self.views
- .push(self.make_long_view(length, offset,
&self.in_progress[cursor..]));
+ self.views.push(Self::make_long_view_checked(
+ length,
+ buffer_index,
+ offset,
+ &self.in_progress[cursor..],
+ ));
+ Ok(())
+ }
+
+ #[inline]
+ fn push_view_from_writer(
+ &mut self,
+ inline_buf: &[u8; 12],
+ inline_len: u8,
+ spill_cursor: Option<usize>,
+ ) -> Result<()> {
+ match spill_cursor {
+ None => {
+ self.views
+ .push(make_view(&inline_buf[..inline_len as usize], 0, 0));
+ }
+ Some(start) => {
+ let end = self.in_progress.len();
+ let length = try_string_view_part("value length", end -
start)?;
+ let offset = try_string_view_part("offset", start)?;
+ let buffer_index =
+ try_string_view_part("buffer count",
self.completed.len())?;
+ self.views.push(Self::make_long_view_checked(
Review Comment:
It is strange to me that this is called `make_long_view_checked` but doesn't
seem to return an error
##########
datafusion/functions/src/strings.rs:
##########
@@ -969,30 +937,70 @@ pub(crate) struct StringViewWriter<'a> {
/// `None` while the row fits inline; becomes `Some(start)` (offset of
/// the row's first byte in `in_progress`) at first spill.
spill_cursor: Option<usize>,
+ error: Option<DataFusionError>,
Review Comment:
why does this need delayed error handling? It seems like the code would be a
lot simpler if the error was returned immediately
If we do need delayed reporting, can we please add a comment about why ?
##########
datafusion/functions/src/strings.rs:
##########
@@ -969,30 +937,70 @@ pub(crate) struct StringViewWriter<'a> {
/// `None` while the row fits inline; becomes `Some(start)` (offset of
/// the row's first byte in `in_progress`) at first spill.
spill_cursor: Option<usize>,
+ error: Option<DataFusionError>,
builder: &'a mut StringViewArrayBuilder,
}
+impl StringViewWriter<'_> {
+ #[inline]
Review Comment:
can we please document in comments what this method is doing (likewise for
write_spilled_bytes)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
