This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23473-17ec0833bdca68add1c359ef04551d1320d10c65 in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit d4160119db8c6b348928af100393c23783caf05e Author: Andy Grove <[email protected]> AuthorDate: Fri Jul 17 09:18:58 2026 -0600 perf: avoid per-row copy in Spark hex byte encoding (#23473) ## Which issue does this PR close? N/A ## Rationale for this change The Spark `hex` function's byte-encoding path (`hex_encode_bytes`) filled a reusable per-row `Vec` and then copied it into a `StringBuilder` for every row. Since each input byte maps to exactly two hex characters, the output can be written once directly into a single value buffer, eliminating the per-row copy. ## What changes are included in this PR? Rewrote `hex_encode_bytes` to write hex digits directly into one growing value buffer with manually-tracked offsets and a `NullBufferBuilder`, constructing the `StringArray` via `new_unchecked` (the buffer contains only ASCII hex digits, so the existing no-validation behavior is preserved). The 256-entry lookup table and all other hex paths are unchanged. ## Are these changes tested? Existing tests (`function::math::hex`) pass unchanged. Benchmark (`datafusion/spark/benches/hex.rs`): - hex_binary/1024·4096·8192: ~13.8% / 14.3% / 13.9% faster - hex_utf8/1024·4096·8192: ~14.1% / 14.9% / 15.0% faster ## Are there any user-facing changes? No Co-authored-by: Andrew Lamb <[email protected]> --- datafusion/spark/src/function/math/hex.rs | 44 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rs index c7b82d5373..a283bd8fa7 100644 --- a/datafusion/spark/src/function/math/hex.rs +++ b/datafusion/spark/src/function/math/hex.rs @@ -18,7 +18,8 @@ use std::str::from_utf8_unchecked; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, StringBuilder}; +use arrow::array::{Array, ArrayRef, NullBufferBuilder, StringArray, StringBuilder}; +use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::datatypes::DataType; use arrow::{ array::{as_dictionary_array, as_largestring_array, as_string_array}, @@ -167,40 +168,59 @@ where I: Iterator<Item = Option<T>>, T: AsRef<[u8]> + 'a, { - let mut builder = StringBuilder::with_capacity(len, len * 64); - let mut buffer = Vec::with_capacity(64); let lookup = if lowercase { &HEX_LOOKUP_LOWER } else { &HEX_LOOKUP_UPPER }; + // Write hex digits directly into one growing value buffer, tracking offsets + // ourselves. Each input byte becomes exactly two output bytes, so there is + // no per-row `String`/`StringBuilder` copy — the hex digits are written once + // into the final buffer. + let mut values: Vec<u8> = Vec::with_capacity(len * 64); + let mut offsets: Vec<i32> = Vec::with_capacity(len + 1); + offsets.push(0); + let mut nulls = NullBufferBuilder::new(len); + for v in iter { if let Some(b) = v { let bytes = b.as_ref(); - buffer.clear(); let additional = bytes .len() .checked_mul(2) .ok_or_else(|| exec_datafusion_err!("hex output size overflow"))?; - buffer.try_reserve(additional).map_err(|e| { + values.try_reserve(additional).map_err(|e| { exec_datafusion_err!( "failed to reserve {additional} bytes for hex output: {e}" ) })?; for &byte in bytes { - buffer.extend_from_slice(&lookup[byte as usize]); - } - // SAFETY: buffer contains only ASCII hex digits, which are valid UTF-8. - unsafe { - builder.append_value(from_utf8_unchecked(&buffer)); + values.extend_from_slice(&lookup[byte as usize]); } + nulls.append_non_null(); } else { - builder.append_null(); + nulls.append_null(); } + offsets.push( + i32::try_from(values.len()).map_err(|_| { + exec_datafusion_err!("hex output exceeds i32 offset range") + })?, + ); } - Ok(Arc::new(builder.finish())) + // SAFETY: the value buffer contains only ASCII hex digits (valid UTF-8) and + // the offsets are monotonically increasing and end at `values.len()`, so the + // array invariants hold. This mirrors the previous `from_utf8_unchecked` + // path and avoids a redundant UTF-8 validation pass over the whole buffer. + let array = unsafe { + StringArray::new_unchecked( + OffsetBuffer::new(offsets.into()), + Buffer::from_vec(values), + nulls.finish(), + ) + }; + Ok(Arc::new(array)) } /// Generic hex encoding for int64 type --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
