This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new d4160119db perf: avoid per-row copy in Spark hex byte encoding (#23473)
d4160119db is described below
commit d4160119db8c6b348928af100393c23783caf05e
Author: Andy Grove <[email protected]>
AuthorDate: Fri Jul 17 09:18:58 2026 -0600
perf: avoid per-row copy in Spark hex byte encoding (#23473)
## Which issue does this PR close?
N/A
## Rationale for this change
The Spark `hex` function's byte-encoding path (`hex_encode_bytes`)
filled a reusable per-row `Vec` and then copied it into a
`StringBuilder` for every row. Since each input byte maps to exactly two
hex characters, the output can be written once directly into a single
value buffer, eliminating the per-row copy.
## What changes are included in this PR?
Rewrote `hex_encode_bytes` to write hex digits directly into one growing
value buffer with manually-tracked offsets and a `NullBufferBuilder`,
constructing the `StringArray` via `new_unchecked` (the buffer contains
only ASCII hex digits, so the existing no-validation behavior is
preserved). The 256-entry lookup table and all other hex paths are
unchanged.
## Are these changes tested?
Existing tests (`function::math::hex`) pass unchanged.
Benchmark (`datafusion/spark/benches/hex.rs`):
- hex_binary/1024·4096·8192: ~13.8% / 14.3% / 13.9% faster
- hex_utf8/1024·4096·8192: ~14.1% / 14.9% / 15.0% faster
## Are there any user-facing changes?
No
Co-authored-by: Andrew Lamb <[email protected]>
---
datafusion/spark/src/function/math/hex.rs | 44 ++++++++++++++++++++++---------
1 file changed, 32 insertions(+), 12 deletions(-)
diff --git a/datafusion/spark/src/function/math/hex.rs
b/datafusion/spark/src/function/math/hex.rs
index c7b82d5373..a283bd8fa7 100644
--- a/datafusion/spark/src/function/math/hex.rs
+++ b/datafusion/spark/src/function/math/hex.rs
@@ -18,7 +18,8 @@
use std::str::from_utf8_unchecked;
use std::sync::Arc;
-use arrow::array::{Array, ArrayRef, StringBuilder};
+use arrow::array::{Array, ArrayRef, NullBufferBuilder, StringArray,
StringBuilder};
+use arrow::buffer::{Buffer, OffsetBuffer};
use arrow::datatypes::DataType;
use arrow::{
array::{as_dictionary_array, as_largestring_array, as_string_array},
@@ -167,40 +168,59 @@ where
I: Iterator<Item = Option<T>>,
T: AsRef<[u8]> + 'a,
{
- let mut builder = StringBuilder::with_capacity(len, len * 64);
- let mut buffer = Vec::with_capacity(64);
let lookup = if lowercase {
&HEX_LOOKUP_LOWER
} else {
&HEX_LOOKUP_UPPER
};
+ // Write hex digits directly into one growing value buffer, tracking
offsets
+ // ourselves. Each input byte becomes exactly two output bytes, so there is
+ // no per-row `String`/`StringBuilder` copy — the hex digits are written
once
+ // into the final buffer.
+ let mut values: Vec<u8> = Vec::with_capacity(len * 64);
+ let mut offsets: Vec<i32> = Vec::with_capacity(len + 1);
+ offsets.push(0);
+ let mut nulls = NullBufferBuilder::new(len);
+
for v in iter {
if let Some(b) = v {
let bytes = b.as_ref();
- buffer.clear();
let additional = bytes
.len()
.checked_mul(2)
.ok_or_else(|| exec_datafusion_err!("hex output size
overflow"))?;
- buffer.try_reserve(additional).map_err(|e| {
+ values.try_reserve(additional).map_err(|e| {
exec_datafusion_err!(
"failed to reserve {additional} bytes for hex output: {e}"
)
})?;
for &byte in bytes {
- buffer.extend_from_slice(&lookup[byte as usize]);
- }
- // SAFETY: buffer contains only ASCII hex digits, which are valid
UTF-8.
- unsafe {
- builder.append_value(from_utf8_unchecked(&buffer));
+ values.extend_from_slice(&lookup[byte as usize]);
}
+ nulls.append_non_null();
} else {
- builder.append_null();
+ nulls.append_null();
}
+ offsets.push(
+ i32::try_from(values.len()).map_err(|_| {
+ exec_datafusion_err!("hex output exceeds i32 offset range")
+ })?,
+ );
}
- Ok(Arc::new(builder.finish()))
+ // SAFETY: the value buffer contains only ASCII hex digits (valid UTF-8)
and
+ // the offsets are monotonically increasing and end at `values.len()`, so
the
+ // array invariants hold. This mirrors the previous `from_utf8_unchecked`
+ // path and avoids a redundant UTF-8 validation pass over the whole buffer.
+ let array = unsafe {
+ StringArray::new_unchecked(
+ OffsetBuffer::new(offsets.into()),
+ Buffer::from_vec(values),
+ nulls.finish(),
+ )
+ };
+ Ok(Arc::new(array))
}
/// Generic hex encoding for int64 type
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
