This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23456-3e058f09c91a18c8f36b71a656231603b33888c2 in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 5b60fcb1107b4d9b4dffa11ff10ed8083b5e3a66 Author: Andy Grove <[email protected]> AuthorDate: Sat Jul 11 00:34:51 2026 -0600 perf: optimize encode in datafusion-functions (#23456) ## Which issue does this PR close? N/A ## Rationale for this change Rewrote the hex path of encode() to write hex directly into one pre-sized buffer via hex::encode_to_slice, eliminating a per-element String allocation and copy per row. ## What changes are included in this PR? Rewrote the hex path of encode() to write hex directly into one pre-sized buffer via hex::encode_to_slice, eliminating a per-element String allocation and copy per row. ## Are these changes tested? Existing tests Benchmark: - hex_encode_1024: 80.934% faster (base 25305ns -> cand 4824ns) - hex_encode_4096: 81.297% faster (base 99386ns -> cand 18587ns) - hex_encode_8192: 80.24% faster (base 200031ns -> cand 39525ns) ## Are there any user-facing changes? No <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --- datafusion/functions/benches/encoding.rs | 25 +++++++++++++++++ datafusion/functions/src/encoding/inner.rs | 45 ++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/datafusion/functions/benches/encoding.rs b/datafusion/functions/benches/encoding.rs index 0b8f0c5c51..451baff518 100644 --- a/datafusion/functions/benches/encoding.rs +++ b/datafusion/functions/benches/encoding.rs @@ -27,10 +27,35 @@ use std::sync::Arc; fn criterion_benchmark(c: &mut Criterion) { let decode = encoding::decode(); + let encode = encoding::encode(); let config_options = Arc::new(ConfigOptions::default()); for size in [1024, 4096, 8192] { let bin_array = Arc::new(create_binary_array::<i32>(size, 0.2)); + + c.bench_function(&format!("hex_encode/{size}"), |b| { + let method = ColumnarValue::Scalar("hex".into()); + let arg_fields = vec![ + Field::new("a", bin_array.data_type().to_owned(), true).into(), + Field::new("b", method.data_type().to_owned(), true).into(), + ]; + let args = vec![ColumnarValue::Array(bin_array.clone()), method]; + let return_field = Field::new("f", DataType::Utf8, true).into(); + + b.iter(|| { + black_box( + encode + .invoke_with_args(ScalarFunctionArgs { + args: args.clone(), + arg_fields: arg_fields.clone(), + number_rows: size, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config_options), + }) + .unwrap(), + ) + }) + }); c.bench_function(&format!("base64_decode/{size}"), |b| { let method = ColumnarValue::Scalar("base64".into()); let encoded = encoding::encode() diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index 877acbb529..027ec8e5e5 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -410,11 +410,7 @@ impl Encoding { .collect(); Ok(Arc::new(array)) } - Self::Hex => { - let array: GenericStringArray<OutputOffset> = - array.iter().map(|x| x.map(hex::encode)).collect(); - Ok(Arc::new(array)) - } + Self::Hex => hex_encode_array::<_, OutputOffset>(array), } } @@ -459,6 +455,45 @@ impl Encoding { } } +/// Hex-encode a binary array into a string array, writing the lowercase hex +/// digits directly into a single pre-sized value buffer. Each input byte maps +/// to exactly two hex characters, so the output size is known up front and no +/// per-element `String` is allocated. +fn hex_encode_array<'a, InputBinaryArray, OutputOffset>( + array: &InputBinaryArray, +) -> Result<ArrayRef> +where + InputBinaryArray: BinaryArrayType<'a>, + OutputOffset: OffsetSizeTrait, +{ + let total_input_bytes: usize = array.iter().flatten().map(|v| v.len()).sum(); + + let mut values = vec![0u8; total_input_bytes * 2]; + let mut offsets = Vec::<OutputOffset>::with_capacity(array.len() + 1); + offsets.push(OutputOffset::zero()); + + let mut pos = 0usize; + for v in array.iter() { + if let Some(v) = v { + let out_len = v.len() * 2; + // The slice is sized to exactly `2 * v.len()`, which is the only + // condition under which `encode_to_slice` can fail, so this cannot + // error. + hex::encode_to_slice(v, &mut values[pos..pos + out_len]) + .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?; + pos += out_len; + } + offsets.push(OutputOffset::usize_as(pos)); + } + + let array = GenericStringArray::<OutputOffset>::try_new( + OffsetBuffer::new(offsets.into()), + Buffer::from_vec(values), + array.nulls().cloned(), + )?; + Ok(Arc::new(array)) +} + fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>( decode: DecodeFunction, input: &InputBinaryArray, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
