lyne7-sc commented on code in PR #19738:
URL: https://github.com/apache/datafusion/pull/19738#discussion_r2682397849
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
Review Comment:
Both `Int64Array` and `Dictionary<Int64>` paths need hex encoding and yield
different iterator types. Perhaps using a generic iterator here could help
share the same encoding logic?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
