lyne7-sc commented on code in PR #19738:
URL: https://github.com/apache/datafusion/pull/19738#discussion_r2682733920
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
Review Comment:
Understood, I've refactored the dictionary handling to explicitly check the
validity bitmap of the values array using `is_valid`.
And I've also incorporated your test case to ensure this edge case is
covered. Thanks for the detailed explanation and the doc link!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
