Jefffrey commented on code in PR #19738:
URL: https://github.com/apache/datafusion/pull/19738#discussion_r2682529393
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
Review Comment:
Ah I didn't see that dictionary used it like that. However that made me
realise this edge case:
```rust
#[test]
fn test_dict_values_null() {
let keys = Int32Array::from(vec![Some(0), None, Some(1)]);
let vals = Int64Array::from(vec![Some(32), None]);
// [32, null, null]
let dict = DictionaryArray::new(keys, Arc::new(vals));
let columnar_value = ColumnarValue::Array(Arc::new(dict));
let result = super::spark_hex(&[columnar_value]).unwrap();
let result = match result {
ColumnarValue::Array(array) => array,
_ => panic!("Expected array"),
};
let result = as_string_array(&result);
let expected = StringArray::from(vec![Some("20"), None, None]);
assert_eq!(&expected, result);
}
```
- When both key & value arrays have nulls
On main this succeeds because the 2nd element is null from the key (aka from
dictionary) but 3rd element is null from the underlying values array.
On this PR it seems it will ignore any nulls in the values and treat it as
non-null.
See doc on dictionaries:
https://arrow.apache.org/docs/format/Columnar.html#dictionary-encoded-layout
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
