Dandandan commented on code in PR #19975:
URL: https://github.com/apache/datafusion/pull/19975#discussion_r2727451991
##########
datafusion/physical-expr-common/src/binary_view_map.rs:
##########
@@ -250,52 +265,92 @@ where
// step 2: insert each value into the set, if not already present
let values = values.as_byte_view::<B>();
+ // Get raw views buffer for direct comparison
+ let input_views = values.views();
+
// Ensure lengths are equivalent
- assert_eq!(values.len(), batch_hashes.len());
+ assert_eq!(values.len(), self.hashes_buffer.len());
+
+ for i in 0..values.len() {
+ let view_u128 = input_views[i];
+ let hash = self.hashes_buffer[i];
- for (value, &hash) in values.iter().zip(batch_hashes.iter()) {
- // handle null value
- let Some(value) = value else {
+ // handle null value via validity bitmap check
+ if !values.is_valid(i) {
let payload = if let Some(&(payload, _offset)) =
self.null.as_ref() {
payload
} else {
let payload = make_payload_fn(None);
- let null_index = self.builder.len();
- self.builder.append_null();
+ let null_index = self.views.len();
+ self.views.push(0);
+ self.nulls.append(false); // false = null in validity
buffer
self.null = Some((payload, null_index));
payload
};
observe_payload_fn(payload);
continue;
- };
+ }
- // get the value as bytes
- let value: &[u8] = value.as_ref();
+ // Extract length from the view (first 4 bytes of u128 in
little-endian)
+ let len = (view_u128 & 0xFFFFFFFF) as u32;
Review Comment:
`view_u128 as u32` does the same thing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
