This is an automated email from the ASF dual-hosted git repository.
Rachelint pushed a commit to branch improve-compare-in-view-map
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/improve-compare-in-view-map by
this push:
new bddc47a826 Skip byte view hashing on input cache hits
bddc47a826 is described below
commit bddc47a826ec7b47bfd8015e9992de21a34faafb
Author: kamille <[email protected]>
AuthorDate: Thu Jul 9 08:37:38 2026 +0800
Skip byte view hashing on input cache hits
---
.../physical-expr-common/src/binary_view_map.rs | 35 ++++++++++++----------
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs
b/datafusion/physical-expr-common/src/binary_view_map.rs
index 2df967e32d..4d44c35916 100644
--- a/datafusion/physical-expr-common/src/binary_view_map.rs
+++ b/datafusion/physical-expr-common/src/binary_view_map.rs
@@ -25,8 +25,7 @@ use arrow::array::{
};
use arrow::buffer::{Buffer, ScalarBuffer};
use arrow::datatypes::{BinaryViewType, ByteViewType, DataType, StringViewType};
-use datafusion_common::hash_utils::RandomState;
-use datafusion_common::hash_utils::create_hashes;
+use datafusion_common::hash_utils::{HashValue, RandomState};
use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};
use std::fmt::Debug;
use std::hash::BuildHasher;
@@ -258,29 +257,15 @@ where
OP: FnMut(V),
B: ByteViewType,
{
- // step 1: compute hashes
- let batch_hashes = &mut self.hashes_buffer;
- batch_hashes.clear();
- batch_hashes.resize(values.len(), 0);
- create_hashes([values], &self.random_state, batch_hashes)
- // hash is supported for all types and create_hashes only
- // returns errors for unsupported types
- .unwrap();
-
- // step 2: insert each value into the set, if not already present
let values = values.as_byte_view::<B>();
// Get raw views buffer for direct comparison
let input_views = values.views();
- // Ensure lengths are equivalent
- assert_eq!(values.len(), self.hashes_buffer.len());
-
let input_has_buffers = !values.data_buffers().is_empty();
self.input_view_to_payload.clear();
for i in 0..values.len() {
let view_u128 = input_views[i];
- let hash = self.hashes_buffer[i];
// handle null value via validity bitmap check
if values.is_null(i) {
@@ -319,6 +304,8 @@ where
continue;
}
+ let hash = self.hash_value::<B>(view_u128, len, values, i);
+
// Check if value already exists
let maybe_payload = {
let completed = &self.completed;
@@ -424,6 +411,22 @@ where
random_state.hash_one(location)
}
+ #[inline(always)]
+ fn hash_value<B: ByteViewType>(
+ &self,
+ view: u128,
+ len: u32,
+ array: &GenericByteViewArray<B>,
+ row: usize,
+ ) -> u64 {
+ if len <= 12 || array.data_buffers().is_empty() {
+ view.hash_one(&self.random_state)
+ } else {
+ let value: &[u8] = unsafe { array.value_unchecked(row).as_ref() };
+ value.hash_one(&self.random_state)
+ }
+ }
+
#[inline(always)]
fn view_equal_to_input<B: ByteViewType, const HAS_BUFFERS: bool>(
exist_view: u128,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
