This is an automated email from the ASF dual-hosted git repository. Rachelint pushed a commit to branch improve-compare-in-view-map in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 9b0db54500caa49ec35704435bb02d37c62b3488 Author: kamille <[email protected]> AuthorDate: Thu Jul 9 08:19:49 2026 +0800 Hash byte view input locations directly --- .../physical-expr-common/src/binary_view_map.rs | 36 +++++++++++++++------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs b/datafusion/physical-expr-common/src/binary_view_map.rs index 4398104b32..2df967e32d 100644 --- a/datafusion/physical-expr-common/src/binary_view_map.rs +++ b/datafusion/physical-expr-common/src/binary_view_map.rs @@ -29,6 +29,7 @@ use datafusion_common::hash_utils::RandomState; use datafusion_common::hash_utils::create_hashes; use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt}; use std::fmt::Debug; +use std::hash::BuildHasher; use std::mem::size_of; use std::sync::Arc; @@ -144,7 +145,7 @@ where /// buffer that stores hash values (reused across batches to save allocations) hashes_buffer: Vec<u64>, /// Maps input buffer locations to payloads for repeated physical views in one batch. - input_view_to_payload: hashbrown::hash_table::HashTable<(u64, V, u64)>, + input_view_to_payload: hashbrown::hash_table::HashTable<(u64, V)>, /// `(payload, null_index)` for the 'null' value, if any /// NOTE null_index is the logical index in the final array, not the index /// in the buffer @@ -302,11 +303,13 @@ where let cached_payload = if input_has_buffers && len > 12 { let input_buffer_location = Self::buffer_location(view_u128); + let input_location_hash = + Self::hash_buffer_location(input_buffer_location, &self.random_state); self.input_view_to_payload - .find(hash, |(cached_location, _, cached_hash)| { - *cached_hash == hash && *cached_location == input_buffer_location + .find(input_location_hash, |(cached_location, _)| { + *cached_location == input_buffer_location }) - .map(|(_, payload, _)| *payload) + .map(|(_, payload)| *payload) } else { None }; @@ -351,18 +354,24 @@ where let payload = if let Some(payload) = maybe_payload { if input_has_buffers && len > 12 { let input_buffer_location = Self::buffer_location(view_u128); + let input_location_hash = Self::hash_buffer_location( + input_buffer_location, + &self.random_state, + ); let entry = self.input_view_to_payload.entry( - hash, - |(cached_location, _, cached_hash)| { - *cached_hash == hash - && *cached_location == input_buffer_location + input_location_hash, + |(cached_location, _)| *cached_location == input_buffer_location, + |(cached_location, _)| { + Self::hash_buffer_location( + *cached_location, + &self.random_state, + ) }, - |(_, _, cached_hash)| *cached_hash, ); match entry { hashbrown::hash_table::Entry::Occupied(_) => {} hashbrown::hash_table::Entry::Vacant(entry) => { - entry.insert((input_buffer_location, payload, hash)); + entry.insert((input_buffer_location, payload)); } } } @@ -410,6 +419,11 @@ where (view >> 64) as u64 } + #[inline(always)] + fn hash_buffer_location(location: u64, random_state: &RandomState) -> u64 { + random_state.hash_one(location) + } + #[inline(always)] fn view_equal_to_input<B: ByteViewType, const HAS_BUFFERS: bool>( exist_view: u128, @@ -570,7 +584,7 @@ where + completed_size + nulls_size + self.hashes_buffer.allocated_size() - + self.input_view_to_payload.capacity() * size_of::<(u64, V, u64)>() + + self.input_view_to_payload.capacity() * size_of::<(u64, V)>() } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
