jayzhan211 commented on code in PR #24573:
URL: https://github.com/apache/datafusion/pull/24573#discussion_r3836133853
##########
datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs:
##########
@@ -1665,20 +1667,19 @@ impl MaterializingSortMergeJoinStream {
// Multiple source batches: map each buffered_batch_idx to a
// contiguous source index, reserving source 0 for a null sentinel.
- let mut batch_idx_to_source: HashMap<usize, usize> = HashMap::new();
+ // A group spans only a handful of buffered batches, so a linear
+ // scan beats hashing here.
let mut source_batches: Vec<usize> = Vec::new();
- for (batch_idx, _, _) in matched_chunks {
- batch_idx_to_source.entry(*batch_idx).or_insert_with(|| {
- let idx = source_batches.len() + 1;
- source_batches.push(*batch_idx);
- idx
- });
- }
-
let mut interleave_indices: Vec<(usize, usize)> =
Vec::with_capacity(total_matched_rows);
for (batch_idx, _, right) in matched_chunks {
- let source = batch_idx_to_source[batch_idx];
+ let source = match source_batches.iter().position(|b| b ==
batch_idx) {
Review Comment:
It seems that linear scan would be a better choice than hash map because the
distinct sources is likely "small", I add the comment to show why linear scan
is preferred
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
