korowa commented on code in PR #2591:
URL: https://github.com/apache/arrow-datafusion/pull/2591#discussion_r882104689
##########
datafusion/core/src/physical_plan/hash_join.rs:
##########
@@ -791,6 +826,109 @@ fn build_join_indexes(
}
}
+fn apply_join_filter(
+ left: &RecordBatch,
+ right: &RecordBatch,
+ join_type: JoinType,
+ left_indices: UInt64Array,
+ right_indices: UInt32Array,
+ filter: &JoinFilter,
+) -> Result<(UInt64Array, UInt32Array)> {
+ if left_indices.is_empty() {
+ return Ok((left_indices, right_indices));
+ };
+
+ let (intermediate_batch, _) = build_batch_from_indices(
+ filter.schema(),
+ left,
+ right,
+ PrimitiveArray::from(left_indices.data().clone()),
+ PrimitiveArray::from(right_indices.data().clone()),
+ filter.column_indices(),
+ )?;
+
+ match join_type {
+ JoinType::Inner | JoinType::Left => {
+ // For both INNER and LEFT joins, input arrays contains only
indices for matched data.
+ // Due to this fact it's correct to simply apply filter to
intermediate batch and return
+ // indices for left/right rows satisfying filter predicate
+ let filter_result = filter
+ .expression()
+ .evaluate(&intermediate_batch)?
+ .into_array(intermediate_batch.num_rows());
+ let mask = as_boolean_array(&filter_result);
+
+ let left_filtered = PrimitiveArray::<UInt64Type>::from(
+ compute::filter(&left_indices, mask)?.data().clone(),
+ );
+ let right_filtered = PrimitiveArray::<UInt32Type>::from(
+ compute::filter(&right_indices, mask)?.data().clone(),
+ );
+
+ Ok((left_filtered, right_filtered))
+ }
+ JoinType::Right | JoinType::Full => {
+ // In case of RIGHT and FULL join, left_indices could contain null
values - these rows,
+ // where no match has been found, should retain in result arrays
(thus join condition is satified)
+ //
+ // So, filter should be applied only to matched rows, and in case
right (outer, batch) index
Review Comment:
Integration tests for more join types added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
