alamb commented on code in PR #4122: URL: https://github.com/apache/arrow-datafusion/pull/4122#discussion_r1017867041
########## datafusion/core/src/physical_optimizer/enforcement.rs: ########## @@ -0,0 +1,2001 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Enforcement optimizer rules are used to make sure the plan's Distribution and Ordering +//! requirements are met by inserting necessary [[RepartitionExec]] and [[SortExec]]. +//! +use crate::error::Result; +use crate::physical_optimizer::PhysicalOptimizerRule; +use crate::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; +use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use crate::physical_plan::joins::{ + CrossJoinExec, HashJoinExec, PartitionMode, SortMergeJoinExec, +}; +use crate::physical_plan::projection::ProjectionExec; +use crate::physical_plan::repartition::RepartitionExec; +use crate::physical_plan::rewrite::TreeNodeRewritable; +use crate::physical_plan::sorts::sort::SortExec; +use crate::physical_plan::windows::WindowAggExec; +use crate::physical_plan::Partitioning; +use crate::physical_plan::{with_new_children_if_necessary, Distribution, ExecutionPlan}; +use crate::prelude::SessionConfig; +use datafusion_expr::logical_plan::JoinType; +use datafusion_physical_expr::equivalence::EquivalenceProperties; +use datafusion_physical_expr::expressions::Column; +use datafusion_physical_expr::expressions::NoOp; +use datafusion_physical_expr::{ + expr_list_eq_strict_order, normalize_expr_with_equivalence_properties, + normalize_sort_expr_with_equivalence_properties, PhysicalExpr, PhysicalSortExpr, +}; +use std::collections::HashMap; +use std::sync::Arc; + +/// BasicEnforcement rule, it ensures the Distribution and Ordering requirements are met +/// in the strictest way. It might add additional [[RepartitionExec]] to the plan tree +/// and give a non-optimal plan, but it can avoid the possible data skew in joins +/// +/// For example for a HashJoin with keys(a, b, c), the required Distribution(a, b, c) can be satisfied by +/// several alternative partitioning ways: [(a, b, c), (a, b), (a, c), (b, c), (a), (b), (c), ( )]. +/// +/// This rule only chooses the exactly match and satisfies the Distribution(a, b, c) by a HashPartition(a, b, c). +#[derive(Default)] +pub struct BasicEnforcement {} + +impl BasicEnforcement { + #[allow(missing_docs)] + pub fn new() -> Self { + Self {} + } +} + +impl PhysicalOptimizerRule for BasicEnforcement { + fn optimize( + &self, + plan: Arc<dyn ExecutionPlan>, + config: &SessionConfig, + ) -> Result<Arc<dyn ExecutionPlan>> { + let target_partitions = config.target_partitions; + let top_down_join_key_reordering = config.top_down_join_key_reordering; + let new_plan = if top_down_join_key_reordering { + // Run a top-down process to adjust input key ordering recursively + adjust_input_keys_down_recursively(plan, vec![])? + } else { + plan + }; + // Distribution and Ordering enforcement need to be applied bottom-up. + new_plan.transform_up(&{ + |plan| { + let adjusted = if !top_down_join_key_reordering { + reorder_join_keys_to_inputs(plan) + } else { + plan + }; + Some(ensure_distribution_and_ordering( + adjusted, + target_partitions, + )) + } + }) + } + + fn name(&self) -> &str { + "BasicEnforcement" + } +} + +/// When the physical planner creates the Joins, the ordering of join keys is from the original query. Review Comment: Specifically I am trying to figure out under what circumstances join column reordering is needed at all. Like if you have a join on `(a, b)` and then you have a aggregate grouped on `b, a` the distributions are compatible aren't they? ########## datafusion/core/src/physical_optimizer/enforcement.rs: ########## @@ -0,0 +1,1739 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Enforcement optimizer rules are used to make sure the plan's Distribution and Ordering +//! requirements are met by inserting necessary [[RepartitionExec]] and [[SortExec]]. +//! +use crate::error::Result; +use crate::physical_optimizer::PhysicalOptimizerRule; +use crate::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; +use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use crate::physical_plan::joins::{ + CrossJoinExec, HashJoinExec, PartitionMode, SortMergeJoinExec, +}; +use crate::physical_plan::projection::ProjectionExec; +use crate::physical_plan::repartition::RepartitionExec; +use crate::physical_plan::rewrite::TreeNodeRewritable; +use crate::physical_plan::sorts::sort::SortExec; +use crate::physical_plan::Partitioning; +use crate::physical_plan::{with_new_children_if_necessary, Distribution, ExecutionPlan}; +use crate::prelude::SessionConfig; +use datafusion_expr::logical_plan::JoinType; +use datafusion_physical_expr::equivalence::EquivalenceProperties; +use datafusion_physical_expr::expressions::Column; +use datafusion_physical_expr::expressions::NoOp; +use datafusion_physical_expr::{ + expr_list_eq_strict_order, normalize_expr_with_equivalence_properties, + normalize_sort_expr_with_equivalence_properties, PhysicalExpr, PhysicalSortExpr, +}; +use std::collections::HashMap; +use std::sync::Arc; + +/// BasicEnforcement rule, it ensures the Distribution and Ordering requirements are met +/// in the strictest way. It might add additional [[RepartitionExec]] to the plan tree +/// and give a non-optimal plan, but it can avoid the possible data skew in joins +/// +/// For example for a HashJoin with keys(a, b, c), the required Distribution(a, b, c) can be satisfied by +/// several alternative partitioning ways: [(a, b, c), (a, b), (a, c), (b, c), (a), (b), (c), ( )]. +/// +/// This rule only chooses the exactly match and satisfies the Distribution(a, b, c) by a HashPartition(a, b, c). +#[derive(Default)] +pub struct BasicEnforcement {} + +impl BasicEnforcement { + #[allow(missing_docs)] + pub fn new() -> Self { + Self {} + } +} + +impl PhysicalOptimizerRule for BasicEnforcement { + fn optimize( + &self, + plan: Arc<dyn ExecutionPlan>, + config: &SessionConfig, + ) -> Result<Arc<dyn ExecutionPlan>> { + let target_partitions = config.target_partitions; + let top_down_join_key_reordering = config.top_down_join_key_reordering; + let new_plan = if top_down_join_key_reordering { + // Run a top-down process to adjust input key ordering recursively + adjust_input_keys_down_recursively(plan, vec![])? + } else { + plan + }; + // Distribution and Ordering enforcement need to be applied bottom-up. + new_plan.transform_up(&{ + |plan| { + let adjusted = if !top_down_join_key_reordering { + reorder_join_keys_to_inputs(plan) + } else { + plan + }; + Some(ensure_distribution_and_ordering( + adjusted, + target_partitions, + )) + } + }) + } + + fn name(&self) -> &str { + "BasicEnforcement" + } +} + +/// When the physical planner creates the Joins, the ordering of join keys is from the original query. +/// That might not match with the output partitioning of the join node's children +/// This method runs a top-down process and try to adjust the output partitioning of the children +/// if the children themselves are Joins or Aggregations. +fn adjust_input_keys_down_recursively( + plan: Arc<dyn crate::physical_plan::ExecutionPlan>, + parent_required: Vec<Arc<dyn PhysicalExpr>>, +) -> Result<Arc<dyn crate::physical_plan::ExecutionPlan>> { + let plan_any = plan.as_any(); + if let Some(HashJoinExec { + left, + right, + on, + filter, + join_type, + mode, + null_equals_null, + .. + }) = plan_any.downcast_ref::<HashJoinExec>() + { + match mode { + PartitionMode::Partitioned => { + let join_key_pairs = extract_join_keys(on); + if let Some(( + JoinKeyPairs { + left_keys, + right_keys, + }, + new_positions, + )) = try_reorder( + join_key_pairs.clone(), + parent_required, + &plan.equivalence_properties(), + ) { + let new_join_on = if !new_positions.is_empty() { + new_join_conditions(&left_keys, &right_keys) + } else { + on.clone() + }; + let new_left = + adjust_input_keys_down_recursively(left.clone(), left_keys)?; + let new_right = + adjust_input_keys_down_recursively(right.clone(), right_keys)?; + Ok(Arc::new(HashJoinExec::try_new( + new_left, + new_right, + new_join_on, + filter.clone(), + join_type, + PartitionMode::Partitioned, + null_equals_null, + )?)) + } else { + let new_left = adjust_input_keys_down_recursively( + left.clone(), + join_key_pairs.left_keys, + )?; + let new_right = adjust_input_keys_down_recursively( + right.clone(), + join_key_pairs.right_keys, + )?; + Ok(Arc::new(HashJoinExec::try_new( + new_left, + new_right, + on.clone(), + filter.clone(), + join_type, + PartitionMode::Partitioned, + null_equals_null, + )?)) + } + } + PartitionMode::CollectLeft => { + let new_left = adjust_input_keys_down_recursively(left.clone(), vec![])?; + let new_right = match join_type { + JoinType::Inner | JoinType::Right => try_push_required_to_right( + parent_required, + right.clone(), + left.schema().fields().len(), + )?, + JoinType::RightSemi | JoinType::RightAnti => { + adjust_input_keys_down_recursively( + right.clone(), + parent_required.clone(), + )? + } + JoinType::Left + | JoinType::LeftSemi + | JoinType::LeftAnti + | JoinType::Full => { + adjust_input_keys_down_recursively(right.clone(), vec![])? + } + }; + + Ok(Arc::new(HashJoinExec::try_new( + new_left, + new_right, + on.clone(), + filter.clone(), + join_type, + PartitionMode::CollectLeft, + null_equals_null, + )?)) + } + } + } else if let Some(CrossJoinExec { left, right, .. }) = + plan_any.downcast_ref::<CrossJoinExec>() + { + let new_left = adjust_input_keys_down_recursively(left.clone(), vec![])?; + let new_right = try_push_required_to_right( + parent_required, + right.clone(), + left.schema().fields().len(), + )?; + Ok(Arc::new(CrossJoinExec::try_new(new_left, new_right)?)) + } else if let Some(SortMergeJoinExec { + left, + right, + on, + join_type, + sort_options, + null_equals_null, + .. + }) = plan_any.downcast_ref::<SortMergeJoinExec>() + { + let join_key_pairs = extract_join_keys(on); + if let Some(( + JoinKeyPairs { + left_keys, + right_keys, + }, + new_positions, + )) = try_reorder( + join_key_pairs.clone(), + parent_required, + &plan.equivalence_properties(), + ) { + let new_join_on = if !new_positions.is_empty() { + new_join_conditions(&left_keys, &right_keys) + } else { + on.clone() + }; + let new_options = if !new_positions.is_empty() { + let mut new_sort_options = vec![]; + for idx in 0..sort_options.len() { + new_sort_options.push(sort_options[new_positions[idx]]) + } + new_sort_options + } else { + sort_options.clone() + }; + + let new_left = adjust_input_keys_down_recursively(left.clone(), left_keys)?; + let new_right = + adjust_input_keys_down_recursively(right.clone(), right_keys)?; + + Ok(Arc::new(SortMergeJoinExec::try_new( + new_left, + new_right, + new_join_on, + *join_type, + new_options, + *null_equals_null, + )?)) + } else { + let new_left = adjust_input_keys_down_recursively( + left.clone(), + join_key_pairs.left_keys, + )?; + let new_right = adjust_input_keys_down_recursively( + right.clone(), + join_key_pairs.right_keys, + )?; + Ok(Arc::new(SortMergeJoinExec::try_new( + new_left, + new_right, + on.clone(), + *join_type, + sort_options.clone(), + *null_equals_null, + )?)) + } + } else if let Some(AggregateExec { + mode, + group_by, + aggr_expr, + input, + input_schema, + .. + }) = plan_any.downcast_ref::<AggregateExec>() + { + if parent_required.is_empty() { + plan.map_children(|plan| adjust_input_keys_down_recursively(plan, vec![])) + } else { + match mode { + AggregateMode::Final => plan.map_children(|plan| { + adjust_input_keys_down_recursively(plan, vec![]) + }), + AggregateMode::FinalPartitioned | AggregateMode::Partial => { + let out_put_columns = group_by + .expr() + .iter() + .enumerate() + .map(|(index, (_col, name))| Column::new(name, index)) + .collect::<Vec<_>>(); + + let out_put_exprs = out_put_columns + .iter() + .map(|c| Arc::new(c.clone()) as Arc<dyn PhysicalExpr>) + .collect::<Vec<_>>(); + + // Check whether the requirements can be satisfied by the Aggregation + if parent_required.len() != out_put_exprs.len() + || expr_list_eq_strict_order(&out_put_exprs, &parent_required) + || !group_by.null_expr().is_empty() + { + plan.map_children(|plan| { + adjust_input_keys_down_recursively(plan, vec![]) + }) + } else { + let new_positions = + expected_expr_positions(&out_put_exprs, &parent_required); + match new_positions { + Some(positions) => { + let mut new_group_exprs = vec![]; + for idx in positions.into_iter() { + new_group_exprs.push(group_by.expr()[idx].clone()); + } + let new_group_by = + PhysicalGroupBy::new_single(new_group_exprs); + match mode { + AggregateMode::FinalPartitioned => { + let new_input = + adjust_input_keys_down_recursively( + input.clone(), + parent_required, + )?; + let new_agg = Arc::new(AggregateExec::try_new( + AggregateMode::FinalPartitioned, + new_group_by, + aggr_expr.clone(), + new_input, + input_schema.clone(), + )?); + + // Need to create a new projection to change the expr ordering back + let mut proj_exprs = out_put_columns + .iter() + .map(|col| { + ( + Arc::new(Column::new( + col.name(), + new_agg + .schema() + .index_of(col.name()) + .unwrap(), + )) + as Arc<dyn PhysicalExpr>, + col.name().to_owned(), + ) + }) + .collect::<Vec<_>>(); + let agg_schema = new_agg.schema(); + let agg_fields = agg_schema.fields(); + for (idx, field) in agg_fields + .iter() + .enumerate() + .skip(out_put_columns.len()) + { + proj_exprs.push(( + Arc::new(Column::new( + field.name().as_str(), + idx, + )) + as Arc<dyn PhysicalExpr>, + field.name().clone(), + )) + } + // TODO merge adjacent Projections if there are + Ok(Arc::new(ProjectionExec::try_new( + proj_exprs, new_agg, + )?)) + } + AggregateMode::Partial => { + let new_input = + adjust_input_keys_down_recursively( + input.clone(), + vec![], + )?; + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::Partial, + new_group_by, + aggr_expr.clone(), + new_input, + input_schema.clone(), + )?)) + } + _ => Ok(plan), + } + } + None => plan.map_children(|plan| { + adjust_input_keys_down_recursively(plan, vec![]) + }), + } + } + } + } + } + } else if let Some(ProjectionExec { expr, .. }) = + plan_any.downcast_ref::<ProjectionExec>() + { + // For Projection, we need to transform the columns to the columns before the Projection + // And then to push down the requirements + // Construct a mapping from new name to the the orginal Column + let mut column_mapping = HashMap::new(); + for (expression, name) in expr.iter() { + if let Some(column) = expression.as_any().downcast_ref::<Column>() { + column_mapping.insert(name.clone(), column.clone()); + }; + } + let new_required: Vec<Arc<dyn PhysicalExpr>> = parent_required + .iter() + .filter_map(|r| { + if let Some(column) = r.as_any().downcast_ref::<Column>() { + column_mapping.get(column.name()) + } else { + None + } + }) + .map(|e| Arc::new(e.clone()) as Arc<dyn PhysicalExpr>) + .collect::<Vec<_>>(); + if new_required.len() == parent_required.len() { + plan.map_children(|plan| { + adjust_input_keys_down_recursively(plan, new_required.clone()) + }) + } else { + plan.map_children(|plan| adjust_input_keys_down_recursively(plan, vec![])) + } + } else if plan_any.downcast_ref::<RepartitionExec>().is_some() + || plan_any.downcast_ref::<CoalescePartitionsExec>().is_some() + { + plan.map_children(|plan| adjust_input_keys_down_recursively(plan, vec![])) + } else { + plan.map_children(|plan| { + adjust_input_keys_down_recursively(plan, parent_required.clone()) + }) + } +} + +fn try_push_required_to_right( + parent_required: Vec<Arc<dyn PhysicalExpr>>, + right: Arc<dyn ExecutionPlan>, + left_columns_len: usize, +) -> Result<Arc<dyn ExecutionPlan>> { + let new_required: Vec<Arc<dyn PhysicalExpr>> = parent_required + .iter() + .filter_map(|r| { + if let Some(col) = r.as_any().downcast_ref::<Column>() { + if col.index() >= left_columns_len { + Some( + Arc::new(Column::new(col.name(), col.index() - left_columns_len)) + as Arc<dyn PhysicalExpr>, + ) + } else { + None + } + } else { + None + } + }) + .collect::<Vec<_>>(); + + // if the parent required are all comming from the right side, the requirements can be pushdown + if new_required.len() == parent_required.len() { + adjust_input_keys_down_recursively(right.clone(), new_required) + } else { + adjust_input_keys_down_recursively(right.clone(), vec![]) + } +} + +/// When the physical planner creates the Joins, the ordering of join keys is from the original query. +/// That might not match with the output partitioning of the join node's children +/// This method will try to change the ordering of the join keys to match with the +/// partitioning of the join nodes' children. +/// If it can not match with both sides, it will try to match with one, either left side or right side. +fn reorder_join_keys_to_inputs( + plan: Arc<dyn crate::physical_plan::ExecutionPlan>, +) -> Arc<dyn crate::physical_plan::ExecutionPlan> { + let plan_any = plan.as_any(); + if let Some(HashJoinExec { + left, + right, + on, + filter, + join_type, + mode, + null_equals_null, + .. + }) = plan_any.downcast_ref::<HashJoinExec>() + { + match mode { + PartitionMode::Partitioned => { + let join_key_pairs = extract_join_keys(on); + if let Some(( + JoinKeyPairs { + left_keys, + right_keys, + }, + new_positions, + )) = reorder_current_join_keys( + join_key_pairs, + Some(left.output_partitioning()), + Some(right.output_partitioning()), + &plan.equivalence_properties(), + ) { + if !new_positions.is_empty() { + let new_join_on = new_join_conditions(&left_keys, &right_keys); + Arc::new( + HashJoinExec::try_new( + left.clone(), + right.clone(), + new_join_on, + filter.clone(), + join_type, + PartitionMode::Partitioned, + null_equals_null, + ) + .unwrap(), + ) + } else { + plan + } + } else { + plan + } + } + _ => plan, + } + } else if let Some(SortMergeJoinExec { + left, + right, + on, + join_type, + sort_options, + null_equals_null, + .. + }) = plan_any.downcast_ref::<SortMergeJoinExec>() + { + let join_key_pairs = extract_join_keys(on); + if let Some(( + JoinKeyPairs { + left_keys, + right_keys, + }, + new_positions, + )) = reorder_current_join_keys( + join_key_pairs, + Some(left.output_partitioning()), + Some(right.output_partitioning()), + &plan.equivalence_properties(), + ) { + if !new_positions.is_empty() { + let new_join_on = new_join_conditions(&left_keys, &right_keys); + let mut new_sort_options = vec![]; + for idx in 0..sort_options.len() { + new_sort_options.push(sort_options[new_positions[idx]]) + } + Arc::new( + SortMergeJoinExec::try_new( + left.clone(), + right.clone(), + new_join_on, + *join_type, + new_sort_options, + *null_equals_null, + ) + .unwrap(), + ) + } else { + plan + } + } else { + plan + } + } else { + plan + } +} + +/// Reorder the current join keys ordering based on either left partition or right partition. +fn reorder_current_join_keys( + join_keys: JoinKeyPairs, + left_partition: Option<Partitioning>, + right_partition: Option<Partitioning>, + equivalence_properties: &EquivalenceProperties, +) -> Option<(JoinKeyPairs, Vec<usize>)> { + match (left_partition.clone(), right_partition.clone()) { + (Some(Partitioning::Hash(left_exprs, _)), _) => { + try_reorder(join_keys.clone(), left_exprs, equivalence_properties).or_else( + || { + reorder_current_join_keys( + join_keys, + None, + right_partition, + equivalence_properties, + ) + }, + ) + } + (_, Some(Partitioning::Hash(right_exprs, _))) => { + try_reorder(join_keys.clone(), right_exprs, equivalence_properties).or_else( + || { + reorder_current_join_keys( + join_keys, + left_partition, + None, + equivalence_properties, + ) + }, + ) + } + _ => None, + } +} + +fn try_reorder( + join_keys: JoinKeyPairs, + expected: Vec<Arc<dyn PhysicalExpr>>, + equivalence_properties: &EquivalenceProperties, +) -> Option<(JoinKeyPairs, Vec<usize>)> { + if join_keys.left_keys.len() != expected.len() { + return None; + } + if expr_list_eq_strict_order(&expected, &join_keys.left_keys) { + return Some((join_keys, vec![])); + } + let new_positions = expected_expr_positions(&join_keys.left_keys, &expected); + match new_positions { + Some(positions) => { + let mut new_right_keys = vec![]; + for pos in positions.iter() { + new_right_keys.push(join_keys.right_keys[*pos].clone()); + } + Some(( + JoinKeyPairs { + left_keys: expected, + right_keys: new_right_keys, + }, + positions, + )) + } + None => { + if !equivalence_properties.classes().is_empty() { + let normalized_expected = expected + .iter() + .map(|e| { + normalize_expr_with_equivalence_properties( + e.clone(), + equivalence_properties.classes(), + ) + }) + .collect::<Vec<_>>(); + let normalized_left_keys = join_keys + .left_keys + .iter() + .map(|e| { + normalize_expr_with_equivalence_properties( + e.clone(), + equivalence_properties.classes(), + ) + }) + .collect::<Vec<_>>(); + if expr_list_eq_strict_order(&normalized_expected, &normalized_left_keys) + { + Some((join_keys, vec![])) + } else { + let new_positions = expected_expr_positions( + &normalized_left_keys, + &normalized_expected, + ); + match new_positions { + Some(positions) => { + let mut new_left_keys = vec![]; + let mut new_right_keys = vec![]; + for pos in positions.iter() { + new_left_keys.push(join_keys.left_keys[*pos].clone()); + new_right_keys.push(join_keys.right_keys[*pos].clone()); + } + Some(( + JoinKeyPairs { + left_keys: new_left_keys, + right_keys: new_right_keys, + }, + positions, + )) + } + None => None, + } + } + } else { + None + } + } + } +} + +/// Return the expected expressions positions. +/// For example, the current expressions are ['c', 'a', 'a', b'], the expected expressions are ['b', 'c', 'a', 'a'], +/// +/// This method will return a Vec [3, 0, 1, 2] +fn expected_expr_positions( + current: &[Arc<dyn PhysicalExpr>], + expected: &[Arc<dyn PhysicalExpr>], +) -> Option<Vec<usize>> { + let mut indexes: Vec<usize> = vec![]; + let mut current = current.to_vec(); + for expr in expected.iter() { + // Find the position of the expected expr in the current expressions + if let Some(expected_position) = current.iter().position(|e| e.eq(expr)) { + current[expected_position] = Arc::new(NoOp::new()); + indexes.push(expected_position); + } else { + return None; + } + } + Some(indexes) +} + +fn extract_join_keys(on: &[(Column, Column)]) -> JoinKeyPairs { + let (left_keys, right_keys) = on + .iter() + .map(|(l, r)| { + ( + Arc::new(l.clone()) as Arc<dyn PhysicalExpr>, + Arc::new(r.clone()) as Arc<dyn PhysicalExpr>, + ) + }) + .unzip(); + JoinKeyPairs { + left_keys, + right_keys, + } +} + +fn new_join_conditions( + new_left_keys: &[Arc<dyn PhysicalExpr>], + new_right_keys: &[Arc<dyn PhysicalExpr>], +) -> Vec<(Column, Column)> { + let new_join_on = new_left_keys + .iter() + .zip(new_right_keys.iter()) + .map(|(l_key, r_key)| { + ( + l_key.as_any().downcast_ref::<Column>().unwrap().clone(), + r_key.as_any().downcast_ref::<Column>().unwrap().clone(), + ) + }) + .collect::<Vec<_>>(); + new_join_on +} + +fn ensure_distribution_and_ordering( + plan: Arc<dyn crate::physical_plan::ExecutionPlan>, + target_partitions: usize, +) -> Arc<dyn crate::physical_plan::ExecutionPlan> { + if plan.children().is_empty() { + return plan; + } + let required_input_distributions = plan.required_input_distribution(); + let required_input_orderings = plan.required_input_ordering(); + let children: Vec<Arc<dyn ExecutionPlan>> = plan.children(); + assert_eq!(children.len(), required_input_distributions.len()); + assert_eq!(children.len(), required_input_orderings.len()); + + // Add RepartitionExec to guarantee output partitioning + let children = children + .into_iter() + .zip(required_input_distributions.into_iter()) + .map(|(child, required)| { + if child + .output_partitioning() + .satisfy(required.clone(), || child.equivalence_properties()) + { + child + } else { + let new_child: Arc<dyn ExecutionPlan> = match required { + Distribution::SinglePartition + if child.output_partitioning().partition_count() > 1 => + { + Arc::new(CoalescePartitionsExec::new(child.clone())) + } + _ => { Review Comment: Isn't that case covered by `if child.output_partitioning().satisfy()` check above? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
