ozankabak commented on code in PR #5171:
URL: https://github.com/apache/arrow-datafusion/pull/5171#discussion_r1097831935
##########
datafusion/core/src/physical_optimizer/sort_enforcement.rs:
##########
@@ -33,48 +33,80 @@ use crate::config::ConfigOptions;
use crate::error::Result;
use crate::physical_optimizer::utils::add_sort_above_child;
use crate::physical_optimizer::PhysicalOptimizerRule;
+use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use crate::physical_plan::rewrite::TreeNodeRewritable;
use crate::physical_plan::sorts::sort::SortExec;
+use
crate::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
+use crate::physical_plan::union::UnionExec;
use crate::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
-use crate::physical_plan::{with_new_children_if_necessary, ExecutionPlan};
+use crate::physical_plan::{with_new_children_if_necessary, Distribution,
ExecutionPlan};
use arrow::datatypes::SchemaRef;
use datafusion_common::{reverse_sort_options, DataFusionError};
use datafusion_physical_expr::utils::{ordering_satisfy,
ordering_satisfy_concrete};
-use datafusion_physical_expr::window::WindowExpr;
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};
-use itertools::izip;
+use itertools::{concat, izip};
use std::iter::zip;
use std::sync::Arc;
/// This rule inspects SortExec's in the given physical plan and removes the
-/// ones it can prove unnecessary.
+/// ones it can prove unnecessary. The boolean flag `parallelize_sorts`
+/// indicates whether we elect to transform CoalescePartitionsExec + SortExec
+/// cascades into SortExec + SortPreservingMergeExec cascades, which enables
+/// us to perform sorting in parallel.
#[derive(Default)]
-pub struct EnforceSorting {}
+pub struct EnforceSorting {
+ parallelize_sorts: bool,
+}
impl EnforceSorting {
#[allow(missing_docs)]
- pub fn new() -> Self {
- Self {}
+ pub fn new(parallelize_sorts: bool) -> Self {
Review Comment:
We will modify the tests to verify both cases, thank you for pointing it
out. I also agree with making this flag a configuration parameter, will do that
too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
