jayzhan211 commented on code in PR #27395:
URL: https://github.com/apache/datafusion/pull/27395#discussion_r4585913692
##########
datafusion/core/tests/physical_optimizer/projection_pushdown.rs:
##########
@@ -2035,3 +2040,431 @@ async fn split_declines_hash_repartition() ->
Result<()> {
);
Ok(())
}
+
+fn limit_split_scan() -> Result<(SchemaRef, Arc<dyn ExecutionPlan>)> {
+ let schema: SchemaRef = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new(
+ "s",
+ DataType::Struct(vec![Field::new("value", DataType::Int64,
true)].into()),
+ true,
+ ),
+ ]));
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(ParquetSource::new(Arc::clone(&schema))),
+ )
+ .with_file(PartitionedFile::new("x".to_string(), 100))
+ .build();
+ Ok((schema, DataSourceExec::from_data_source(config)))
+}
+
+fn limit_split_exprs(schema: &SchemaRef) -> Result<Vec<ProjectionExpr>> {
+ let leaf: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
+ "get_field",
+ datafusion_functions::core::get_field(),
+ vec![
+ col("s", schema)?,
+ Arc::new(Literal::new(ScalarValue::from("value"))),
+ ],
+ Arc::new(Field::new("v", DataType::Int64, true)),
+ Arc::new(ConfigOptions::default()),
+ ));
+ Ok(vec![
+ ProjectionExpr {
+ expr: leaf,
+ alias: "v".to_string(),
+ },
+ ProjectionExpr {
+ expr: binary(
+ col("id", schema)?,
+ Operator::Plus,
+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),
+ schema,
+ )?,
+ alias: "w".to_string(),
+ },
+ ])
+}
+
+#[tokio::test]
+async fn split_descends_through_global_limit() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ // `skip` and `fetch` differ so that swapping them would show.
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(scan, 1,
Some(3)));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// The no-leaf `EXPLAIN` in the `.slt` cannot pin this, because the plan is
the
+/// same either way.
+#[tokio::test]
+async fn split_declines_global_limit_without_leaf() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(scan, 1,
Some(3)));
+ let exprs = vec![limit_split_exprs(&schema)?.swap_remove(1)];
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(exprs, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// `LocalLimitExec` is kept out of the host set. The corpus never strands a
+/// leaf above one, but that is what the corpus shows, not a proof that nothing
+/// can, so this pins the exclusion.
Review Comment:
Optional cleanup: "corpus" isn't defined anywhere, so a reader can't tell
which queries it means. Suggested wording:
```suggestion
/// `LocalLimitExec` is left out of `split_host_child`: no SQL query strands
a
/// leaf expression above one today. This pins that choice.
```
##########
datafusion/physical-optimizer/src/projection_pushdown.rs:
##########
@@ -490,29 +486,56 @@ fn leaf_column_name(index: usize) -> String {
format!("{LEAF_COLUMN_PREFIX}_{index}")
}
+fn split_host_child(node: &Arc<dyn ExecutionPlan>) -> Option<&Arc<dyn
ExecutionPlan>> {
Review Comment:
Optional cleanup: with the old doc gone, nothing says what a node has to
satisfy to be allowed here, and the tests now say "host" without a definition.
One short reason would keep the next person from adding, say, a `SortExec` arm,
whose sort expressions would point at the wrong columns after the split:
```suggestion
/// Returns the input of `node` if the split may place the leaf projection
below
/// it. The split gives `node` an input with different columns, so `node`
must
/// not reference its input's columns and must output them unchanged, or the
/// outer projection's column indices would point at the wrong columns.
fn split_host_child(node: &Arc<dyn ExecutionPlan>) -> Option<&Arc<dyn
ExecutionPlan>> {
```
##########
datafusion/core/tests/physical_optimizer/projection_pushdown.rs:
##########
@@ -2035,3 +2040,431 @@ async fn split_declines_hash_repartition() ->
Result<()> {
);
Ok(())
}
+
+fn limit_split_scan() -> Result<(SchemaRef, Arc<dyn ExecutionPlan>)> {
+ let schema: SchemaRef = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new(
+ "s",
+ DataType::Struct(vec![Field::new("value", DataType::Int64,
true)].into()),
+ true,
+ ),
+ ]));
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(ParquetSource::new(Arc::clone(&schema))),
+ )
+ .with_file(PartitionedFile::new("x".to_string(), 100))
+ .build();
+ Ok((schema, DataSourceExec::from_data_source(config)))
+}
+
+fn limit_split_exprs(schema: &SchemaRef) -> Result<Vec<ProjectionExpr>> {
+ let leaf: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
+ "get_field",
+ datafusion_functions::core::get_field(),
+ vec![
+ col("s", schema)?,
+ Arc::new(Literal::new(ScalarValue::from("value"))),
+ ],
+ Arc::new(Field::new("v", DataType::Int64, true)),
+ Arc::new(ConfigOptions::default()),
+ ));
+ Ok(vec![
+ ProjectionExpr {
+ expr: leaf,
+ alias: "v".to_string(),
+ },
+ ProjectionExpr {
+ expr: binary(
+ col("id", schema)?,
+ Operator::Plus,
+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),
+ schema,
+ )?,
+ alias: "w".to_string(),
+ },
+ ])
+}
+
+#[tokio::test]
+async fn split_descends_through_global_limit() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ // `skip` and `fetch` differ so that swapping them would show.
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(scan, 1,
Some(3)));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// The no-leaf `EXPLAIN` in the `.slt` cannot pin this, because the plan is
the
+/// same either way.
+#[tokio::test]
+async fn split_declines_global_limit_without_leaf() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(scan, 1,
Some(3)));
+ let exprs = vec![limit_split_exprs(&schema)?.swap_remove(1)];
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(exprs, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// `LocalLimitExec` is kept out of the host set. The corpus never strands a
+/// leaf above one, but that is what the corpus shows, not a proof that nothing
+/// can, so this pins the exclusion.
+#[tokio::test]
+async fn split_declines_local_limit() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(LocalLimitExec::new(scan, 2));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - LocalLimitExec: fetch=2
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - LocalLimitExec: fetch=2
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// A scan with more than one file group: a single `OFFSET` already stacks two
+/// hosts, so stopping at the first would leave the scan reading the struct.
+#[tokio::test]
+async fn split_descends_through_limit_and_coalesce() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let coalesce: Arc<dyn ExecutionPlan> =
Arc::new(CoalescePartitionsExec::new(scan));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(coalesce, 1, Some(3)));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - CoalescePartitionsExec
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - CoalescePartitionsExec
+ - DataSourceExec: file_groups={1 group: [[x]]},
projection=[id, get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// `fetch = None`, which plain SQL does not reach: an `OFFSET` with no `LIMIT`
+/// is not pushed into the scan, so only a hand-built plan covers it.
+#[tokio::test]
+async fn split_descends_through_global_limit_without_fetch() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let limit: Arc<dyn ExecutionPlan> = Arc::new(GlobalLimitExec::new(scan, 2,
None));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=2, fetch=None
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=2, fetch=None
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// A host carrying a `fetch` of its own has to keep it across the rebuild. The
+/// `.slt` plans show a fetch on this node, but `LimitPushdown` puts it there
+/// after the split, so they do not cover this.
+#[tokio::test]
+async fn split_keeps_the_coalesce_fetch() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ let coalesce: Arc<dyn ExecutionPlan> =
+ Arc::new(CoalescePartitionsExec::new(scan).with_fetch(Some(4)));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(coalesce, 1, Some(3)));
+ let projection: Arc<dyn ExecutionPlan> =
+ Arc::new(ProjectionExec::try_new(limit_split_exprs(&schema)?, limit)?);
+
+ assert_snapshot!(
+ OptimizationTest::new(projection, ProjectionPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - CoalescePartitionsExec: fetch=4
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
s], file_type=parquet
+ output:
+ Ok:
+ - ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+ - GlobalLimitExec: skip=1, fetch=3
+ - CoalescePartitionsExec: fetch=4
+ - DataSourceExec: file_groups={1 group: [[x]]},
projection=[id, get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+ "
+ );
+ Ok(())
+}
+
+/// Six numbers have to come back in the right places, not two. The corpus
+/// reaches a chain this deep but never with a leaf to move.
Review Comment:
Optional cleanup: same "corpus" wording here. The snapshot already shows the
chain, so the comment only needs to say what is checked:
```suggestion
/// Each of the three limits keeps its own `skip` and `fetch` after the
rebuild.
```
##########
datafusion/physical-optimizer/src/projection_pushdown.rs:
##########
@@ -490,29 +486,56 @@ fn leaf_column_name(index: usize) -> String {
format!("{LEAF_COLUMN_PREFIX}_{index}")
}
+fn split_host_child(node: &Arc<dyn ExecutionPlan>) -> Option<&Arc<dyn
ExecutionPlan>> {
+ if let Some(repartition) = node.downcast_ref::<RepartitionExec>() {
+ // Hash and Range partitioning reference columns by index, which would
+ // have to be remapped against the rewritten child schema. Round robin
+ // carries no column references, and is what `EnsureRequirements`
+ // inserts here.
+ if !matches!(repartition.partitioning(),
Partitioning::RoundRobinBatch(_)) {
+ return None;
+ }
+ Some(repartition.input())
+ } else if let Some(limit) = node.downcast_ref::<GlobalLimitExec>() {
+ Some(limit.input())
+ } else {
+ // A scan with more than one file group puts this between the limit
+ // and the source.
+ Some(node.downcast_ref::<CoalescePartitionsExec>()?.input())
+ }
+}
+
+struct SplitHostChain<'a> {
+ /// Outermost first, so the rebuild walks this in reverse.
+ hosts: Vec<&'a Arc<dyn ExecutionPlan>>,
+ child: &'a Arc<dyn ExecutionPlan>,
+}
+
+fn split_host_chain(node: &Arc<dyn ExecutionPlan>) ->
Option<SplitHostChain<'_>> {
+ let mut hosts = vec![];
+ let mut current = node;
+ while let Some(child) = split_host_child(current) {
+ hosts.push(current);
+ current = child;
+ }
+ (!hosts.is_empty()).then_some(SplitHostChain {
+ hosts,
+ child: current,
+ })
+}
+
/// Splits a `ProjectionExec` that mixes leaf expressions with ones that must
/// stay in place, moving the leaves into a projection the source absorbs.
Review Comment:
Optional cleanup: main's doc here explained why the split is needed at all,
and that reason is gone now. A short version that also covers the new limit
case:
```suggestion
/// Splits a `ProjectionExec` that mixes leaf expressions with ones that must
/// stay in place, moving the leaves into a projection the source absorbs.
/// Without the split the leaves stay above the source: no projection swaps
past
/// a `GlobalLimitExec`, and none with a computed expression swaps past a
/// `RepartitionExec`.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]