alamb commented on code in PR #27395:
URL: https://github.com/apache/datafusion/pull/27395#discussion_r4573617141
##########
datafusion/physical-optimizer/src/projection_pushdown.rs:
##########
@@ -67,11 +69,16 @@ impl PhysicalOptimizerRule for ProjectionPushdown {
.map(|t| t.data)?;
// Push what can already descend, then split the projections that are
- // stuck above a `RepartitionExec`. The split needs no cleanup pass of
- // its own: the outer projection it leaves either keeps every
expression
- // a plain column, and is then no narrower than the widened child, or
it
- // narrows and holds a computed one. `RepartitionExec` declines both,
it
- // is not removable because it renames, and nothing else moved.
+ // stuck above a chain of the nodes `split_host_child` accepts. The
Review Comment:
Can you please update these comments to not refer to implementation details
elsewere -- pelase try and rephrase in your own words
##########
datafusion/physical-optimizer/src/projection_pushdown.rs:
##########
@@ -490,29 +497,111 @@ fn leaf_column_name(index: usize) -> String {
format!("{LEAF_COLUMN_PREFIX}_{index}")
}
+/// Returns the child of `node` if a projection may be split beneath it.
+///
+/// Every arm evaluates no expression of its own and has exactly one child, so
+/// widening that child with the leaf columns cannot change what it does. Each
+/// also hands its child's schema straight through, which is what lets the
+/// outer expressions keep their column indices when the split descends past
+/// several of these in a row.
+///
+/// That last property holds by construction rather than by luck: none of the
+/// three `compute_properties` takes a schema, and each clones its input's
+/// equivalence properties, so a host's schema *is* its child's. There is no
+/// guard here for it. A fourth arm that projects would need one, because
+/// `rewrite_outer_expr` remaps the outer column indices and rebases lambda
+/// parameters against the width of the chain's bottom schema, and a host that
+/// changed the width would leave both against the wrong one.
+fn split_host_child(node: &Arc<dyn ExecutionPlan>) -> Option<&Arc<dyn
ExecutionPlan>> {
+ if let Some(repartition) = node.downcast_ref::<RepartitionExec>() {
+ // Hash and Range partitioning reference columns by index, which would
+ // have to be remapped against the rewritten child schema. Round robin
+ // carries no column references, and is what `EnsureRequirements`
+ // inserts here.
+ if !matches!(repartition.partitioning(),
Partitioning::RoundRobinBatch(_)) {
+ return None;
+ }
+ Some(repartition.input())
+ } else if let Some(limit) = node.downcast_ref::<GlobalLimitExec>() {
+ // `LocalLimitExec` qualifies structurally — no expressions of its own,
+ // one child, and `replace_children` keeps its fetch — and the planner
+ // builds one under a `GlobalLimitExec` when the input has more than
one
+ // partition (`physical_planner.rs`, the `Limit` branch).
+ // The corpus does enter this function at a `LocalLimitExec`, but only
+ // with no leaf to move, never with one stranded above it, so it is
left
+ // out. `CooperativeExec` and `BufferExec` meet the same three
+ // conditions, and so did `CoalesceBatchesExec` before it was
deprecated
+ // in 52.0.0: these arms are an allowlist, not whatever those
conditions
+ // admit. `split_declines_local_limit` pins it.
+ Some(limit.input())
+ } else {
+ // A scan with more than one file group puts this between the limit and
+ // the source. Its own `try_swapping_with_projection` takes a
projection
+ // only if it narrows, which a leaf projection need not do; the split
is
+ // still worth it here, because what descends is exactly what the
source
+ // prunes to.
+ Some(node.downcast_ref::<CoalescePartitionsExec>()?.input())
+ }
+}
+
+/// The hosts a split descends past, and the node that has to absorb the leaf
+/// projection.
+struct SplitHostChain<'a> {
+ /// Outermost first, so the rebuild walks this in reverse.
+ hosts: Vec<&'a Arc<dyn ExecutionPlan>>,
+ /// The first node that is not a host.
+ child: &'a Arc<dyn ExecutionPlan>,
+}
+
+/// Walks the chain of hosts beneath a projection.
+///
+/// `None` when the projection sits on no host at all. A single `OFFSET` on a
+/// multi-file table already stacks two of them, a `GlobalLimitExec` over a
+/// `CoalescePartitionsExec`, so stopping at the first one would leave the leaf
+/// expression stranded exactly where it is today.
+fn split_host_chain(node: &Arc<dyn ExecutionPlan>) ->
Option<SplitHostChain<'_>> {
+ let mut hosts = vec![];
+ let mut current = node;
+ while let Some(child) = split_host_child(current) {
+ hosts.push(current);
+ current = child;
+ }
+ // A termination guard, not an optimization. With no host, `current` is the
+ // projection's own input. When that is a `ProjectionExec` the two cannot
+ // collapse into (the `counted_volatile` queries in
`projection_pushdown.slt`),
+ // `ProjectionExec::try_swapping_with_projection` returns the lower
+ // projection stacked on top of it, so the split commits, and
+ // `transform_down` then visits that lower projection and splits it again.
+ (!hosts.is_empty()).then_some(SplitHostChain {
+ hosts,
+ child: current,
+ })
+}
+
/// Splits a `ProjectionExec` that mixes leaf expressions with ones that must
/// stay in place, moving the leaves into a projection the source absorbs.
///
-/// `benefits_from_input_partitioning` is true as soon as any expression is
-/// `KeepInPlace`, which both inserts a `RepartitionExec` below the projection
-/// and stops that projection descending past it, so a `MoveTowardsLeafNodes`
-/// expression sharing it never reaches the source. Splitting leaves the
-/// computation above the repartition, still parallelised.
+/// Two nodes strand a `MoveTowardsLeafNodes` expression above the source this
+/// way, and neither lets the whole projection past:
Review Comment:
please reword
##########
datafusion/core/tests/physical_optimizer/projection_pushdown.rs:
##########
@@ -2035,3 +2038,355 @@ async fn split_declines_hash_repartition() ->
Result<()> {
);
Ok(())
}
+
+/// Schema and parquet scan the `GlobalLimitExec` tests below share: a plain
+/// column beside a struct whose `value` field the leaf expression reads.
+fn limit_split_scan() -> Result<(SchemaRef, Arc<dyn ExecutionPlan>)> {
+ let schema: SchemaRef = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new(
+ "s",
+ DataType::Struct(vec![Field::new("value", DataType::Int64,
true)].into()),
+ true,
+ ),
+ ]));
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(ParquetSource::new(Arc::clone(&schema))),
+ )
+ .with_file(PartitionedFile::new("x".to_string(), 100))
+ .build();
+ Ok((schema, DataSourceExec::from_data_source(config)))
+}
+
+/// `[get_field(s, 'value') as v, id + 1 as w]`: one leaf expression beside one
+/// that has to stay in place.
+fn limit_split_exprs(schema: &SchemaRef) -> Result<Vec<ProjectionExpr>> {
+ let leaf: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
+ "get_field",
+ datafusion_functions::core::get_field(),
+ vec![
+ col("s", schema)?,
+ Arc::new(Literal::new(ScalarValue::from("value"))),
+ ],
+ Arc::new(Field::new("v", DataType::Int64, true)),
+ Arc::new(ConfigOptions::default()),
+ ));
+ Ok(vec![
+ ProjectionExpr {
+ expr: leaf,
+ alias: "v".to_string(),
+ },
+ ProjectionExpr {
+ expr: binary(
+ col("id", schema)?,
+ Operator::Plus,
+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),
+ schema,
+ )?,
+ alias: "w".to_string(),
+ },
+ ])
+}
+
+fn optimized_plan_text(plan: Arc<dyn ExecutionPlan>) -> Result<String> {
+ let optimized =
+ ProjectionPushdown::new().optimize(plan, &ConfigOptions::default())?;
+ Ok(displayable(optimized.as_ref())
+ .indent(true)
+ .to_string()
+ .trim()
+ .to_string())
+}
+
+#[tokio::test]
+async fn split_descends_through_global_limit() -> Result<()> {
+ let (schema, scan) = limit_split_scan()?;
+ // What a source that pushes `OFFSET` down builds: the skip becomes a
Review Comment:
I don't understand these comments -- they are trying to describe in english
the plan shape
I think these tests would be much more readable if instead of trying to
describe this in textual comments, you instead add a snapshot of the orignal
and the optimized plan
Sort of like
```
assert_snapshot!(input_and_optimized_plan_text(projection)?,
@r"
--- Input Plan
...
--- OutputPlan
ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
GlobalLimitExec: skip=1, fetch=3
DataSourceExec: file_groups={1 group: [[x]]}, projection=[id,
get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
"
```
More background on the rationale here:
https://andrew.nerdnetworks.org/blog/2026/11/13/transcript-testing/
##########
datafusion/sqllogictest/test_files/file_row_index.slt:
##########
@@ -314,6 +314,191 @@ SELECT file_row_index() + 1, column1 FROM parquet_table
ORDER BY column1
# `RESET` restores the global default rather than the harness value, which the
# harness rejects as leftover configuration.
+# A leaf expression must also reach the scan when the source pushes an `OFFSET`
Review Comment:
can you please reword these comments to explain the case they are teseting,
not details about the code that implements it?
##########
datafusion/physical-optimizer/src/projection_pushdown.rs:
##########
@@ -490,29 +497,111 @@ fn leaf_column_name(index: usize) -> String {
format!("{LEAF_COLUMN_PREFIX}_{index}")
}
+/// Returns the child of `node` if a projection may be split beneath it.
+///
+/// Every arm evaluates no expression of its own and has exactly one child, so
+/// widening that child with the leaf columns cannot change what it does. Each
+/// also hands its child's schema straight through, which is what lets the
+/// outer expressions keep their column indices when the split descends past
+/// several of these in a row.
+///
+/// That last property holds by construction rather than by luck: none of the
+/// three `compute_properties` takes a schema, and each clones its input's
+/// equivalence properties, so a host's schema *is* its child's. There is no
+/// guard here for it. A fourth arm that projects would need one, because
+/// `rewrite_outer_expr` remaps the outer column indices and rebases lambda
+/// parameters against the width of the chain's bottom schema, and a host that
+/// changed the width would leave both against the wrong one.
+fn split_host_child(node: &Arc<dyn ExecutionPlan>) -> Option<&Arc<dyn
ExecutionPlan>> {
+ if let Some(repartition) = node.downcast_ref::<RepartitionExec>() {
+ // Hash and Range partitioning reference columns by index, which would
Review Comment:
likewise, these comments is far too verbise -- please rewrite these in your
own words rather than leaving this re-explanation of the code in
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]