timsaucer commented on code in PR #27428:
URL: https://github.com/apache/datafusion/pull/27428#discussion_r4603653519


##########
datafusion/ffi/src/execution_plan.rs:
##########
@@ -1005,6 +1174,161 @@ pub mod tests {
         Ok(())
     }
 
+    /// A foreign plan reports the same input requirements as the plan it 
wraps,
+    /// so optimizer rules on the other side of the boundary see them.
+    #[test]
+    fn test_ffi_execution_plan_input_requirements_round_trip() -> Result<()> {
+        use arrow::compute::SortOptions;
+        use datafusion_physical_expr::expressions::Column;
+        use datafusion_physical_expr_common::sort_expr::{
+            LexRequirement, PhysicalSortRequirement,
+        };
+
+        let schema = Arc::new(arrow::datatypes::Schema::new(vec![
+            arrow::datatypes::Field::new("a", 
arrow::datatypes::DataType::Int32, false),
+        ]));
+        let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;
+        let child =
+            || Arc::new(EmptyExec::new(Arc::clone(&schema))) as Arc<dyn 
ExecutionPlan>;
+        let foreign = |plan: EmptyExec| -> Result<Arc<dyn ExecutionPlan>> {
+            let mut ffi = FFI_ExecutionPlan::new(Arc::new(plan), None);
+            ffi.library_marker_id = crate::mock_foreign_marker_id;
+            (&ffi).try_into()
+        };
+
+        // Defaults survive unchanged.
+        let default_plan = foreign(
+            EmptyExec::new(Arc::clone(&schema)).with_children(vec![child(), 
child()]),
+        )?;
+        assert_eq!(
+            format!("{:?}", default_plan.input_distribution_requirements()),
+            format!(
+                "{:?}",
+                InputDistributionRequirements::new(vec![
+                    Distribution::UnspecifiedDistribution;
+                    2
+                ])
+            )
+        );
+        assert_eq!(default_plan.required_input_ordering(), vec![None, None]);
+        assert_eq!(default_plan.maintains_input_order(), vec![false, false]);
+        assert_eq!(
+            default_plan.benefits_from_input_partitioning(),
+            vec![true, true]
+        );
+
+        // Non-default requirements, including co-partitioning and soft
+        // ordering, survive unchanged.
+        let ordering = OrderingRequirements::new_soft(
+            LexRequirement::new([PhysicalSortRequirement::new(
+                Arc::clone(&a),
+                Some(SortOptions::new(true, false)),
+            )])
+            .expect("non-empty"),
+        );
+        let reqs = TestInputRequirements {
+            distribution: InputDistributionRequirements::co_partitioned(vec![
+                Distribution::KeyPartitioned(vec![Arc::clone(&a)]),
+                Distribution::KeyPartitioned(vec![Arc::clone(&a)]),
+            ]),
+            ordering: vec![Some(ordering.clone()), None],
+            maintains_input_order: vec![true, false],
+            benefits_from_input_partitioning: vec![false, true],
+        };
+        let plan = foreign(
+            EmptyExec::new(Arc::clone(&schema))
+                .with_children(vec![child(), child()])
+                .with_input_requirements(reqs.clone()),
+        )?;
+        assert!(plan.is::<ForeignExecutionPlan>());
+        assert_eq!(
+            format!("{:?}", plan.input_distribution_requirements()),
+            format!("{:?}", reqs.distribution)
+        );
+        #[expect(deprecated, reason = "the deprecated method must agree too")]
+        let per_child = plan.required_input_distribution();
+        assert_eq!(
+            format!("{per_child:?}"),
+            format!("{:?}", reqs.distribution.clone().into_per_child())
+        );
+        assert_eq!(plan.required_input_ordering(), vec![Some(ordering), None]);
+        assert_eq!(plan.maintains_input_order(), vec![true, false]);
+        assert_eq!(plan.benefits_from_input_partitioning(), vec![false, true]);
+
+        Ok(())
+    }
+
+    /// `EnsureRequirements` running on the consumer side of the boundary sees
+    /// that a foreign `FinalPartitioned` aggregate needs its input
+    /// hash-partitioned, and inserts the repartition. Without the requirement
+    /// crossing FFI it adds only a round-robin repartition, producing a plan
+    /// that groups the same key in several partitions.
+    #[cfg(test)]
+    #[tokio::test]
+    async fn test_ensure_requirements_sees_foreign_plan_requirements() -> 
Result<()> {
+        use datafusion::arrow::array::Int32Array;
+        use datafusion::arrow::datatypes::{DataType, Field, Schema};
+        use datafusion::arrow::record_batch::RecordBatch;
+        use datafusion::common::tree_node::{Transformed, TreeNode};

Review Comment:
   Should all the use statements within these test methods be up at the `mod` 
level instead? 



##########
docs/source/library-user-guide/upgrading/56.0.0.md:
##########
@@ -308,6 +308,24 @@ serialized physical plan reconstructed with local type 
identities. See the
 pattern. `ForeignSession::query_planner`, `optimize`, and `physical_optimizers`
 continue to forward to the owning session across the FFI boundary.
 
+### `FFI_ExecutionPlan` carries input requirements
+
+`FFI_ExecutionPlan` has new callbacks for `input_distribution_requirements`,
+`required_input_ordering`, `maintains_input_order` and
+`benefits_from_input_partitioning`, so a `ForeignExecutionPlan` now reports the
+same input requirements as the plan it wraps. Previously these fell back to the
+`ExecutionPlan` defaults, so physical optimizer rules running across the FFI
+boundary (for example with a foreign `QueryPlanner`) could produce plans that
+did not satisfy them.
+
+`FFI_PhysicalExpr` also has a new `eq` callback, so two `ForeignPhysicalExpr`s
+wrapping equal expressions from the same library now compare equal. Previously
+they compared by identity, so a foreign plan's requirements never matched its
+children's partitioning or ordering on the host.
+
+Both structs have a new layout; rebuild FFI providers and consumers against the
+same release.

Review Comment:
   I've seen my agents creating these kinds of notes in the upgrade guide, 
except I don't think they actually help anyone during upgrading. These are more 
like release note changes rather than instructions for how a user might need to 
upgrade.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to