andygrove commented on code in PR #2875:
URL: 
https://github.com/apache/datafusion-ballista/pull/2875#discussion_r4601658854


##########
ballista/scheduler/src/state/execution_stage.rs:
##########
@@ -234,6 +236,13 @@ pub struct RunningStage {
     pub task_failure_numbers: Vec<usize>,
     /// Combined metrics of the already finished tasks in the stage, If it is 
None, no task is finished yet.
     pub stage_metrics: Option<Vec<MetricsSet>>,
+    /// How many operator metrics sets `plan` has, which is how many a task
+    /// of this stage reports. A task that reports a different number ran a
+    /// plan of a different shape and its metrics can't be merged.
+    pub plan_metrics_len: usize,

Review Comment:
   Small suggestion. `plan_metrics_len` and `skipped_metrics_warned` look like 
internal bookkeeping. Would `pub(crate)` work for these? `RunningStage` is 
reachable through `pub mod state`, so keeping them crate-private avoids growing 
the public surface.



##########
ballista/scheduler/src/planner/range_shuffle_tests.rs:
##########
@@ -462,3 +479,109 @@ async fn 
consumer_of_a_range_shuffle_resolves_to_one_partition_per_range() -> Re
     assert_eq!(reader.properties().output_partitioning(), &partitioning);
     Ok(())
 }
+
+// --- operator metrics, as executors report them 
------------------------------
+
+/// Runs `graph` to completion, running every task the way an executor does and
+/// reporting its real operator metrics back. Checks that the metrics of every
+/// task merge into its stage and that each finished stage has one metrics set
+/// per operator of its plan. Returns `(stage id, tasks, metrics sets)` for 
each
+/// stage.
+async fn run_graph_reporting_metrics(
+    mut graph: StaticExecutionGraph,
+) -> Result<Vec<(usize, usize, usize)>> {
+    let work_dir = TempDir::new()?;
+    let executor = mock_executor("executor-0".to_string());
+    let mut tasks: HashMap<usize, usize> = HashMap::new();
+    while !graph.is_successful() {
+        let task = graph
+            .pop_next_task(&executor.id)?
+            .expect("a job that has not succeeded has a task to run");
+        let (stage_id, task_id) = (task.key.stage_id, task.key.task_id);
+        let (partitions, metrics) = run_task(
+            &task.plan,
+            stage_id,
+            task_id,
+            &task.global_input_partition_ids,
+            work_dir.path(),
+        )
+        .await?;
+
+        // The graph skips metrics it can't merge, so merge them into a copy of
+        // the stage first to fail on a skipped task, with the reason.
+        let ExecutionStage::Running(stage) = &graph.stages()[&stage_id] else {
+            panic!("stage {stage_id} is not running");
+        };
+        stage
+            .clone()
+            .update_task_metrics(task_id, metrics.clone())?;
+
+        let status = TaskStatus {
+            task_id: task_id as u32,
+            job_id: task.key.job_id.clone().into(),
+            stage_id: stage_id as u32,
+            stage_attempt_num: task.stage_attempt_num as u32,
+            metrics,
+            status: Some(task_status::Status::Successful(SuccessfulTask {
+                executor_id: executor.id.clone(),
+                partitions,
+                ..Default::default()
+            })),
+            ..Default::default()
+        };
+        graph.update_task_status(&executor, vec![status], 1, 1)?;
+        *tasks.entry(stage_id).or_default() += 1;
+    }
+
+    let mut stages: Vec<_> = graph
+        .stages()
+        .iter()
+        .map(|(stage_id, stage)| {
+            let ExecutionStage::Successful(stage) = stage else {
+                panic!("stage {stage_id} is {}", stage.variant_name());
+            };
+            let plan_metrics = collect_plan_metrics(stage.plan.as_ref()).len();
+            assert_eq!(stage.stage_metrics.len(), plan_metrics, "stage 
{stage_id}");
+            (*stage_id, tasks[stage_id], plan_metrics)
+        })
+        .collect();
+    stages.sort();
+    Ok(stages)
+}
+
+/// Every task of a global sort reports metrics that match its stage plan.
+#[tokio::test]
+async fn global_sort_tasks_report_metrics_that_match_the_stage_plan() -> 
Result<()> {
+    let plan = global_sort_plan(source(&input_rows(4, 50)), 
range_partitioning());
+    let graph = graph_from_physical(plan, SessionConfig::new_with_ballista());
+
+    let stages = run_graph_reporting_metrics(graph).await?;
+
+    // (stage id, tasks, metrics sets)
+    assert_eq!(stages, vec![(1, 4, 2), (2, 3, 3)]);
+    Ok(())
+}
+
+/// Every task of a join, aggregate and sort reports metrics that match its
+/// stage plan.
+#[tokio::test]
+async fn join_aggregate_tasks_report_metrics_that_match_the_stage_plan() -> 
Result<()> {
+    let ctx =
+        
SessionContext::new_with_config(SessionConfig::new().with_target_partitions(4));
+    for table in ["a", "b"] {
+        let rows = MemTable::try_new(schema(), batches(&input_rows(4, 50)))?;
+        ctx.register_table(table, Arc::new(rows))?;
+    }
+    let plan = ctx
+        .sql("SELECT a.k, count(*), sum(b.id) FROM a JOIN b ON a.k = b.k GROUP 
BY a.k ORDER BY a.k")
+        .await?
+        .create_physical_plan()
+        .await?;
+    let graph = graph_from_physical(plan, SessionConfig::new_with_ballista());
+
+    let stages = run_graph_reporting_metrics(graph).await?;
+
+    // (stage id, tasks, metrics sets)
+    assert_eq!(stages, vec![(1, 4, 2), (2, 4, 5), (3, 4, 5), (4, 1, 3)]);

Review Comment:
   Optional thought. The exact `(stage, tasks, metrics sets)` tuples depend on 
the plan DataFusion produces, so they might need updating on an unrelated 
DataFusion bump. `run_graph_reporting_metrics` already asserts the invariant we 
care about. What do you think about relaxing this to something like checking 
the stage count and that each stage ran at least one task?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to