andygrove commented on code in PR #2884:
URL: 
https://github.com/apache/datafusion-ballista/pull/2884#discussion_r4607336635


##########
ballista/core/src/execution_plans/distributed_query.rs:
##########
@@ -1206,87 +1213,84 @@ async fn execute_query_push(
                     job_id,
                     pipelined_output_locations,
                     PipelinedFetch {
-                        max_message_size,
-                        scheduler_url,
                         flight_proxy,
-                        customize_endpoint,
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                        registration_timeout,
+                        ..fetch.clone()
                     },
                     failure,
                     scheduler,
                 );
             }
             Some(job_status::Status::Running(_)) => {
-                if has_status_change {
-                    info!("Job {job_id} is running...");
-                }
+                log_job_wait(&job_id, &status, &prev_status);
                 prev_status = status;
             }
             Some(job_status::Status::Failed(err)) => {
-                let msg = format!("Job {} failed: {}", job_id, err.error);
-                error!("{msg}");
-                break Err(DataFusionError::Execution(msg));
+                break Err(job_failed(&job_id, err.error));
             }
-            Some(job_status::Status::Successful(SuccessfulJob {
-                queued_at,
-                started_at,
-                ended_at,
-                partition_location,
-                ..
-            })) => {
-                // Calculate job execution time (server-side execution)
-                let job_execution_ms = ended_at.saturating_sub(started_at);
-                let duration = Duration::from_millis(job_execution_ms);
-
-                info!("Job {job_id} finished executing in {duration:?} ");
-
-                // Calculate scheduling time (server-side queue time)
-                // This includes network latency and actual queue time
-                let scheduling_ms = started_at.saturating_sub(queued_at);
-
-                // Calculate total query time (end-to-end from client 
perspective)
-                let total_elapsed = query_start_time.elapsed();
-                let total_ms = total_elapsed.as_millis();
-
-                // Set timing metrics
-                let metric_job_execution = MetricBuilder::new(&metrics)
-                    .gauge("job_execution_time_ms", partition);
-                metric_job_execution.set(job_execution_ms as usize);
-
-                let metric_scheduling =
-                    MetricBuilder::new(&metrics).gauge("job_scheduling_in_ms", 
partition);
-                metric_scheduling.set(scheduling_ms as usize);
-
-                let metric_total_time =
-                    MetricBuilder::new(&metrics).gauge("total_query_time_ms", 
partition);
-                metric_total_time.set(total_ms as usize);
-
-                // Note: data_transfer_time_ms is not set here because 
partition fetching
-                // happens lazily when the stream is consumed, not during 
execute_query.
-                // This could be added in a future enhancement by wrapping the 
stream.
-
-                let streams = partition_location.into_iter().map(move 
|partition| {
-                    let f = fetch_partition(
-                        partition,
-                        max_message_size,
-                        true,
-                        scheduler_url.clone(),
-                        flight_proxy.clone(),
-                        customize_endpoint.clone(),
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                    );
-
-                    futures::stream::once(f).try_flatten()
-                });
+            Some(job_status::Status::Successful(successful_job)) => {
+                break Ok(finish_successful_job(
+                    &job_id,
+                    successful_job,
+                    query_start_time,
+                    &metrics,
+                    partition,
+                    PipelinedFetch {
+                        flight_proxy,
+                        ..fetch
+                    },
+                ));
+            }
+        }
+    }
+}
+
+/// Polls a job's status every `poll_interval` until it finishes, and returns
+/// the job once it has succeeded.
+///
+/// Pipelined jobs are not supported here, because their results stream while
+/// the job runs. Use [DistributedQueryExec] to read them.
+pub async fn poll_job_until_successful(
+    scheduler: &mut SchedulerClient,
+    job_id: JobId,
+    poll_interval: Duration,
+) -> Result<SuccessfulJob> {
+    let mut prev_status: Option<job_status::Status> = None;
+
+    loop {
+        let GetJobStatusResult { status, .. } = scheduler
+            .get_job_status(GetJobStatusParams {
+                job_id: job_id.clone().into(),
+            })
+            .await
+            .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?
+            .into_inner();
 
-                break Ok(futures::stream::iter(streams).flatten().boxed());
+        let status = status.and_then(|s| s.status);
+        match status {
+            Some(
+                job_status::Status::Running(RunningJob {
+                    pipelined_output_locations,
+                    ..
+                })
+                | job_status::Status::Successful(SuccessfulJob {
+                    pipelined_output_locations,
+                    ..
+                }),
+            ) if !pipelined_output_locations.is_empty() => {
+                return Err(DataFusionError::NotImplemented(format!(

Review Comment:
   When this sees a running pipelined job it returns an error, but the job is 
still running on the cluster. Since nobody reads the final stage, it holds its 
vcores until the producer hits 
`ballista.pipelined.registration_timeout_seconds`. Could we call `cancel_job` 
before returning in the `Running` case? `FakeScheduler` already records 
cancellations, so `poll_job_until_successful_rejects_pipelined_jobs` could 
assert on `scheduler.cancelled()` too.



##########
ballista/core/src/execution_plans/distributed_query.rs:
##########
@@ -985,136 +1078,55 @@ async fn execute_query_pull(
                     job_id,
                     pipelined_output_locations,
                     PipelinedFetch {
-                        max_message_size,
-                        scheduler_url,
                         flight_proxy,
-                        customize_endpoint,
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                        registration_timeout,
+                        ..fetch.clone()

Review Comment:
   Tiny one. This arm uses `..fetch.clone()` but it `break`s right after, so 
`..fetch` should compile just like the `Successful` arm. Same at line 1217.



##########
ballista/core/src/execution_plans/distributed_query.rs:
##########
@@ -1206,87 +1213,84 @@ async fn execute_query_push(
                     job_id,
                     pipelined_output_locations,
                     PipelinedFetch {
-                        max_message_size,
-                        scheduler_url,
                         flight_proxy,
-                        customize_endpoint,
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                        registration_timeout,
+                        ..fetch.clone()
                     },
                     failure,
                     scheduler,
                 );
             }
             Some(job_status::Status::Running(_)) => {
-                if has_status_change {
-                    info!("Job {job_id} is running...");
-                }
+                log_job_wait(&job_id, &status, &prev_status);
                 prev_status = status;
             }
             Some(job_status::Status::Failed(err)) => {
-                let msg = format!("Job {} failed: {}", job_id, err.error);
-                error!("{msg}");
-                break Err(DataFusionError::Execution(msg));
+                break Err(job_failed(&job_id, err.error));
             }
-            Some(job_status::Status::Successful(SuccessfulJob {
-                queued_at,
-                started_at,
-                ended_at,
-                partition_location,
-                ..
-            })) => {
-                // Calculate job execution time (server-side execution)
-                let job_execution_ms = ended_at.saturating_sub(started_at);
-                let duration = Duration::from_millis(job_execution_ms);
-
-                info!("Job {job_id} finished executing in {duration:?} ");
-
-                // Calculate scheduling time (server-side queue time)
-                // This includes network latency and actual queue time
-                let scheduling_ms = started_at.saturating_sub(queued_at);
-
-                // Calculate total query time (end-to-end from client 
perspective)
-                let total_elapsed = query_start_time.elapsed();
-                let total_ms = total_elapsed.as_millis();
-
-                // Set timing metrics
-                let metric_job_execution = MetricBuilder::new(&metrics)
-                    .gauge("job_execution_time_ms", partition);
-                metric_job_execution.set(job_execution_ms as usize);
-
-                let metric_scheduling =
-                    MetricBuilder::new(&metrics).gauge("job_scheduling_in_ms", 
partition);
-                metric_scheduling.set(scheduling_ms as usize);
-
-                let metric_total_time =
-                    MetricBuilder::new(&metrics).gauge("total_query_time_ms", 
partition);
-                metric_total_time.set(total_ms as usize);
-
-                // Note: data_transfer_time_ms is not set here because 
partition fetching
-                // happens lazily when the stream is consumed, not during 
execute_query.
-                // This could be added in a future enhancement by wrapping the 
stream.
-
-                let streams = partition_location.into_iter().map(move 
|partition| {
-                    let f = fetch_partition(
-                        partition,
-                        max_message_size,
-                        true,
-                        scheduler_url.clone(),
-                        flight_proxy.clone(),
-                        customize_endpoint.clone(),
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                    );
-
-                    futures::stream::once(f).try_flatten()
-                });
+            Some(job_status::Status::Successful(successful_job)) => {
+                break Ok(finish_successful_job(
+                    &job_id,
+                    successful_job,
+                    query_start_time,
+                    &metrics,
+                    partition,
+                    PipelinedFetch {
+                        flight_proxy,
+                        ..fetch
+                    },
+                ));
+            }
+        }
+    }
+}
+
+/// Polls a job's status every `poll_interval` until it finishes, and returns
+/// the job once it has succeeded.
+///
+/// Pipelined jobs are not supported here, because their results stream while
+/// the job runs. Use [DistributedQueryExec] to read them.
+pub async fn poll_job_until_successful(

Review Comment:
   Small question on the new public surface. `poll_job_until_successful` only 
accepts the intercepted `SchedulerClient`, which is why the Substrait example 
had to switch to `with_interceptor`. Would it be worth making it generic over 
the client's transport, so embedders with their own `SchedulerGrpcClient<T>` 
can use it too? Happy either way if you'd rather keep it simple for now.



##########
ballista/core/src/execution_plans/distributed_query.rs:
##########
@@ -1206,87 +1213,84 @@ async fn execute_query_push(
                     job_id,
                     pipelined_output_locations,
                     PipelinedFetch {
-                        max_message_size,
-                        scheduler_url,
                         flight_proxy,
-                        customize_endpoint,
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                        registration_timeout,
+                        ..fetch.clone()
                     },
                     failure,
                     scheduler,
                 );
             }
             Some(job_status::Status::Running(_)) => {
-                if has_status_change {
-                    info!("Job {job_id} is running...");
-                }
+                log_job_wait(&job_id, &status, &prev_status);
                 prev_status = status;
             }
             Some(job_status::Status::Failed(err)) => {
-                let msg = format!("Job {} failed: {}", job_id, err.error);
-                error!("{msg}");
-                break Err(DataFusionError::Execution(msg));
+                break Err(job_failed(&job_id, err.error));
             }
-            Some(job_status::Status::Successful(SuccessfulJob {
-                queued_at,
-                started_at,
-                ended_at,
-                partition_location,
-                ..
-            })) => {
-                // Calculate job execution time (server-side execution)
-                let job_execution_ms = ended_at.saturating_sub(started_at);
-                let duration = Duration::from_millis(job_execution_ms);
-
-                info!("Job {job_id} finished executing in {duration:?} ");
-
-                // Calculate scheduling time (server-side queue time)
-                // This includes network latency and actual queue time
-                let scheduling_ms = started_at.saturating_sub(queued_at);
-
-                // Calculate total query time (end-to-end from client 
perspective)
-                let total_elapsed = query_start_time.elapsed();
-                let total_ms = total_elapsed.as_millis();
-
-                // Set timing metrics
-                let metric_job_execution = MetricBuilder::new(&metrics)
-                    .gauge("job_execution_time_ms", partition);
-                metric_job_execution.set(job_execution_ms as usize);
-
-                let metric_scheduling =
-                    MetricBuilder::new(&metrics).gauge("job_scheduling_in_ms", 
partition);
-                metric_scheduling.set(scheduling_ms as usize);
-
-                let metric_total_time =
-                    MetricBuilder::new(&metrics).gauge("total_query_time_ms", 
partition);
-                metric_total_time.set(total_ms as usize);
-
-                // Note: data_transfer_time_ms is not set here because 
partition fetching
-                // happens lazily when the stream is consumed, not during 
execute_query.
-                // This could be added in a future enhancement by wrapping the 
stream.
-
-                let streams = partition_location.into_iter().map(move 
|partition| {
-                    let f = fetch_partition(
-                        partition,
-                        max_message_size,
-                        true,
-                        scheduler_url.clone(),
-                        flight_proxy.clone(),
-                        customize_endpoint.clone(),
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                    );
-
-                    futures::stream::once(f).try_flatten()
-                });
+            Some(job_status::Status::Successful(successful_job)) => {
+                break Ok(finish_successful_job(
+                    &job_id,
+                    successful_job,
+                    query_start_time,
+                    &metrics,
+                    partition,
+                    PipelinedFetch {
+                        flight_proxy,
+                        ..fetch
+                    },
+                ));
+            }
+        }
+    }
+}
+
+/// Polls a job's status every `poll_interval` until it finishes, and returns
+/// the job once it has succeeded.
+///
+/// Pipelined jobs are not supported here, because their results stream while
+/// the job runs. Use [DistributedQueryExec] to read them.
+pub async fn poll_job_until_successful(
+    scheduler: &mut SchedulerClient,
+    job_id: JobId,
+    poll_interval: Duration,
+) -> Result<SuccessfulJob> {
+    let mut prev_status: Option<job_status::Status> = None;
+
+    loop {
+        let GetJobStatusResult { status, .. } = scheduler
+            .get_job_status(GetJobStatusParams {
+                job_id: job_id.clone().into(),
+            })
+            .await
+            .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?
+            .into_inner();
 
-                break Ok(futures::stream::iter(streams).flatten().boxed());
+        let status = status.and_then(|s| s.status);
+        match status {
+            Some(
+                job_status::Status::Running(RunningJob {
+                    pipelined_output_locations,
+                    ..
+                })
+                | job_status::Status::Successful(SuccessfulJob {
+                    pipelined_output_locations,
+                    ..
+                }),
+            ) if !pipelined_output_locations.is_empty() => {
+                return Err(DataFusionError::NotImplemented(format!(
+                    "Job {job_id} is pipelined, which 
poll_job_until_successful does not support; use DistributedQueryExec"
+                )));
             }
-        };
+            Some(job_status::Status::Failed(err)) => {
+                return Err(job_failed(&job_id, err.error));
+            }
+            Some(job_status::Status::Successful(successful)) => return 
Ok(successful),

Review Comment:
   Not a big deal, but the Substrait example used to log `Job ... finished 
executing in ...`, and this helper doesn't log on success. Could it log here 
like `finish_successful_job` does?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to