marvelefe commented on code in PR #2884:
URL: 
https://github.com/apache/datafusion-ballista/pull/2884#discussion_r4621033317


##########
ballista/core/src/execution_plans/distributed_query.rs:
##########
@@ -1206,87 +1213,84 @@ async fn execute_query_push(
                     job_id,
                     pipelined_output_locations,
                     PipelinedFetch {
-                        max_message_size,
-                        scheduler_url,
                         flight_proxy,
-                        customize_endpoint,
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                        registration_timeout,
+                        ..fetch.clone()
                     },
                     failure,
                     scheduler,
                 );
             }
             Some(job_status::Status::Running(_)) => {
-                if has_status_change {
-                    info!("Job {job_id} is running...");
-                }
+                log_job_wait(&job_id, &status, &prev_status);
                 prev_status = status;
             }
             Some(job_status::Status::Failed(err)) => {
-                let msg = format!("Job {} failed: {}", job_id, err.error);
-                error!("{msg}");
-                break Err(DataFusionError::Execution(msg));
+                break Err(job_failed(&job_id, err.error));
             }
-            Some(job_status::Status::Successful(SuccessfulJob {
-                queued_at,
-                started_at,
-                ended_at,
-                partition_location,
-                ..
-            })) => {
-                // Calculate job execution time (server-side execution)
-                let job_execution_ms = ended_at.saturating_sub(started_at);
-                let duration = Duration::from_millis(job_execution_ms);
-
-                info!("Job {job_id} finished executing in {duration:?} ");
-
-                // Calculate scheduling time (server-side queue time)
-                // This includes network latency and actual queue time
-                let scheduling_ms = started_at.saturating_sub(queued_at);
-
-                // Calculate total query time (end-to-end from client 
perspective)
-                let total_elapsed = query_start_time.elapsed();
-                let total_ms = total_elapsed.as_millis();
-
-                // Set timing metrics
-                let metric_job_execution = MetricBuilder::new(&metrics)
-                    .gauge("job_execution_time_ms", partition);
-                metric_job_execution.set(job_execution_ms as usize);
-
-                let metric_scheduling =
-                    MetricBuilder::new(&metrics).gauge("job_scheduling_in_ms", 
partition);
-                metric_scheduling.set(scheduling_ms as usize);
-
-                let metric_total_time =
-                    MetricBuilder::new(&metrics).gauge("total_query_time_ms", 
partition);
-                metric_total_time.set(total_ms as usize);
-
-                // Note: data_transfer_time_ms is not set here because 
partition fetching
-                // happens lazily when the stream is consumed, not during 
execute_query.
-                // This could be added in a future enhancement by wrapping the 
stream.
-
-                let streams = partition_location.into_iter().map(move 
|partition| {
-                    let f = fetch_partition(
-                        partition,
-                        max_message_size,
-                        true,
-                        scheduler_url.clone(),
-                        flight_proxy.clone(),
-                        customize_endpoint.clone(),
-                        use_tls,
-                        io_retries_times,
-                        io_retry_wait_time_ms,
-                    );
-
-                    futures::stream::once(f).try_flatten()
-                });
+            Some(job_status::Status::Successful(successful_job)) => {
+                break Ok(finish_successful_job(
+                    &job_id,
+                    successful_job,
+                    query_start_time,
+                    &metrics,
+                    partition,
+                    PipelinedFetch {
+                        flight_proxy,
+                        ..fetch
+                    },
+                ));
+            }
+        }
+    }
+}
+
+/// Polls a job's status every `poll_interval` until it finishes, and returns
+/// the job once it has succeeded.
+///
+/// Pipelined jobs are not supported here, because their results stream while
+/// the job runs. Use [DistributedQueryExec] to read them.
+pub async fn poll_job_until_successful(

Review Comment:
   I'm leaning towards keeping `poll_job_until_successful` on `SchedulerClient` 
for now. It already carries Ballista's metadata interceptor, and from my 
checks, nothing in the repo needs a generic version yet. 
   
   Happy to add it in a follow-up if you'd like - I'll create its own issue. 
Just let me know



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to