This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 55189de57c0 adding 3 logging statements to display that a job is still
waiting in the Dataproc wait_for_job method (#56523)
55189de57c0 is described below
commit 55189de57c01235c1cab238b23246fdc4b2358ad
Author: jakepage11 <[email protected]>
AuthorDate: Fri Oct 10 10:02:20 2025 -0700
adding 3 logging statements to display that a job is still waiting in the
Dataproc wait_for_job method (#56523)
---
providers/google/src/airflow/providers/google/cloud/hooks/dataproc.py | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/providers/google/src/airflow/providers/google/cloud/hooks/dataproc.py
b/providers/google/src/airflow/providers/google/cloud/hooks/dataproc.py
index 872673aae6b..388ea22e24d 100644
--- a/providers/google/src/airflow/providers/google/cloud/hooks/dataproc.py
+++ b/providers/google/src/airflow/providers/google/cloud/hooks/dataproc.py
@@ -912,12 +912,15 @@ class DataprocHook(GoogleBaseHook):
state = None
start = time.monotonic()
while state not in (JobStatus.State.ERROR, JobStatus.State.DONE,
JobStatus.State.CANCELLED):
+ self.log.debug("Waiting for job %s to complete", job_id)
if timeout and start + timeout < time.monotonic():
raise AirflowException(f"Timeout: dataproc job {job_id} is not
ready after {timeout}s")
+ self.log.debug("Sleeping for %s seconds", wait_time)
time.sleep(wait_time)
try:
job = self.get_job(project_id=project_id, region=region,
job_id=job_id)
state = job.status.state
+ self.log.debug("Job %s is in state %s", job_id, state)
except ServerError as err:
self.log.info("Retrying. Dataproc API returned server error
when waiting for job: %s", err)
