This is an automated email from the ASF dual-hosted git repository.
pankaj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6412b06a7b Fix KPO task hanging when pod fails to start within
specified timeout (#37514)
6412b06a7b is described below
commit 6412b06a7b35a0743656dd3b2160f390f40108c2
Author: Pankaj Koti <[email protected]>
AuthorDate: Sun Feb 18 18:04:46 2024 +0530
Fix KPO task hanging when pod fails to start within specified timeout
(#37514)
I am observing an issue wrt to the recent deferrable KPO changes in
PR https://github.com/apache/airflow/pull/37279 and
https://github.com/apache/airflow/pull/37454,
where when the pod fails to start within a specified timeout value,
the KPO task is hanging forever whereas it is expected to fail after
the timeout. This PR fixes the issue by correcting a logical error
for detecting if elapsed timeout has occured for raising the timeout
trigger event.
---
airflow/providers/cncf/kubernetes/operators/pod.py | 10 ++++++++--
airflow/providers/cncf/kubernetes/triggers/pod.py | 2 ++
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py
b/airflow/providers/cncf/kubernetes/operators/pod.py
index 61442a6014..93fc2b59e7 100644
--- a/airflow/providers/cncf/kubernetes/operators/pod.py
+++ b/airflow/providers/cncf/kubernetes/operators/pod.py
@@ -704,6 +704,10 @@ class KubernetesPodOperator(BaseOperator):
)
if event["status"] in ("error", "failed", "timeout"):
+ # fetch some logs when pod is failed
+ if self.get_logs:
+ self.write_logs(self.pod)
+
if self.do_xcom_push:
_ = self.extract_xcom(pod=self.pod)
@@ -729,6 +733,10 @@ class KubernetesPodOperator(BaseOperator):
self.invoke_defer_method()
elif event["status"] == "success":
+ # fetch some logs when pod is executed successfully
+ if self.get_logs:
+ self.write_logs(self.pod)
+
if self.do_xcom_push:
xcom_sidecar_output = self.extract_xcom(pod=self.pod)
return xcom_sidecar_output
@@ -741,8 +749,6 @@ class KubernetesPodOperator(BaseOperator):
def _clean(self, event: dict[str, Any]):
if event["status"] == "running":
return
- if self.get_logs:
- self.write_logs(self.pod)
istio_enabled = self.is_istio_enabled(self.pod)
# Skip await_pod_completion when the event is 'timeout' due to the pod
can hang
# on the ErrImagePull or ContainerCreating step and it will never
complete
diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py
b/airflow/providers/cncf/kubernetes/triggers/pod.py
index 8c411e4e4a..7cfb34a97e 100644
--- a/airflow/providers/cncf/kubernetes/triggers/pod.py
+++ b/airflow/providers/cncf/kubernetes/triggers/pod.py
@@ -190,6 +190,7 @@ class KubernetesPodTrigger(BaseTrigger):
"message": message,
}
)
+ return
except Exception as e:
yield TriggerEvent(
{
@@ -223,6 +224,7 @@ class KubernetesPodTrigger(BaseTrigger):
return self.define_container_state(pod)
self.log.info("Still waiting for pod to start. The pod state is
%s", pod.status.phase)
await asyncio.sleep(self.poll_interval)
+ delta = datetime.datetime.now(tz=datetime.timezone.utc) -
self.trigger_start_time
raise PodLaunchTimeoutException("Pod did not leave 'Pending' phase
within specified timeout")
async def _wait_for_container_completion(self) -> TriggerEvent: