This is an automated email from the ASF dual-hosted git repository.

pankaj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 6412b06a7b Fix KPO task hanging when pod fails to start within 
specified timeout (#37514)
6412b06a7b is described below

commit 6412b06a7b35a0743656dd3b2160f390f40108c2
Author: Pankaj Koti <[email protected]>
AuthorDate: Sun Feb 18 18:04:46 2024 +0530

    Fix KPO task hanging when pod fails to start within specified timeout 
(#37514)
    
    I am observing an issue wrt to the recent deferrable KPO changes in
    PR https://github.com/apache/airflow/pull/37279 and 
https://github.com/apache/airflow/pull/37454,
    where when the pod fails to start within a specified timeout value,
    the KPO task is hanging forever whereas it is expected to fail after
    the timeout. This PR fixes the issue by correcting a logical error
    for detecting if elapsed timeout has occured for raising the timeout
    trigger event.
---
 airflow/providers/cncf/kubernetes/operators/pod.py | 10 ++++++++--
 airflow/providers/cncf/kubernetes/triggers/pod.py  |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py 
b/airflow/providers/cncf/kubernetes/operators/pod.py
index 61442a6014..93fc2b59e7 100644
--- a/airflow/providers/cncf/kubernetes/operators/pod.py
+++ b/airflow/providers/cncf/kubernetes/operators/pod.py
@@ -704,6 +704,10 @@ class KubernetesPodOperator(BaseOperator):
                 )
 
             if event["status"] in ("error", "failed", "timeout"):
+                # fetch some logs when pod is failed
+                if self.get_logs:
+                    self.write_logs(self.pod)
+
                 if self.do_xcom_push:
                     _ = self.extract_xcom(pod=self.pod)
 
@@ -729,6 +733,10 @@ class KubernetesPodOperator(BaseOperator):
                     self.invoke_defer_method()
 
             elif event["status"] == "success":
+                # fetch some logs when pod is executed successfully
+                if self.get_logs:
+                    self.write_logs(self.pod)
+
                 if self.do_xcom_push:
                     xcom_sidecar_output = self.extract_xcom(pod=self.pod)
                     return xcom_sidecar_output
@@ -741,8 +749,6 @@ class KubernetesPodOperator(BaseOperator):
     def _clean(self, event: dict[str, Any]):
         if event["status"] == "running":
             return
-        if self.get_logs:
-            self.write_logs(self.pod)
         istio_enabled = self.is_istio_enabled(self.pod)
         # Skip await_pod_completion when the event is 'timeout' due to the pod 
can hang
         # on the ErrImagePull or ContainerCreating step and it will never 
complete
diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py 
b/airflow/providers/cncf/kubernetes/triggers/pod.py
index 8c411e4e4a..7cfb34a97e 100644
--- a/airflow/providers/cncf/kubernetes/triggers/pod.py
+++ b/airflow/providers/cncf/kubernetes/triggers/pod.py
@@ -190,6 +190,7 @@ class KubernetesPodTrigger(BaseTrigger):
                     "message": message,
                 }
             )
+            return
         except Exception as e:
             yield TriggerEvent(
                 {
@@ -223,6 +224,7 @@ class KubernetesPodTrigger(BaseTrigger):
                 return self.define_container_state(pod)
             self.log.info("Still waiting for pod to start. The pod state is 
%s", pod.status.phase)
             await asyncio.sleep(self.poll_interval)
+            delta = datetime.datetime.now(tz=datetime.timezone.utc) - 
self.trigger_start_time
         raise PodLaunchTimeoutException("Pod did not leave 'Pending' phase 
within specified timeout")
 
     async def _wait_for_container_completion(self) -> TriggerEvent:

Reply via email to