shahar1 commented on code in PR #75658:
URL: https://github.com/apache/airflow/pull/75658#discussion_r4619472223


##########
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/kueue.py:
##########
@@ -0,0 +1,293 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Wait for a Kubernetes Job run through Kueue."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import AsyncIterator, Iterable
+from enum import Enum
+from functools import cached_property
+from typing import TYPE_CHECKING, Any
+
+from kubernetes.client.rest import ApiException
+
+from airflow.providers.cncf.kubernetes.hooks.kubernetes import (
+    JOB_FINAL_STATUS_CONDITION_TYPES,
+    AsyncKubernetesHook,
+    KubernetesHook,
+)
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import 
generic_api_retry
+from airflow.providers.cncf.kubernetes.utils.container import 
container_is_running, container_is_terminated
+from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager, 
XComRetrievalError
+from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
+from airflow.triggers.base import BaseTrigger, TriggerEvent
+
+if TYPE_CHECKING:
+    from kubernetes.client import CoreV1Api, models as k8s
+
+
+class KueueJobState(str, Enum):
+    """State of a Job run through Kueue."""
+
+    SUSPENDED = "suspended"
+    RUNNING = "running"
+    FINISHED = "finished"
+
+
+JOB_STATE_MESSAGES = {
+    KueueJobState.SUSPENDED: "is suspended: waiting for admission by Kueue, or 
preempted by Kueue",
+    KueueJobState.RUNNING: "is admitted by Kueue: its pods are starting or 
running",
+    KueueJobState.FINISHED: "is finished",
+}
+
+
+def get_job_state(job: k8s.V1Job) -> KueueJobState:
+    """Return the state of a Job run through Kueue; finished means Complete or 
Failed."""
+    conditions = (job.status.conditions if job.status else None) or []
+    if any(c.type in JOB_FINAL_STATUS_CONDITION_TYPES and c.status == "True" 
for c in conditions):
+        return KueueJobState.FINISHED
+    # The Job is created suspended; Kueue resumes it on admission and suspends 
it again on preemption
+    if job.spec and job.spec.suspend:
+        return KueueJobState.SUSPENDED
+    return KueueJobState.RUNNING
+
+
+def build_pod_label_selector(job: k8s.V1Job) -> str:
+    """
+    Return the label selector of the pods of the Job, built from its 
``spec.selector``.
+
+    Kubernetes adds the ``batch.kubernetes.io/job-name`` label to the pods 
only without ``manualSelector``,
+    while ``spec.selector`` matches them in both modes.
+    """
+    selector = job.spec.selector
+    requirements = [f"{key}={value}" for key, value in 
sorted((selector.match_labels or {}).items())]
+    for expression in selector.match_expressions or []:
+        values = ",".join(expression.values or [])
+        requirements.append(
+            {
+                "In": f"{expression.key} in ({values})",
+                "NotIn": f"{expression.key} notin ({values})",
+                "Exists": expression.key,
+                "DoesNotExist": f"!{expression.key}",
+            }[expression.operator]
+        )
+    return ",".join(requirements)
+
+
+def build_job_event(job: k8s.V1Job, xcom_results: dict[str, str]) -> dict[str, 
Any]:
+    """
+    Build the event of a finished Job, as passed to 
``KubernetesStartKueueJobOperator.execute_complete``.
+
+    :param job: the finished Job.
+    :param xcom_results: the raw XComs read from the pods, by pod name.
+    """
+    error_message = KubernetesHook.is_job_failed(job=job)
+    return {
+        "name": job.metadata.name,
+        "namespace": job.metadata.namespace,
+        "status": "error" if error_message else "success",
+        "message": f"Job failed with error: {error_message}"
+        if error_message
+        else "Job completed successfully",
+        "job": job.to_dict(),
+        "pod_label_selector": build_pod_label_selector(job),
+        "xcom_result": xcom_results,
+    }
+
+
+def get_succeeded_pod_xcoms(pods: Iterable[k8s.V1Pod], xcom_results: dict[str, 
str]) -> list[str]:
+    """Return the XComs read from the pods that succeeded, skipping failed and 
preempted pods."""
+    return [
+        xcom_results[pod.metadata.name]
+        for pod in pods
+        if pod.metadata.name in xcom_results and pod.status and 
pod.status.phase == "Succeeded"
+    ]
+
+
+@generic_api_retry
+def list_job_pods(client: CoreV1Api, namespace: str, label_selector: str, 
job_uid: str) -> list[k8s.V1Pod]:
+    """
+    Return the pods the Job has now: those matched by 
``build_pod_label_selector`` that the Job owns.
+
+    With ``manualSelector``, the selector may also match the pods of other 
Jobs, such as the same task
+    in another Dag run; the Job controller tells them apart by owner too.
+    """
+    pods = client.list_namespaced_pod(namespace=namespace, 
label_selector=label_selector).items
+    return [pod for pod in pods if any(owner.uid == job_uid for owner in 
pod.metadata.owner_references or [])]
+
+
+@generic_api_retry
+def is_pod_deleted(client: CoreV1Api, pod: k8s.V1Pod) -> bool:
+    """Return whether the pod no longer exists."""
+    try:
+        client.read_namespaced_pod(name=pod.metadata.name, 
namespace=pod.metadata.namespace)
+    except ApiException as e:
+        if e.status == 404:
+            return True
+        raise
+    return False
+
+
+def collect_xcom_results(
+    client: CoreV1Api,
+    pod_manager: PodManager,
+    job: k8s.V1Job,
+    base_container_name: str,
+    xcom_results: dict[str, str],
+) -> None:
+    """
+    Read into ``xcom_results`` the XCom of each pod of the Job whose base 
container has ended.
+
+    Reading the XCom also stops the XCom sidecar: until then the pod, and so 
the Job, cannot finish.
+    Pods already read are skipped, so this is called at each poll.
+    """
+    for pod in list_job_pods(client, job.metadata.namespace, 
build_pod_label_selector(job), job.metadata.uid):
+        if (
+            pod.metadata.name in xcom_results
+            or not container_is_terminated(pod, base_container_name)
+            or not container_is_running(pod, 
PodDefaults.SIDECAR_CONTAINER_NAME)
+        ):
+            continue
+        try:
+            xcom_results[pod.metadata.name] = pod_manager.extract_xcom(pod, 
ignore_kill_failure=False)
+        except XComRetrievalError as e:
+            pod_manager.log.warning("Cannot read the XCom of the pod %s: %s", 
pod.metadata.name, e)
+        except ApiException as e:
+            # Skip a pod deleted meanwhile, e.g. by a preemption: 
kubernetes.stream reports any failed
+            # exec with status 0, so check that the pod is gone. Raise other 
failures, such as a
+            # forbidden exec: skipped, the sidecar would keep the Job running 
forever.
+            if e.status != 404 and not (e.status == 0 and 
is_pod_deleted(client, pod)):

Review Comment:
   **minor**: a pod that is being deleted, but still exists, fails the task 
here. `is_pod_deleted` returns `True` only on a 404. Kubernetes deletes pods 
gracefully, so `read_namespaced_pod` can still return a pod whose sidecar has 
already stopped, with `metadata.deletion_timestamp` set.
   
   If Kueue preempts the Job between the Job read of this poll and the exec, 
the exec fails with status 0, `is_pod_deleted` returns `False`, and the 
exception is raised again. Neither `_wait_until_job_finished` nor 
`KubernetesKueueJobTrigger.run` catches it. The task then fails during a 
preemption, the Job stays in its queue, and a retry creates a second Job.
   
   The window is short: one poll, for a pod whose base container has ended and 
whose XCom is not read yet. So I don't see it as blocking. Treating a pod that 
has a `deletion_timestamp` as gone would close it:
   
   ```python
       try:
           pod = client.read_namespaced_pod(name=pod.metadata.name, 
namespace=pod.metadata.namespace)
       except ApiException as e:
           if e.status == 404:
               return True
           raise
       return pod.metadata.deletion_timestamp is not None
   ```
   
   *Flagged by copilot (adversarial review); cross-checked.*
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to