shahar1 commented on code in PR #75658: URL: https://github.com/apache/airflow/pull/75658#discussion_r4619472223
########## providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/kueue.py: ########## @@ -0,0 +1,293 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Wait for a Kubernetes Job run through Kueue.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator, Iterable +from enum import Enum +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from kubernetes.client.rest import ApiException + +from airflow.providers.cncf.kubernetes.hooks.kubernetes import ( + JOB_FINAL_STATUS_CONDITION_TYPES, + AsyncKubernetesHook, + KubernetesHook, +) +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import generic_api_retry +from airflow.providers.cncf.kubernetes.utils.container import container_is_running, container_is_terminated +from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager, XComRetrievalError +from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults +from airflow.triggers.base import BaseTrigger, TriggerEvent + +if TYPE_CHECKING: + from kubernetes.client import CoreV1Api, models as k8s + + +class KueueJobState(str, Enum): + """State of a Job run through Kueue.""" + + SUSPENDED = "suspended" + RUNNING = "running" + FINISHED = "finished" + + +JOB_STATE_MESSAGES = { + KueueJobState.SUSPENDED: "is suspended: waiting for admission by Kueue, or preempted by Kueue", + KueueJobState.RUNNING: "is admitted by Kueue: its pods are starting or running", + KueueJobState.FINISHED: "is finished", +} + + +def get_job_state(job: k8s.V1Job) -> KueueJobState: + """Return the state of a Job run through Kueue; finished means Complete or Failed.""" + conditions = (job.status.conditions if job.status else None) or [] + if any(c.type in JOB_FINAL_STATUS_CONDITION_TYPES and c.status == "True" for c in conditions): + return KueueJobState.FINISHED + # The Job is created suspended; Kueue resumes it on admission and suspends it again on preemption + if job.spec and job.spec.suspend: + return KueueJobState.SUSPENDED + return KueueJobState.RUNNING + + +def build_pod_label_selector(job: k8s.V1Job) -> str: + """ + Return the label selector of the pods of the Job, built from its ``spec.selector``. + + Kubernetes adds the ``batch.kubernetes.io/job-name`` label to the pods only without ``manualSelector``, + while ``spec.selector`` matches them in both modes. + """ + selector = job.spec.selector + requirements = [f"{key}={value}" for key, value in sorted((selector.match_labels or {}).items())] + for expression in selector.match_expressions or []: + values = ",".join(expression.values or []) + requirements.append( + { + "In": f"{expression.key} in ({values})", + "NotIn": f"{expression.key} notin ({values})", + "Exists": expression.key, + "DoesNotExist": f"!{expression.key}", + }[expression.operator] + ) + return ",".join(requirements) + + +def build_job_event(job: k8s.V1Job, xcom_results: dict[str, str]) -> dict[str, Any]: + """ + Build the event of a finished Job, as passed to ``KubernetesStartKueueJobOperator.execute_complete``. + + :param job: the finished Job. + :param xcom_results: the raw XComs read from the pods, by pod name. + """ + error_message = KubernetesHook.is_job_failed(job=job) + return { + "name": job.metadata.name, + "namespace": job.metadata.namespace, + "status": "error" if error_message else "success", + "message": f"Job failed with error: {error_message}" + if error_message + else "Job completed successfully", + "job": job.to_dict(), + "pod_label_selector": build_pod_label_selector(job), + "xcom_result": xcom_results, + } + + +def get_succeeded_pod_xcoms(pods: Iterable[k8s.V1Pod], xcom_results: dict[str, str]) -> list[str]: + """Return the XComs read from the pods that succeeded, skipping failed and preempted pods.""" + return [ + xcom_results[pod.metadata.name] + for pod in pods + if pod.metadata.name in xcom_results and pod.status and pod.status.phase == "Succeeded" + ] + + +@generic_api_retry +def list_job_pods(client: CoreV1Api, namespace: str, label_selector: str, job_uid: str) -> list[k8s.V1Pod]: + """ + Return the pods the Job has now: those matched by ``build_pod_label_selector`` that the Job owns. + + With ``manualSelector``, the selector may also match the pods of other Jobs, such as the same task + in another Dag run; the Job controller tells them apart by owner too. + """ + pods = client.list_namespaced_pod(namespace=namespace, label_selector=label_selector).items + return [pod for pod in pods if any(owner.uid == job_uid for owner in pod.metadata.owner_references or [])] + + +@generic_api_retry +def is_pod_deleted(client: CoreV1Api, pod: k8s.V1Pod) -> bool: + """Return whether the pod no longer exists.""" + try: + client.read_namespaced_pod(name=pod.metadata.name, namespace=pod.metadata.namespace) + except ApiException as e: + if e.status == 404: + return True + raise + return False + + +def collect_xcom_results( + client: CoreV1Api, + pod_manager: PodManager, + job: k8s.V1Job, + base_container_name: str, + xcom_results: dict[str, str], +) -> None: + """ + Read into ``xcom_results`` the XCom of each pod of the Job whose base container has ended. + + Reading the XCom also stops the XCom sidecar: until then the pod, and so the Job, cannot finish. + Pods already read are skipped, so this is called at each poll. + """ + for pod in list_job_pods(client, job.metadata.namespace, build_pod_label_selector(job), job.metadata.uid): + if ( + pod.metadata.name in xcom_results + or not container_is_terminated(pod, base_container_name) + or not container_is_running(pod, PodDefaults.SIDECAR_CONTAINER_NAME) + ): + continue + try: + xcom_results[pod.metadata.name] = pod_manager.extract_xcom(pod, ignore_kill_failure=False) + except XComRetrievalError as e: + pod_manager.log.warning("Cannot read the XCom of the pod %s: %s", pod.metadata.name, e) + except ApiException as e: + # Skip a pod deleted meanwhile, e.g. by a preemption: kubernetes.stream reports any failed + # exec with status 0, so check that the pod is gone. Raise other failures, such as a + # forbidden exec: skipped, the sidecar would keep the Job running forever. + if e.status != 404 and not (e.status == 0 and is_pod_deleted(client, pod)): Review Comment: **minor**: a pod that is being deleted, but still exists, fails the task here. `is_pod_deleted` returns `True` only on a 404. Kubernetes deletes pods gracefully, so `read_namespaced_pod` can still return a pod whose sidecar has already stopped, with `metadata.deletion_timestamp` set. If Kueue preempts the Job between the Job read of this poll and the exec, the exec fails with status 0, `is_pod_deleted` returns `False`, and the exception is raised again. Neither `_wait_until_job_finished` nor `KubernetesKueueJobTrigger.run` catches it. The task then fails during a preemption, the Job stays in its queue, and a retry creates a second Job. The window is short: one poll, for a pod whose base container has ended and whose XCom is not read yet. So I don't see it as blocking. Treating a pod that has a `deletion_timestamp` as gone would close it: ```python try: pod = client.read_namespaced_pod(name=pod.metadata.name, namespace=pod.metadata.namespace) except ApiException as e: if e.status == 404: return True raise return pod.metadata.deletion_timestamp is not None ``` *Flagged by copilot (adversarial review); cross-checked.* -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
