This is an automated email from the ASF dual-hosted git repository.

amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 6120f7a0c06 Support durable execution for AWS BatchOperator (#77158)
6120f7a0c06 is described below

commit 6120f7a0c0621777888cd2982fb8c92df30f6c7d
Author: Amogh Desai <[email protected]>
AuthorDate: Thu Nov 19 17:27:54 2026 +0530

    Support durable execution for AWS BatchOperator (#77158)
---
 providers/amazon/docs/changelog.rst                |   8 ++
 providers/amazon/docs/operators/batch.rst          |  26 +++++
 .../providers/amazon/aws/operators/batch.py        |  62 ++++++++--
 .../tests/unit/amazon/aws/operators/test_batch.py  | 129 ++++++++++++++++++++-
 4 files changed, 214 insertions(+), 11 deletions(-)

diff --git a/providers/amazon/docs/changelog.rst 
b/providers/amazon/docs/changelog.rst
index ee50e6561c0..e1cc68887ec 100644
--- a/providers/amazon/docs/changelog.rst
+++ b/providers/amazon/docs/changelog.rst
@@ -29,6 +29,14 @@ Changelog
 9.41.0
 ......
 
+.. note::
+
+    ``BatchOperator`` supports durable execution and ``durable`` defaults to 
``True``. A retry now
+    reconnects to the Batch job the previous try submitted instead of 
submitting a new one. This
+    applies to the synchronous path only. Pass ``durable=False`` to keep the 
old behaviour.
+    Reconnecting needs Airflow 3.3 or later. On Airflow 3.3 and 3.4 most 
clears keep the stored job
+    id, so a cleared task whose job already succeeded returns without 
submitting a new one.
+
 .. note::
 
     ``SageMakerTrainingOperator`` supports durable execution and ``durable`` 
defaults to ``True``.
diff --git a/providers/amazon/docs/operators/batch.rst 
b/providers/amazon/docs/operators/batch.rst
index b992e26e4b0..c4581669c87 100644
--- a/providers/amazon/docs/operators/batch.rst
+++ b/providers/amazon/docs/operators/batch.rst
@@ -51,6 +51,32 @@ use 
:class:`~airflow.providers.amazon.aws.operators.batch.BatchOperator`.
     :start-after: [START howto_operator_batch]
     :end-before: [END howto_operator_batch]
 
+Durable execution
+=================
+
+By default ``BatchOperator`` saves the Batch job id to the
+:doc:`task state store <apache-airflow:core-concepts/task-state-store>` right 
after submitting the
+job. If the worker crashes and the task is retried, the operator checks that 
job first:
+
+* still active (``SUBMITTED`` through ``RUNNING``): reconnects and keeps 
waiting on it
+* already ``SUCCEEDED``: returns the job id without submitting again
+* ``FAILED``, or no longer returned by AWS Batch: submits a new job
+
+This applies to the synchronous path only; a deferrable run is unchanged.
+
+Reconnecting only helps when the worker dies without a chance to clean up, 
such as an OOM kill or a
+lost node. On SIGTERM (for example a pod eviction) or on 
``execution_timeout``, ``on_kill`` terminates
+the Batch job, so the retry submits a new one.
+
+Clearing a task can keep the stored job id. On Airflow 3.3 every clear keeps 
it. On 3.4 only clearing
+task instances through the UI or REST API discards it. When the id is kept and 
that job already
+succeeded, the cleared task returns without submitting a new job. From Airflow 
3.5, every clear
+discards it by default. See 
:doc:`apache-airflow:core-concepts/resumable-tasks` for the clear paths,
+and set ``durable=False`` if every clear must submit a new job.
+
+Set ``durable=False`` to submit a new job on every try. Durable execution 
needs Airflow 3.3 or
+later; on earlier versions the operator always submits a new job.
+
 .. _howto/operator:BatchCreateComputeEnvironmentOperator:
 
 Create an AWS Batch compute environment
diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py 
b/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
index 610ab9e5952..7831cd5d09a 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
@@ -28,7 +28,7 @@ from __future__ import annotations
 
 from collections.abc import Sequence
 from datetime import timedelta
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
 from airflow.providers.amazon.aws.links.batch import (
@@ -46,13 +46,15 @@ from airflow.providers.amazon.aws.utils import 
trim_none_values, validate_execut
 from airflow.providers.amazon.aws.utils.deferral import should_defer
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
 from airflow.providers.amazon.aws.utils.task_log_fetcher import 
AwsTaskLogFetcher
-from airflow.providers.common.compat.sdk import AirflowException, conf
+from airflow.providers.common.compat.sdk import AirflowException, 
ResumableJobMixin, conf
 
 if TYPE_CHECKING:
+    from pydantic import JsonValue
+
     from airflow.sdk import Context
 
 
-class BatchOperator(AwsBaseOperator[BatchClientHook]):
+class BatchOperator(ResumableJobMixin, AwsBaseOperator[BatchClientHook]):
     """
     Execute a job on AWS Batch.
 
@@ -101,6 +103,8 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
     :param awslogs_fetch_interval: The interval with which cloudwatch logs are 
to be fetched, 30 sec.
     :param poll_interval: (Deferrable mode only) Time in seconds to wait 
between polling.
     :param submit_job_timeout: Execution timeout in seconds for submitted 
batch job.
+    :param durable: If True (default), a retry reconnects to the job submitted 
by the previous try
+        instead of submitting a new one. Synchronous mode only; requires 
Airflow 3.3+.
 
     .. note::
         Any custom waiters must return a waiter for these calls:
@@ -112,6 +116,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
     """
 
     aws_hook_class = BatchClientHook
+    external_id_key = "batch_job_id"
 
     ui_color = "#c3dae0"
     arn: str | None = None
@@ -175,8 +180,11 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
         awslogs_enabled: bool = False,
         awslogs_fetch_interval: timedelta = timedelta(seconds=30),
         submit_job_timeout: int | None = None,
+        durable: bool | None = None,
         **kwargs,
     ) -> None:
+        if durable is not None:
+            kwargs["durable"] = durable
         super().__init__(**kwargs)
         self.job_id = job_id
         self.job_name = job_name
@@ -220,9 +228,8 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
         :raises ValueError: if job_id is not found
         :raises AirflowException: if job submission fails or job ends in 
failure state
         """
-        self.submit_job(context)
-
         if should_defer(deferrable=self.deferrable, 
wait_for_completion=self.wait_for_completion):
+            self.submit_job(context)
             if not self.job_id:
                 raise ValueError("AWS Batch job - job_id was not found")
 
@@ -256,9 +263,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
 
             raise AirflowException(f"Unexpected status: {job_status}")
 
-        if self.wait_for_completion:
-            self.monitor_job(context)
-
+        self.execute_resumable(context)
         return self.job_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None 
= None) -> str:
@@ -302,7 +307,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
         response = self.hook.client.terminate_job(jobId=self.job_id, 
reason="Task killed by the user")
         self.log.info("AWS Batch job (%s) terminated: %s", self.job_id, 
response)
 
-    def submit_job(self, context: Context):
+    def submit_job(self, context: Context) -> str:
         """
         Submit an AWS Batch job.
 
@@ -356,6 +361,45 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
 
         self.job_id = response["jobId"]
         self.log.info("AWS Batch job (%s) started: %s", self.job_id, response)
+        self._persist_job_details_link(context)
+        return self.job_id
+
+    def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+        jobs = self.hook.client.describe_jobs(jobs=[cast("str", 
external_id)])["jobs"]
+        return jobs[0]["status"] if jobs else "NOT_FOUND"
+
+    def is_job_active(self, status: str) -> bool:
+        return status in self.hook.INTERMEDIATE_STATES
+
+    def is_job_succeeded(self, status: str) -> bool:
+        return status == self.hook.SUCCESS_STATE
+
+    def poll_until_complete(self, external_id: JsonValue, context: Context) -> 
None:
+        self._attach_to_job(external_id, context)
+        if self.wait_for_completion:
+            self.monitor_job(context)
+
+    def get_job_result(self, external_id: JsonValue, context: Context) -> str 
| None:
+        # Not attached yet only when the stored job already succeeded, so 
monitor_job never ran.
+        if external_id is not None and self.job_id != external_id:
+            self._attach_to_job(external_id, context)
+            self._persist_links(context)
+            self._persist_cloudwatch_link(context)
+        return self.job_id
+
+    def _attach_to_job(self, external_id: JsonValue, context: Context) -> None:
+        # None comes from subclasses whose submit_job override predates it 
returning the job id;
+        # they still set self.job_id, so keep it.
+        if external_id is None:
+            if self.durable:
+                self.log.warning("submit_job returned no job id, so a retry 
will submit a new job")
+            return
+        if self.job_id == external_id:
+            return
+        self.job_id = cast("str", external_id)
+        self._persist_job_details_link(context)
+
+    def _persist_job_details_link(self, context: Context) -> None:
         BatchJobDetailsLink.persist(
             context=context,
             operator=self,
diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py 
b/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
index 1cd8dac185e..797c35fbda3 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
@@ -33,6 +33,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
 )
 from airflow.providers.common.compat.sdk import AirflowException, TaskDeferred
 
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS
 from unit.amazon.aws.utils.test_deferral import expect_deferral_warning
 from unit.amazon.aws.utils.test_template_fields import validate_template_fields
 
@@ -91,7 +92,7 @@ class TestBatchOperator:
         assert self.batch.job_id is None
         self.batch.job_id = JOB_ID
 
-        self.mock_context = mock.MagicMock()
+        self.mock_context = {"ti": mock.MagicMock()}
 
     def test_init(self):
         assert self.batch.job_id == JOB_ID
@@ -418,7 +419,7 @@ class TestBatchOperator:
             wait_for_completion=False,
         )
 
-        batch.execute(None)
+        batch.execute({})
 
         expected_args = {
             "jobQueue": "queue",
@@ -730,6 +731,130 @@ class TestBatchOperator:
             mock_get_job_all_awslogs_info.assert_called_once_with("12345")
 
 
[email protected](not AIRFLOW_V_3_3_PLUS, reason="durable execution requires 
Airflow 3.3+")
+class TestBatchOperatorDurable:
+    NEW_JOB_ID = "11111111-2222-3333-4444-555555555555"
+
+    @pytest.fixture
+    def client(self):
+        with 
mock.patch("airflow.providers.amazon.aws.hooks.batch_client.AwsBaseHook.get_client_type")
 as m:
+            m.return_value.submit_job.return_value = {"jobName": JOB_NAME, 
"jobId": self.NEW_JOB_ID}
+            yield m.return_value
+
+    @pytest.fixture
+    def task_store(self):
+        store = mock.MagicMock(spec_set=["get", "set"])
+        store.get.return_value = None
+        return store
+
+    @staticmethod
+    def _make_operator(operator_class=BatchOperator, **kwargs):
+        return operator_class(
+            task_id="task",
+            job_name=JOB_NAME,
+            job_queue="queue",
+            job_definition="hello-world",
+            do_xcom_push=False,
+            **kwargs,
+        )
+
+    @staticmethod
+    def _context(task_store):
+        return {"ti": mock.MagicMock(spec_set=["stats_tags"], stats_tags={}), 
"task_state_store": task_store}
+
+    
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_fresh_submit_persists_job_id(self, monitor_mock, link_mock, 
client, task_store):
+        result = self._make_operator().execute(self._context(task_store))
+
+        client.submit_job.assert_called_once()
+        link_mock.assert_called_once()
+        task_store.set.assert_called_once_with("batch_job_id", self.NEW_JOB_ID)
+        monitor_mock.assert_called_once()
+        assert result == self.NEW_JOB_ID
+
+    @pytest.mark.parametrize("status", ["SUBMITTED", "PENDING", "RUNNABLE", 
"STARTING", "RUNNING"])
+    
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_reconnects_to_active_job(self, monitor_mock, link_mock, client, 
task_store, status):
+        task_store.get.return_value = JOB_ID
+        client.describe_jobs.return_value = {"jobs": [{"jobId": JOB_ID, 
"status": status}]}
+        operator = self._make_operator()
+
+        result = operator.execute(self._context(task_store))
+
+        client.describe_jobs.assert_called_once_with(jobs=[JOB_ID])
+        client.submit_job.assert_not_called()
+        task_store.set.assert_not_called()
+        monitor_mock.assert_called_once()
+        assert link_mock.call_args.kwargs["job_id"] == JOB_ID
+        assert operator.job_id == JOB_ID
+        assert result == JOB_ID
+
+    @mock.patch.object(BatchOperator, "_persist_cloudwatch_link", 
autospec=True)
+    @mock.patch.object(BatchOperator, "_persist_links", autospec=True)
+    
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_already_succeeded_skips_submit_and_wait(
+        self, monitor_mock, link_mock, persist_links_mock, 
persist_cloudwatch_mock, client, task_store
+    ):
+        task_store.get.return_value = JOB_ID
+        client.describe_jobs.return_value = {"jobs": [{"jobId": JOB_ID, 
"status": "SUCCEEDED"}]}
+
+        result = self._make_operator().execute(self._context(task_store))
+
+        client.submit_job.assert_not_called()
+        monitor_mock.assert_not_called()
+        assert link_mock.call_args.kwargs["job_id"] == JOB_ID
+        persist_links_mock.assert_called_once()
+        persist_cloudwatch_mock.assert_called_once()
+        assert result == JOB_ID
+
+    @pytest.mark.parametrize(
+        "describe_response",
+        [
+            pytest.param({"jobs": [{"jobId": JOB_ID, "status": "FAILED"}]}, 
id="failed"),
+            pytest.param({"jobs": []}, id="not-found"),
+        ],
+    )
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_resubmits_when_stored_job_not_active(self, monitor_mock, client, 
task_store, describe_response):
+        task_store.get.return_value = JOB_ID
+        client.describe_jobs.return_value = describe_response
+
+        result = self._make_operator().execute(self._context(task_store))
+
+        client.submit_job.assert_called_once()
+        task_store.set.assert_called_once_with("batch_job_id", self.NEW_JOB_ID)
+        assert result == self.NEW_JOB_ID
+
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_durable_false_does_not_use_task_state_store(self, monitor_mock, 
client, task_store):
+        self._make_operator(durable=False).execute(self._context(task_store))
+
+        client.submit_job.assert_called_once()
+        task_store.get.assert_not_called()
+        task_store.set.assert_not_called()
+
+    @pytest.mark.parametrize("durable", [True, False])
+    @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+    def test_submit_job_override_returning_none_keeps_job_id(
+        self, monitor_mock, client, task_store, caplog, durable
+    ):
+        class LegacySubmitBatchOperator(BatchOperator):
+            def submit_job(self, context):
+                super().submit_job(context)
+
+        operator = 
self._make_operator(operator_class=LegacySubmitBatchOperator, durable=durable)
+
+        result = operator.execute(self._context(task_store))
+
+        monitor_mock.assert_called_once()
+        assert operator.job_id == self.NEW_JOB_ID
+        assert result == self.NEW_JOB_ID
+        assert ("submit_job returned no job id, so a retry will submit a new 
job" in caplog) is durable
+
+
 class TestBatchCreateComputeEnvironmentOperator:
     warn_message = "The `status_retries` parameter is unused and should be 
removed"
 

Reply via email to