amoghrajesh commented on code in PR #77158:
URL: https://github.com/apache/airflow/pull/77158#discussion_r4612223918
##########
providers/amazon/tests/unit/amazon/aws/operators/test_batch.py:
##########
@@ -730,6 +731,118 @@ def test_execute_complete_failure_without_logs(
mock_get_job_all_awslogs_info.assert_called_once_with("12345")
[email protected](not AIRFLOW_V_3_3_PLUS, reason="durable execution requires
Airflow 3.3+")
+class TestBatchOperatorDurable:
+ NEW_JOB_ID = "11111111-2222-3333-4444-555555555555"
+
+ @pytest.fixture
+ def client(self):
+ with
mock.patch("airflow.providers.amazon.aws.hooks.batch_client.AwsBaseHook.get_client_type")
as m:
+ m.return_value.submit_job.return_value = {"jobName": JOB_NAME,
"jobId": self.NEW_JOB_ID}
+ yield m.return_value
+
+ @pytest.fixture
+ def task_store(self):
+ store = mock.MagicMock(spec_set=["get", "set"])
+ store.get.return_value = None
+ return store
+
+ @staticmethod
+ def _make_operator(operator_class=BatchOperator, **kwargs):
+ return operator_class(
+ task_id="task",
+ job_name=JOB_NAME,
+ job_queue="queue",
+ job_definition="hello-world",
+ do_xcom_push=False,
+ **kwargs,
+ )
+
+ @staticmethod
+ def _context(task_store):
+ return {"ti": mock.MagicMock(spec_set=["stats_tags"], stats_tags={}),
"task_state_store": task_store}
+
+
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_fresh_submit_persists_job_id(self, monitor_mock, link_mock,
client, task_store):
+ result = self._make_operator().execute(self._context(task_store))
+
+ client.submit_job.assert_called_once()
+ link_mock.assert_called_once()
+ task_store.set.assert_called_once_with("batch_job_id", self.NEW_JOB_ID)
+ monitor_mock.assert_called_once()
+ assert result == self.NEW_JOB_ID
+
+
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_reconnects_to_running_job(self, monitor_mock, link_mock, client,
task_store):
+ task_store.get.return_value = JOB_ID
+ client.describe_jobs.return_value = {"jobs": [{"jobId": JOB_ID,
"status": "RUNNING"}]}
Review Comment:
Handled in 05eb0fdb31e
##########
providers/amazon/docs/operators/batch.rst:
##########
@@ -51,6 +51,22 @@ use
:class:`~airflow.providers.amazon.aws.operators.batch.BatchOperator`.
:start-after: [START howto_operator_batch]
:end-before: [END howto_operator_batch]
+Durable execution
+=================
+
+By default ``BatchOperator`` saves the Batch job id to the
+:doc:`task state store <apache-airflow:core-concepts/task-state-store>` right
after submitting the
+job. If the worker crashes and the task is retried, the operator checks that
job first:
+
+* still active (``SUBMITTED`` through ``RUNNING``): reconnects and keeps
waiting on it
+* already ``SUCCEEDED``: returns the job id without submitting again
Review Comment:
Handled in 05eb0fdb31e
##########
providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py:
##########
@@ -363,6 +368,40 @@ def submit_job(self, context: Context):
aws_partition=self.hook.conn_partition,
job_id=self.job_id,
)
+ return self.job_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ jobs = self.hook.client.describe_jobs(jobs=[cast("str",
external_id)])["jobs"]
+ return jobs[0]["status"] if jobs else "NOT_FOUND"
+
+ def is_job_active(self, status: str) -> bool:
+ return status in self.hook.INTERMEDIATE_STATES
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == self.hook.SUCCESS_STATE
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self._attach_to_job(external_id, context)
+ if self.wait_for_completion:
+ self.monitor_job(context)
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> str
| None:
+ self._attach_to_job(external_id, context)
+ return self.job_id
+
+ def _attach_to_job(self, external_id: JsonValue, context: Context) -> None:
+ # None comes from subclasses whose submit_job override predates it
returning the job id;
+ # they still set self.job_id, so keep it.
+ if external_id is None or self.job_id == external_id:
+ return
+ self.job_id = cast("str", external_id)
+ BatchJobDetailsLink.persist(
Review Comment:
Handled in 05eb0fdb31e
##########
providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py:
##########
@@ -363,6 +368,40 @@ def submit_job(self, context: Context):
aws_partition=self.hook.conn_partition,
job_id=self.job_id,
)
+ return self.job_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ jobs = self.hook.client.describe_jobs(jobs=[cast("str",
external_id)])["jobs"]
+ return jobs[0]["status"] if jobs else "NOT_FOUND"
+
+ def is_job_active(self, status: str) -> bool:
+ return status in self.hook.INTERMEDIATE_STATES
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == self.hook.SUCCESS_STATE
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self._attach_to_job(external_id, context)
+ if self.wait_for_completion:
+ self.monitor_job(context)
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> str
| None:
+ self._attach_to_job(external_id, context)
+ return self.job_id
Review Comment:
Handled in 05eb0fdb31e
##########
providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py:
##########
@@ -363,6 +368,40 @@ def submit_job(self, context: Context):
aws_partition=self.hook.conn_partition,
job_id=self.job_id,
)
+ return self.job_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ jobs = self.hook.client.describe_jobs(jobs=[cast("str",
external_id)])["jobs"]
+ return jobs[0]["status"] if jobs else "NOT_FOUND"
+
+ def is_job_active(self, status: str) -> bool:
+ return status in self.hook.INTERMEDIATE_STATES
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == self.hook.SUCCESS_STATE
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self._attach_to_job(external_id, context)
+ if self.wait_for_completion:
+ self.monitor_job(context)
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> str
| None:
+ self._attach_to_job(external_id, context)
+ return self.job_id
+
+ def _attach_to_job(self, external_id: JsonValue, context: Context) -> None:
+ # None comes from subclasses whose submit_job override predates it
returning the job id;
+ # they still set self.job_id, so keep it.
Review Comment:
Handled in 05eb0fdb31e
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]