This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 842402c7eee Test reconnect on_kill and document SageMaker durable
execution (#76964)
842402c7eee is described below
commit 842402c7eeef1698a19030931832e82a81f87795
Author: Amogh Desai <[email protected]>
AuthorDate: Wed Nov 18 15:36:35 2026 +0530
Test reconnect on_kill and document SageMaker durable execution (#76964)
---
providers/amazon/docs/operators/sagemaker.rst | 61 ++++++++++++++++++++++
.../providers/amazon/aws/operators/sagemaker.py | 10 ++--
.../aws/operators/test_sagemaker_training.py | 25 ++++++++-
3 files changed, 89 insertions(+), 7 deletions(-)
diff --git a/providers/amazon/docs/operators/sagemaker.rst
b/providers/amazon/docs/operators/sagemaker.rst
index e9e703f4f01..89526a81d6a 100644
--- a/providers/amazon/docs/operators/sagemaker.rst
+++ b/providers/amazon/docs/operators/sagemaker.rst
@@ -67,6 +67,67 @@ To create an Amazon Sagemaker training job you can use
:start-after: [START howto_operator_sagemaker_training]
:end-before: [END howto_operator_sagemaker_training]
+Durable execution
+"""""""""""""""""
+
+``SageMakerTrainingOperator`` submits a training job and then polls it to
completion on the
+worker. By default the operator runs in a *durable* mode that makes this
crash-safe: the training
+job name is persisted to :doc:`task state store
<apache-airflow:core-concepts/task-state-store>`
+before polling begins, so if the worker crashes or is preempted and the task
is retried, the
+operator reconnects to the job that is already running in SageMaker instead of
starting a new one.
+
+On retry the operator checks the prior job's status:
+
+* if it is ``InProgress``, the operator reconnects and continues polling
+* if it is ``Completed``, the operator returns its result without resubmitting
+* if it is ``Failed``, ``Stopped`` or ``Stopping``, or the name is no longer
found, the operator
+ submits a new job
+
+Unlike Glue, a job that is still ``Stopping`` gets a new submit rather than a
reconnect. With
+``print_log=False``, the wait treats ``Stopped`` as done, so reconnecting
would let the task succeed
+without a finished training run. ``cancel_on_kill`` leaves the job in exactly
that state when a
+timed out attempt is killed. A new submit is renamed with a timestamp suffix
when
+``check_if_job_exists=True`` and ``action_if_job_exists="timestamp"`` (both
defaults), because the
+earlier job still holds its name. With ``check_if_job_exists=False``,
SageMaker rejects the new
+submit.
+Spot training resume through ``CheckpointConfig`` is unaffected, since the
checkpoint
+``S3Uri`` does not depend on the job name.
+
+On Airflow 3.3, clearing a task keeps the stored job name, so the next attempt
is treated the same
+as a retry. A task whose job already completed returns the existing result
without starting a new
+job.
+
+From Airflow 3.4, clearing a single task instance (UI, REST API or
``airflowctl``) deletes the
+stored job name by default, so the next attempt submits a new job. Set
``keep_task_state`` when
+clearing to keep it, and the next attempt then reconnects to the job or
returns its result. Clearing
+a whole Dag run, marking a task as failed or success, and ``airflow tasks
clear`` /
+``airflow dags clear`` still keep the job name before Airflow 3.5. From
Airflow 3.5, every way of
+clearing deletes it by default.
+
+Clearing a running task leaves its job running by default, so the next attempt
starts a second one
+and both are billed. To avoid that, set ``keep_task_state`` when clearing to
reconnect, or set
+``cancel_on_kill=True`` to stop the old job.
+
+See :doc:`apache-airflow:core-concepts/resumable-tasks` for the ways of
clearing, and for the
+``[state_store] clear_on_success`` setting.
+
+To opt out and always start a new job on retry, set ``durable=False``:
+
+.. code-block:: python
+
+ training = SageMakerTrainingOperator(
+ task_id="train_model",
+ config=training_config,
+ durable=False,
+ )
+
+Durable execution applies to the synchronous path. When ``deferrable=True`` and
+``wait_for_completion=True`` are both set, ``durable`` has no effect.
+
+Durable execution requires Airflow 3.3 or newer, since it relies on the task
state store. Below
+3.3, ``durable`` has no effect either way: setting it explicitly only emits a
warning, and the
+operator always starts a new job on retry, exactly as before this feature
existed.
+
.. _howto/operator:SageMakerModelOperator:
Create an Amazon SageMaker model
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
index d27678e4444..9a8b2cd7712 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
@@ -21,7 +21,7 @@ import json
import time
import urllib
from collections.abc import Callable, Sequence
-from typing import TYPE_CHECKING, Any, ClassVar, cast
+from typing import TYPE_CHECKING, Any, ClassVar
from botocore.exceptions import ClientError
@@ -1326,9 +1326,7 @@ class SageMakerTrainingOperator(ResumableJobMixin,
SageMakerBaseOperator):
return self._execute_deferrable(context)
self.execute_resumable(context)
- # On reconnect the mixin returns poll_until_complete's value and never
calls
- # get_job_result, so the result is read off the operator instead.
- return cast("dict[str, dict]", self._result)
+ return self.get_job_result(self._training_job_name, context)
def submit_job(self, context: Context) -> str:
# Not in execute(): on a reconnecting retry the S3Operations uploads
would raise on the
@@ -1368,8 +1366,8 @@ class SageMakerTrainingOperator(ResumableJobMixin,
SageMakerBaseOperator):
raise
def is_job_active(self, status: str) -> bool:
- # Narrower than non_terminal_states on purpose: a "Stopping" job never
produces a model, and
- # wait_for_training_job accepts the resulting "Stopped" as done when
print_log is off.
+ # Narrower than non_terminal_states on purpose: wait_for_training_job
accepts the "Stopped" a
+ # "Stopping" job ends in as done when print_log is off, so
reconnecting would report success.
return status == "InProgress"
def is_job_succeeded(self, status: str) -> bool:
diff --git
a/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
index 3c5c9319372..cae0dd02b98 100644
---
a/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
+++
b/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
@@ -185,14 +185,19 @@ class TestSageMakerTrainingOperator:
@mock.patch.object(SageMakerHook, "create_training_job")
@mock.patch.object(sagemaker, "serialize", return_value="")
@mock.patch.object(SageMakerHook, "configure_s3_resources")
+ @mock.patch.object(SageMakerHook, "stop_training_job")
def test_durable_reconnects_to_a_running_job(
- self, mock_configure_s3, _, mock_training, mock_desc, mock_poll
+ self, mock_stop, mock_configure_s3, _, mock_training, mock_desc,
mock_poll
):
mock_desc.return_value = {"TrainingJobStatus": "InProgress"}
+ self.sagemaker.cancel_on_kill = True
store = self._task_state_store("previously_running_job")
+ mock_poll.side_effect = lambda *a, **k: self.sagemaker.on_kill()
result = self.sagemaker.execute({"task_state_store": store})
+ mock_stop.assert_called_once_with("previously_running_job")
+
mock_training.assert_not_called()
mock_configure_s3.assert_not_called()
# autospec leaves the hook instance as args[0], so the job name is
args[1].
@@ -200,6 +205,24 @@ class TestSageMakerTrainingOperator:
assert result == {"Training": ""}
assert self.sagemaker.serialized_training_data == ""
+ @pytest.mark.skipif(
+ not AIRFLOW_V_3_3_PLUS,
+ reason="ResumableJobMixin reconnect requires task_state_store,
available in Airflow 3.3+",
+ )
+ @mock.patch.object(SageMakerHook, "wait_for_training_job")
+ @mock.patch.object(SageMakerHook, "describe_training_job")
+ def test_execute_returns_overridden_serialize_result(self, mock_desc, _):
+ class TrimmedTrainingOperator(SageMakerTrainingOperator):
+ def serialize_result(self, job_name: str) -> dict[str, dict]:
+ return {"trimmed": {"job_name": job_name}}
+
+ mock_desc.return_value = {"TrainingJobStatus": "InProgress"}
+ op = TrimmedTrainingOperator(task_id="test_sagemaker_operator",
config=CREATE_TRAINING_PARAMS.copy())
+
+ result = op.execute({"task_state_store":
self._task_state_store("previously_running_job")})
+
+ assert result == {"trimmed": {"job_name": "previously_running_job"}}
+
@pytest.mark.skipif(
not AIRFLOW_V_3_3_PLUS,
reason="ResumableJobMixin reconnect requires task_state_store,
available in Airflow 3.3+",