This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0fb56a641c4 Let deferrable operators reuse the durable reconnect
decision tree (#77161)
0fb56a641c4 is described below
commit 0fb56a641c4a013c4b23b58f70ccc2b6cf97c117
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Nov 17 11:57:46 2026 +0530
Let deferrable operators reuse the durable reconnect decision tree (#77161)
---
task-sdk/docs/api.rst | 2 +
task-sdk/docs/resumable-job-mixin.rst | 35 ++++-
task-sdk/src/airflow/sdk/__init__.py | 4 +-
task-sdk/src/airflow/sdk/__init__.pyi | 6 +-
.../src/airflow/sdk/bases/resumablejobmixin.py | 154 ++++++++++++---------
.../tests/task_sdk/bases/test_resumablejobmixin.py | 59 ++++++++
6 files changed, 187 insertions(+), 73 deletions(-)
diff --git a/task-sdk/docs/api.rst b/task-sdk/docs/api.rst
index b1a72b1c871..6799ac3d5df 100644
--- a/task-sdk/docs/api.rst
+++ b/task-sdk/docs/api.rst
@@ -106,6 +106,8 @@ Bases
.. autoclass:: airflow.sdk.ResumableJobMixin
+.. autoclass:: airflow.sdk.ResumeDecision
+
.. autoapiclass:: airflow.sdk.BaseHook
Callbacks
diff --git a/task-sdk/docs/resumable-job-mixin.rst
b/task-sdk/docs/resumable-job-mixin.rst
index 52e0b004792..fb4f95db76a 100644
--- a/task-sdk/docs/resumable-job-mixin.rst
+++ b/task-sdk/docs/resumable-job-mixin.rst
@@ -28,10 +28,9 @@ the external job identifier to task state store before
polling begins. If the wo
or the host is preempted, the next retry reconnects to the already running job
instead of submitting
a duplicate.
-This mixin is not a replacement for deferrable operators. Deferrable operators
free the
-worker slot during polling and are the recommended approach when a Triggerer
is available.
-Use this mixin when you want crash safety on an existing synchronous operator
without
-migrating to the deferrable pattern, or when your deployment does not include
a Triggerer.
+The mixin works with both synchronous and deferrable operators. A synchronous
operator calls
+``execute_resumable``. A deferrable operator calls two smaller methods before
``defer()``, see
+:ref:`sdk-resumable-job-mixin-deferrable`.
For guidance on choosing between deferrable, resumable, and async approaches,
see
:ref:`apache-airflow:concepts-resumable-tasks`.
@@ -153,6 +152,34 @@ Example
def get_job_result(self, external_id: JsonValue, context):
return None
+.. _sdk-resumable-job-mixin-deferrable:
+
+Deferrable operators
+--------------------
+
+A deferrable operator can't use ``execute_resumable``, because the Triggerer
does the polling.
+Call these two methods before ``defer()`` instead:
+
+* ``get_resume_decision(context)`` checks the stored job and returns a
+ :class:`~airflow.sdk.ResumeDecision`. Its ``action`` is ``"submit"``,
``"reconnect"`` or
+ ``"already_succeeded"``. It never submits or polls.
+* ``persist_job_id(context, job_id)`` saves a new job's id so a retry can find
it.
+
+.. code-block:: python
+
+ def execute(self, context):
+ decision = self.get_resume_decision(context)
+ if decision.action == "already_succeeded":
+ return self.get_job_result(decision.external_id, context)
+
+ if decision.action == "reconnect":
+ job_id = decision.external_id
+ else:
+ job_id = self.submit_job(context)
+ self.persist_job_id(context, job_id)
+
+ self.defer(trigger=MyBatchTrigger(job_id=job_id),
method_name="execute_complete")
+
.. _sdk-resumable-job-mixin-resume-on-retry:
Disabling crash recovery per task
diff --git a/task-sdk/src/airflow/sdk/__init__.py
b/task-sdk/src/airflow/sdk/__init__.py
index e9f85bfb7ce..b5554920f3c 100644
--- a/task-sdk/src/airflow/sdk/__init__.py
+++ b/task-sdk/src/airflow/sdk/__init__.py
@@ -82,6 +82,7 @@ __all__ = [
"ProductMapper",
"QuarterWindow",
"ResumableJobMixin",
+ "ResumeDecision",
"RetryAction",
"RetryDecision",
"RetryPolicy",
@@ -143,7 +144,7 @@ if TYPE_CHECKING:
cross_downstream,
)
from airflow.sdk.bases.operatorlink import BaseOperatorLink
- from airflow.sdk.bases.resumablejobmixin import ResumableJobMixin
+ from airflow.sdk.bases.resumablejobmixin import ResumableJobMixin,
ResumeDecision
from airflow.sdk.bases.sensor import BaseSensorOperator, PokeReturnValue
from airflow.sdk.bases.skipmixin import SkipMixin
from airflow.sdk.bases.xcom import BaseXCom
@@ -314,6 +315,7 @@ __lazy_imports: dict[str, tuple[str, str]] = {
"ProductMapper": (".definitions.partition_mappers.product",
"ProductMapper"),
"QuarterWindow": (".definitions.partition_mappers.window",
"QuarterWindow"),
"ResumableJobMixin": (".bases.resumablejobmixin", "ResumableJobMixin"),
+ "ResumeDecision": (".bases.resumablejobmixin", "ResumeDecision"),
"RetryAction": (".definitions.retry_policy", "RetryAction"),
"RetryDecision": (".definitions.retry_policy", "RetryDecision"),
"RetryPolicy": (".definitions.retry_policy", "RetryPolicy"),
diff --git a/task-sdk/src/airflow/sdk/__init__.pyi
b/task-sdk/src/airflow/sdk/__init__.pyi
index 33892acda22..fa906c44af3 100644
--- a/task-sdk/src/airflow/sdk/__init__.pyi
+++ b/task-sdk/src/airflow/sdk/__init__.pyi
@@ -35,7 +35,10 @@ from airflow.sdk.bases.operator import (
cross_downstream as cross_downstream,
)
from airflow.sdk.bases.operatorlink import BaseOperatorLink as BaseOperatorLink
-from airflow.sdk.bases.resumablejobmixin import ResumableJobMixin as
ResumableJobMixin
+from airflow.sdk.bases.resumablejobmixin import (
+ ResumableJobMixin as ResumableJobMixin,
+ ResumeDecision as ResumeDecision,
+)
from airflow.sdk.bases.sensor import (
BaseSensorOperator as BaseSensorOperator,
PokeReturnValue as PokeReturnValue,
@@ -207,6 +210,7 @@ __all__ = [
"RetryPolicy",
"RetryRule",
"ResumableJobMixin",
+ "ResumeDecision",
"RollupMapper",
"SecretCache",
"SegmentWindow",
diff --git a/task-sdk/src/airflow/sdk/bases/resumablejobmixin.py
b/task-sdk/src/airflow/sdk/bases/resumablejobmixin.py
index 347a700a349..b1164f9787b 100644
--- a/task-sdk/src/airflow/sdk/bases/resumablejobmixin.py
+++ b/task-sdk/src/airflow/sdk/bases/resumablejobmixin.py
@@ -17,7 +17,7 @@
from __future__ import annotations
from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple
from opentelemetry import trace
@@ -33,6 +33,13 @@ if TYPE_CHECKING:
tracer = trace.get_tracer(__name__)
+class ResumeDecision(NamedTuple):
+ """Result of :meth:`ResumableJobMixin.get_resume_decision`."""
+
+ action: Literal["reconnect", "already_succeeded", "submit"]
+ external_id: JsonValue | None = None
+
+
class ResumableJobMixin(ABC):
"""
Mixin for operators that submit one long-running job to an external system
and poll for completion.
@@ -51,7 +58,8 @@ class ResumableJobMixin(ABC):
and it does not stream logs from the remote system (the operator controls
that separately).
Usage: call ``execute_resumable(context)`` from the operator's
``execute()`` when reconnection
- is supported.
+ is supported. An operator that cannot poll inline (e.g. one that defers)
calls
+ ``get_resume_decision(context)`` and ``persist_job_id(context,
external_id)`` instead.
Subclasses must implement all the methods specific to their external
system. The mixin owns
only ``execute_resumable()`` and the task_state_store read/write logic.
@@ -122,88 +130,100 @@ class ResumableJobMixin(ABC):
self.poll_until_complete(external_id, context)
return self.get_job_result(external_id, context)
+ decision = self.get_resume_decision(context)
+ if decision.action == "reconnect":
+ return self.poll_until_complete(decision.external_id, context)
+ if decision.action == "already_succeeded":
+ return self.get_job_result(decision.external_id, context)
+
+ external_id = self.submit_job(context)
+ self.persist_job_id(context, external_id)
+ self.poll_until_complete(external_id, context)
+ return self.get_job_result(external_id, context)
+
+ def get_resume_decision(self, context: Context) -> ResumeDecision:
+ """
+ Decide whether to reconnect to the stored job, take its result, or
submit a new one.
+
+ Never submits or polls, so a deferrable operator can call it before
``defer()``.
+ """
+ if not self.durable:
+ return ResumeDecision("submit")
+
ti = context.get("ti")
stats_tags = {**(ti.stats_tags if ti is not None else {}), "operator":
type(self).__name__}
- reconnect_to: Any = None
- already_succeeded_id: Any = None
-
with tracer.start_as_current_span("resumable_job.resume_decision") as
span:
span.set_attribute("operator", type(self).__name__)
span.set_attribute("resumable.external_id_key",
self.external_id_key)
task_state_store = context.get("task_state_store")
-
if task_state_store is None:
span.set_attribute("resumable.decision", "no_task_state_store")
self.log.warning(
"task_state_store not available in context, crash recovery
is disabled for this run"
)
- else:
- external_id = task_state_store.get(self.external_id_key)
- if external_id:
- stats.incr("resumable_job.reconnect_attempt",
tags=stats_tags)
-
- status = self.get_job_status(external_id, context)
-
- span.set_attribute("resumable.external_id",
str(external_id))
- span.set_attribute("resumable.prior_status", status)
-
- if self.is_job_active(status):
- # Job is still running, skip submission and reconnect
to it.
- span.set_attribute("resumable.decision", "reconnect")
- stats.incr("resumable_job.reconnect_success",
tags=stats_tags)
- self.log.info(
- "Reconnecting to existing job",
- external_id_key=self.external_id_key,
- external_id=external_id,
- status=status,
- )
- reconnect_to = external_id
- elif self.is_job_succeeded(status):
- # Job already finished successfully, skip polling and
return result directly.
- span.set_attribute("resumable.decision",
"already_succeeded")
- stats.incr("resumable_job.already_succeeded",
tags=stats_tags)
- self.log.info(
- "Job already completed successfully, skipping
resubmission",
- external_id_key=self.external_id_key,
- external_id=external_id,
- )
- already_succeeded_id = external_id
- else:
- # Job is in a terminal failed state, fall through and
submit a new job.
- span.set_attribute("resumable.decision",
"terminal_resubmit")
- stats.incr("resumable_job.terminal_resubmit",
tags=stats_tags)
- self.log.warning(
- "Prior job in terminal state, resubmitting fresh",
- external_id_key=self.external_id_key,
- external_id=external_id,
- status=status,
- )
- else:
- span.set_attribute("resumable.decision", "fresh_submit")
- stats.incr("resumable_job.fresh_submit", tags=stats_tags)
- self.log.debug(
- "No stored external ID found; submitting fresh job",
- external_id_key=self.external_id_key,
- )
-
- if reconnect_to is not None:
- return self.poll_until_complete(reconnect_to, context)
- if already_succeeded_id is not None:
- return self.get_job_result(already_succeeded_id, context)
- external_id = self.submit_job(context)
+ return ResumeDecision("submit")
+
+ external_id = task_state_store.get(self.external_id_key)
+ if external_id is None:
+ span.set_attribute("resumable.decision", "fresh_submit")
+ stats.incr("resumable_job.fresh_submit", tags=stats_tags)
+ self.log.debug(
+ "No stored external ID found; submitting fresh job",
+ external_id_key=self.external_id_key,
+ )
+ return ResumeDecision("submit")
+
+ stats.incr("resumable_job.reconnect_attempt", tags=stats_tags)
+ status = self.get_job_status(external_id, context)
+ span.set_attribute("resumable.external_id", str(external_id))
+ span.set_attribute("resumable.prior_status", status)
+
+ if self.is_job_active(status):
+ span.set_attribute("resumable.decision", "reconnect")
+ stats.incr("resumable_job.reconnect_success", tags=stats_tags)
+ self.log.info(
+ "Reconnecting to existing job",
+ external_id_key=self.external_id_key,
+ external_id=external_id,
+ status=status,
+ )
+ return ResumeDecision("reconnect", external_id)
+
+ if self.is_job_succeeded(status):
+ span.set_attribute("resumable.decision", "already_succeeded")
+ stats.incr("resumable_job.already_succeeded", tags=stats_tags)
+ self.log.info(
+ "Job already completed successfully, skipping
resubmission",
+ external_id_key=self.external_id_key,
+ external_id=external_id,
+ )
+ return ResumeDecision("already_succeeded", external_id)
- if task_state_store is not None and external_id is not None:
- task_state_store.set(self.external_id_key, external_id)
- self.log.debug(
- "Persisted external ID to task store",
+ span.set_attribute("resumable.decision", "terminal_resubmit")
+ stats.incr("resumable_job.terminal_resubmit", tags=stats_tags)
+ self.log.warning(
+ "Prior job in terminal state, resubmitting fresh",
external_id_key=self.external_id_key,
external_id=external_id,
+ status=status,
)
-
- self.poll_until_complete(external_id, context)
- return self.get_job_result(external_id, context)
+ return ResumeDecision("submit")
+
+ def persist_job_id(self, context: Context, external_id: JsonValue) -> None:
+ """Store a newly submitted job's external ID so a retry can reconnect
to it."""
+ if not self.durable or external_id is None:
+ return
+ task_state_store = context.get("task_state_store")
+ if task_state_store is None:
+ return
+ task_state_store.set(self.external_id_key, external_id)
+ self.log.debug(
+ "Persisted external ID to task store",
+ external_id_key=self.external_id_key,
+ external_id=external_id,
+ )
@abstractmethod
def submit_job(self, context: Context) -> JsonValue:
diff --git a/task-sdk/tests/task_sdk/bases/test_resumablejobmixin.py
b/task-sdk/tests/task_sdk/bases/test_resumablejobmixin.py
index 4d9487cb3f8..12cfb5b2607 100644
--- a/task-sdk/tests/task_sdk/bases/test_resumablejobmixin.py
+++ b/task-sdk/tests/task_sdk/bases/test_resumablejobmixin.py
@@ -28,6 +28,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter
import InMemorySpanE
from airflow.sdk import ResumableJobMixin
from airflow.sdk.bases.operator import BaseOperator
+from airflow.sdk.bases.resumablejobmixin import ResumeDecision
if TYPE_CHECKING:
from pydantic import JsonValue
@@ -160,6 +161,16 @@ class TestRetryWithDifferentJobStatuses:
assert op.submitted_ids == []
assert op.polled_ids == ["job-001"]
+ def test_reconnects_to_stored_id_zero(self):
+ op = ConcreteResumableOperator(task_id="test_task")
+ op._status_map["0"] = "RUNNING"
+ task_state = FakeTaskState({"test_job_id": 0})
+
+ op.execute_resumable(make_context(task_state))
+
+ assert op.submitted_ids == []
+ assert op.polled_ids == ["0"]
+
def test_returns_result_immediately_without_polling(self):
op = ConcreteResumableOperator(task_id="test_task")
op._status_map["job-001"] = "SUCCEEDED"
@@ -581,3 +592,51 @@ class TestAbstractMethodEnforcement:
assert partial_cls.__abstractmethods__ == frozenset({missing_method})
with pytest.raises(TypeError):
partial_cls(task_id="test_task")
+
+
+class TestGetResumeDecision:
+ @pytest.mark.parametrize(
+ ("stored", "status", "expected"),
+ [
+ (None, None, ResumeDecision("submit")),
+ ("job-001", "RUNNING", ResumeDecision("reconnect", "job-001")),
+ ("job-001", "SUCCEEDED", ResumeDecision("already_succeeded",
"job-001")),
+ ("job-001", "FAILED", ResumeDecision("submit")),
+ ],
+ )
+ def test_decides_without_submitting_or_polling(self, stored, status,
expected):
+ op = ConcreteResumableOperator(task_id="test_task")
+ if status:
+ op._status_map[stored] = status
+ task_state = FakeTaskState({"test_job_id": stored} if stored else None)
+
+ assert op.get_resume_decision(make_context(task_state)) == expected
+ assert op.submitted_ids == []
+ assert op.polled_ids == []
+
+ def test_submit_without_reading_store_when_not_durable(self):
+ op = ConcreteResumableOperator(task_id="test_task", durable=False)
+ task_state = MagicMock(spec=FakeTaskState)
+
+ assert op.get_resume_decision(make_context(task_state)) ==
ResumeDecision("submit")
+ task_state.get.assert_not_called()
+
+
+class TestPersistJobId:
+ def test_persisted_id_is_reconnected_on_retry(self):
+ op = ConcreteResumableOperator(task_id="test_task")
+ op._status_map["job-001"] = "RUNNING"
+ task_state = FakeTaskState()
+
+ op.persist_job_id(make_context(task_state), "job-001")
+
+ assert op.get_resume_decision(make_context(task_state)) ==
ResumeDecision("reconnect", "job-001")
+
+ @pytest.mark.parametrize(("durable", "external_id"), [(False, "job-001"),
(True, None)])
+ def test_skips_persist(self, durable, external_id):
+ op = ConcreteResumableOperator(task_id="test_task", durable=durable)
+ task_state = FakeTaskState()
+
+ op.persist_job_id(make_context(task_state), external_id)
+
+ assert task_state._store == {}