This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d099acc6a40 Detect supervisor-subprocess trigger count mismatch in
Triggerer (#68792)
d099acc6a40 is described below
commit d099acc6a4061d1f364785d3b6224c99c5cea704
Author: Jihee Lim <[email protected]>
AuthorDate: Sun Nov 15 02:33:59 2026 +0900
Detect supervisor-subprocess trigger count mismatch in Triggerer (#68792)
* Detect supervisor-subprocess trigger count mismatch in Triggerer
* Fix false-positive path in trigger count mismatch check
Remove the incorrect difference_update on msg.failures (which would
break for serialization failures where the trigger is still running).
Instead, include creation-failure IDs in finished_ids so the supervisor
naturally removes them from running_triggers via the existing finished
processing path.
Co-Authored-By: Claude Opus 4.6 <[email protected]>
* Recover lost Triggerer triggers instead of shutting down
* Distinguish a missing running_ids from an empty one
* Avoid reporting a finished trigger twice in the Triggerer
* Clear releasing_triggers when recovering a lost Triggerer trigger
* Finalize trigger logs when recovering unhandled triggers
* Drop the bugfix newsfragment for the Triggerer recovery fix
contributing-docs/18_contribution_workflow.rst limits airflow-core
newsfragments to significant changes; an ordinary bug fix does not get one.
Claude-Session: https://claude.ai/code/session_01BVx6gBBL7XtC13QVHEbCVh
---------
Co-authored-by: Claude Opus 4.6 <[email protected]>
Co-authored-by: Shahar Epstein <[email protected]>
---
.../src/airflow/jobs/triggerer_job_runner.py | 61 ++++++--
airflow-core/tests/unit/jobs/test_triggerer_job.py | 163 +++++++++++++++++++++
.../observability/metrics/metrics_template.yaml | 7 +
3 files changed, 221 insertions(+), 10 deletions(-)
diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py
b/airflow-core/src/airflow/jobs/triggerer_job_runner.py
index afb50da0ef2..ef16d50a28d 100644
--- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py
+++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py
@@ -446,6 +446,8 @@ class messages:
# Format of list[str] is the exc traceback format
failures: list[tuple[int, list[str] | None]] | None = None
finished: list[int] | None = None
+ # Ids the runner has a live coroutine for; None when the message
carries no information
+ running_ids: set[int] | None = None
class TriggerStateSync(BaseModel):
type: Literal["TriggerStateSync"] = "TriggerStateSync"
@@ -722,15 +724,9 @@ class TriggerRunnerSupervisor(WatchedSubprocess):
self.running_triggers.discard(id)
self.cancelling_triggers.discard(id)
self.releasing_triggers.discard(id)
- if factory := self.logger_cache.pop(id, None):
- try:
- factory.upload_to_remote()
- except Exception:
- log.exception("Failed to upload trigger logs to
remote", trigger_id=id)
- finally:
- # Close the FD explicitly even if upload raised,
otherwise the file
- # handle leaks for every failed upload.
- factory.close()
+ self.finalize_trigger_log(id)
+
+ self.check_for_unhandled_triggers(msg.running_ids)
# Drain the persist confirmations accumulated since the last sync.
events_persisted: list[int] = []
@@ -916,6 +912,46 @@ class TriggerRunnerSupervisor(WatchedSubprocess):
with attempt, create_session() as session:
Trigger.clean_unused(session=session)
+ def finalize_trigger_log(self, trigger_id: int) -> None:
+ """
+ Upload and close the log file of a trigger that no longer runs here.
+
+ Every path that drops an id from ``running_triggers`` has to call it:
re-creation
+ reuses the log path, so a leftover entry is lost.
+ """
+ if factory := self.logger_cache.pop(trigger_id, None):
+ try:
+ factory.upload_to_remote()
+ except Exception:
+ log.exception("Failed to upload trigger logs to remote",
trigger_id=trigger_id)
+ finally:
+ # Close the FD explicitly even if upload raised, otherwise the
file handle
+ # leaks for every failed upload. close() flushes, so it can
raise too.
+ try:
+ factory.close()
+ except Exception:
+ log.exception("Failed to close trigger log file",
trigger_id=trigger_id)
+
+ def check_for_unhandled_triggers(self, running_ids: set[int] | None) ->
None:
+ """
+ Re-create triggers we track as running that the runner has no
coroutine for.
+
+ Only valid between finished-removal and to_create-addition in
``_handle_request``, where the
+ two sides agree. Dropping the leftovers lets :meth:`update_triggers`
rebuild them.
+ """
+ if running_ids is None:
+ return
+ unhandled = self.running_triggers - running_ids
+ if not unhandled:
+ return
+ log.error("Triggers have no coroutine in the runner; re-creating",
trigger_ids=sorted(unhandled))
+ self.running_triggers -= unhandled
+ self.cancelling_triggers -= unhandled
+ self.releasing_triggers -= unhandled
+ for trigger_id in unhandled:
+ self.finalize_trigger_log(trigger_id)
+ stats.incr("triggers.state_mismatch", len(unhandled),
tags=prune_dict({"team_name": self.team_name}))
+
def handle_failed_triggers(self):
"""
Handle "failed" triggers. - ones that errored or exited before they
sent an event.
@@ -1789,6 +1825,7 @@ class TriggerRunner:
# Copy out of our dequeues in threadsafe manner to sync state with
parent
events_to_send: list[TriggerEventEntry] = []
failures_to_send: list[tuple[int, list[str] | None]] = []
+ finished_to_send = list(finished_ids)
while self.events:
events_to_send.append(self.events.popleft())
@@ -1797,11 +1834,14 @@ class TriggerRunner:
trigger_id, exc = self.failed_triggers.popleft()
tb = format_exception(type(exc), exc, exc.__traceback__) if exc
else None
failures_to_send.append((trigger_id, tb))
+ if trigger_id not in self.triggers and trigger_id not in
finished_to_send:
+ finished_to_send.append(trigger_id)
return messages.TriggerStateChanges(
events=events_to_send if events_to_send else None,
- finished=finished_ids if finished_ids else None,
+ finished=finished_to_send if finished_to_send else None,
failures=failures_to_send if failures_to_send else None,
+ running_ids=set(self.triggers),
)
def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) ->
messages.TriggerStateChanges:
@@ -1830,6 +1870,7 @@ class TriggerRunner:
events=events_to_send if events_to_send else None,
finished=msg.finished,
failures=msg.failures,
+ running_ids=msg.running_ids,
)
async def sync_state_to_supervisor(self, finished_ids: list[int]) -> None:
diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py
b/airflow-core/tests/unit/jobs/test_triggerer_job.py
index 2a1f6265f65..03adf3ade98 100644
--- a/airflow-core/tests/unit/jobs/test_triggerer_job.py
+++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py
@@ -4569,3 +4569,166 @@ class TestOwnerOfTriggersCreatedWithoutOne:
TriggerRunnerSupervisor._record_missing_owner(trigger)
assert (trigger.dag_id, trigger.owner_ref) == ("d", {"ti_id": "x"})
+
+
+class TestCheckForUnhandledTriggers:
+ """Tests for recovery of triggers the runner has no coroutine for."""
+
+ @pytest.mark.parametrize(
+ ("running_triggers", "running_ids", "expected_after"),
+ [
+ pytest.param({1, 2, 3}, {1, 2, 3}, {1, 2, 3}, id="in-sync"),
+ pytest.param(set(), set(), set(), id="no-triggers"),
+ pytest.param({1, 2, 3}, {1, 2}, {1, 2}, id="one-unhandled"),
+ pytest.param({1, 2, 3}, set(), set(), id="all-unhandled"),
+ ],
+ )
+ def test_unhandled_triggers_are_dropped(
+ self, jobless_supervisor, mocker, running_triggers, running_ids,
expected_after
+ ):
+ incr = mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr",
autospec=True)
+ jobless_supervisor.running_triggers = set(running_triggers)
+ jobless_supervisor.cancelling_triggers = set(running_triggers)
+
+ jobless_supervisor.check_for_unhandled_triggers(running_ids)
+
+ unhandled = running_triggers - running_ids
+ assert jobless_supervisor.running_triggers == expected_after
+ assert jobless_supervisor.cancelling_triggers == expected_after
+ assert (
+ mocker.call("triggers.state_mismatch", len(unhandled), tags={}) in
incr.call_args_list
+ ) is bool(unhandled)
+
+ def test_dropped_trigger_is_recreated_next_loop(self, jobless_supervisor,
mocker):
+ """Dropping an unhandled id is what lets update_triggers rebuild its
workload."""
+ build = mocker.patch.object(
+ TriggerRunnerSupervisor, "build_trigger_workloads", autospec=True,
return_value=[]
+ )
+ jobless_supervisor.running_triggers = {1, 2, 3}
+ # A trigger released to another triggerer stays in running_triggers,
so it can turn up
+ # unhandled too, and releasing_triggers also gates re-creation via
known_trigger_ids.
+ jobless_supervisor.releasing_triggers = {3}
+
+ jobless_supervisor.check_for_unhandled_triggers({1})
+ jobless_supervisor.update_triggers({1, 2, 3})
+
+ assert build.call_args.args[1] == {2, 3}
+
+ def test_recovered_trigger_log_is_finalized(self, jobless_supervisor,
mocker):
+ """Re-creation reuses the log path, so recovery has to upload and
close it first."""
+ mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr",
autospec=True)
+ factory = MagicMock(spec=TriggerLoggingFactory)
+ jobless_supervisor.logger_cache[42] = factory
+ jobless_supervisor.running_triggers = {42}
+
+ jobless_supervisor.check_for_unhandled_triggers(set())
+
+ factory.upload_to_remote.assert_called_once()
+ factory.close.assert_called_once()
+ assert 42 not in jobless_supervisor.logger_cache
+
+ def test_recovery_survives_a_log_file_that_cannot_be_closed(self,
jobless_supervisor, mocker):
+ """An unwritable log file must not strand the remaining ids, nor reach
the runner."""
+ incr = mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr",
autospec=True)
+ factories = {}
+ for trigger_id in (1, 2):
+ factory = MagicMock(spec=TriggerLoggingFactory)
+ factory.close.side_effect = OSError("no space left on device")
+ factories[trigger_id] = factory
+ jobless_supervisor.logger_cache.update(factories)
+ jobless_supervisor.running_triggers = {1, 2}
+
+ jobless_supervisor.check_for_unhandled_triggers(set())
+
+ assert not jobless_supervisor.logger_cache
+ assert all(factory.close.called for factory in factories.values())
+ assert mocker.call("triggers.state_mismatch", 2, tags={}) in
incr.call_args_list
+
+ def test_message_without_running_ids_is_ignored(self, jobless_supervisor):
+ """``None`` means the message carried no information, which must not
read as "nothing running"."""
+ jobless_supervisor.running_triggers = {1, 2, 3}
+ jobless_supervisor.cancelling_triggers = {1, 2, 3}
+
+ jobless_supervisor.check_for_unhandled_triggers(None)
+
+ assert jobless_supervisor.running_triggers == {1, 2, 3}
+ assert jobless_supervisor.cancelling_triggers == {1, 2, 3}
+
+ def test_handle_request_checks_before_adding_to_create(self,
jobless_supervisor, mocker):
+ """The check fires after finished processing but before to_create is
added to running_triggers."""
+ mocker.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True)
+ jobless_supervisor.running_triggers = {1, 2}
+ jobless_supervisor.creating_triggers.append(
+ mocker.MagicMock(id=3),
+ )
+
+ jobless_supervisor._handle_request(
+ messages.TriggerStateChanges(
+ events=None,
+ failures=None,
+ finished=[1],
+ running_ids={2},
+ ),
+ log=MagicMock(spec=FilteringBoundLogger),
+ req_id=1,
+ )
+
+ assert jobless_supervisor.running_triggers == {2, 3}
+
+ def test_creation_failure_reported_in_finished(self, jobless_supervisor,
mocker):
+ """A creation failure appears in both failures and finished, so
running_triggers stays in sync."""
+ mocker.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True)
+ jobless_supervisor.running_triggers = {1, 2, 3}
+
+ jobless_supervisor._handle_request(
+ messages.TriggerStateChanges(
+ events=None,
+ failures=[(3, ["Traceback..."])],
+ finished=[3],
+ running_ids={1, 2},
+ ),
+ log=MagicMock(spec=FilteringBoundLogger),
+ req_id=1,
+ )
+
+ assert 3 not in jobless_supervisor.running_triggers
+
+
+class TestCreationFailureInFinished:
+ """Tests that creation failures (not in self.triggers) are included in
finished_ids."""
+
+ def test_creation_failure_included_in_finished(self):
+ runner = TriggerRunner()
+ runner.failed_triggers.append((42, ValueError("bad classpath")))
+
+ msg = runner.process_trigger_events(finished_ids=[10])
+
+ assert msg.finished == [10, 42]
+ assert msg.running_ids == set()
+
+ def test_serialization_failure_not_in_finished(self):
+ runner = TriggerRunner()
+ runner.triggers = {42: {"task": MagicMock(), "is_watcher": False,
"name": "t", "events": 0}}
+ runner.failed_triggers.append((42, ValueError("not serializable")))
+
+ msg = runner.process_trigger_events(finished_ids=[])
+
+ assert msg.finished is None
+ assert msg.running_ids == {42}
+
+ def test_failure_already_in_finished_ids_not_duplicated(self):
+ runner = TriggerRunner()
+ runner.failed_triggers.append((42, ValueError("exited without an
event")))
+
+ msg = runner.process_trigger_events(finished_ids=[42])
+
+ assert msg.finished == [42]
+
+ def test_caller_finished_ids_not_mutated(self):
+ runner = TriggerRunner()
+ runner.failed_triggers.append((42, ValueError("bad classpath")))
+ finished_ids = [10]
+
+ runner.process_trigger_events(finished_ids=finished_ids)
+
+ assert finished_ids == [10]
diff --git
a/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
b/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
index cfe767fd4d3..e3c5c3401e0 100644
---
a/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
+++
b/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
@@ -387,6 +387,13 @@ metrics:
legacy_name: "-"
name_variables: []
+ - name: "triggers.state_mismatch"
+ description: "Number of triggers the Triggerer tracked as running that the
trigger runner
+ had no coroutine for, and which were re-created"
+ type: "counter"
+ legacy_name: "-"
+ name_variables: []
+
- name: "triggers.succeeded"
description: "Number of triggers that have fired at least one event.
Tagged with trigger_kind (asset_watcher, callback or task)."