This is an automated email from the ASF dual-hosted git repository.

shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new d099acc6a40 Detect supervisor-subprocess trigger count mismatch in 
Triggerer (#68792)
d099acc6a40 is described below

commit d099acc6a4061d1f364785d3b6224c99c5cea704
Author: Jihee Lim <[email protected]>
AuthorDate: Sun Nov 15 02:33:59 2026 +0900

    Detect supervisor-subprocess trigger count mismatch in Triggerer (#68792)
    
    * Detect supervisor-subprocess trigger count mismatch in Triggerer
    
    * Fix false-positive path in trigger count mismatch check
    
    Remove the incorrect difference_update on msg.failures (which would
    break for serialization failures where the trigger is still running).
    
    Instead, include creation-failure IDs in finished_ids so the supervisor
    naturally removes them from running_triggers via the existing finished
    processing path.
    
    Co-Authored-By: Claude Opus 4.6 <[email protected]>
    
    * Recover lost Triggerer triggers instead of shutting down
    
    * Distinguish a missing running_ids from an empty one
    
    * Avoid reporting a finished trigger twice in the Triggerer
    
    * Clear releasing_triggers when recovering a lost Triggerer trigger
    
    * Finalize trigger logs when recovering unhandled triggers
    
    * Drop the bugfix newsfragment for the Triggerer recovery fix
    
    contributing-docs/18_contribution_workflow.rst limits airflow-core 
newsfragments to significant changes; an ordinary bug fix does not get one.
    
    Claude-Session: https://claude.ai/code/session_01BVx6gBBL7XtC13QVHEbCVh
    
    ---------
    
    Co-authored-by: Claude Opus 4.6 <[email protected]>
    Co-authored-by: Shahar Epstein <[email protected]>
---
 .../src/airflow/jobs/triggerer_job_runner.py       |  61 ++++++--
 airflow-core/tests/unit/jobs/test_triggerer_job.py | 163 +++++++++++++++++++++
 .../observability/metrics/metrics_template.yaml    |   7 +
 3 files changed, 221 insertions(+), 10 deletions(-)

diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py 
b/airflow-core/src/airflow/jobs/triggerer_job_runner.py
index afb50da0ef2..ef16d50a28d 100644
--- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py
+++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py
@@ -446,6 +446,8 @@ class messages:
         # Format of list[str] is the exc traceback format
         failures: list[tuple[int, list[str] | None]] | None = None
         finished: list[int] | None = None
+        # Ids the runner has a live coroutine for; None when the message 
carries no information
+        running_ids: set[int] | None = None
 
     class TriggerStateSync(BaseModel):
         type: Literal["TriggerStateSync"] = "TriggerStateSync"
@@ -722,15 +724,9 @@ class TriggerRunnerSupervisor(WatchedSubprocess):
                 self.running_triggers.discard(id)
                 self.cancelling_triggers.discard(id)
                 self.releasing_triggers.discard(id)
-                if factory := self.logger_cache.pop(id, None):
-                    try:
-                        factory.upload_to_remote()
-                    except Exception:
-                        log.exception("Failed to upload trigger logs to 
remote", trigger_id=id)
-                    finally:
-                        # Close the FD explicitly even if upload raised, 
otherwise the file
-                        # handle leaks for every failed upload.
-                        factory.close()
+                self.finalize_trigger_log(id)
+
+            self.check_for_unhandled_triggers(msg.running_ids)
 
             # Drain the persist confirmations accumulated since the last sync.
             events_persisted: list[int] = []
@@ -916,6 +912,46 @@ class TriggerRunnerSupervisor(WatchedSubprocess):
             with attempt, create_session() as session:
                 Trigger.clean_unused(session=session)
 
+    def finalize_trigger_log(self, trigger_id: int) -> None:
+        """
+        Upload and close the log file of a trigger that no longer runs here.
+
+        Every path that drops an id from ``running_triggers`` has to call it: 
re-creation
+        reuses the log path, so a leftover entry is lost.
+        """
+        if factory := self.logger_cache.pop(trigger_id, None):
+            try:
+                factory.upload_to_remote()
+            except Exception:
+                log.exception("Failed to upload trigger logs to remote", 
trigger_id=trigger_id)
+            finally:
+                # Close the FD explicitly even if upload raised, otherwise the 
file handle
+                # leaks for every failed upload. close() flushes, so it can 
raise too.
+                try:
+                    factory.close()
+                except Exception:
+                    log.exception("Failed to close trigger log file", 
trigger_id=trigger_id)
+
+    def check_for_unhandled_triggers(self, running_ids: set[int] | None) -> 
None:
+        """
+        Re-create triggers we track as running that the runner has no 
coroutine for.
+
+        Only valid between finished-removal and to_create-addition in 
``_handle_request``, where the
+        two sides agree. Dropping the leftovers lets :meth:`update_triggers` 
rebuild them.
+        """
+        if running_ids is None:
+            return
+        unhandled = self.running_triggers - running_ids
+        if not unhandled:
+            return
+        log.error("Triggers have no coroutine in the runner; re-creating", 
trigger_ids=sorted(unhandled))
+        self.running_triggers -= unhandled
+        self.cancelling_triggers -= unhandled
+        self.releasing_triggers -= unhandled
+        for trigger_id in unhandled:
+            self.finalize_trigger_log(trigger_id)
+        stats.incr("triggers.state_mismatch", len(unhandled), 
tags=prune_dict({"team_name": self.team_name}))
+
     def handle_failed_triggers(self):
         """
         Handle "failed" triggers. - ones that errored or exited before they 
sent an event.
@@ -1789,6 +1825,7 @@ class TriggerRunner:
         # Copy out of our dequeues in threadsafe manner to sync state with 
parent
         events_to_send: list[TriggerEventEntry] = []
         failures_to_send: list[tuple[int, list[str] | None]] = []
+        finished_to_send = list(finished_ids)
 
         while self.events:
             events_to_send.append(self.events.popleft())
@@ -1797,11 +1834,14 @@ class TriggerRunner:
             trigger_id, exc = self.failed_triggers.popleft()
             tb = format_exception(type(exc), exc, exc.__traceback__) if exc 
else None
             failures_to_send.append((trigger_id, tb))
+            if trigger_id not in self.triggers and trigger_id not in 
finished_to_send:
+                finished_to_send.append(trigger_id)
 
         return messages.TriggerStateChanges(
             events=events_to_send if events_to_send else None,
-            finished=finished_ids if finished_ids else None,
+            finished=finished_to_send if finished_to_send else None,
             failures=failures_to_send if failures_to_send else None,
+            running_ids=set(self.triggers),
         )
 
     def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) -> 
messages.TriggerStateChanges:
@@ -1830,6 +1870,7 @@ class TriggerRunner:
             events=events_to_send if events_to_send else None,
             finished=msg.finished,
             failures=msg.failures,
+            running_ids=msg.running_ids,
         )
 
     async def sync_state_to_supervisor(self, finished_ids: list[int]) -> None:
diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py 
b/airflow-core/tests/unit/jobs/test_triggerer_job.py
index 2a1f6265f65..03adf3ade98 100644
--- a/airflow-core/tests/unit/jobs/test_triggerer_job.py
+++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py
@@ -4569,3 +4569,166 @@ class TestOwnerOfTriggersCreatedWithoutOne:
         TriggerRunnerSupervisor._record_missing_owner(trigger)
 
         assert (trigger.dag_id, trigger.owner_ref) == ("d", {"ti_id": "x"})
+
+
+class TestCheckForUnhandledTriggers:
+    """Tests for recovery of triggers the runner has no coroutine for."""
+
+    @pytest.mark.parametrize(
+        ("running_triggers", "running_ids", "expected_after"),
+        [
+            pytest.param({1, 2, 3}, {1, 2, 3}, {1, 2, 3}, id="in-sync"),
+            pytest.param(set(), set(), set(), id="no-triggers"),
+            pytest.param({1, 2, 3}, {1, 2}, {1, 2}, id="one-unhandled"),
+            pytest.param({1, 2, 3}, set(), set(), id="all-unhandled"),
+        ],
+    )
+    def test_unhandled_triggers_are_dropped(
+        self, jobless_supervisor, mocker, running_triggers, running_ids, 
expected_after
+    ):
+        incr = mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr", 
autospec=True)
+        jobless_supervisor.running_triggers = set(running_triggers)
+        jobless_supervisor.cancelling_triggers = set(running_triggers)
+
+        jobless_supervisor.check_for_unhandled_triggers(running_ids)
+
+        unhandled = running_triggers - running_ids
+        assert jobless_supervisor.running_triggers == expected_after
+        assert jobless_supervisor.cancelling_triggers == expected_after
+        assert (
+            mocker.call("triggers.state_mismatch", len(unhandled), tags={}) in 
incr.call_args_list
+        ) is bool(unhandled)
+
+    def test_dropped_trigger_is_recreated_next_loop(self, jobless_supervisor, 
mocker):
+        """Dropping an unhandled id is what lets update_triggers rebuild its 
workload."""
+        build = mocker.patch.object(
+            TriggerRunnerSupervisor, "build_trigger_workloads", autospec=True, 
return_value=[]
+        )
+        jobless_supervisor.running_triggers = {1, 2, 3}
+        # A trigger released to another triggerer stays in running_triggers, 
so it can turn up
+        # unhandled too, and releasing_triggers also gates re-creation via 
known_trigger_ids.
+        jobless_supervisor.releasing_triggers = {3}
+
+        jobless_supervisor.check_for_unhandled_triggers({1})
+        jobless_supervisor.update_triggers({1, 2, 3})
+
+        assert build.call_args.args[1] == {2, 3}
+
+    def test_recovered_trigger_log_is_finalized(self, jobless_supervisor, 
mocker):
+        """Re-creation reuses the log path, so recovery has to upload and 
close it first."""
+        mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr", 
autospec=True)
+        factory = MagicMock(spec=TriggerLoggingFactory)
+        jobless_supervisor.logger_cache[42] = factory
+        jobless_supervisor.running_triggers = {42}
+
+        jobless_supervisor.check_for_unhandled_triggers(set())
+
+        factory.upload_to_remote.assert_called_once()
+        factory.close.assert_called_once()
+        assert 42 not in jobless_supervisor.logger_cache
+
+    def test_recovery_survives_a_log_file_that_cannot_be_closed(self, 
jobless_supervisor, mocker):
+        """An unwritable log file must not strand the remaining ids, nor reach 
the runner."""
+        incr = mocker.patch("airflow.jobs.triggerer_job_runner.stats.incr", 
autospec=True)
+        factories = {}
+        for trigger_id in (1, 2):
+            factory = MagicMock(spec=TriggerLoggingFactory)
+            factory.close.side_effect = OSError("no space left on device")
+            factories[trigger_id] = factory
+        jobless_supervisor.logger_cache.update(factories)
+        jobless_supervisor.running_triggers = {1, 2}
+
+        jobless_supervisor.check_for_unhandled_triggers(set())
+
+        assert not jobless_supervisor.logger_cache
+        assert all(factory.close.called for factory in factories.values())
+        assert mocker.call("triggers.state_mismatch", 2, tags={}) in 
incr.call_args_list
+
+    def test_message_without_running_ids_is_ignored(self, jobless_supervisor):
+        """``None`` means the message carried no information, which must not 
read as "nothing running"."""
+        jobless_supervisor.running_triggers = {1, 2, 3}
+        jobless_supervisor.cancelling_triggers = {1, 2, 3}
+
+        jobless_supervisor.check_for_unhandled_triggers(None)
+
+        assert jobless_supervisor.running_triggers == {1, 2, 3}
+        assert jobless_supervisor.cancelling_triggers == {1, 2, 3}
+
+    def test_handle_request_checks_before_adding_to_create(self, 
jobless_supervisor, mocker):
+        """The check fires after finished processing but before to_create is 
added to running_triggers."""
+        mocker.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True)
+        jobless_supervisor.running_triggers = {1, 2}
+        jobless_supervisor.creating_triggers.append(
+            mocker.MagicMock(id=3),
+        )
+
+        jobless_supervisor._handle_request(
+            messages.TriggerStateChanges(
+                events=None,
+                failures=None,
+                finished=[1],
+                running_ids={2},
+            ),
+            log=MagicMock(spec=FilteringBoundLogger),
+            req_id=1,
+        )
+
+        assert jobless_supervisor.running_triggers == {2, 3}
+
+    def test_creation_failure_reported_in_finished(self, jobless_supervisor, 
mocker):
+        """A creation failure appears in both failures and finished, so 
running_triggers stays in sync."""
+        mocker.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True)
+        jobless_supervisor.running_triggers = {1, 2, 3}
+
+        jobless_supervisor._handle_request(
+            messages.TriggerStateChanges(
+                events=None,
+                failures=[(3, ["Traceback..."])],
+                finished=[3],
+                running_ids={1, 2},
+            ),
+            log=MagicMock(spec=FilteringBoundLogger),
+            req_id=1,
+        )
+
+        assert 3 not in jobless_supervisor.running_triggers
+
+
+class TestCreationFailureInFinished:
+    """Tests that creation failures (not in self.triggers) are included in 
finished_ids."""
+
+    def test_creation_failure_included_in_finished(self):
+        runner = TriggerRunner()
+        runner.failed_triggers.append((42, ValueError("bad classpath")))
+
+        msg = runner.process_trigger_events(finished_ids=[10])
+
+        assert msg.finished == [10, 42]
+        assert msg.running_ids == set()
+
+    def test_serialization_failure_not_in_finished(self):
+        runner = TriggerRunner()
+        runner.triggers = {42: {"task": MagicMock(), "is_watcher": False, 
"name": "t", "events": 0}}
+        runner.failed_triggers.append((42, ValueError("not serializable")))
+
+        msg = runner.process_trigger_events(finished_ids=[])
+
+        assert msg.finished is None
+        assert msg.running_ids == {42}
+
+    def test_failure_already_in_finished_ids_not_duplicated(self):
+        runner = TriggerRunner()
+        runner.failed_triggers.append((42, ValueError("exited without an 
event")))
+
+        msg = runner.process_trigger_events(finished_ids=[42])
+
+        assert msg.finished == [42]
+
+    def test_caller_finished_ids_not_mutated(self):
+        runner = TriggerRunner()
+        runner.failed_triggers.append((42, ValueError("bad classpath")))
+        finished_ids = [10]
+
+        runner.process_trigger_events(finished_ids=finished_ids)
+
+        assert finished_ids == [10]
diff --git 
a/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
 
b/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
index cfe767fd4d3..e3c5c3401e0 100644
--- 
a/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
+++ 
b/shared/observability/src/airflow_shared/observability/metrics/metrics_template.yaml
@@ -387,6 +387,13 @@ metrics:
     legacy_name: "-"
     name_variables: []
 
+  - name: "triggers.state_mismatch"
+    description: "Number of triggers the Triggerer tracked as running that the 
trigger runner
+    had no coroutine for, and which were re-created"
+    type: "counter"
+    legacy_name: "-"
+    name_variables: []
+
   - name: "triggers.succeeded"
     description: "Number of triggers that have fired at least one event.
     Tagged with trigger_kind (asset_watcher, callback or task)."

Reply via email to