vincbeck commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4635566007


##########
airflow-core/src/airflow/api_fastapi/core_api/datamodels/monitor.py:
##########
@@ -75,10 +86,25 @@ class DagProcessorInfoResponse(BaseInfoResponse):
     instances: list[DagProcessorInstanceInfoResponse] | None = None
 
 
+class ExecutorInfoResponse(BaseInfoResponse):
+    """
+    Executor info serializer for responses.
+
+    Only present when at least one configured executor implements a health 
check; an executor that
+    doesn't isn't a gap in coverage, so a deployment where nothing reports 
simply has no ``executor``
+    key at all, rather than one that always reads unhealthy.
+    """
+
+    latest_executor_heartbeat: str | None
+    detailed_status: DetailedHealthStatus | None
+    instances: list[ExecutorInstanceInfoResponse] | None = None
+
+
 class HealthInfoResponse(BaseModel):
     """Health serializer for responses."""
 
     metadatabase: BaseInfoResponse
     scheduler: SchedulerInfoResponse
     triggerer: TriggererInfoResponse
     dag_processor: DagProcessorInfoResponse | None = None
+    executor: ExecutorInfoResponse | None = None

Review Comment:
   You need to regenerate the OpenAPI spec (`v2-rest-api-generated.yaml`) and 
the UI client (`openapi-gen`), `ExecutorInfoResponse` is in neither. The prek 
hooks should do it.



##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_executor_health_columns_to_job.py:
##########
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add health_check_result and owner_job_id to Job.
+
+Revision ID: feb8b1e719ee
+Revises: c64b10719069
+Create Date: 2026-11-05 09:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+revision = "feb8b1e719ee"

Review Comment:
   `_REVISION_HEADS_MAP` in `airflow/utils/db.py` still maps `3.5.0` to 
`c64b10719069`, it should be `feb8b1e719ee`. `docs/migrations-ref.rst` needs an 
update too (the `update-migration-references` prek hook does it).



##########
airflow-core/tests/unit/api/common/test_airflow_health.py:
##########
@@ -662,4 +668,299 @@ def 
test_get_airflow_health_mixed_triggerers_include_team_names(self, testing_te
                 "team_names": [testing_team.name],
             },
         ]
-        assert health_status["triggerer"]["latest_triggerer_heartbeat"] == 
alive_heartbeat.isoformat()

Review Comment:
   Why removing this assert? It looks unrelated to this PR



##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -435,6 +454,41 @@ def __init__(
         self.executors: list[BaseExecutor] = executors if executors else 
ExecutorLoader.init_executors()
         self.executor: BaseExecutor = self.executors[0]
 
+        # Each executor's get_health() runs on its own daemon thread, never on 
the scheduler loop's own
+        # thread: a check that talks to a broker or a cluster control plane 
can be slow or hang, and
+        # must not be able to delay task scheduling. A pool isn't needed since 
each executor is only
+        # ever checked by one in-flight call at a time (see 
_check_executor_health); a daemon thread
+        # also means a hung check can't block process exit the way a 
ThreadPoolExecutor worker would.
+        self._executor_health_check_futures: dict[str, tuple[Future, float]] = 
{}
+        # A probe that ignores the timeout passed to it would otherwise occupy 
its slot forever: once
+        # a pending future has been sitting this many intervals, it's reported 
as stalled instead of
+        # silently skipped on every later tick (see _check_executor_health).
+        self._executor_health_check_stall_after_intervals = 3
+        self._stalled_health_checks: set[str] = set()
+
+        self._executor_health_check_interval = conf.getfloat(
+            "scheduler", "executor_health_check_interval", fallback=30.0
+        )
+        # Must be comfortably below the check interval so a probe that honours 
it always finishes
+        # before the next tick fires; half the interval is the starting point.
+        self._executor_health_check_timeout = 
self._executor_health_check_interval / 2
+
+        # Only executors that actually override get_health() get a Job row at 
all: one that was never
+        # going to report anything isn't a gap in coverage (see 
_configured_executor_keys in
+        # airflow_health.py, which applies this same rule from the 
/monitor/health side). An executor
+        # with no name is also skipped: _executor_health_key has nothing 
stable to key its row under.
+        # getattr rather than a plain attribute access: a test double standing 
in for an executor
+        # (e.g. a bare, non-speced MagicMock) has no real get_health of its 
own to compare, and should
+        # be treated the same as one that doesn't override it, not raise 
AttributeError.
+        from airflow.executors.base_executor import BaseExecutor as 
_BaseExecutor
+
+        self._executor_health_keys: dict[str, str | None] = {
+            key: executor.team_name
+            for executor in self.executors
+            if getattr(type(executor), "get_health", _BaseExecutor.get_health) 
is not _BaseExecutor.get_health

Review Comment:
   Why `getattr`? We should not change production code to handle a bare 
`MagicMock`, the tests should use `spec=BaseExecutor` instead.



##########
airflow-core/newsfragments/76410.significant.rst:
##########
@@ -0,0 +1,18 @@
+Added scaffolding for per-executor health reporting

Review Comment:
   Do we need a newsfragment? The last paragraph says behavior is unchanged for 
every existing executor, so there is nothing users need to read before 
upgrading.



##########
airflow-core/src/airflow/api/common/airflow_health.py:
##########
@@ -250,6 +377,28 @@ def get_airflow_health() -> dict[str, Any]:
         dag_processor_status = HealthStatus.UNHEALTHY
         dag_processor_detailed_status = DetailedHealthStatus.DOWN
 
+    # Omitted entirely, not just reported unhealthy, when no configured 
executor implements a health
+    # check: a reporting mechanism nothing has opted into isn't a gap in 
coverage, and showing it as
+    # "unhealthy" on every deployment that hasn't adopted it yet would be a 
false alarm, not a signal.
+    report_executor_health = bool(_configured_executor_keys())
+    if report_executor_health:
+        try:
+            executor_jobs = get_jobs_health(EXECUTOR_JOB_TYPE)
+            executor_detailed_status = _executor_detailed_status(executor_jobs)
+            executor_status = (
+                HealthStatus.UNHEALTHY
+                if executor_detailed_status == DetailedHealthStatus.DOWN
+                else HealthStatus.HEALTHY
+            )
+            if executor_jobs and executor_jobs[0].latest_heartbeat:
+                latest_executor_heartbeat = 
executor_jobs[0].latest_heartbeat.isoformat()

Review Comment:
   `latest_executor_heartbeat` is refreshed by the scheduler on every 
heartbeat, not by the check. So nothing in the response says when 
`health_check_result` was last computed. Should we record when the last check 
finished?



##########
airflow-core/src/airflow/api/common/airflow_health.py:
##########
@@ -170,6 +223,76 @@ def _dag_processor_detailed_status(jobs: list[Job]) -> 
DetailedHealthStatus:
     return _coverage_status(expected, covered)
 
 
+@lru_cache
+def _configured_executor_keys() -> set[str]:
+    """
+    Return the key each configured executor that implements a health check 
would be recorded under.
+
+    Most executors don't override 
:meth:`~airflow.executors.base_executor.BaseExecutor.get_health`,
+    so this deliberately excludes them rather than counting every configured 
executor: an executor
+    that was never going to report anything isn't a gap in coverage, and a 
deployment where nothing
+    implements a health check should see no ``executor`` block at all in 
``/monitor/health``, not an
+    ``unhealthy`` one.
+
+    Cached: this calls 
:meth:`~airflow.executors.executor_loader.ExecutorLoader.import_executor_cls`,
+    which imports every configured executor class, and the configured 
executors can't change while the
+    process runs. Without caching, an API server would redo those imports -- 
and reproduce any import
+    failure's warning and traceback -- on every single call to 
``/monitor/health``. Team validation is
+    skipped (``validate_teams=False``), the same as ``connections.py`` does, 
so a database hiccup on
+    the first call can't get cached as "no executors".
+    """
+    from airflow.executors.base_executor import BaseExecutor
+
+    try:
+        names = ExecutorLoader.get_executor_names(validate_teams=False)
+    except Exception:
+        # A health probe must not fail on malformed executor config; callers 
fall back to liveness only.
+        log.warning("Could not read the executor configuration", exc_info=True)
+        return set()
+
+    keys = set()
+    for name in names:
+        try:
+            executor_cls, _ = ExecutorLoader.import_executor_cls(name)
+        except Exception:
+            log.warning("Could not import configured executor %s", 
name.module_path, exc_info=True)
+            continue
+        if executor_cls.get_health is not BaseExecutor.get_health:

Review Comment:
   nit: this is the same check as in `SchedulerJobRunner.__init__`, any chance 
to factorize? Maybe a classmethod in `BaseExecutor`



##########
airflow-core/tests/unit/api/common/test_airflow_health.py:
##########
@@ -662,4 +668,299 @@ def 
test_get_airflow_health_mixed_triggerers_include_team_names(self, testing_te
                 "team_names": [testing_team.name],
             },
         ]
-        assert health_status["triggerer"]["latest_triggerer_heartbeat"] == 
alive_heartbeat.isoformat()
+
+
+# -- Executor health: no configured executor overrides get_health() by 
default, so these tests patch
+# -- _configured_executor_keys directly rather than relying on a real executor 
class, exactly as the
+# -- bundle/team partition tests above patch _configured_bundle_teams.
+
+
+def _executor(
+    *,
+    alive: bool,
+    healthy: bool | None = True,
+    team_name: str | None = None,
+    executor: str | None = None,
+    owner_job_id: int = 1,
+) -> MagicMock:
+    job = _mock_job(
+        hostname=f"host-{owner_job_id}",
+        heartbeat=datetime(2024, 2, 1),
+        alive=alive,
+        team_names=[team_name] if team_name else [],
+    )
+    job.executor_class = executor or f"executor-{team_name}-{'alive' if alive 
else 'stale'}"
+    job.owner_job_id = owner_job_id
+    if healthy is None:
+        job.health_check_result = HealthCheckResult.UNKNOWN
+    else:
+        job.health_check_result = HealthCheckResult.HEALTHY if healthy else 
HealthCheckResult.UNHEALTHY
+    return job
+
+
+def _patch_executor_keys(keys: set[str]):
+    return 
patch("airflow.api.common.airflow_health._configured_executor_keys", 
return_value=keys)
+
+
+def _patch_live_scheduler_ids(ids: set[int]):
+    """Patch which owning-scheduler ids count as alive, standing in for a real 
SchedulerJob query."""
+    return patch("airflow.api.common.airflow_health._live_scheduler_job_ids", 
return_value=ids)
+
+
+class _HealthCheckingExecutor(BaseExecutor):
+    """A stand-in for a concrete executor (e.g. Celery) that has implemented 
get_health()."""
+
+    def get_health(self, timeout: float) -> bool | None:
+        return True
+
+
+class TestConfiguredExecutorKeys:
+    """``_configured_executor_keys`` only counts an executor that actually 
implements get_health()."""
+
+    @pytest.fixture(autouse=True)
+    def clear_executor_caches(self):
+        _configured_executor_keys.cache_clear()
+        _configured_executor_names.cache_clear()
+        yield
+        _configured_executor_keys.cache_clear()
+        _configured_executor_names.cache_clear()
+
+    def test_empty_when_no_executor_overrides_get_health(self):
+        names = 
[ExecutorName(module_path="airflow.executors.local_executor.LocalExecutor", 
alias="Local")]
+        with 
patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names", 
return_value=names):

Review Comment:
   nit: we usually use the `@mock.patch` decorator rather than the context 
manager. Same in the other tests below



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to