vincbeck commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4635566007
##########
airflow-core/src/airflow/api_fastapi/core_api/datamodels/monitor.py:
##########
@@ -75,10 +86,25 @@ class DagProcessorInfoResponse(BaseInfoResponse):
instances: list[DagProcessorInstanceInfoResponse] | None = None
+class ExecutorInfoResponse(BaseInfoResponse):
+ """
+ Executor info serializer for responses.
+
+ Only present when at least one configured executor implements a health
check; an executor that
+ doesn't isn't a gap in coverage, so a deployment where nothing reports
simply has no ``executor``
+ key at all, rather than one that always reads unhealthy.
+ """
+
+ latest_executor_heartbeat: str | None
+ detailed_status: DetailedHealthStatus | None
+ instances: list[ExecutorInstanceInfoResponse] | None = None
+
+
class HealthInfoResponse(BaseModel):
"""Health serializer for responses."""
metadatabase: BaseInfoResponse
scheduler: SchedulerInfoResponse
triggerer: TriggererInfoResponse
dag_processor: DagProcessorInfoResponse | None = None
+ executor: ExecutorInfoResponse | None = None
Review Comment:
You need to regenerate the OpenAPI spec (`v2-rest-api-generated.yaml`) and
the UI client (`openapi-gen`), `ExecutorInfoResponse` is in neither. The prek
hooks should do it.
##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_executor_health_columns_to_job.py:
##########
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add health_check_result and owner_job_id to Job.
+
+Revision ID: feb8b1e719ee
+Revises: c64b10719069
+Create Date: 2026-11-05 09:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+revision = "feb8b1e719ee"
Review Comment:
`_REVISION_HEADS_MAP` in `airflow/utils/db.py` still maps `3.5.0` to
`c64b10719069`, it should be `feb8b1e719ee`. `docs/migrations-ref.rst` needs an
update too (the `update-migration-references` prek hook does it).
##########
airflow-core/tests/unit/api/common/test_airflow_health.py:
##########
@@ -662,4 +668,299 @@ def
test_get_airflow_health_mixed_triggerers_include_team_names(self, testing_te
"team_names": [testing_team.name],
},
]
- assert health_status["triggerer"]["latest_triggerer_heartbeat"] ==
alive_heartbeat.isoformat()
Review Comment:
Why removing this assert? It looks unrelated to this PR
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -435,6 +454,41 @@ def __init__(
self.executors: list[BaseExecutor] = executors if executors else
ExecutorLoader.init_executors()
self.executor: BaseExecutor = self.executors[0]
+ # Each executor's get_health() runs on its own daemon thread, never on
the scheduler loop's own
+ # thread: a check that talks to a broker or a cluster control plane
can be slow or hang, and
+ # must not be able to delay task scheduling. A pool isn't needed since
each executor is only
+ # ever checked by one in-flight call at a time (see
_check_executor_health); a daemon thread
+ # also means a hung check can't block process exit the way a
ThreadPoolExecutor worker would.
+ self._executor_health_check_futures: dict[str, tuple[Future, float]] =
{}
+ # A probe that ignores the timeout passed to it would otherwise occupy
its slot forever: once
+ # a pending future has been sitting this many intervals, it's reported
as stalled instead of
+ # silently skipped on every later tick (see _check_executor_health).
+ self._executor_health_check_stall_after_intervals = 3
+ self._stalled_health_checks: set[str] = set()
+
+ self._executor_health_check_interval = conf.getfloat(
+ "scheduler", "executor_health_check_interval", fallback=30.0
+ )
+ # Must be comfortably below the check interval so a probe that honours
it always finishes
+ # before the next tick fires; half the interval is the starting point.
+ self._executor_health_check_timeout =
self._executor_health_check_interval / 2
+
+ # Only executors that actually override get_health() get a Job row at
all: one that was never
+ # going to report anything isn't a gap in coverage (see
_configured_executor_keys in
+ # airflow_health.py, which applies this same rule from the
/monitor/health side). An executor
+ # with no name is also skipped: _executor_health_key has nothing
stable to key its row under.
+ # getattr rather than a plain attribute access: a test double standing
in for an executor
+ # (e.g. a bare, non-speced MagicMock) has no real get_health of its
own to compare, and should
+ # be treated the same as one that doesn't override it, not raise
AttributeError.
+ from airflow.executors.base_executor import BaseExecutor as
_BaseExecutor
+
+ self._executor_health_keys: dict[str, str | None] = {
+ key: executor.team_name
+ for executor in self.executors
+ if getattr(type(executor), "get_health", _BaseExecutor.get_health)
is not _BaseExecutor.get_health
Review Comment:
Why `getattr`? We should not change production code to handle a bare
`MagicMock`, the tests should use `spec=BaseExecutor` instead.
##########
airflow-core/newsfragments/76410.significant.rst:
##########
@@ -0,0 +1,18 @@
+Added scaffolding for per-executor health reporting
Review Comment:
Do we need a newsfragment? The last paragraph says behavior is unchanged for
every existing executor, so there is nothing users need to read before
upgrading.
##########
airflow-core/src/airflow/api/common/airflow_health.py:
##########
@@ -250,6 +377,28 @@ def get_airflow_health() -> dict[str, Any]:
dag_processor_status = HealthStatus.UNHEALTHY
dag_processor_detailed_status = DetailedHealthStatus.DOWN
+ # Omitted entirely, not just reported unhealthy, when no configured
executor implements a health
+ # check: a reporting mechanism nothing has opted into isn't a gap in
coverage, and showing it as
+ # "unhealthy" on every deployment that hasn't adopted it yet would be a
false alarm, not a signal.
+ report_executor_health = bool(_configured_executor_keys())
+ if report_executor_health:
+ try:
+ executor_jobs = get_jobs_health(EXECUTOR_JOB_TYPE)
+ executor_detailed_status = _executor_detailed_status(executor_jobs)
+ executor_status = (
+ HealthStatus.UNHEALTHY
+ if executor_detailed_status == DetailedHealthStatus.DOWN
+ else HealthStatus.HEALTHY
+ )
+ if executor_jobs and executor_jobs[0].latest_heartbeat:
+ latest_executor_heartbeat =
executor_jobs[0].latest_heartbeat.isoformat()
Review Comment:
`latest_executor_heartbeat` is refreshed by the scheduler on every
heartbeat, not by the check. So nothing in the response says when
`health_check_result` was last computed. Should we record when the last check
finished?
##########
airflow-core/src/airflow/api/common/airflow_health.py:
##########
@@ -170,6 +223,76 @@ def _dag_processor_detailed_status(jobs: list[Job]) ->
DetailedHealthStatus:
return _coverage_status(expected, covered)
+@lru_cache
+def _configured_executor_keys() -> set[str]:
+ """
+ Return the key each configured executor that implements a health check
would be recorded under.
+
+ Most executors don't override
:meth:`~airflow.executors.base_executor.BaseExecutor.get_health`,
+ so this deliberately excludes them rather than counting every configured
executor: an executor
+ that was never going to report anything isn't a gap in coverage, and a
deployment where nothing
+ implements a health check should see no ``executor`` block at all in
``/monitor/health``, not an
+ ``unhealthy`` one.
+
+ Cached: this calls
:meth:`~airflow.executors.executor_loader.ExecutorLoader.import_executor_cls`,
+ which imports every configured executor class, and the configured
executors can't change while the
+ process runs. Without caching, an API server would redo those imports --
and reproduce any import
+ failure's warning and traceback -- on every single call to
``/monitor/health``. Team validation is
+ skipped (``validate_teams=False``), the same as ``connections.py`` does,
so a database hiccup on
+ the first call can't get cached as "no executors".
+ """
+ from airflow.executors.base_executor import BaseExecutor
+
+ try:
+ names = ExecutorLoader.get_executor_names(validate_teams=False)
+ except Exception:
+ # A health probe must not fail on malformed executor config; callers
fall back to liveness only.
+ log.warning("Could not read the executor configuration", exc_info=True)
+ return set()
+
+ keys = set()
+ for name in names:
+ try:
+ executor_cls, _ = ExecutorLoader.import_executor_cls(name)
+ except Exception:
+ log.warning("Could not import configured executor %s",
name.module_path, exc_info=True)
+ continue
+ if executor_cls.get_health is not BaseExecutor.get_health:
Review Comment:
nit: this is the same check as in `SchedulerJobRunner.__init__`, any chance
to factorize? Maybe a classmethod in `BaseExecutor`
##########
airflow-core/tests/unit/api/common/test_airflow_health.py:
##########
@@ -662,4 +668,299 @@ def
test_get_airflow_health_mixed_triggerers_include_team_names(self, testing_te
"team_names": [testing_team.name],
},
]
- assert health_status["triggerer"]["latest_triggerer_heartbeat"] ==
alive_heartbeat.isoformat()
+
+
+# -- Executor health: no configured executor overrides get_health() by
default, so these tests patch
+# -- _configured_executor_keys directly rather than relying on a real executor
class, exactly as the
+# -- bundle/team partition tests above patch _configured_bundle_teams.
+
+
+def _executor(
+ *,
+ alive: bool,
+ healthy: bool | None = True,
+ team_name: str | None = None,
+ executor: str | None = None,
+ owner_job_id: int = 1,
+) -> MagicMock:
+ job = _mock_job(
+ hostname=f"host-{owner_job_id}",
+ heartbeat=datetime(2024, 2, 1),
+ alive=alive,
+ team_names=[team_name] if team_name else [],
+ )
+ job.executor_class = executor or f"executor-{team_name}-{'alive' if alive
else 'stale'}"
+ job.owner_job_id = owner_job_id
+ if healthy is None:
+ job.health_check_result = HealthCheckResult.UNKNOWN
+ else:
+ job.health_check_result = HealthCheckResult.HEALTHY if healthy else
HealthCheckResult.UNHEALTHY
+ return job
+
+
+def _patch_executor_keys(keys: set[str]):
+ return
patch("airflow.api.common.airflow_health._configured_executor_keys",
return_value=keys)
+
+
+def _patch_live_scheduler_ids(ids: set[int]):
+ """Patch which owning-scheduler ids count as alive, standing in for a real
SchedulerJob query."""
+ return patch("airflow.api.common.airflow_health._live_scheduler_job_ids",
return_value=ids)
+
+
+class _HealthCheckingExecutor(BaseExecutor):
+ """A stand-in for a concrete executor (e.g. Celery) that has implemented
get_health()."""
+
+ def get_health(self, timeout: float) -> bool | None:
+ return True
+
+
+class TestConfiguredExecutorKeys:
+ """``_configured_executor_keys`` only counts an executor that actually
implements get_health()."""
+
+ @pytest.fixture(autouse=True)
+ def clear_executor_caches(self):
+ _configured_executor_keys.cache_clear()
+ _configured_executor_names.cache_clear()
+ yield
+ _configured_executor_keys.cache_clear()
+ _configured_executor_names.cache_clear()
+
+ def test_empty_when_no_executor_overrides_get_health(self):
+ names =
[ExecutorName(module_path="airflow.executors.local_executor.LocalExecutor",
alias="Local")]
+ with
patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names",
return_value=names):
Review Comment:
nit: we usually use the `@mock.patch` decorator rather than the context
manager. Same in the other tests below
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]