JoelEmmanuelCloud commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4644214125
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2174,6 +2437,21 @@ def _run_scheduler_loop(self) -> None:
non_fatal=True,
)
+ timers.call_regular_interval(
+ # A floor, not just self.job.heartrate directly: that can be
configured down to 0 (and
+ # tests commonly do), and a zero delay re-queues itself at "now"
on every single pass
+ # through the event scheduler, which never lets a non-blocking
run() return.
+ delay=max(self.job.heartrate, 1.0),
+ action=self._touch_executor_heartbeats,
+ non_fatal=True,
+ )
+
+ timers.call_regular_interval(
+ delay=self._executor_health_check_interval,
Review Comment:
good catch, fixed. both timers only get registered now if there's actually
something to check and the interval is above 0, same as
stale_bundle_cleanup_interval does it.
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +513,182 @@ def __init__(
def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
stats.incr("scheduler_heartbeat", 1, 1)
+ def _check_executor_health(self) -> None:
+ """
+ Kick off a health check for each reporting executor, in the background.
+
+ Called on its own timer (``[scheduler]
executor_health_check_interval``), independent of the
+ scheduling loop and of each executor's own
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+ A new check for a given executor is skipped while its previous check
is still running, so a
+ slow or hung dependency can only ever delay that executor's own health
being updated, never
+ pile up work or block this (or any other) scheduler tick.
+
+ ``get_health()`` is passed a timeout, but an implementation that
ignores it can still leave its
+ future pending forever. Once a pending future is a few intervals old,
it's reported as stalled:
+ the scheduler records UNHEALTHY itself, since a check that hasn't
answered isn't still-healthy,
+ and logs and emits a metric so the stall is visible. If the stuck call
does eventually return,
+ its result overwrites this one.
+ """
+ for executor in self.executors:
+ key = _executor_health_key(executor)
+ if key is None or key not in self._executor_health_keys:
+ # No name, or doesn't override get_health(): there's nothing
to check or record.
+ continue
+ future, started = self._executor_health_check_futures.get(key,
(None, 0.0))
+ if future is not None and not future.done():
+ stall_after = (
+ self._executor_health_check_interval *
self._executor_health_check_stall_after_intervals
+ )
+ if time.monotonic() - started > stall_after and key not in
self._stalled_health_checks:
+ self._stalled_health_checks.add(key)
+ self.log.warning("Health check for executor %s has stopped
responding", key)
+ stats.incr(
+ "scheduler.executor_health_check.stalled",
Review Comment:
added it to the registry.
##########
airflow-core/src/airflow/api_fastapi/core_api/datamodels/monitor.py:
##########
@@ -75,10 +86,25 @@ class DagProcessorInfoResponse(BaseInfoResponse):
instances: list[DagProcessorInstanceInfoResponse] | None = None
+class ExecutorInfoResponse(BaseInfoResponse):
+ """
+ Executor info serializer for responses.
+
+ Only present when at least one configured executor implements a health
check; an executor that
+ doesn't isn't a gap in coverage, so a deployment where nothing reports
simply has no ``executor``
+ key at all, rather than one that always reads unhealthy.
+ """
+
+ latest_executor_heartbeat: str | None
+ detailed_status: DetailedHealthStatus | None
+ instances: list[ExecutorInstanceInfoResponse] | None = None
+
+
class HealthInfoResponse(BaseModel):
"""Health serializer for responses."""
metadatabase: BaseInfoResponse
scheduler: SchedulerInfoResponse
triggerer: TriggererInfoResponse
dag_processor: DagProcessorInfoResponse | None = None
+ executor: ExecutorInfoResponse | None = None
Review Comment:
regenerated it.
##########
airflow-core/tests/unit/jobs/test_scheduler_job_executor_health.py:
##########
@@ -0,0 +1,280 @@
+from __future__ import annotations
Review Comment:
added it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]