ferruzzi commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4605762507
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -425,6 +471,141 @@ def __init__(
def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
stats.incr("scheduler_heartbeat", 1, 1)
+ def _check_executor_health(self) -> None:
+ """
+ Kick off a health check for each reporting executor, in the background.
+
+ Called on its own timer (``[scheduler]
executor_health_check_interval``), independent of the
+ scheduling loop and of each executor's own
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+ A new check for a given executor is skipped while its previous check
is still running, so a
+ slow or hung dependency can only ever delay that executor's own health
being updated, never
+ pile up work or block this (or any other) scheduler tick.
+
+ ``get_health()`` is passed a timeout, but an implementation that
ignores it can still leave its
+ future pending forever. That future's own worker slot is never
reclaimed -- the pool is sized
+ one worker per executor, so a stuck probe can only ever cost that
executor's own slot -- but
+ without this check it would silently stop being checked again, with no
sign anything was
+ wrong. Once a pending future is a few intervals old, it's reported as
stalled instead.
+ """
+ for executor in self.executors:
+ key = _executor_health_key(executor)
+ if key not in self._executor_health_keys:
+ # Doesn't override get_health(); there's nothing to check or
record.
+ continue
+ future, started = self._executor_health_check_futures.get(key,
(None, 0.0))
+ if future is not None and not future.done():
+ stall_after = (
+ self._executor_health_check_interval *
self._executor_health_check_stall_after_intervals
+ )
+ if time.monotonic() - started > stall_after and key not in
self._stalled_health_checks:
+ self._stalled_health_checks.add(key)
+ self.log.warning("Health check for executor %s has stopped
responding", key)
+ stats.incr("scheduler.executor_health_check.stalled",
tags={"executor": key})
+ else:
+ self.log.debug(
+ "Skipping health check for executor %s, the previous
check hasn't finished", key
+ )
+ continue
+ self._stalled_health_checks.discard(key)
+ future =
self._executor_health_check_pool.submit(self._run_executor_health_check,
executor, key)
+ self._executor_health_check_futures[key] = (future,
time.monotonic())
+
+ def _run_executor_health_check(self, executor: BaseExecutor, key: str) ->
None:
+ """Run one executor's get_health() off the scheduler thread and record
an explicit result."""
+ try:
+ healthy =
executor.get_health(timeout=self._executor_health_check_timeout)
+ except Exception:
+ self.log.exception("Health check for executor %s raised an
exception", key)
+ healthy = None
Review Comment:
I think if it raises, it needs to be UNHEALTHY. As written, an executor
whose live rows are all UNKNOWN is dropped from `expected`. Which means a
check that raises on every cycle never counts against the executor.
I think we need to treat a check that raises as a failed check and
if the executor is down and always raises, it gets recorded as UNKNOWN which
then gets reported as HEALTHY. Keep 'None` for UNKNOWN and a failed check as
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]