JoelEmmanuelCloud commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4645807429


##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2174,6 +2419,21 @@ def _run_scheduler_loop(self) -> None:
             non_fatal=True,
         )
 
+        if self._executor_health_keys:
+            if self.job.heartrate > 0:

Review Comment:
   good point. _executor_health_keys is empty now when the interval is 0, so 
nothing gets created in the first place.



##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +495,182 @@ def __init__(
     def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
         stats.incr("scheduler_heartbeat", 1, 1)
 
+    def _check_executor_health(self) -> None:
+        """
+        Kick off a health check for each reporting executor, in the background.
+
+        Called on its own timer (``[scheduler] 
executor_health_check_interval``), independent of the
+        scheduling loop and of each executor's own 
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+        A new check for a given executor is skipped while its previous check 
is still running, so a
+        slow or hung dependency can only ever delay that executor's own health 
being updated, never
+        pile up work or block this (or any other) scheduler tick.
+
+        ``get_health()`` is passed a timeout, but an implementation that 
ignores it can still leave its
+        future pending forever. Once a pending future is a few intervals old, 
it's reported as stalled:
+        the scheduler records UNHEALTHY itself, since a check that hasn't 
answered isn't still-healthy,
+        and logs and emits a metric so the stall is visible. If the stuck call 
does eventually return,
+        its result overwrites this one.
+        """
+        for executor in self.executors:
+            key = _executor_health_key(executor)
+            if key is None or key not in self._executor_health_keys:
+                # No name, or doesn't override get_health(): there's nothing 
to check or record.
+                continue

Review Comment:
   you're right, simplified.



##########
airflow-core/src/airflow/api_fastapi/core_api/datamodels/monitor.py:
##########
@@ -75,10 +86,25 @@ class DagProcessorInfoResponse(BaseInfoResponse):
     instances: list[DagProcessorInstanceInfoResponse] | None = None
 
 
+class ExecutorInfoResponse(BaseInfoResponse):
+    """
+    Executor info serializer for responses.
+
+    Only present when at least one configured executor implements a health 
check; an executor that
+    doesn't isn't a gap in coverage, so a deployment where nothing reports 
simply has no ``executor``
+    key at all, rather than one that always reads unhealthy.
+    """

Review Comment:
   trimmed it to one line.



##########
airflow-core/tests/unit/api/common/test_airflow_health.py:
##########
@@ -663,3 +669,297 @@ def 
test_get_airflow_health_mixed_triggerers_include_team_names(self, testing_te
             },
         ]
         assert health_status["triggerer"]["latest_triggerer_heartbeat"] == 
alive_heartbeat.isoformat()
+
+
+# -- Executor health: no configured executor overrides get_health() by 
default, so these tests patch
+# -- _configured_executor_keys directly rather than relying on a real executor 
class, exactly as the
+# -- bundle/team partition tests above patch _configured_bundle_teams.
+
+
+def _executor(
+    *,
+    alive: bool,
+    healthy: bool | None = True,
+    team_name: str | None = None,
+    executor: str | None = None,
+    owner_job_id: int = 1,
+) -> MagicMock:
+    job = _mock_job(
+        hostname=f"host-{owner_job_id}",
+        heartbeat=datetime(2024, 2, 1),
+        alive=alive,
+        team_names=[team_name] if team_name else [],
+    )
+    job.executor_class = executor or f"executor-{team_name}-{'alive' if alive 
else 'stale'}"
+    job.owner_job_id = owner_job_id
+    if healthy is None:
+        job.health_check_result = HealthCheckResult.UNKNOWN
+    else:
+        job.health_check_result = HealthCheckResult.HEALTHY if healthy else 
HealthCheckResult.UNHEALTHY
+    return job
+
+
+def _patch_executor_keys(keys: set[str]):
+    return 
patch("airflow.api.common.airflow_health._configured_executor_keys", 
return_value=keys)
+
+
+def _patch_live_scheduler_ids(ids: set[int]):
+    """Patch which owning-scheduler ids count as alive, standing in for a real 
SchedulerJob query."""
+    return patch("airflow.api.common.airflow_health._live_scheduler_job_ids", 
return_value=ids)
+
+
+class _HealthCheckingExecutor(BaseExecutor):
+    """A stand-in for a concrete executor (e.g. Celery) that has implemented 
get_health()."""
+
+    def get_health(self, timeout: float) -> bool | None:
+        return True
+
+
+class TestConfiguredExecutorKeys:
+    """``_configured_executor_keys`` only counts an executor that actually 
implements get_health()."""
+
+    @pytest.fixture(autouse=True)
+    def clear_executor_caches(self):
+        _configured_executor_keys.cache_clear()
+        _configured_executor_names.cache_clear()
+        yield
+        _configured_executor_keys.cache_clear()
+        _configured_executor_names.cache_clear()
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names")
+    def test_empty_when_no_executor_overrides_get_health(self, 
mock_get_executor_names):
+        mock_get_executor_names.return_value = [
+            
ExecutorName(module_path="airflow.executors.local_executor.LocalExecutor", 
alias="Local")
+        ]
+        assert _configured_executor_keys() == set()
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.import_executor_cls")
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names")
+    def test_includes_an_executor_that_overrides_get_health(
+        self, mock_get_executor_names, mock_import_executor_cls
+    ):
+        names = [
+            ExecutorName(
+                
module_path=f"{_HealthCheckingExecutor.__module__}.{_HealthCheckingExecutor.__qualname__}",
+                alias="Celery",
+            )
+        ]
+        mock_get_executor_names.return_value = names
+        mock_import_executor_cls.return_value = (_HealthCheckingExecutor, None)
+
+        assert _configured_executor_keys() == {repr(names[0])}
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.import_executor_cls")
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names")
+    def test_module_path_configured_executor_has_no_alias_but_still_matches(
+        self, mock_get_executor_names, mock_import_executor_cls
+    ):
+        """The key must agree with the scheduler side even when the executor 
has no alias at all."""
+        names = [
+            ExecutorName(
+                
module_path=f"{_HealthCheckingExecutor.__module__}.{_HealthCheckingExecutor.__qualname__}",
+            )
+        ]
+        mock_get_executor_names.return_value = names
+        mock_import_executor_cls.return_value = (_HealthCheckingExecutor, None)
+
+        assert _configured_executor_keys() == {repr(names[0])}
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names", 
side_effect=Exception)
+    def test_survives_unreadable_executor_config(self, 
mock_get_executor_names):
+        assert _configured_executor_keys() == set()
+
+
+class TestExecutorInstanceHealthDisplayName:
+    """The API reports the name the user configured, not the internal 
executor_class key."""
+
+    @pytest.fixture(autouse=True)
+    def clear_executor_caches(self):
+        _configured_executor_names.cache_clear()
+        yield
+        _configured_executor_names.cache_clear()
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names")
+    def test_reports_alias_when_the_key_matches_a_configured_executor(self, 
mock_get_executor_names):
+        name = ExecutorName(
+            
module_path=f"{_HealthCheckingExecutor.__module__}.{_HealthCheckingExecutor.__qualname__}",
+            alias="Celery",
+        )
+        job = _executor(alive=True, executor=repr(name))
+        mock_get_executor_names.return_value = [name]
+
+        instance = _executor_instance_health(job)
+
+        assert instance["executor"] == "Celery"
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names")
+    def 
test_reports_module_path_when_the_configured_executor_has_no_alias(self, 
mock_get_executor_names):
+        name = ExecutorName(
+            
module_path=f"{_HealthCheckingExecutor.__module__}.{_HealthCheckingExecutor.__qualname__}",
+        )
+        job = _executor(alive=True, executor=repr(name))
+        mock_get_executor_names.return_value = [name]
+
+        instance = _executor_instance_health(job)
+
+        assert instance["executor"] == name.module_path
+
+    
@patch("airflow.api.common.airflow_health.ExecutorLoader.get_executor_names", 
return_value=[])
+    def test_falls_back_to_the_raw_key_when_nothing_configured_matches(self, 
mock_get_executor_names):
+        job = _executor(alive=True, executor=":SomeExecutor:")
+
+        instance = _executor_instance_health(job)
+
+        assert instance["executor"] == ":SomeExecutor:"
+
+
+class TestExecutorDetailedStatus:
+    """Each executor that implements a health check is its own coverage 
partition."""
+
+    @pytest.mark.parametrize(
+        ("keys", "jobs", "expected"),
+        [
+            pytest.param(
+                {"Celery"},
+                [_executor(alive=True, executor="Celery")],
+                DetailedHealthStatus.HEALTHY,
+                id="the_only_configured_executor_is_alive",
+            ),
+            pytest.param(
+                {"Celery", "Kubernetes"},
+                [_executor(alive=True, executor="Celery"), 
_executor(alive=True, executor="Kubernetes")],
+                DetailedHealthStatus.HEALTHY,
+                id="both_configured_executors_are_alive",
+            ),
+            pytest.param(
+                {"Celery", "Kubernetes"},
+                [_executor(alive=True, executor="Celery")],
+                DetailedHealthStatus.DEGRADED,
+                id="one_of_two_configured_executors_reports",
+            ),
+            pytest.param(
+                {"Celery"},
+                [_executor(alive=False, executor="Celery")],
+                DetailedHealthStatus.DOWN,
+                id="the_only_configured_executor_is_stale",
+            ),
+            pytest.param({"Celery"}, [], DetailedHealthStatus.DOWN, 
id="no_executor_has_ever_reported"),
+            pytest.param(
+                {"Celery"},
+                [_executor(alive=True, healthy=None, executor="Celery")],
+                DetailedHealthStatus.HEALTHY,
+                id="first_check_still_pending_is_not_a_coverage_gap",
+            ),
+        ],
+    )
+    def test_coverage(self, keys, jobs, expected):
+        with _patch_executor_keys(keys), _patch_live_scheduler_ids({1}):

Review Comment:
   converted those too.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to