JoelEmmanuelCloud commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4647367153
##########
airflow-core/src/airflow/api/common/airflow_health.py:
##########
@@ -250,6 +372,28 @@ def get_airflow_health() -> dict[str, Any]:
dag_processor_status = HealthStatus.UNHEALTHY
dag_processor_detailed_status = DetailedHealthStatus.DOWN
+ # Omitted entirely, not just reported unhealthy, when no configured
executor implements a health
+ # check: a reporting mechanism nothing has opted into isn't a gap in
coverage, and showing it as
+ # "unhealthy" on every deployment that hasn't adopted it yet would be a
false alarm, not a signal.
+ report_executor_health = bool(_configured_executor_keys())
Review Comment:
good catch. _configured_executor_keys() now returns empty too when the
interval is 0, so the executor block is omitted the same way it is when nothing
implements get_health().
##########
airflow-core/src/airflow/config_templates/config.yml:
##########
@@ -3020,6 +3020,18 @@ scheduler:
type: float
example: ~
default: "120.0"
+ executor_health_check_interval:
+ description: |
+ How often, in seconds, the scheduler checks each configured executor's
health via
+ ``get_health()``, reported on ``/monitor/health``. Each check runs on
its own background
+ thread so a slow or unresponsive executor backend cannot delay task
scheduling; this setting
+ also bounds how long a single check is allowed to take, since each
call is given half this
Review Comment:
added that.
##########
airflow-core/tests/unit/jobs/test_scheduler_job_executor_health.py:
##########
@@ -0,0 +1,297 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import time
+from concurrent.futures import Future
+
+import pytest
+from sqlalchemy import select
+
+from airflow.executors.base_executor import BaseExecutor
+from airflow.executors.executor_utils import ExecutorName
+from airflow.jobs.job import EXECUTOR_JOB_TYPE, HealthCheckResult, Job,
JobState
+from airflow.jobs.scheduler_job_runner import SchedulerJobRunner,
_executor_health_key
+
+from tests_common.test_utils.config import conf_vars
+from tests_common.test_utils.db import clear_db_jobs
+
+pytestmark = pytest.mark.db_test
+
+
+class _StubExecutor(BaseExecutor):
+ mock_module_path = "stub.executor.path"
+ mock_alias = "stub_executor"
+
+ def __init__(self, result=True, team_name=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.name = ExecutorName(module_path=self.mock_module_path,
alias=self.mock_alias)
+ self.team_name = team_name
+ self.result = result
+ self.calls: list[float] = []
+
+ def get_health(self, timeout: float) -> bool | None:
+ self.calls.append(timeout)
+ if isinstance(self.result, Exception):
+ raise self.result
+ return self.result
+
+
[email protected](autouse=True)
+def clean_jobs():
+ clear_db_jobs()
+ yield
+ clear_db_jobs()
+
+
+def _make_runner(executor, session):
+ scheduler_job = Job()
+ session.add(scheduler_job)
+ session.commit()
+ runner = SchedulerJobRunner(job=scheduler_job, executors=[executor])
+ return runner
+
+
+def _get_executor_job(session, owner_job_id, executor_key):
+ return session.scalar(
+ select(Job).where(
+ Job.job_type == EXECUTOR_JOB_TYPE,
+ Job.owner_job_id == owner_job_id,
+ Job.executor_class == executor_key,
+ )
+ )
+
+
+def
test_touch_executor_heartbeats_creates_a_row_for_a_reporting_executor(session):
+ executor = _StubExecutor()
+ runner = _make_runner(executor, session)
+ key = _executor_health_key(executor)
+
+ runner._touch_executor_heartbeats(session=session)
+
+ job = _get_executor_job(session, runner.job.id, key)
+ assert job is not None
+ assert job.health_check_result == HealthCheckResult.UNKNOWN
+ assert job.latest_heartbeat is not None
+ assert job.hostname != key
+
+
+def
test_touch_executor_heartbeats_skips_executors_that_do_not_override_get_health(session):
+ executor = BaseExecutor()
+ executor.name =
ExecutorName(module_path="airflow.executors.local_executor.LocalExecutor",
alias="Local")
+ runner = _make_runner(executor, session)
+
+ runner._touch_executor_heartbeats(session=session)
+
+ assert session.scalar(select(Job).where(Job.job_type ==
EXECUTOR_JOB_TYPE)) is None
+
+
+def
test_touch_executor_heartbeats_moves_the_heartbeat_without_changing_the_result(session):
+ executor = _StubExecutor()
+ runner = _make_runner(executor, session)
+ key = _executor_health_key(executor)
+
+ runner._touch_executor_heartbeats(session=session)
+ first = _get_executor_job(session, runner.job.id, key)
+ first.health_check_result = HealthCheckResult.HEALTHY
+ first_heartbeat = first.latest_heartbeat
+ session.commit()
+
+ runner._touch_executor_heartbeats(session=session)
+ second = _get_executor_job(session, runner.job.id, key)
+
+ assert second.id == first.id
+ assert second.health_check_result == HealthCheckResult.HEALTHY
+ assert second.latest_heartbeat >= first_heartbeat
+
+
+def test_run_executor_health_check_records_healthy(session):
Review Comment:
merged them into one parametrized test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]