shahar1 commented on code in PR #77032:
URL: https://github.com/apache/airflow/pull/77032#discussion_r4639478759


##########
devel-common/src/tests_common/test_utils/integration_setup.py:
##########
@@ -16,28 +16,46 @@
 # under the License.
 from __future__ import annotations
 
+import contextlib
+import datetime
 import json
 import logging
 import os
+import signal
+import socket
 import subprocess
+import threading
 import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from typing import Any
 
-from sqlalchemy import func, select
+from sqlalchemy import func, select, tuple_, update
 
-from airflow.dag_processing.bundles.manager import DagBundlesManager
+from airflow.dag_processing.bundles.manager import DagBundlesManager, 
_load_bundle_config_snapshot
 from airflow.dag_processing.dagbag import DagBag
-from airflow.models import DagRun
+from airflow.models import DagModel, DagRun, TaskInstance
 from airflow.models.serialized_dag import SerializedDagModel
 from airflow.sdk import timezone
 from airflow.serialization.definitions.dag import SerializedDAG
 from airflow.utils.session import create_session
-from airflow.utils.state import State
+from airflow.utils.state import DagRunState, State
+from airflow.utils.types import DagRunTriggeredByType, DagRunType
 
 from tests_common.test_utils.dag import create_scheduler_dag
 from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS, 
AIRFLOW_V_3_1_PLUS
 
 log = logging.getLogger(__name__)
 
+# breeze service names and ports.
+OTEL_COLLECTOR_HOST = "breeze-otel-collector"
+OTEL_COLLECTOR_PORT = 4318
+POSTGRES_CONTAINER = "breeze-postgres-1"

Review Comment:
   **major** — this name is correct only for breeze in the main checkout. In a 
linked git worktree, breeze isolates Docker resources by default, so the 
Compose project is `breeze-<worktree>-<hash>` (`get_default_project_name()` in 
`dev/breeze/src/airflow_breeze/utils/path_utils.py`) and the Postgres container 
is `<project>-postgres-1`. `run_psql` reaches the worktree's own `postgres` 
service, but `docker restart breeze-postgres-1` restarts the main checkout's 
database if it is up, or fails otherwise. The fallback message then asks the 
user to restart that same wrong container.
   
   Could you resolve the container from the current Compose project instead of 
hard-coding it? For example, pass the project name into the container, or look 
it up by the `com.docker.compose.service=postgres` label within the current 
project. Agents mostly run in worktrees, so the skill's main audience would hit 
this first.



##########
airflow-core/tests/integration/performance/test_perf.py:
##########
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Performance integration test for the scheduler.
+Creates a backlog of dag runs, starts the Airflow processes and measures how 
long they take to
+finish it, once per test case of the code under test. See README.md.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+import time
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from airflow.configuration import conf
+from airflow.utils.state import State
+
+from tests_common.test_utils.integration_setup import (
+    OTEL_COLLECTOR_HOST,
+    OTEL_COLLECTOR_PORT,
+    ProgressReporter,
+    compute_usable_cpu_count,
+    create_queued_dag_runs,
+    ensure_postgres_max_connections,
+    export_setting,
+    print_to_terminal,
+    serialize_and_get_dags,
+    start_process,
+    start_processes,
+    terminate_processes,
+    wait_for_dag_runs,
+    wait_for_otel_collector,
+)
+
+if TYPE_CHECKING:
+    from airflow.serialization.definitions.dag import SerializedDAG
+
+log = logging.getLogger("integration.perf.test_perf")
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+DAG_FOLDER = os.path.join(TEST_DIR, "dags")
+POLL_USAGE_SCRIPT = os.path.join(TEST_DIR, "poll_usage.py")
+
+# "true" or "false" per test case, read by the code under test.
+NEW_CODE_ENV_VAR = "PERF_TEST__NEW_CODE"
+
+# Per Airflow process; Airflow's defaults times the API server workers exceed 
Postgres's limit.
+DB_POOL_SIZE = 3
+DB_MAX_OVERFLOW = 5
+# Per Celery child, which runs one task at a time.
+RESULT_BACKEND_POOL_SIZE = 1
+RESULT_BACKEND_MAX_OVERFLOW = 1
+# psql, the test process and the usage poller.
+DB_CONNECTION_HEADROOM = 20
+
+CPUS = compute_usable_cpu_count()
+
+# The first three are process counts; the rest are Airflow settings, None 
keeps Airflow's default.
+PROFILES: dict[str, dict[str, Any]] = {
+    # Sized so the scheduler is the only constraint: every other cap is far 
above what the workers
+    # can execute.
+    "dynamic": {
+        # More slots per worker rather than more workers: each worker process 
costs connections.
+        "worker_count": 3,
+        "worker_concurrency": max(2, CPUS * 2 // 3),
+        # One worker saturates well before the scheduler does.
+        "api_server_workers": max(4, CPUS // 3),
+        "parallelism": 4096,
+        "default_pool_task_slot_count": 4096,
+        "max_active_tasks_per_dag": 4096,
+        # Many runs of one Dag; the default of 16 would throttle them.
+        "max_active_runs_per_dag": 4096,
+        "max_tis_per_query": None,
+    },
+    # Fill in as needed; the process counts start at the tools' defaults.
+    "custom": {
+        "worker_count": 1,
+        "worker_concurrency": CPUS,
+        "api_server_workers": conf.getint("api", "workers"),
+        "parallelism": None,
+        "default_pool_task_slot_count": None,
+        "max_active_tasks_per_dag": None,
+        "max_active_runs_per_dag": None,
+        "max_tis_per_query": None,
+    },
+}
+
+
+def compute_required_db_connections(sizing: dict[str, Any]) -> int:
+    """Peak connections the deployment can open against the metadata DB."""
+    # Tasks use the Execution API and never connect.
+    scheduler = 1
+    airflow_processes = scheduler + sizing["api_server_workers"] + 
sizing["worker_count"]
+    per_airflow_process = DB_POOL_SIZE + DB_MAX_OVERFLOW
+    # One result-backend pool per Celery child.
+    celery_children = sizing["worker_count"] * sizing["worker_concurrency"]
+    per_celery_child = RESULT_BACKEND_POOL_SIZE + RESULT_BACKEND_MAX_OVERFLOW
+    return (
+        airflow_processes * per_airflow_process + celery_children * 
per_celery_child + DB_CONNECTION_HEADROOM
+    )
+
+
[email protected]("otel")
[email protected]("redis")
[email protected]("postgres")
+class TestPerformance:
+    """One environment per class, one deployment per test case."""
+
+    profile = os.environ.get("PERF_TEST__PROFILE", "dynamic")  # key of 
PROFILES; sizes the deployment
+    sizing: dict[str, Any]
+    dags: dict[str, SerializedDAG] = {}
+
+    @classmethod
+    def setup_class(cls):
+        if cls.profile not in PROFILES:
+            pytest.fail(f"profile {cls.profile!r} is not one of 
{sorted(PROFILES)}")
+        cls.sizing = PROFILES[cls.profile]
+        log.info("profile %s: %s", cls.profile, cls.sizing)
+
+        wait_for_otel_collector(OTEL_COLLECTOR_HOST, OTEL_COLLECTOR_PORT)
+        # Exceeding the limit kills the scheduler mid-run.
+        
ensure_postgres_max_connections(compute_required_db_connections(cls.sizing))
+
+        # The pytest plugin strips AIRFLOW__* variables; scheduler and API 
server must share the secret.
+        os.environ["AIRFLOW__API_AUTH__JWT_SECRET"] = 
"test-secret-key-for-testing"
+        os.environ["AIRFLOW__API_AUTH__JWT_ISSUER"] = "airflow"
+
+        os.environ["AIRFLOW__CORE__EXECUTOR"] = "CeleryExecutor"
+        os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = DAG_FOLDER
+        os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False"
+        os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = "/dev/null"
+        os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "False"
+
+        os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE"] = 
str(DB_POOL_SIZE)
+        os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW"] = 
str(DB_MAX_OVERFLOW)
+        
os.environ["AIRFLOW__CELERY__RESULT_BACKEND_SQLALCHEMY_ENGINE_OPTIONS"] = 
json.dumps(
+            {"pool_size": RESULT_BACKEND_POOL_SIZE, "max_overflow": 
RESULT_BACKEND_MAX_OVERFLOW}
+        )
+        # The default spawns a ProcessPoolExecutor per publish batch, on the 
scheduler's heartbeat.
+        os.environ["AIRFLOW__CELERY__SYNC_PARALLELISM"] = "1"
+        # The default of one second fails publishes behind a busy broker.
+        os.environ["AIRFLOW__CELERY__OPERATION_TIMEOUT"] = "300"
+
+        # Before db migrate, which creates the default pool with this slot 
count.
+        export_setting("AIRFLOW__CORE__PARALLELISM", cls.sizing["parallelism"])
+        export_setting(
+            "AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT", 
cls.sizing["default_pool_task_slot_count"]
+        )
+        export_setting("AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG", 
cls.sizing["max_active_tasks_per_dag"])
+        export_setting("AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG", 
cls.sizing["max_active_runs_per_dag"])
+        export_setting("AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY", 
cls.sizing["max_tis_per_query"])
+
+        os.environ["AIRFLOW__METRICS__OTEL_ON"] = "True"
+        os.environ["AIRFLOW__METRICS__OTEL_HOST"] = OTEL_COLLECTOR_HOST
+        os.environ["AIRFLOW__METRICS__OTEL_PORT"] = str(OTEL_COLLECTOR_PORT)
+        # breeze's OTEL_METRIC_EXPORT_INTERVAL=30000 overrides the Airflow 
option; Prometheus still
+        # scrapes every 30s.
+        os.environ["AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS"] = "1000"
+        os.environ["OTEL_METRIC_EXPORT_INTERVAL"] = "1000"
+        # The dashboard reads the tagged names; legacy names add one series 
family per task.
+        os.environ["AIRFLOW__METRICS__LEGACY_NAMES_ON"] = "False"
+        os.environ["AIRFLOW__TRACES__OTEL_ON"] = "True"
+        os.environ["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf"
+        os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = (
+            f"http://{OTEL_COLLECTOR_HOST}:{OTEL_COLLECTOR_PORT}/v1/traces";
+        )
+
+        # Once per class: the scheduler ignores finished runs, and run ids 
carry the test case.
+        subprocess.run(["airflow", "db", "reset", "--yes"], check=True, 
env=os.environ.copy())
+        subprocess.run(["airflow", "db", "migrate"], check=True, 
env=os.environ.copy())
+
+        # Every Dag in the folder; the test picks by id.
+        cls.dags = serialize_and_get_dags(dag_folder=DAG_FOLDER)
+        assert cls.dags, f"no Dags found in {DAG_FOLDER}"
+
+    def start_usage_poller(self, processes: dict[str, subprocess.Popen]) -> 
subprocess.Popen:
+        """Start poll_usage.py, which pushes CPU, memory and worker slot 
gauges for ``processes``."""
+        slot_budget = self.sizing["worker_count"] * 
self.sizing["worker_concurrency"]
+        command = [sys.executable, POLL_USAGE_SCRIPT, "--slot-budget", 
str(slot_budget)]
+        for role, process in processes.items():
+            command.append(f"{role}={process.pid}")
+        # Own process group, or terminate_processes would kill pytest with it.
+        return start_process(command)
+
+    def execute_dag_runs(
+        self,
+        *,
+        run_id_prefix: str,
+        dag_ids: list[str],
+        dag_run_count: int,
+        run_conf: dict[str, Any],
+        timeout_seconds: int,
+    ) -> None:
+        """Create the dag runs, start the processes, wait for the runs to 
finish, stop the processes."""
+        dags = {dag_id: self.dags[dag_id] for dag_id in dag_ids}
+        # Created before any process exists, so every test case starts from 
the same rows, all
+        # eligible at once, and CLI triggering cannot cap the live runs at its 
own rate. This is the
+        # scheduler under full load, not a production trickle.
+        runs = create_queued_dag_runs(
+            dags, run_id_prefix=run_id_prefix, dag_run_count=dag_run_count, 
run_conf=run_conf
+        )
+        processes = start_processes(
+            worker_count=self.sizing["worker_count"],
+            worker_concurrency=self.sizing["worker_concurrency"],
+            api_server_workers=self.sizing["api_server_workers"],
+        )
+        started_at = time.monotonic()

Review Comment:
   **minor** — by the time this line runs, `start_processes()` has already 
started the scheduler and slept 5 s, with every queued run eligible, so the 
scheduling and execution done in that window are not counted. The offset is 
about the same for both cases, but it skews small loads and the absolute 
number. Taking the start time when the scheduler starts (for example, have 
`start_processes` record it) would count the whole run.



##########
.agents/skills/airflow-performance-harness/SKILL.md:
##########
@@ -0,0 +1,67 @@
+---
+name: airflow-performance-harness
+description: Measure a scheduler or executor-path change under load with the 
performance harness in airflow-core/tests/integration/performance. Use when 
asked to benchmark, load test or compare the performance of a change to the 
scheduler, the executors or task scheduling. Covers putting the change behind 
the flag, shaping the load, running the harness in breeze, reading its 
dashboard, and dropping the scratch changes afterwards.
+---
+
+<!-- SPDX-License-Identifier: Apache-2.0
+     https://www.apache.org/licenses/LICENSE-2.0 -->
+
+# Measure a change with the performance harness
+
+The harness in 
[`airflow-core/tests/integration/performance`](../../../airflow-core/tests/integration/performance/README.md)
+is a scratch environment, not a test suite. It runs the Airflow processes 
inside breeze against a
+backlog of dag runs, once with the original code and once with the change, and 
the comparison is

Review Comment:
   **nit** — AGENTS.md, Naming:
   
   > Write **Dag** (title case) in all prose.
   
   So "Dag runs" here, and in the `test_perf.py` docstrings at lines 19, 214 
and 259.



##########
devel-common/src/tests_common/test_utils/integration_setup.py:
##########
@@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str, 
SerializedDAG]:
     ]
 
     os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] = 
json.dumps(TESTING_BUNDLE_CONFIG)
-    # Initial add
+    # The manager caches the bundle config per process, and the pytest plugin 
already loaded it with
+    # the default list; without this the sync below deactivates the "testing" 
bundle just added, and
+    # the scheduler marks its Dags stale a minute later.
+    _load_bundle_config_snapshot.cache_clear()
     manager = DagBundlesManager()
     manager.sync_bundles_to_db()
 
     return dag_dict
 
 
-def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int):
-    # max_wait_time, is the timeout for the DAG run to complete. The value is 
in seconds.
-    start_time = timezone.utcnow().timestamp()
-
-    while timezone.utcnow().timestamp() - start_time < max_wait_time:
+def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) -> 
dict[tuple[str, str], str | None]:
+    """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or 
``max_wait_time`` elapses; return the states."""
+    # One budget for all runs, not one per run.
+    deadline = time.monotonic() + max_wait_time
+    states: dict[tuple[str, str], str | None] = dict.fromkeys(runs)
+    while True:
         with create_session() as session:
-            dag_run = session.scalar(
-                select(DagRun).where(
-                    DagRun.dag_id == dag_id,
-                    DagRun.run_id == run_id,
+            rows = session.execute(
+                select(DagRun.dag_id, DagRun.run_id, DagRun.state).where(
+                    tuple_(DagRun.dag_id, DagRun.run_id).in_(runs)
                 )
-            )
+            ).all()
+        for dag_id, run_id, state in rows:
+            states[(dag_id, run_id)] = state
+        if all(state in (State.SUCCESS, State.FAILED) for state in 
states.values()):
+            break
+        if time.monotonic() >= deadline:
+            break
+        # Otherwise the loop spins and loads the database under test.
+        time.sleep(2)
+    return states
+
+
+def terminate_process(proc: subprocess.Popen, timeout: int = 30, *, 
process_group: bool = False) -> None:
+    """Stop ``proc``, with SIGKILL as the fallback if it is still alive after 
``timeout``."""
+
+    def send(sig: signal.Signals) -> None:
+        # Children (uvicorn workers, prefork pool) would be orphaned if only 
the parent was signalled.
+        if process_group:
+            with contextlib.suppress(ProcessLookupError):
+                os.killpg(os.getpgid(proc.pid), sig)
+        else:
+            proc.send_signal(sig)
+
+    send(signal.SIGTERM)
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        send(signal.SIGKILL)
+        proc.wait()
+
+
+def print_to_terminal(text: str) -> None:
+    """Write ``text`` as plain text to the controlling terminal, or to stdout 
when there is none."""
+    # Plain text past pytest's capture, distinct from the processes' logging.
+    try:
+        with open("/dev/tty", "w") as tty:
+            tty.write(text + "\n")
+    except OSError:
+        # No terminal: stdout, shown live by pytest -s.
+        print(text, flush=True)
+
+
+def compute_usable_cpu_count() -> int:
+    """CPUs this process may actually run on."""
+    # Honours a cpuset, which os.cpu_count() does not. Linux only.
+    if hasattr(os, "sched_getaffinity"):
+        return len(os.sched_getaffinity(0))
+    # The whole host.
+    return os.cpu_count() or 8
+
+
+def export_setting(env_var: str, value: int | None) -> None:
+    """Export an Airflow setting for the subprocesses; ``None`` keeps 
Airflow's default."""
+    if value is not None:
+        os.environ[env_var] = str(value)
+
+
+def run_psql(sql: str) -> str:
+    """Run ``sql`` against the breeze Postgres as its superuser and return the 
output."""
+    result = subprocess.run(
+        ["psql", "-h", "postgres", "-U", "postgres", "-d", "postgres", "-tAc", 
sql],
+        env={**os.environ, "PGPASSWORD": "airflow"},
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout.strip()
+
+
+def restart_postgres_container() -> bool:
+    """Restart the breeze Postgres container; False when that is not possible 
from here."""
+    # breeze mounts the Docker CLI and socket; FileNotFoundError means another 
environment.
+    try:
+        restart = subprocess.run(["docker", "restart", POSTGRES_CONTAINER], 
capture_output=True, check=False)
+    except FileNotFoundError:
+        return False
+    return restart.returncode == 0
 
-            if dag_run is None:
-                time.sleep(5)
-                continue
 
-            dag_run_state = dag_run.state
-            log.debug("DAG Run state: %s.", dag_run_state)
+def ensure_postgres_max_connections(required: int) -> None:
+    """Raise the metadata DB's ``max_connections`` to ``required`` if it is 
lower, restarting it once."""
+    try:
+        current = int(run_psql("show max_connections"))
+    except FileNotFoundError:
+        raise RuntimeError("psql not found: this helper runs inside the breeze 
environment")
+    if current >= required:
+        return
+
+    # Persisted in the data volume; takes effect at server start, hence the 
restart.
+    run_psql(f"ALTER SYSTEM SET max_connections = {required}")
+    log.info(
+        "Raised postgres max_connections from %s to %s; restarting %s", 
current, required, POSTGRES_CONTAINER
+    )
+    if restart_postgres_container():
+        deadline = time.monotonic() + 60
+        while time.monotonic() < deadline:
+            with contextlib.suppress(subprocess.CalledProcessError):
+                if int(run_psql("show max_connections")) >= required:
+                    return
+            time.sleep(2)
+    raise RuntimeError(
+        f"postgres max_connections={current} is below the {required} needed. 
The new value is "
+        "persisted and applies after a restart, which could not be done from 
here: run "
+        f"`docker restart {POSTGRES_CONTAINER}` on the host, then re-run the 
test."
+    )
 
-            if dag_run_state in [State.SUCCESS, State.FAILED]:
-                break
-    return dag_run_state
 
+def start_process(command: list[str]) -> subprocess.Popen:
+    """Start ``command`` with the current environment, as the leader of its 
own process group."""
+    # Children (uvicorn workers, prefork pool) are signalled with the parent.
+    return subprocess.Popen(command, env=os.environ.copy(), 
start_new_session=True)
+
+
+def wait_for_api_server(process: subprocess.Popen, port: int = 
API_SERVER_PORT, timeout: int = 120) -> None:
+    """Block until the API server answers its execution health route, or 
``timeout`` passes."""
+    # uvicorn accepts connections before its workers can answer; a task 
starting in that gap times
+    # out and fails.
+    health_url = f"http://localhost:{port}/execution/health";
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError(f"api_server exited early with code 
{process.returncode}")
+        with contextlib.suppress(urllib.error.URLError, TimeoutError, 
ConnectionError):
+            with urllib.request.urlopen(health_url, timeout=5) as response:
+                if response.status == 200:
+                    return
+        time.sleep(1)
+    raise RuntimeError(f"api_server did not answer {health_url} within 
{timeout}s")
+
+
+def api_server_command(*, port: int, workers: int) -> list[str]:

Review Comment:
   **nit** — AGENTS.md, Coding Standards:
   
   > Name functions and methods with action verbs: `get_`, `extract_`, `find_`, 
`compute_`, `build_`, etc. Avoid noun-only names like `_serialize_keys` or 
`_base_names` — they read as attributes, not callables. Predicates (`is_`, 
`has_`) are the one exception.
   
   This applies to `api_server_command` and `celery_worker_command` here, and 
to `gauge` and `data_point` in `poll_usage.py`. For example: 
`build_api_server_command`, `build_celery_worker_command`, `build_gauge`, 
`build_data_point`.



##########
devel-common/src/tests_common/test_utils/integration_setup.py:
##########
@@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str, 
SerializedDAG]:
     ]
 
     os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] = 
json.dumps(TESTING_BUNDLE_CONFIG)
-    # Initial add
+    # The manager caches the bundle config per process, and the pytest plugin 
already loaded it with
+    # the default list; without this the sync below deactivates the "testing" 
bundle just added, and
+    # the scheduler marks its Dags stale a minute later.
+    _load_bundle_config_snapshot.cache_clear()
     manager = DagBundlesManager()
     manager.sync_bundles_to_db()
 
     return dag_dict
 
 
-def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int):
-    # max_wait_time, is the timeout for the DAG run to complete. The value is 
in seconds.
-    start_time = timezone.utcnow().timestamp()
-
-    while timezone.utcnow().timestamp() - start_time < max_wait_time:
+def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) -> 
dict[tuple[str, str], str | None]:
+    """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or 
``max_wait_time`` elapses; return the states."""
+    # One budget for all runs, not one per run.
+    deadline = time.monotonic() + max_wait_time
+    states: dict[tuple[str, str], str | None] = dict.fromkeys(runs)
+    while True:
         with create_session() as session:
-            dag_run = session.scalar(
-                select(DagRun).where(
-                    DagRun.dag_id == dag_id,
-                    DagRun.run_id == run_id,
+            rows = session.execute(
+                select(DagRun.dag_id, DagRun.run_id, DagRun.state).where(
+                    tuple_(DagRun.dag_id, DagRun.run_id).in_(runs)
                 )
-            )
+            ).all()
+        for dag_id, run_id, state in rows:
+            states[(dag_id, run_id)] = state
+        if all(state in (State.SUCCESS, State.FAILED) for state in 
states.values()):
+            break
+        if time.monotonic() >= deadline:
+            break
+        # Otherwise the loop spins and loads the database under test.
+        time.sleep(2)
+    return states
+
+
+def terminate_process(proc: subprocess.Popen, timeout: int = 30, *, 
process_group: bool = False) -> None:
+    """Stop ``proc``, with SIGKILL as the fallback if it is still alive after 
``timeout``."""
+
+    def send(sig: signal.Signals) -> None:
+        # Children (uvicorn workers, prefork pool) would be orphaned if only 
the parent was signalled.
+        if process_group:
+            with contextlib.suppress(ProcessLookupError):
+                os.killpg(os.getpgid(proc.pid), sig)

Review Comment:
   **minor** — `start_new_session=True` makes the pgid equal to the pid, so the 
group can be signalled by `proc.pid` directly. `os.getpgid(proc.pid)` raises 
`ProcessLookupError` once `poll()` has reaped the leader, and both 
`start_processes` and `execute_dag_runs` call `poll()` before cleanup. When 
that happens, surviving group members, such as uvicorn workers or prefork 
children, are never signalled. I checked this with a leader that exits and 
leaves a child behind: `getpgid` raised and the child stayed alive, while 
`os.killpg(proc.pid, SIGTERM)` stopped it.
   
   ```suggestion
                   os.killpg(proc.pid, sig)
   ```
   
   Related: `proc.wait(timeout=...)` only waits for the leader. If the leader 
exits on SIGTERM while its children keep running, SIGKILL is never sent. 
Checking the group (for example `os.killpg(proc.pid, 0)` until it raises) 
before skipping the SIGKILL would close that gap.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to