shahar1 commented on code in PR #77032: URL: https://github.com/apache/airflow/pull/77032#discussion_r4639478759
########## devel-common/src/tests_common/test_utils/integration_setup.py: ########## @@ -16,28 +16,46 @@ # under the License. from __future__ import annotations +import contextlib +import datetime import json import logging import os +import signal +import socket import subprocess +import threading import time +import urllib.error +import urllib.request +from collections.abc import Sequence +from typing import Any -from sqlalchemy import func, select +from sqlalchemy import func, select, tuple_, update -from airflow.dag_processing.bundles.manager import DagBundlesManager +from airflow.dag_processing.bundles.manager import DagBundlesManager, _load_bundle_config_snapshot from airflow.dag_processing.dagbag import DagBag -from airflow.models import DagRun +from airflow.models import DagModel, DagRun, TaskInstance from airflow.models.serialized_dag import SerializedDagModel from airflow.sdk import timezone from airflow.serialization.definitions.dag import SerializedDAG from airflow.utils.session import create_session -from airflow.utils.state import State +from airflow.utils.state import DagRunState, State +from airflow.utils.types import DagRunTriggeredByType, DagRunType from tests_common.test_utils.dag import create_scheduler_dag from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_1_PLUS log = logging.getLogger(__name__) +# breeze service names and ports. +OTEL_COLLECTOR_HOST = "breeze-otel-collector" +OTEL_COLLECTOR_PORT = 4318 +POSTGRES_CONTAINER = "breeze-postgres-1" Review Comment: **major** — this name is correct only for breeze in the main checkout. In a linked git worktree, breeze isolates Docker resources by default, so the Compose project is `breeze-<worktree>-<hash>` (`get_default_project_name()` in `dev/breeze/src/airflow_breeze/utils/path_utils.py`) and the Postgres container is `<project>-postgres-1`. `run_psql` reaches the worktree's own `postgres` service, but `docker restart breeze-postgres-1` restarts the main checkout's database if it is up, or fails otherwise. The fallback message then asks the user to restart that same wrong container. Could you resolve the container from the current Compose project instead of hard-coding it? For example, pass the project name into the container, or look it up by the `com.docker.compose.service=postgres` label within the current project. Agents mostly run in worktrees, so the skill's main audience would hit this first. ########## airflow-core/tests/integration/performance/test_perf.py: ########## @@ -0,0 +1,281 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Performance integration test for the scheduler. +Creates a backlog of dag runs, starts the Airflow processes and measures how long they take to +finish it, once per test case of the code under test. See README.md. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +import sys +import time +from typing import TYPE_CHECKING, Any + +import pytest + +from airflow.configuration import conf +from airflow.utils.state import State + +from tests_common.test_utils.integration_setup import ( + OTEL_COLLECTOR_HOST, + OTEL_COLLECTOR_PORT, + ProgressReporter, + compute_usable_cpu_count, + create_queued_dag_runs, + ensure_postgres_max_connections, + export_setting, + print_to_terminal, + serialize_and_get_dags, + start_process, + start_processes, + terminate_processes, + wait_for_dag_runs, + wait_for_otel_collector, +) + +if TYPE_CHECKING: + from airflow.serialization.definitions.dag import SerializedDAG + +log = logging.getLogger("integration.perf.test_perf") + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +DAG_FOLDER = os.path.join(TEST_DIR, "dags") +POLL_USAGE_SCRIPT = os.path.join(TEST_DIR, "poll_usage.py") + +# "true" or "false" per test case, read by the code under test. +NEW_CODE_ENV_VAR = "PERF_TEST__NEW_CODE" + +# Per Airflow process; Airflow's defaults times the API server workers exceed Postgres's limit. +DB_POOL_SIZE = 3 +DB_MAX_OVERFLOW = 5 +# Per Celery child, which runs one task at a time. +RESULT_BACKEND_POOL_SIZE = 1 +RESULT_BACKEND_MAX_OVERFLOW = 1 +# psql, the test process and the usage poller. +DB_CONNECTION_HEADROOM = 20 + +CPUS = compute_usable_cpu_count() + +# The first three are process counts; the rest are Airflow settings, None keeps Airflow's default. +PROFILES: dict[str, dict[str, Any]] = { + # Sized so the scheduler is the only constraint: every other cap is far above what the workers + # can execute. + "dynamic": { + # More slots per worker rather than more workers: each worker process costs connections. + "worker_count": 3, + "worker_concurrency": max(2, CPUS * 2 // 3), + # One worker saturates well before the scheduler does. + "api_server_workers": max(4, CPUS // 3), + "parallelism": 4096, + "default_pool_task_slot_count": 4096, + "max_active_tasks_per_dag": 4096, + # Many runs of one Dag; the default of 16 would throttle them. + "max_active_runs_per_dag": 4096, + "max_tis_per_query": None, + }, + # Fill in as needed; the process counts start at the tools' defaults. + "custom": { + "worker_count": 1, + "worker_concurrency": CPUS, + "api_server_workers": conf.getint("api", "workers"), + "parallelism": None, + "default_pool_task_slot_count": None, + "max_active_tasks_per_dag": None, + "max_active_runs_per_dag": None, + "max_tis_per_query": None, + }, +} + + +def compute_required_db_connections(sizing: dict[str, Any]) -> int: + """Peak connections the deployment can open against the metadata DB.""" + # Tasks use the Execution API and never connect. + scheduler = 1 + airflow_processes = scheduler + sizing["api_server_workers"] + sizing["worker_count"] + per_airflow_process = DB_POOL_SIZE + DB_MAX_OVERFLOW + # One result-backend pool per Celery child. + celery_children = sizing["worker_count"] * sizing["worker_concurrency"] + per_celery_child = RESULT_BACKEND_POOL_SIZE + RESULT_BACKEND_MAX_OVERFLOW + return ( + airflow_processes * per_airflow_process + celery_children * per_celery_child + DB_CONNECTION_HEADROOM + ) + + [email protected]("otel") [email protected]("redis") [email protected]("postgres") +class TestPerformance: + """One environment per class, one deployment per test case.""" + + profile = os.environ.get("PERF_TEST__PROFILE", "dynamic") # key of PROFILES; sizes the deployment + sizing: dict[str, Any] + dags: dict[str, SerializedDAG] = {} + + @classmethod + def setup_class(cls): + if cls.profile not in PROFILES: + pytest.fail(f"profile {cls.profile!r} is not one of {sorted(PROFILES)}") + cls.sizing = PROFILES[cls.profile] + log.info("profile %s: %s", cls.profile, cls.sizing) + + wait_for_otel_collector(OTEL_COLLECTOR_HOST, OTEL_COLLECTOR_PORT) + # Exceeding the limit kills the scheduler mid-run. + ensure_postgres_max_connections(compute_required_db_connections(cls.sizing)) + + # The pytest plugin strips AIRFLOW__* variables; scheduler and API server must share the secret. + os.environ["AIRFLOW__API_AUTH__JWT_SECRET"] = "test-secret-key-for-testing" + os.environ["AIRFLOW__API_AUTH__JWT_ISSUER"] = "airflow" + + os.environ["AIRFLOW__CORE__EXECUTOR"] = "CeleryExecutor" + os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = DAG_FOLDER + os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False" + os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = "/dev/null" + os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "False" + + os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE"] = str(DB_POOL_SIZE) + os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW"] = str(DB_MAX_OVERFLOW) + os.environ["AIRFLOW__CELERY__RESULT_BACKEND_SQLALCHEMY_ENGINE_OPTIONS"] = json.dumps( + {"pool_size": RESULT_BACKEND_POOL_SIZE, "max_overflow": RESULT_BACKEND_MAX_OVERFLOW} + ) + # The default spawns a ProcessPoolExecutor per publish batch, on the scheduler's heartbeat. + os.environ["AIRFLOW__CELERY__SYNC_PARALLELISM"] = "1" + # The default of one second fails publishes behind a busy broker. + os.environ["AIRFLOW__CELERY__OPERATION_TIMEOUT"] = "300" + + # Before db migrate, which creates the default pool with this slot count. + export_setting("AIRFLOW__CORE__PARALLELISM", cls.sizing["parallelism"]) + export_setting( + "AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT", cls.sizing["default_pool_task_slot_count"] + ) + export_setting("AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG", cls.sizing["max_active_tasks_per_dag"]) + export_setting("AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG", cls.sizing["max_active_runs_per_dag"]) + export_setting("AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY", cls.sizing["max_tis_per_query"]) + + os.environ["AIRFLOW__METRICS__OTEL_ON"] = "True" + os.environ["AIRFLOW__METRICS__OTEL_HOST"] = OTEL_COLLECTOR_HOST + os.environ["AIRFLOW__METRICS__OTEL_PORT"] = str(OTEL_COLLECTOR_PORT) + # breeze's OTEL_METRIC_EXPORT_INTERVAL=30000 overrides the Airflow option; Prometheus still + # scrapes every 30s. + os.environ["AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS"] = "1000" + os.environ["OTEL_METRIC_EXPORT_INTERVAL"] = "1000" + # The dashboard reads the tagged names; legacy names add one series family per task. + os.environ["AIRFLOW__METRICS__LEGACY_NAMES_ON"] = "False" + os.environ["AIRFLOW__TRACES__OTEL_ON"] = "True" + os.environ["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf" + os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = ( + f"http://{OTEL_COLLECTOR_HOST}:{OTEL_COLLECTOR_PORT}/v1/traces" + ) + + # Once per class: the scheduler ignores finished runs, and run ids carry the test case. + subprocess.run(["airflow", "db", "reset", "--yes"], check=True, env=os.environ.copy()) + subprocess.run(["airflow", "db", "migrate"], check=True, env=os.environ.copy()) + + # Every Dag in the folder; the test picks by id. + cls.dags = serialize_and_get_dags(dag_folder=DAG_FOLDER) + assert cls.dags, f"no Dags found in {DAG_FOLDER}" + + def start_usage_poller(self, processes: dict[str, subprocess.Popen]) -> subprocess.Popen: + """Start poll_usage.py, which pushes CPU, memory and worker slot gauges for ``processes``.""" + slot_budget = self.sizing["worker_count"] * self.sizing["worker_concurrency"] + command = [sys.executable, POLL_USAGE_SCRIPT, "--slot-budget", str(slot_budget)] + for role, process in processes.items(): + command.append(f"{role}={process.pid}") + # Own process group, or terminate_processes would kill pytest with it. + return start_process(command) + + def execute_dag_runs( + self, + *, + run_id_prefix: str, + dag_ids: list[str], + dag_run_count: int, + run_conf: dict[str, Any], + timeout_seconds: int, + ) -> None: + """Create the dag runs, start the processes, wait for the runs to finish, stop the processes.""" + dags = {dag_id: self.dags[dag_id] for dag_id in dag_ids} + # Created before any process exists, so every test case starts from the same rows, all + # eligible at once, and CLI triggering cannot cap the live runs at its own rate. This is the + # scheduler under full load, not a production trickle. + runs = create_queued_dag_runs( + dags, run_id_prefix=run_id_prefix, dag_run_count=dag_run_count, run_conf=run_conf + ) + processes = start_processes( + worker_count=self.sizing["worker_count"], + worker_concurrency=self.sizing["worker_concurrency"], + api_server_workers=self.sizing["api_server_workers"], + ) + started_at = time.monotonic() Review Comment: **minor** — by the time this line runs, `start_processes()` has already started the scheduler and slept 5 s, with every queued run eligible, so the scheduling and execution done in that window are not counted. The offset is about the same for both cases, but it skews small loads and the absolute number. Taking the start time when the scheduler starts (for example, have `start_processes` record it) would count the whole run. ########## .agents/skills/airflow-performance-harness/SKILL.md: ########## @@ -0,0 +1,67 @@ +--- +name: airflow-performance-harness +description: Measure a scheduler or executor-path change under load with the performance harness in airflow-core/tests/integration/performance. Use when asked to benchmark, load test or compare the performance of a change to the scheduler, the executors or task scheduling. Covers putting the change behind the flag, shaping the load, running the harness in breeze, reading its dashboard, and dropping the scratch changes afterwards. +--- + +<!-- SPDX-License-Identifier: Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0 --> + +# Measure a change with the performance harness + +The harness in [`airflow-core/tests/integration/performance`](../../../airflow-core/tests/integration/performance/README.md) +is a scratch environment, not a test suite. It runs the Airflow processes inside breeze against a +backlog of dag runs, once with the original code and once with the change, and the comparison is Review Comment: **nit** — AGENTS.md, Naming: > Write **Dag** (title case) in all prose. So "Dag runs" here, and in the `test_perf.py` docstrings at lines 19, 214 and 259. ########## devel-common/src/tests_common/test_utils/integration_setup.py: ########## @@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str, SerializedDAG]: ] os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] = json.dumps(TESTING_BUNDLE_CONFIG) - # Initial add + # The manager caches the bundle config per process, and the pytest plugin already loaded it with + # the default list; without this the sync below deactivates the "testing" bundle just added, and + # the scheduler marks its Dags stale a minute later. + _load_bundle_config_snapshot.cache_clear() manager = DagBundlesManager() manager.sync_bundles_to_db() return dag_dict -def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int): - # max_wait_time, is the timeout for the DAG run to complete. The value is in seconds. - start_time = timezone.utcnow().timestamp() - - while timezone.utcnow().timestamp() - start_time < max_wait_time: +def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) -> dict[tuple[str, str], str | None]: + """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or ``max_wait_time`` elapses; return the states.""" + # One budget for all runs, not one per run. + deadline = time.monotonic() + max_wait_time + states: dict[tuple[str, str], str | None] = dict.fromkeys(runs) + while True: with create_session() as session: - dag_run = session.scalar( - select(DagRun).where( - DagRun.dag_id == dag_id, - DagRun.run_id == run_id, + rows = session.execute( + select(DagRun.dag_id, DagRun.run_id, DagRun.state).where( + tuple_(DagRun.dag_id, DagRun.run_id).in_(runs) ) - ) + ).all() + for dag_id, run_id, state in rows: + states[(dag_id, run_id)] = state + if all(state in (State.SUCCESS, State.FAILED) for state in states.values()): + break + if time.monotonic() >= deadline: + break + # Otherwise the loop spins and loads the database under test. + time.sleep(2) + return states + + +def terminate_process(proc: subprocess.Popen, timeout: int = 30, *, process_group: bool = False) -> None: + """Stop ``proc``, with SIGKILL as the fallback if it is still alive after ``timeout``.""" + + def send(sig: signal.Signals) -> None: + # Children (uvicorn workers, prefork pool) would be orphaned if only the parent was signalled. + if process_group: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(proc.pid), sig) + else: + proc.send_signal(sig) + + send(signal.SIGTERM) + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + send(signal.SIGKILL) + proc.wait() + + +def print_to_terminal(text: str) -> None: + """Write ``text`` as plain text to the controlling terminal, or to stdout when there is none.""" + # Plain text past pytest's capture, distinct from the processes' logging. + try: + with open("/dev/tty", "w") as tty: + tty.write(text + "\n") + except OSError: + # No terminal: stdout, shown live by pytest -s. + print(text, flush=True) + + +def compute_usable_cpu_count() -> int: + """CPUs this process may actually run on.""" + # Honours a cpuset, which os.cpu_count() does not. Linux only. + if hasattr(os, "sched_getaffinity"): + return len(os.sched_getaffinity(0)) + # The whole host. + return os.cpu_count() or 8 + + +def export_setting(env_var: str, value: int | None) -> None: + """Export an Airflow setting for the subprocesses; ``None`` keeps Airflow's default.""" + if value is not None: + os.environ[env_var] = str(value) + + +def run_psql(sql: str) -> str: + """Run ``sql`` against the breeze Postgres as its superuser and return the output.""" + result = subprocess.run( + ["psql", "-h", "postgres", "-U", "postgres", "-d", "postgres", "-tAc", sql], + env={**os.environ, "PGPASSWORD": "airflow"}, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def restart_postgres_container() -> bool: + """Restart the breeze Postgres container; False when that is not possible from here.""" + # breeze mounts the Docker CLI and socket; FileNotFoundError means another environment. + try: + restart = subprocess.run(["docker", "restart", POSTGRES_CONTAINER], capture_output=True, check=False) + except FileNotFoundError: + return False + return restart.returncode == 0 - if dag_run is None: - time.sleep(5) - continue - dag_run_state = dag_run.state - log.debug("DAG Run state: %s.", dag_run_state) +def ensure_postgres_max_connections(required: int) -> None: + """Raise the metadata DB's ``max_connections`` to ``required`` if it is lower, restarting it once.""" + try: + current = int(run_psql("show max_connections")) + except FileNotFoundError: + raise RuntimeError("psql not found: this helper runs inside the breeze environment") + if current >= required: + return + + # Persisted in the data volume; takes effect at server start, hence the restart. + run_psql(f"ALTER SYSTEM SET max_connections = {required}") + log.info( + "Raised postgres max_connections from %s to %s; restarting %s", current, required, POSTGRES_CONTAINER + ) + if restart_postgres_container(): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + with contextlib.suppress(subprocess.CalledProcessError): + if int(run_psql("show max_connections")) >= required: + return + time.sleep(2) + raise RuntimeError( + f"postgres max_connections={current} is below the {required} needed. The new value is " + "persisted and applies after a restart, which could not be done from here: run " + f"`docker restart {POSTGRES_CONTAINER}` on the host, then re-run the test." + ) - if dag_run_state in [State.SUCCESS, State.FAILED]: - break - return dag_run_state +def start_process(command: list[str]) -> subprocess.Popen: + """Start ``command`` with the current environment, as the leader of its own process group.""" + # Children (uvicorn workers, prefork pool) are signalled with the parent. + return subprocess.Popen(command, env=os.environ.copy(), start_new_session=True) + + +def wait_for_api_server(process: subprocess.Popen, port: int = API_SERVER_PORT, timeout: int = 120) -> None: + """Block until the API server answers its execution health route, or ``timeout`` passes.""" + # uvicorn accepts connections before its workers can answer; a task starting in that gap times + # out and fails. + health_url = f"http://localhost:{port}/execution/health" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"api_server exited early with code {process.returncode}") + with contextlib.suppress(urllib.error.URLError, TimeoutError, ConnectionError): + with urllib.request.urlopen(health_url, timeout=5) as response: + if response.status == 200: + return + time.sleep(1) + raise RuntimeError(f"api_server did not answer {health_url} within {timeout}s") + + +def api_server_command(*, port: int, workers: int) -> list[str]: Review Comment: **nit** — AGENTS.md, Coding Standards: > Name functions and methods with action verbs: `get_`, `extract_`, `find_`, `compute_`, `build_`, etc. Avoid noun-only names like `_serialize_keys` or `_base_names` — they read as attributes, not callables. Predicates (`is_`, `has_`) are the one exception. This applies to `api_server_command` and `celery_worker_command` here, and to `gauge` and `data_point` in `poll_usage.py`. For example: `build_api_server_command`, `build_celery_worker_command`, `build_gauge`, `build_data_point`. ########## devel-common/src/tests_common/test_utils/integration_setup.py: ########## @@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str, SerializedDAG]: ] os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] = json.dumps(TESTING_BUNDLE_CONFIG) - # Initial add + # The manager caches the bundle config per process, and the pytest plugin already loaded it with + # the default list; without this the sync below deactivates the "testing" bundle just added, and + # the scheduler marks its Dags stale a minute later. + _load_bundle_config_snapshot.cache_clear() manager = DagBundlesManager() manager.sync_bundles_to_db() return dag_dict -def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int): - # max_wait_time, is the timeout for the DAG run to complete. The value is in seconds. - start_time = timezone.utcnow().timestamp() - - while timezone.utcnow().timestamp() - start_time < max_wait_time: +def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) -> dict[tuple[str, str], str | None]: + """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or ``max_wait_time`` elapses; return the states.""" + # One budget for all runs, not one per run. + deadline = time.monotonic() + max_wait_time + states: dict[tuple[str, str], str | None] = dict.fromkeys(runs) + while True: with create_session() as session: - dag_run = session.scalar( - select(DagRun).where( - DagRun.dag_id == dag_id, - DagRun.run_id == run_id, + rows = session.execute( + select(DagRun.dag_id, DagRun.run_id, DagRun.state).where( + tuple_(DagRun.dag_id, DagRun.run_id).in_(runs) ) - ) + ).all() + for dag_id, run_id, state in rows: + states[(dag_id, run_id)] = state + if all(state in (State.SUCCESS, State.FAILED) for state in states.values()): + break + if time.monotonic() >= deadline: + break + # Otherwise the loop spins and loads the database under test. + time.sleep(2) + return states + + +def terminate_process(proc: subprocess.Popen, timeout: int = 30, *, process_group: bool = False) -> None: + """Stop ``proc``, with SIGKILL as the fallback if it is still alive after ``timeout``.""" + + def send(sig: signal.Signals) -> None: + # Children (uvicorn workers, prefork pool) would be orphaned if only the parent was signalled. + if process_group: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(proc.pid), sig) Review Comment: **minor** — `start_new_session=True` makes the pgid equal to the pid, so the group can be signalled by `proc.pid` directly. `os.getpgid(proc.pid)` raises `ProcessLookupError` once `poll()` has reaped the leader, and both `start_processes` and `execute_dag_runs` call `poll()` before cleanup. When that happens, surviving group members, such as uvicorn workers or prefork children, are never signalled. I checked this with a leader that exits and leaves a child behind: `getpgid` raised and the child stayed alive, while `os.killpg(proc.pid, SIGTERM)` stopped it. ```suggestion os.killpg(proc.pid, sig) ``` Related: `proc.wait(timeout=...)` only waits for the leader. If the leader exits on SIGTERM while its children keep running, SIGKILL is never sent. Checking the group (for example `os.killpg(proc.pid, 0)` until it raises) before skipping the SIGKILL would close that gap. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
