xBis7 commented on code in PR #77032:
URL: https://github.com/apache/airflow/pull/77032#discussion_r4642129860
##########
devel-common/src/tests_common/test_utils/integration_setup.py:
##########
@@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str,
SerializedDAG]:
]
os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] =
json.dumps(TESTING_BUNDLE_CONFIG)
- # Initial add
+ # The manager caches the bundle config per process, and the pytest plugin
already loaded it with
+ # the default list; without this the sync below deactivates the "testing"
bundle just added, and
+ # the scheduler marks its Dags stale a minute later.
+ _load_bundle_config_snapshot.cache_clear()
manager = DagBundlesManager()
manager.sync_bundles_to_db()
return dag_dict
-def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int):
- # max_wait_time, is the timeout for the DAG run to complete. The value is
in seconds.
- start_time = timezone.utcnow().timestamp()
-
- while timezone.utcnow().timestamp() - start_time < max_wait_time:
+def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) ->
dict[tuple[str, str], str | None]:
+ """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or
``max_wait_time`` elapses; return the states."""
+ # One budget for all runs, not one per run.
+ deadline = time.monotonic() + max_wait_time
+ states: dict[tuple[str, str], str | None] = dict.fromkeys(runs)
+ while True:
with create_session() as session:
- dag_run = session.scalar(
- select(DagRun).where(
- DagRun.dag_id == dag_id,
- DagRun.run_id == run_id,
+ rows = session.execute(
+ select(DagRun.dag_id, DagRun.run_id, DagRun.state).where(
+ tuple_(DagRun.dag_id, DagRun.run_id).in_(runs)
)
- )
+ ).all()
+ for dag_id, run_id, state in rows:
+ states[(dag_id, run_id)] = state
+ if all(state in (State.SUCCESS, State.FAILED) for state in
states.values()):
+ break
+ if time.monotonic() >= deadline:
+ break
+ # Otherwise the loop spins and loads the database under test.
+ time.sleep(2)
+ return states
+
+
+def terminate_process(proc: subprocess.Popen, timeout: int = 30, *,
process_group: bool = False) -> None:
+ """Stop ``proc``, with SIGKILL as the fallback if it is still alive after
``timeout``."""
+
+ def send(sig: signal.Signals) -> None:
+ # Children (uvicorn workers, prefork pool) would be orphaned if only
the parent was signalled.
+ if process_group:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(os.getpgid(proc.pid), sig)
Review Comment:
Fixed
##########
devel-common/src/tests_common/test_utils/integration_setup.py:
##########
@@ -155,43 +211,317 @@ def serialize_and_get_dags(dag_folder) -> dict[str,
SerializedDAG]:
]
os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] =
json.dumps(TESTING_BUNDLE_CONFIG)
- # Initial add
+ # The manager caches the bundle config per process, and the pytest plugin
already loaded it with
+ # the default list; without this the sync below deactivates the "testing"
bundle just added, and
+ # the scheduler marks its Dags stale a minute later.
+ _load_bundle_config_snapshot.cache_clear()
manager = DagBundlesManager()
manager.sync_bundles_to_db()
return dag_dict
-def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int):
- # max_wait_time, is the timeout for the DAG run to complete. The value is
in seconds.
- start_time = timezone.utcnow().timestamp()
-
- while timezone.utcnow().timestamp() - start_time < max_wait_time:
+def wait_for_dag_runs(runs: list[tuple[str, str]], max_wait_time: int) ->
dict[tuple[str, str], str | None]:
+ """Wait until every ``(dag_id, run_id)`` in ``runs`` finishes or
``max_wait_time`` elapses; return the states."""
+ # One budget for all runs, not one per run.
+ deadline = time.monotonic() + max_wait_time
+ states: dict[tuple[str, str], str | None] = dict.fromkeys(runs)
+ while True:
with create_session() as session:
- dag_run = session.scalar(
- select(DagRun).where(
- DagRun.dag_id == dag_id,
- DagRun.run_id == run_id,
+ rows = session.execute(
+ select(DagRun.dag_id, DagRun.run_id, DagRun.state).where(
+ tuple_(DagRun.dag_id, DagRun.run_id).in_(runs)
)
- )
+ ).all()
+ for dag_id, run_id, state in rows:
+ states[(dag_id, run_id)] = state
+ if all(state in (State.SUCCESS, State.FAILED) for state in
states.values()):
+ break
+ if time.monotonic() >= deadline:
+ break
+ # Otherwise the loop spins and loads the database under test.
+ time.sleep(2)
+ return states
+
+
+def terminate_process(proc: subprocess.Popen, timeout: int = 30, *,
process_group: bool = False) -> None:
+ """Stop ``proc``, with SIGKILL as the fallback if it is still alive after
``timeout``."""
+
+ def send(sig: signal.Signals) -> None:
+ # Children (uvicorn workers, prefork pool) would be orphaned if only
the parent was signalled.
+ if process_group:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(os.getpgid(proc.pid), sig)
+ else:
+ proc.send_signal(sig)
+
+ send(signal.SIGTERM)
+ try:
+ proc.wait(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ send(signal.SIGKILL)
+ proc.wait()
+
+
+def print_to_terminal(text: str) -> None:
+ """Write ``text`` as plain text to the controlling terminal, or to stdout
when there is none."""
+ # Plain text past pytest's capture, distinct from the processes' logging.
+ try:
+ with open("/dev/tty", "w") as tty:
+ tty.write(text + "\n")
+ except OSError:
+ # No terminal: stdout, shown live by pytest -s.
+ print(text, flush=True)
+
+
+def compute_usable_cpu_count() -> int:
+ """CPUs this process may actually run on."""
+ # Honours a cpuset, which os.cpu_count() does not. Linux only.
+ if hasattr(os, "sched_getaffinity"):
+ return len(os.sched_getaffinity(0))
+ # The whole host.
+ return os.cpu_count() or 8
+
+
+def export_setting(env_var: str, value: int | None) -> None:
+ """Export an Airflow setting for the subprocesses; ``None`` keeps
Airflow's default."""
+ if value is not None:
+ os.environ[env_var] = str(value)
+
+
+def run_psql(sql: str) -> str:
+ """Run ``sql`` against the breeze Postgres as its superuser and return the
output."""
+ result = subprocess.run(
+ ["psql", "-h", "postgres", "-U", "postgres", "-d", "postgres", "-tAc",
sql],
+ env={**os.environ, "PGPASSWORD": "airflow"},
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return result.stdout.strip()
+
+
+def restart_postgres_container() -> bool:
+ """Restart the breeze Postgres container; False when that is not possible
from here."""
+ # breeze mounts the Docker CLI and socket; FileNotFoundError means another
environment.
+ try:
+ restart = subprocess.run(["docker", "restart", POSTGRES_CONTAINER],
capture_output=True, check=False)
+ except FileNotFoundError:
+ return False
+ return restart.returncode == 0
- if dag_run is None:
- time.sleep(5)
- continue
- dag_run_state = dag_run.state
- log.debug("DAG Run state: %s.", dag_run_state)
+def ensure_postgres_max_connections(required: int) -> None:
+ """Raise the metadata DB's ``max_connections`` to ``required`` if it is
lower, restarting it once."""
+ try:
+ current = int(run_psql("show max_connections"))
+ except FileNotFoundError:
+ raise RuntimeError("psql not found: this helper runs inside the breeze
environment")
+ if current >= required:
+ return
+
+ # Persisted in the data volume; takes effect at server start, hence the
restart.
+ run_psql(f"ALTER SYSTEM SET max_connections = {required}")
+ log.info(
+ "Raised postgres max_connections from %s to %s; restarting %s",
current, required, POSTGRES_CONTAINER
+ )
+ if restart_postgres_container():
+ deadline = time.monotonic() + 60
+ while time.monotonic() < deadline:
+ with contextlib.suppress(subprocess.CalledProcessError):
+ if int(run_psql("show max_connections")) >= required:
+ return
+ time.sleep(2)
+ raise RuntimeError(
+ f"postgres max_connections={current} is below the {required} needed.
The new value is "
+ "persisted and applies after a restart, which could not be done from
here: run "
+ f"`docker restart {POSTGRES_CONTAINER}` on the host, then re-run the
test."
+ )
- if dag_run_state in [State.SUCCESS, State.FAILED]:
- break
- return dag_run_state
+def start_process(command: list[str]) -> subprocess.Popen:
+ """Start ``command`` with the current environment, as the leader of its
own process group."""
+ # Children (uvicorn workers, prefork pool) are signalled with the parent.
+ return subprocess.Popen(command, env=os.environ.copy(),
start_new_session=True)
+
+
+def wait_for_api_server(process: subprocess.Popen, port: int =
API_SERVER_PORT, timeout: int = 120) -> None:
+ """Block until the API server answers its execution health route, or
``timeout`` passes."""
+ # uvicorn accepts connections before its workers can answer; a task
starting in that gap times
+ # out and fails.
+ health_url = f"http://localhost:{port}/execution/health"
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if process.poll() is not None:
+ raise RuntimeError(f"api_server exited early with code
{process.returncode}")
+ with contextlib.suppress(urllib.error.URLError, TimeoutError,
ConnectionError):
+ with urllib.request.urlopen(health_url, timeout=5) as response:
+ if response.status == 200:
+ return
+ time.sleep(1)
+ raise RuntimeError(f"api_server did not answer {health_url} within
{timeout}s")
+
+
+def api_server_command(*, port: int, workers: int) -> list[str]:
Review Comment:
Fixed
##########
airflow-core/tests/integration/performance/test_perf.py:
##########
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Performance integration test for the scheduler.
+Creates a backlog of dag runs, starts the Airflow processes and measures how
long they take to
+finish it, once per test case of the code under test. See README.md.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+import time
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from airflow.configuration import conf
+from airflow.utils.state import State
+
+from tests_common.test_utils.integration_setup import (
+ OTEL_COLLECTOR_HOST,
+ OTEL_COLLECTOR_PORT,
+ ProgressReporter,
+ compute_usable_cpu_count,
+ create_queued_dag_runs,
+ ensure_postgres_max_connections,
+ export_setting,
+ print_to_terminal,
+ serialize_and_get_dags,
+ start_process,
+ start_processes,
+ terminate_processes,
+ wait_for_dag_runs,
+ wait_for_otel_collector,
+)
+
+if TYPE_CHECKING:
+ from airflow.serialization.definitions.dag import SerializedDAG
+
+log = logging.getLogger("integration.perf.test_perf")
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+DAG_FOLDER = os.path.join(TEST_DIR, "dags")
+POLL_USAGE_SCRIPT = os.path.join(TEST_DIR, "poll_usage.py")
+
+# "true" or "false" per test case, read by the code under test.
+NEW_CODE_ENV_VAR = "PERF_TEST__NEW_CODE"
+
+# Per Airflow process; Airflow's defaults times the API server workers exceed
Postgres's limit.
+DB_POOL_SIZE = 3
+DB_MAX_OVERFLOW = 5
+# Per Celery child, which runs one task at a time.
+RESULT_BACKEND_POOL_SIZE = 1
+RESULT_BACKEND_MAX_OVERFLOW = 1
+# psql, the test process and the usage poller.
+DB_CONNECTION_HEADROOM = 20
+
+CPUS = compute_usable_cpu_count()
+
+# The first three are process counts; the rest are Airflow settings, None
keeps Airflow's default.
+PROFILES: dict[str, dict[str, Any]] = {
+ # Sized so the scheduler is the only constraint: every other cap is far
above what the workers
+ # can execute.
+ "dynamic": {
+ # More slots per worker rather than more workers: each worker process
costs connections.
+ "worker_count": 3,
+ "worker_concurrency": max(2, CPUS * 2 // 3),
+ # One worker saturates well before the scheduler does.
+ "api_server_workers": max(4, CPUS // 3),
+ "parallelism": 4096,
+ "default_pool_task_slot_count": 4096,
+ "max_active_tasks_per_dag": 4096,
+ # Many runs of one Dag; the default of 16 would throttle them.
+ "max_active_runs_per_dag": 4096,
+ "max_tis_per_query": None,
+ },
+ # Fill in as needed; the process counts start at the tools' defaults.
+ "custom": {
+ "worker_count": 1,
+ "worker_concurrency": CPUS,
+ "api_server_workers": conf.getint("api", "workers"),
+ "parallelism": None,
+ "default_pool_task_slot_count": None,
+ "max_active_tasks_per_dag": None,
+ "max_active_runs_per_dag": None,
+ "max_tis_per_query": None,
+ },
+}
+
+
+def compute_required_db_connections(sizing: dict[str, Any]) -> int:
+ """Peak connections the deployment can open against the metadata DB."""
+ # Tasks use the Execution API and never connect.
+ scheduler = 1
+ airflow_processes = scheduler + sizing["api_server_workers"] +
sizing["worker_count"]
+ per_airflow_process = DB_POOL_SIZE + DB_MAX_OVERFLOW
+ # One result-backend pool per Celery child.
+ celery_children = sizing["worker_count"] * sizing["worker_concurrency"]
+ per_celery_child = RESULT_BACKEND_POOL_SIZE + RESULT_BACKEND_MAX_OVERFLOW
+ return (
+ airflow_processes * per_airflow_process + celery_children *
per_celery_child + DB_CONNECTION_HEADROOM
+ )
+
+
[email protected]("otel")
[email protected]("redis")
[email protected]("postgres")
+class TestPerformance:
+ """One environment per class, one deployment per test case."""
+
+ profile = os.environ.get("PERF_TEST__PROFILE", "dynamic") # key of
PROFILES; sizes the deployment
+ sizing: dict[str, Any]
+ dags: dict[str, SerializedDAG] = {}
+
+ @classmethod
+ def setup_class(cls):
+ if cls.profile not in PROFILES:
+ pytest.fail(f"profile {cls.profile!r} is not one of
{sorted(PROFILES)}")
+ cls.sizing = PROFILES[cls.profile]
+ log.info("profile %s: %s", cls.profile, cls.sizing)
+
+ wait_for_otel_collector(OTEL_COLLECTOR_HOST, OTEL_COLLECTOR_PORT)
+ # Exceeding the limit kills the scheduler mid-run.
+
ensure_postgres_max_connections(compute_required_db_connections(cls.sizing))
+
+ # The pytest plugin strips AIRFLOW__* variables; scheduler and API
server must share the secret.
+ os.environ["AIRFLOW__API_AUTH__JWT_SECRET"] =
"test-secret-key-for-testing"
+ os.environ["AIRFLOW__API_AUTH__JWT_ISSUER"] = "airflow"
+
+ os.environ["AIRFLOW__CORE__EXECUTOR"] = "CeleryExecutor"
+ os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = DAG_FOLDER
+ os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False"
+ os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = "/dev/null"
+ os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "False"
+
+ os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE"] =
str(DB_POOL_SIZE)
+ os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW"] =
str(DB_MAX_OVERFLOW)
+
os.environ["AIRFLOW__CELERY__RESULT_BACKEND_SQLALCHEMY_ENGINE_OPTIONS"] =
json.dumps(
+ {"pool_size": RESULT_BACKEND_POOL_SIZE, "max_overflow":
RESULT_BACKEND_MAX_OVERFLOW}
+ )
+ # The default spawns a ProcessPoolExecutor per publish batch, on the
scheduler's heartbeat.
+ os.environ["AIRFLOW__CELERY__SYNC_PARALLELISM"] = "1"
+ # The default of one second fails publishes behind a busy broker.
+ os.environ["AIRFLOW__CELERY__OPERATION_TIMEOUT"] = "300"
+
+ # Before db migrate, which creates the default pool with this slot
count.
+ export_setting("AIRFLOW__CORE__PARALLELISM", cls.sizing["parallelism"])
+ export_setting(
+ "AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT",
cls.sizing["default_pool_task_slot_count"]
+ )
+ export_setting("AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG",
cls.sizing["max_active_tasks_per_dag"])
+ export_setting("AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG",
cls.sizing["max_active_runs_per_dag"])
+ export_setting("AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY",
cls.sizing["max_tis_per_query"])
+
+ os.environ["AIRFLOW__METRICS__OTEL_ON"] = "True"
+ os.environ["AIRFLOW__METRICS__OTEL_HOST"] = OTEL_COLLECTOR_HOST
+ os.environ["AIRFLOW__METRICS__OTEL_PORT"] = str(OTEL_COLLECTOR_PORT)
+ # breeze's OTEL_METRIC_EXPORT_INTERVAL=30000 overrides the Airflow
option; Prometheus still
+ # scrapes every 30s.
+ os.environ["AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS"] = "1000"
+ os.environ["OTEL_METRIC_EXPORT_INTERVAL"] = "1000"
+ # The dashboard reads the tagged names; legacy names add one series
family per task.
+ os.environ["AIRFLOW__METRICS__LEGACY_NAMES_ON"] = "False"
+ os.environ["AIRFLOW__TRACES__OTEL_ON"] = "True"
+ os.environ["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf"
+ os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = (
+ f"http://{OTEL_COLLECTOR_HOST}:{OTEL_COLLECTOR_PORT}/v1/traces"
+ )
+
+ # Once per class: the scheduler ignores finished runs, and run ids
carry the test case.
+ subprocess.run(["airflow", "db", "reset", "--yes"], check=True,
env=os.environ.copy())
+ subprocess.run(["airflow", "db", "migrate"], check=True,
env=os.environ.copy())
+
+ # Every Dag in the folder; the test picks by id.
+ cls.dags = serialize_and_get_dags(dag_folder=DAG_FOLDER)
+ assert cls.dags, f"no Dags found in {DAG_FOLDER}"
+
+ def start_usage_poller(self, processes: dict[str, subprocess.Popen]) ->
subprocess.Popen:
+ """Start poll_usage.py, which pushes CPU, memory and worker slot
gauges for ``processes``."""
+ slot_budget = self.sizing["worker_count"] *
self.sizing["worker_concurrency"]
+ command = [sys.executable, POLL_USAGE_SCRIPT, "--slot-budget",
str(slot_budget)]
+ for role, process in processes.items():
+ command.append(f"{role}={process.pid}")
+ # Own process group, or terminate_processes would kill pytest with it.
+ return start_process(command)
+
+ def execute_dag_runs(
+ self,
+ *,
+ run_id_prefix: str,
+ dag_ids: list[str],
+ dag_run_count: int,
+ run_conf: dict[str, Any],
+ timeout_seconds: int,
+ ) -> None:
+ """Create the dag runs, start the processes, wait for the runs to
finish, stop the processes."""
+ dags = {dag_id: self.dags[dag_id] for dag_id in dag_ids}
+ # Created before any process exists, so every test case starts from
the same rows, all
+ # eligible at once, and CLI triggering cannot cap the live runs at its
own rate. This is the
+ # scheduler under full load, not a production trickle.
+ runs = create_queued_dag_runs(
+ dags, run_id_prefix=run_id_prefix, dag_run_count=dag_run_count,
run_conf=run_conf
+ )
+ processes = start_processes(
+ worker_count=self.sizing["worker_count"],
+ worker_concurrency=self.sizing["worker_concurrency"],
+ api_server_workers=self.sizing["api_server_workers"],
+ )
+ started_at = time.monotonic()
Review Comment:
Fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]