This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 583369271f5 Fix generate-providers-metadata hang by using spawn pools
(#69763)
583369271f5 is described below
commit 583369271f5ae2098e9d800a0a427cf2eeb4edd4
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jul 12 20:29:54 2026 +0200
Fix generate-providers-metadata hang by using spawn pools (#69763)
The multiprocessing pools in the providers-metadata generation flow used the
platform-default start method (fork on Linux). Before the pools are created
the
parent process has already used GitPython — which opens persistent
`git cat-file --batch` subprocesses and is not fork-safe — and holds open
network sockets from the version and constraints downloads. Forking that
state
into the workers left them with broken inherited file descriptors, so the
pool
deadlocked and the command hung forever, most reliably when
--refresh-constraints-and-airflow-releases forces the parent through git
and the
network before forking.
Switching these pools to the spawn start method gives each worker a clean
interpreter with no inherited git subprocesses or sockets.
---
.../src/airflow_breeze/commands/release_management_commands.py | 9 ++++++---
dev/breeze/src/airflow_breeze/utils/provider_dependencies.py | 9 +++++++--
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index aed5d0e1ec9..5859b545e47 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -34,7 +34,7 @@ from copy import deepcopy
from datetime import datetime
from enum import Enum
from functools import partial
-from multiprocessing import Pool
+from multiprocessing import get_context
from pathlib import Path
from subprocess import DEVNULL
from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple
@@ -3611,7 +3611,10 @@ def generate_providers_metadata(
)
console_print("\n[info]Checking provider.yaml versions[1:] against PyPI
for stale entries...[/]\n")
- with Pool() as pypi_pool:
+ # "spawn" (not the platform-default fork): the parent has already used
GitPython and
+ # opened network sockets before reaching here, and forking that state into
workers
+ # deadlocks. See get_all_constraint_files_and_airflow_releases for the
same reasoning.
+ with get_context("spawn").Pool() as pypi_pool:
pruned_per_provider =
pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids)
total_pruned = 0
for pid, pruned in zip(package_ids, pruned_per_provider):
@@ -3636,7 +3639,7 @@ def generate_providers_metadata(
airflow_release_dates=airflow_release_dates,
current_metadata=current_metadata,
)
- with Pool() as pool:
+ with get_context("spawn").Pool() as pool:
results = pool.map(
partial_generate_providers_metadata,
package_ids,
diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
index a985a65b55d..d67e0ec9bfc 100644
--- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
@@ -26,7 +26,7 @@ import urllib.error
import urllib.request
from collections.abc import Generator
from functools import cache, partial
-from multiprocessing import Pool
+from multiprocessing import get_context
from pathlib import Path
from threading import Lock
from typing import NamedTuple
@@ -276,7 +276,12 @@ def get_all_constraint_files_and_airflow_releases(
airflow_release_dates_path.write_text(json.dumps(airflow_release_dates,
indent=2))
console_print(f"[info]Airflow release dates saved in:
{airflow_release_dates_path}[/]")
with ci_group("Downloading constraints for all Airflow versions for
all historical Python versions"):
- with Pool() as pool:
+ # Use the "spawn" start method rather than the platform default:
GitPython
+ # (used in the workers via get_tag_date) opens persistent `git
cat-file --batch`
+ # subprocesses and is not fork-safe, and the parent already holds
open network
+ # sockets from the version/constraints downloads. Forking that
state into workers
+ # deadlocks; spawn gives each worker a clean interpreter.
+ with get_context("spawn").Pool() as pool:
# We use partial to pass the common parameters to the function
get_constraints_for_python_version_partial = partial(
get_constraints_for_python_version,
