This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 41619107e37 Fix import airflow slowdown from reading every package's
metadata (#76890)
41619107e37 is described below
commit 41619107e376d9307b84f4dd9400ff101cd0702b
Author: Andrew Chang <[email protected]>
AuthorDate: Tue Nov 17 12:29:16 2026 +0800
Fix import airflow slowdown from reading every package's metadata (#76890)
---
.../src/airflow_shared/module_loading/__init__.py | 21 +++++-----
.../tests/module_loading/test_entry_points.py | 45 ++++++++++++++++++----
.../tests/plugins_manager/test_plugins_manager.py | 10 ++++-
3 files changed, 57 insertions(+), 19 deletions(-)
diff --git
a/shared/module_loading/src/airflow_shared/module_loading/__init__.py
b/shared/module_loading/src/airflow_shared/module_loading/__init__.py
index adfbae8acf4..3a65d9fc250 100644
--- a/shared/module_loading/src/airflow_shared/module_loading/__init__.py
+++ b/shared/module_loading/src/airflow_shared/module_loading/__init__.py
@@ -164,11 +164,6 @@ def is_valid_dotpath(path: str) -> bool:
return bool(re.match(pattern, path))
-def _normalize_dist_name(name: str) -> str:
- # PEP 503 normalization, the same key importlib.metadata.entry_points()
deduplicates on.
- return re.sub(r"[-_.]+", "-", name).lower()
-
-
@functools.cache
def _get_grouped_entry_points() -> dict[str, list[EPnD]]:
mapping: dict[str, list[EPnD]] = defaultdict(list)
@@ -178,12 +173,18 @@ def _get_grouped_entry_points() -> dict[str, list[EPnD]]:
# A distribution is yielded once per sys.path entry it is found on
(e.g. RHEL/Fedora
# venvs list both lib64/ and lib/ site-packages), possibly in
different versions.
# Only the first one is importable, so it is the only one whose
entry points count.
- dist_metadata = dist.metadata
- if dist_metadata and (name := dist_metadata.get("Name")):
- normalized_name = _normalize_dist_name(name)
- if normalized_name in seen_dist_names:
+ # No public API returns the name without parsing METADATA.
_normalized_name is private,
+ # but Python's own importlib.metadata.entry_points() deduplicates
on it too, and it comes
+ # from the .dist-info directory name. If it cannot be read, the
distribution is still
+ # loaded, only without deduplication. The exception type differs
between versions.
+ try:
+ name = dist._normalized_name # type: ignore[attr-defined]
+ except Exception:
+ name = None
+ if name:
+ if name in seen_dist_names:
continue
- seen_dist_names.add(normalized_name)
+ seen_dist_names.add(name)
for e in dist.entry_points:
# importlib_metadata >= 10 types this as its IDistribution
protocol, which older
# releases and the stdlib lack; a cast would be redundant on
those.
diff --git a/shared/module_loading/tests/module_loading/test_entry_points.py
b/shared/module_loading/tests/module_loading/test_entry_points.py
index ff22042aefe..a9a65798bd7 100644
--- a/shared/module_loading/tests/module_loading/test_entry_points.py
+++ b/shared/module_loading/tests/module_loading/test_entry_points.py
@@ -17,6 +17,7 @@
# under the License.
from __future__ import annotations
+import re
from collections.abc import Iterable
from unittest import mock
@@ -29,7 +30,8 @@ class MockDistribution:
def __init__(
self, name: str, entry_points: Iterable[metadata.EntryPoint], version:
str = "1.0.0"
) -> None:
- self.metadata: dict[str, str] | None = {"Name": name}
+ self.metadata = {"Name": name}
+ self._normalized_name = re.sub(r"[-_.]+", "_", name).lower()
self.entry_points = entry_points
self.version = version
@@ -97,17 +99,46 @@ def
test_entry_points_with_dist_same_dist_in_two_versions_returns_first_only():
]
-# No metadata file at all, or metadata without a Name field (the key is
absent, not None).
[email protected]("dist_metadata", [None, {"Version": "1.0.0"}],
ids=["no-metadata", "no-name"])
-def
test_entry_points_with_dist_without_name_are_not_deduplicated(dist_metadata):
+def test_entry_points_with_dist_deduplicates_without_reading_metadata():
+ distributions = [
+ MockDistribution("apache-airflow-providers-foo",
[metadata.EntryPoint("a", "b", "group_x")]),
+ MockDistribution("apache_airflow_providers_foo",
[metadata.EntryPoint("c", "d", "group_x")]),
+ MockDistribution("apache-airflow-providers-bar",
[metadata.EntryPoint("e", "f", "group_x")]),
+ ]
+
+ with (
+ mock.patch("airflow_shared.module_loading.metadata",
MockMetadata(distributions)),
+ mock.patch.object(
+ MockDistribution, "metadata", create=True,
new_callable=mock.PropertyMock
+ ) as metadata_property,
+ ):
+ entries = list(entry_points_with_dist("group_x"))
+
+ assert [ep.name for ep, _ in entries] == ["a", "e"]
+ metadata_property.assert_not_called()
+
+
[email protected](
+ "error",
+ [TypeError, KeyError, FileNotFoundError, AttributeError],
+ ids=["stdlib", "importlib-metadata-8", "importlib-metadata-9-no-file",
"no-attribute"],
+)
+def test_entry_points_with_dist_without_name_are_not_deduplicated(error):
distributions = [
MockDistribution("unnamed", [metadata.EntryPoint("a", "b",
"group_x")]),
MockDistribution("unnamed", [metadata.EntryPoint("c", "d",
"group_x")]),
]
- for dist in distributions:
- dist.metadata = dist_metadata
- with mock.patch("airflow_shared.module_loading.metadata",
MockMetadata(distributions)):
+ with (
+ mock.patch("airflow_shared.module_loading.metadata",
MockMetadata(distributions)),
+ mock.patch.object(
+ MockDistribution,
+ "_normalized_name",
+ create=True,
+ new_callable=mock.PropertyMock,
+ side_effect=error,
+ ),
+ ):
entries = list(entry_points_with_dist("group_x"))
assert [ep.name for ep, _ in entries] == ["a", "c"]
diff --git
a/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
b/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
index 414259908ae..16e89829eaf 100644
--- a/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
+++ b/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
@@ -24,6 +24,7 @@ from unittest import mock
import pytest
+from airflow_shared.module_loading import _get_grouped_entry_points
from airflow_shared.plugins_manager import (
AirflowPlugin,
EntryPointSource,
@@ -41,8 +42,12 @@ def mock_metadata_distribution(mocker):
else:
patch_fq = "importlib.metadata.distributions"
- with mock.patch(patch_fq, *args, **kwargs) as m:
- yield m
+ _get_grouped_entry_points.cache_clear()
+ try:
+ with mock.patch(patch_fq, *args, **kwargs) as m:
+ yield m
+ finally:
+ _get_grouped_entry_points.cache_clear()
return wrapper
@@ -87,6 +92,7 @@ class TestPluginsManager:
"""
mock_dist = mock.Mock()
mock_dist.metadata = {"Name": "test-dist"}
+ mock_dist._normalized_name = "test_dist"
mock_entrypoint = mock.Mock()
mock_entrypoint.name = "test-entrypoint"