This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bdcb7eddd85 Skip Kubernetes volume internals when discovering Dag and
plugin files (#76186)
bdcb7eddd85 is described below
commit bdcb7eddd8505a33c2b513cf208fdd2f0a06fdd2
Author: Snehashis Lenka <[email protected]>
AuthorDate: Sun Nov 1 02:13:42 2026 -0800
Skip Kubernetes volume internals when discovering Dag and plugin files
(#76186)
ConfigMap, Secret and projected volumes keep their files in a hidden
"..<timestamp>" directory linked as "..data", and expose each entry as
a symlink through it. Walking both reached that directory twice, so the
recursive-loop check failed discovery for any such mounted folder.
Co-authored-by: Claude Opus 5.5 <[email protected]>
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../src/airflow_shared/module_loading/file_discovery.py | 3 +++
.../tests/module_loading/test_file_discovery.py | 17 +++++++++++++++++
2 files changed, 20 insertions(+)
diff --git
a/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
b/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
index 00660816b25..4837a6bbb9b 100644
--- a/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
+++ b/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
@@ -174,6 +174,9 @@ def _find_path_from_directory(
# evaluation order of patterns is important with negation
# so that later patterns can override earlier patterns
+ # Kubernetes volume internals; the visible entries are symlinks
through "..data" to the same files
+ if "..data" in dirs:
+ dirs[:] = [subdir for subdir in dirs if not
subdir.startswith("..")]
dirs[:] = [subdir for subdir in dirs if not
ignore_rule_type.match(Path(root) / subdir, patterns)]
# explicit loop for infinite recursion detection since we are
following symlinks in this walk
for sd in dirs:
diff --git a/shared/module_loading/tests/module_loading/test_file_discovery.py
b/shared/module_loading/tests/module_loading/test_file_discovery.py
index 388ad79c239..71dcfe5b818 100644
--- a/shared/module_loading/tests/module_loading/test_file_discovery.py
+++ b/shared/module_loading/tests/module_loading/test_file_discovery.py
@@ -73,6 +73,23 @@ class TestFindPathFromDirectory:
with pytest.raises(RuntimeError, match=error_message):
list(find_path_from_directory(test_dir, ignore_list_file,
ignore_file_syntax="glob"))
+ @pytest.mark.parametrize("ignore_file_syntax", ["glob", "regexp"])
+ def test_find_path_from_directory_skips_kubernetes_volume_internals(self,
tmp_path, ignore_file_syntax):
+ """Files of a Kubernetes ConfigMap or Secret volume are found once,
via their visible symlinks."""
+ # the kubelet keeps the files in a timestamped directory, linked as
"..data"
+ timestamped_dir = tmp_path / "..2026_10_31_12_00_00.123456789"
+ (timestamped_dir / "subdir").mkdir(parents=True)
+ (timestamped_dir / ".airflowignore").write_text("ignored.py")
+ for name in ("plugin.py", "ignored.py", "subdir/nested.py"):
+ (timestamped_dir / name).touch()
+ (tmp_path / "..data").symlink_to(timestamped_dir.name)
+ for name in (".airflowignore", "plugin.py", "ignored.py", "subdir"):
+ (tmp_path / name).symlink_to(Path("..data", name))
+
+ found = sorted(find_path_from_directory(tmp_path, ".airflowignore",
ignore_file_syntax))
+
+ assert found == [str(tmp_path / "plugin.py"), str(tmp_path / "subdir"
/ "nested.py")]
+
def
test_find_path_from_directory_keeps_inherited_rules_on_symlink_switch(self,
tmp_path):
"""Ignore rules inherited by a subdirectory still apply if the Dag
folder symlink is switched mid-walk."""
for revision in ("rev1", "rev2"):
