This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch v3-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-3-test by this push:
new d4e4e576aa3 [v3-3-test] Resolve nested provider paths for conn-fields
check from the real directory tree (#70261) (#70367)
d4e4e576aa3 is described below
commit d4e4e576aa35f6f5ec8a7143ac86b0d6db621dbe
Author: Wei Lee <[email protected]>
AuthorDate: Fri Jul 24 17:16:34 2026 +0800
[v3-3-test] Resolve nested provider paths for conn-fields check from the
real directory tree (#70261) (#70367)
(cherry picked from commit 030efa6f74a50c54e4d4e28df6d8d98eb227c7c4)
---
scripts/ci/prek/check_provider_yaml_files.py | 53 +++++++++++---
.../ci/prek/test_check_provider_yaml_files.py | 84 ++++++++++++++++++++++
2 files changed, 127 insertions(+), 10 deletions(-)
diff --git a/scripts/ci/prek/check_provider_yaml_files.py
b/scripts/ci/prek/check_provider_yaml_files.py
index 0c74234e064..5e8d8da7a98 100755
--- a/scripts/ci/prek/check_provider_yaml_files.py
+++ b/scripts/ci/prek/check_provider_yaml_files.py
@@ -16,28 +16,61 @@
# specific language governing permissions and limitations
# under the License.
# /// script
-# requires-python = ">=3.10,<3.11"
+# requires-python = ">=3.10"
# dependencies = [
# "rich>=13.6.0",
# ]
# ///
from __future__ import annotations
+import pathlib
import sys
from common_prek_utils import (
+ get_provider_base_dir_from_path,
initialize_breeze_prek,
run_command_via_breeze_run,
validate_cmd_result,
)
-initialize_breeze_prek(__name__, __file__)
-files_to_test = sys.argv[1:]
-cmd_result = run_command_via_breeze_run(
- ["python3",
"/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py",
*files_to_test],
- backend="sqlite",
- warn_image_upgrade_needed=True,
- extra_env={"PYTHONWARNINGS": "default"},
-)
-validate_cmd_result(cmd_result, include_ci_env_check=True)
+def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]:
+ """
+ Accept a mix of provider.yaml paths and Python source files.
+
+ When a Python source file is passed (e.g. a hook whose
+ ``get_connection_form_widgets()`` was edited), map it to the
+ ``provider.yaml`` at the root of the same provider package so the
+ conn-fields check runs even when only the hook changes.
+
+ All paths are relative to the ``providers/`` directory, as supplied by
+ prek. Rather than guessing how many path segments make up the provider
+ package name, this walks up the real directory tree (via
+ ``get_provider_base_dir_from_path``) until it finds the actual
+ ``provider.yaml`` file, so nested/namespace provider packages
+ (e.g. ``apache/beam``, ``ibm/mq``) resolve correctly without maintaining
+ a list of known namespace prefixes.
+ """
+ result: set[str] = set()
+ for f in raw_files:
+ p = pathlib.PurePosixPath(f)
+ if p.name == "provider.yaml":
+ result.add(f)
+ else:
+ provider_dir = get_provider_base_dir_from_path(pathlib.Path(f))
+ if provider_dir is not None:
+ result.add((provider_dir / "provider.yaml").as_posix())
+ return sorted(result)
+
+
+if __name__ == "__main__":
+ initialize_breeze_prek(__name__, __file__)
+
+ files_to_test = _resolve_provider_yaml_files(sys.argv[1:])
+ cmd_result = run_command_via_breeze_run(
+ ["python3",
"/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py",
*files_to_test],
+ backend="sqlite",
+ warn_image_upgrade_needed=True,
+ extra_env={"PYTHONWARNINGS": "default"},
+ )
+ validate_cmd_result(cmd_result, include_ci_env_check=True)
diff --git a/scripts/tests/ci/prek/test_check_provider_yaml_files.py
b/scripts/tests/ci/prek/test_check_provider_yaml_files.py
new file mode 100644
index 00000000000..590ad73efb2
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_provider_yaml_files.py
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from ci.prek.check_provider_yaml_files import _resolve_provider_yaml_files
+
+
+def _touch(path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.touch()
+
+
+class TestResolveProviderYamlFiles:
+ @pytest.mark.parametrize(
+ "file_names, raw_files, expected",
+ (
+ (
+ ["samba/provider.yaml"],
+ ["samba/provider.yaml"],
+ ["samba/provider.yaml"],
+ ),
+ (
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ ],
+ ["samba/src/airflow/providers/samba/hooks/samba.py"],
+ ["samba/provider.yaml"],
+ ),
+ (
+ [
+ "ibm/mq/provider.yaml",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ ["ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py"],
+ ["ibm/mq/provider.yaml"],
+ ),
+ (
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ "ibm/mq/provider.yaml",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ ["ibm/mq/provider.yaml", "samba/provider.yaml"],
+ ),
+ (["unrelated/hooks/hook.py"], ["unrelated/hooks/hook.py"], []),
+ ),
+ ids=[
+ "provider_yaml_path_preserved",
+ "top_level_provider_hook_file_resolves",
+ "nested_namespace_provider_hook_file_resolves_without_known_list",
+ "mixed_input_dedups_and_sorts",
+ "no_provider_found",
+ ],
+ )
+ def test__resolve_provider_yaml_files(self, tmp_path, monkeypatch,
file_names, raw_files, expected):
+ monkeypatch.chdir(tmp_path)
+ for file_name in file_names:
+ _touch(tmp_path / file_name)
+
+ result = _resolve_provider_yaml_files(raw_files)
+
+ assert result == expected
