This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d414ff941b0 Improve cross-provider dependency detection (#45784)
d414ff941b0 is described below
commit d414ff941b0f378159822662dc3511fd1aace5ab
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jan 18 21:25:36 2025 +0100
Improve cross-provider dependency detection (#45784)
When another provider is only used in example dags, we should not
make it "cross-provider" dependency - this happened to "edge" provider
having "common.compat" as cross-provider dependency.
It has been previously only implemented for "standard" provider but
this is a universal rule that could be applied to any other provider.
---
generated/provider_dependencies.json | 4 +---
scripts/ci/pre_commit/update_providers_dependencies.py | 12 ++++++++----
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 8797be6c641..c0a2e488d0f 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -535,9 +535,7 @@
"plugin-class":
"airflow.providers.edge.plugins.edge_executor_plugin.EdgeExecutorPlugin"
}
],
- "cross-providers-deps": [
- "common.compat"
- ],
+ "cross-providers-deps": [],
"excluded-python-versions": [],
"state": "not-ready"
},
diff --git a/scripts/ci/pre_commit/update_providers_dependencies.py
b/scripts/ci/pre_commit/update_providers_dependencies.py
index debe6155212..f5c9c018abb 100755
--- a/scripts/ci/pre_commit/update_providers_dependencies.py
+++ b/scripts/ci/pre_commit/update_providers_dependencies.py
@@ -206,11 +206,15 @@ def check_if_different_provider_used(file_path: Path) ->
None:
warnings.append(f"The provider {imported_provider} from
{file_path} cannot be found.")
continue
- if imported_provider == "standard":
- # Standard -- i.e. BashOperator is used in a lot of example dags,
but we don't want to mark this
+ if "/example_dags/" in file_path.as_posix():
+ # If provider is used in a example dags, we don't want to mark this
# as a provider cross dependency
- if file_path.name == "celery_executor_utils.py" or
"/example_dags/" in file_path.as_posix():
- continue
+ continue
+ if imported_provider == "standard" and file_path.name ==
"celery_executor_utils.py":
+ # some common standard operators are pre-imported in celery when
it starts in order to speed
+ # up the task startup time - but it does not mean that standard
provider is a cross-provider
+ # dependency of the celery executor
+ continue
if imported_provider:
if file_provider != imported_provider:
ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider)
