This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ae25a52ae3 Fix dependencies for celery and opentelemetry for Python
3.8 (#33579)
ae25a52ae3 is described below
commit ae25a52ae342c9e0bc3afdb21d613447c3687f6c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Aug 21 17:49:17 2023 +0200
Fix dependencies for celery and opentelemetry for Python 3.8 (#33579)
We used to have problems with `pip` backtracking when we relaxed
too much open-telemetry dependencies. It turned out that the
backtracting was only happening on Python 3.8 and that it was
ultimately caused by conflict between importlib_metadata between
Airflow and newer versions of opentelemetry (we had <5 for Python
3.8, they had >6 for all versions. The reason for limiting it in
Airflow was Celery that was not working well with importlib 5.
Since Celery 5.3 solved the problems (released 6th of June) we can
now relax the importlib_metadata limit and set Celery to version >=
5.3.0) which nicely resolves the conflict and there is no more
backtracking when trying to install newer versions of opentelemetry
for Python 3.8.
Fixes: #33577
---
airflow/providers/celery/provider.yaml | 2 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 2 +-
generated/provider_dependencies.json | 2 +-
setup.cfg | 8 ++------
setup.py | 2 +-
5 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/airflow/providers/celery/provider.yaml
b/airflow/providers/celery/provider.yaml
index d6d0b52c25..bf7de8d95a 100644
--- a/airflow/providers/celery/provider.yaml
+++ b/airflow/providers/celery/provider.yaml
@@ -45,7 +45,7 @@ dependencies:
# Uses Celery for CeleryExecutor, and we also know that Kubernetes Python
client follows SemVer
#
(https://docs.celeryq.dev/en/stable/contributing.html?highlight=semver#versions).
# Make sure that the limit here is synchronized with [celery] extra in the
airflow core
- - celery>=5.2.3,<6
+ - celery>=5.3.0,<6
- flower>=1.0.0
- google-re2>=1.0
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py
b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 2314f3aa9f..c9627c5ffb 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -87,7 +87,7 @@ def get_package_setup_metadata_hash() -> str:
try:
from importlib.metadata import distribution # type:
ignore[attr-defined]
except ImportError:
- from importlib_metadata import distribution # type: ignore[no-redef]
+ from importlib_metadata import distribution # type: ignore[no-redef,
assignment]
prefix = "Package config hash: "
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 34a70d4a4d..15ee5d67e9 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -245,7 +245,7 @@
"celery": {
"deps": [
"apache-airflow>=2.4.0",
- "celery>=5.2.3,<6",
+ "celery>=5.3.0,<6",
"flower>=1.0.0",
"google-re2>=1.0"
],
diff --git a/setup.cfg b/setup.cfg
index 399ff79bd9..c2ff2567f8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -102,10 +102,7 @@ install_requires =
graphviz>=0.12
gunicorn>=20.1.0
httpx
- # Importlib-metadata 5 is breaking Celery import due to regression it
introduced
- # This was tracked and fixed in https://github.com/celery/celery/pull/7785
but it is not released yet
- # We can remove the < 5.0.0 limitation when Celery 5.3.0 gets released and
we bump celery to >= 5.3.0
- importlib_metadata>=1.7,<5.0.0;python_version<"3.9"
+ importlib_metadata>=1.7;python_version<"3.9"
importlib_resources>=5.2;python_version<"3.9"
itsdangerous>=2.0
jinja2>=3.0.0
@@ -118,8 +115,7 @@ install_requires =
markupsafe>=1.1.1
marshmallow-oneofschema>=2.0.1
mdit-py-plugins>=0.3.0
- # Pip can not find a version that satisfies constraints if
opentelemetry-api is not pinned.
- opentelemetry-api==1.15.0
+ opentelemetry-api>=1.15.0
opentelemetry-exporter-otlp
packaging>=14.0
pathspec>=0.9.0
diff --git a/setup.py b/setup.py
index 9dd5ecbf8f..9a2daded8e 100644
--- a/setup.py
+++ b/setup.py
@@ -281,7 +281,7 @@ celery = [
# limiting minimum airflow version supported in celery provider due to the
# potential breaking changes in Airflow Core as well (celery is added as
extra, so Airflow
# core is not hard-limited via install-requires, only by extra).
- "celery>=5.2.3,<6"
+ "celery>=5.3.0,<6"
]
cgroups = [
# Cgroupspy 0.2.2 added Python 3.10 compatibility
