This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e61640e2a1b Aggregate CI-image dependency groups so providers can
register non-default extras with a one-line change (#67130)
e61640e2a1b is described below
commit e61640e2a1bceb3ee864d2dd552f623ac76bfb2d
Author: Bugra Ozturk <[email protected]>
AuthorDate: Tue May 19 00:07:21 2026 +0200
Aggregate CI-image dependency groups so providers can register non-default
extras with a one-line change (#67130)
---
Dockerfile | 12 +++---
Dockerfile.ci | 12 +++---
contributing-docs/12_provider_distributions.rst | 43 ++++++++++++++++++++++
.../utils/constraints_version_check.py | 8 ++--
pyproject.toml | 13 +++++++
.../docker/install_airflow_when_building_images.sh | 12 +++---
6 files changed, 78 insertions(+), 22 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 532dd9c6dc7..1afcd732052 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1228,8 +1228,8 @@ function install_from_sources() {
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- uv sync --all-packages --resolution highest --group dev --group docs
--group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --resolution highest --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
set +x
@@ -1241,8 +1241,8 @@ function install_from_sources() {
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- if ! uv sync --all-packages --frozen --group dev --group docs --group
docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ if ! uv sync --all-packages --frozen --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
@@ -1257,8 +1257,8 @@ function install_from_sources() {
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv
sync without --frozen).${COLOR_RESET}"
echo
set -x
- uv sync --all-packages --group dev --group docs --group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 0c4e7f5a7fc..c8d686aa0e3 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -933,8 +933,8 @@ function install_from_sources() {
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- uv sync --all-packages --resolution highest --group dev --group docs
--group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --resolution highest --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
set +x
@@ -946,8 +946,8 @@ function install_from_sources() {
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- if ! uv sync --all-packages --frozen --group dev --group docs --group
docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ if ! uv sync --all-packages --frozen --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
@@ -962,8 +962,8 @@ function install_from_sources() {
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv
sync without --frozen).${COLOR_RESET}"
echo
set -x
- uv sync --all-packages --group dev --group docs --group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
diff --git a/contributing-docs/12_provider_distributions.rst
b/contributing-docs/12_provider_distributions.rst
index 832d89128a8..49a987cba21 100644
--- a/contributing-docs/12_provider_distributions.rst
+++ b/contributing-docs/12_provider_distributions.rst
@@ -117,6 +117,49 @@ you should do it before you make a PR with such changed
dependency changes
Also, you should rebuild the image ``breeze ci-image build`` or answer ``y``
when you are asked to rebuild the
image for the new dependencies to be used in the Breeze CI environment.
+Non-default provider extras
+---------------------------
+
+Some providers depend on packages that cannot be installed in CI by default —
for example a
+proprietary client library (IBM MQ's ``ibmmq``) or a native library that
requires system packages
+(Google's ``leveldb``/``plyvel`` needs ``libleveldb-dev``). Pulling these into
the default
+``uv sync`` would break CI on every runner that doesn't have the prerequisite
installed.
+
+For these cases, declare the dependency as an extra on the provider and
register it as its own
+group at the root, without adding it to ``dev``:
+
+1. **In the provider's ``pyproject.toml``** — keep the package under
``[project.optional-dependencies]``
+ so users can opt in with ``pip install
apache-airflow-providers-<id>[<extra>]``.
+
+2. **In the root ``pyproject.toml``** — add a dedicated entry under
``[dependency-groups]``,
+ then include it from the ``ci-image`` aggregate group. The ``ci-image``
group is the single
+ source of truth referenced by ``Dockerfile``, ``Dockerfile.ci``,
+ ``scripts/docker/install_airflow_when_building_images.sh``, and the breeze
constraints
+ checker — adding your group there is the only change required for the CI
image to pick it up:
+
+ .. code:: toml
+
+ [dependency-groups]
+ my-non-default-extra = [
+ "some-package>=1.2.3",
+ ]
+
+ ci-image = [
+ {include-group = "dev"},
+ {include-group = "docs"},
+ {include-group = "docs-gen"},
+ {include-group = "leveldb"},
+ {include-group = "my-non-default-extra"},
+ ]
+
+3. **If system libraries are required**, add them to
``scripts/docker/install_os_dependencies.sh``
+ so the CI image has the prerequisites before ``uv sync`` runs.
+
+Because the new group is *not* part of ``dev``, a plain ``uv sync`` on a
contributor's machine
+will not try to install it. The CI image installs it via ``ci-image``;
provider unit tests that
+import the proprietary or hard-to-build module should mock it (see
``providers/google/leveldb``
+for the established pattern).
+
Provider's cross-dependencies
-----------------------------
diff --git a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py
b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py
index 75c5ee22af5..ced0564d855 100755
--- a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py
+++ b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py
@@ -593,10 +593,10 @@ def explain_package_upgrade(
additional_args = []
if airflow_constraints_mode == "constraints-source-providers":
# In case of source constraints we also need to add all development
dependencies
- # to reflect exactly what is installed in the CI image by default
- additional_args.extend(
- ["--group", "dev", "--group", "docs", "--group", "docs-gen",
"--group", "leveldb"]
- )
+ # to reflect exactly what is installed in the CI image by default. The
``ci-image``
+ # group aggregates dev/docs/docs-gen plus any hard-to-install provider
extras
+ # (see root pyproject.toml).
+ additional_args.extend(["--group", "ci-image"])
with (
preserve_pyproject_file(AIRFLOW_ROOT_PATH / "pyproject.toml") as
airflow_pyproject,
preserve_pyproject_file(AIRFLOW_ROOT_PATH / "uv.lock"),
diff --git a/pyproject.toml b/pyproject.toml
index b09e1331701..77ec982702a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1371,6 +1371,19 @@ leveldb = [
"plyvel>=1.5.1"
]
+# Aggregate of every group the CI image installs by default. Single source of
+# truth referenced from the Dockerfiles, the image install script, and the
+# breeze constraints checker. When a provider needs to register a
+# hard-to-install extra (proprietary client, needs system libs, etc.), add a
+# dedicated [dependency-groups] entry above and include it here — no Dockerfile
+# or breeze edits required. See
contributing-docs/12_provider_distributions.rst.
+ci-image = [
+ {include-group = "dev"},
+ {include-group = "docs"},
+ {include-group = "docs-gen"},
+ {include-group = "leveldb"},
+]
+
[tool.uv]
# Bump this only when the project actually relies on a newer uv feature/fix.
It is a
# minimum contributors must install, NOT the uv CI pins to — keeping it in
lockstep
diff --git a/scripts/docker/install_airflow_when_building_images.sh
b/scripts/docker/install_airflow_when_building_images.sh
index 85f879786f3..772465678a8 100644
--- a/scripts/docker/install_airflow_when_building_images.sh
+++ b/scripts/docker/install_airflow_when_building_images.sh
@@ -57,8 +57,8 @@ function install_from_sources() {
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- uv sync --all-packages --resolution highest --group dev --group docs
--group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --resolution highest --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
set +x
@@ -70,8 +70,8 @@ function install_from_sources() {
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
- if ! uv sync --all-packages --frozen --group dev --group docs --group
docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ if ! uv sync --all-packages --frozen --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
@@ -86,8 +86,8 @@ function install_from_sources() {
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv
sync without --frozen).${COLOR_RESET}"
echo
set -x
- uv sync --all-packages --group dev --group docs --group docs-gen \
- --group leveldb ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
+ uv sync --all-packages --group ci-image \
+ ${extra_sync_flags} --no-binary-package lxml
--no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
