This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new dbd91ee2de Add "devel-ci" extra with all possible dependencies to
wheel (#37345)
dbd91ee2de is described below
commit dbd91ee2de8b68fe8865bd726973c1c5e78889fc
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Feb 11 23:49:07 2024 +0100
Add "devel-ci" extra with all possible dependencies to wheel (#37345)
This is the next step of optimization of image caching. Since we
do not have regular devel-* dependencies in the "wheel" installation any
more, when we are using it to install from branch tip, we really
installed plain airflow with no dependencies. This did not improve
caching speed too much. Instead we now have a special (not advertised)
devel-ci extra that consists of all 3rd-party dependencies we can have
for Airflow - both development and production.
This will bring much better optimisation in case pyproject.toml changes.
---
Dockerfile | 5 +++--
Dockerfile.ci | 7 ++++---
docs/apache-airflow/extra-packages-ref.rst | 2 +-
hatch_build.py | 9 +++++++++
pyproject.toml | 2 +-
scripts/docker/install_airflow_dependencies_from_branch_tip.sh | 5 +++--
6 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 2ced62a655..77a8e4e8d6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -455,8 +455,9 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
common::install_pip_version
- # Uninstall airflow to keep only the dependencies. In the future when
planned https://github.com/pypa/pip/issues/11440
- # is implemented in pip we might be able to use this flag and skip the
remove step.
+ # Uninstall airflow and providers to keep only the dependencies. In the
future when
+ # planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
+ # flag and skip the remove step.
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes
2>/dev/null || true
set +x
echo
diff --git a/Dockerfile.ci b/Dockerfile.ci
index a80eed2ac3..4456ff89c0 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -415,8 +415,9 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
common::install_pip_version
- # Uninstall airflow to keep only the dependencies. In the future when
planned https://github.com/pypa/pip/issues/11440
- # is implemented in pip we might be able to use this flag and skip the
remove step.
+ # Uninstall airflow and providers to keep only the dependencies. In the
future when
+ # planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
+ # flag and skip the remove step.
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes
2>/dev/null || true
set +x
echo
@@ -1062,7 +1063,7 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \
ARG AIRFLOW_REPO=apache/airflow
ARG AIRFLOW_BRANCH=main
# Airflow Extras installed
-ARG AIRFLOW_EXTRAS="all"
+ARG AIRFLOW_EXTRAS="devel-ci"
ARG ADDITIONAL_AIRFLOW_EXTRAS=""
# Allows to override constraints source
ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow"
diff --git a/docs/apache-airflow/extra-packages-ref.rst
b/docs/apache-airflow/extra-packages-ref.rst
index ea813aeb49..839184c215 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -407,7 +407,7 @@ to get minimal, complete test environment with usual tools
and dependencies need
| devel-all-dbs | ``pip install -e '.[devel-all-dbs]'`` |
Adds all libraries needed to test database providers |
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+
| devel-all | ``pip install -e '.[devel-all]'`` |
Everything needed for development including Hadoop, all devel extras, |
-| | |
all doc extras. Generally: all possible dependencies |
+| | |
all doc extras. Generally: all possible dependencies except providers |
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+
| devel-ci | ``pip install -e '.[devel-ci]'`` |
All dependencies required for CI tests (same as ``devel-all``) |
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+
diff --git a/hatch_build.py b/hatch_build.py
index 2d6a4ed861..34a9ef8e27 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -153,12 +153,21 @@ class CustomBuildHook(BuildHookInterface[BuilderConfig]):
Any modifications to the build data will be seen by the build target.
"""
if version == "standard":
+ all_possible_non_airlfow_dependencies = []
+ for extra, deps in
self.metadata.core.optional_dependencies.items():
+ for dep in deps:
+ if not dep.startswith("apache-airflow"):
+ all_possible_non_airlfow_dependencies.append(dep)
# remove devel dependencies from optional dependencies for
standard packages
self.metadata.core._optional_dependencies = {
key: value
for (key, value) in
self.metadata.core.optional_dependencies.items()
if not key.startswith("devel") and key not in ["doc",
"doc-gen"]
}
+ # This is the special dependency in wheel package that is used to
install all possible
+ # 3rd-party dependencies for airflow for the CI image. It is
exposed in the wheel package
+ # because we want to use for building the image cache from GitHub
URL.
+ self.metadata.core._optional_dependencies["devel-ci"] =
all_possible_non_airlfow_dependencies
# Replace editable dependencies with provider dependencies for
provider packages
for dependency_id in DEPENDENCIES.keys():
if DEPENDENCIES[dependency_id]["state"] != "ready":
diff --git a/pyproject.toml b/pyproject.toml
index 36411e4f3f..ffa54b8d19 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -487,7 +487,7 @@ cassandra = [
crypto = [
]
druid = [
- "pache-airflow[apache-druid]",
+ "apache-airflow[apache-druid]",
]
gcp = [
"apache-airflow[google]",
diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
index 7a58f2ac15..f8bfabd1e2 100644
--- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
+++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -54,8 +54,9 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
common::install_pip_version
- # Uninstall airflow to keep only the dependencies. In the future when
planned https://github.com/pypa/pip/issues/11440
- # is implemented in pip we might be able to use this flag and skip the
remove step.
+ # Uninstall airflow and providers to keep only the dependencies. In the
future when
+ # planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
+ # flag and skip the remove step.
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes
2>/dev/null || true
set +x
echo
