This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 39037cd0a7c54151815a91f6f5702f3ad3778a12 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Feb 11 23:49:07 2024 +0100 Add "devel-ci" extra with all possible dependencies to wheel (#37345) This is the next step of optimization of image caching. Since we do not have regular devel-* dependencies in the "wheel" installation any more, when we are using it to install from branch tip, we really installed plain airflow with no dependencies. This did not improve caching speed too much. Instead we now have a special (not advertised) devel-ci extra that consists of all 3rd-party dependencies we can have for Airflow - both development and production. This will bring much better optimisation in case pyproject.toml changes. (cherry picked from commit dbd91ee2de8b68fe8865bd726973c1c5e78889fc) --- Dockerfile | 5 +++-- Dockerfile.ci | 7 ++++--- docs/apache-airflow/extra-packages-ref.rst | 2 +- hatch_build.py | 9 +++++++++ pyproject.toml | 2 +- scripts/docker/install_airflow_dependencies_from_branch_tip.sh | 5 +++-- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e64fc58d5..b8ad6ce276 100644 --- a/Dockerfile +++ b/Dockerfile @@ -484,8 +484,9 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz"; common::install_pip_version - # Uninstall airflow to keep only the dependencies. In the future when planned https://github.com/pypa/pip/issues/11440 - # is implemented in pip we might be able to use this flag and skip the remove step. + # Uninstall airflow and providers to keep only the dependencies. In the future when + # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this + # flag and skip the remove step. pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo diff --git a/Dockerfile.ci b/Dockerfile.ci index f9657e42e5..2608b14ff4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -444,8 +444,9 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz"; common::install_pip_version - # Uninstall airflow to keep only the dependencies. In the future when planned https://github.com/pypa/pip/issues/11440 - # is implemented in pip we might be able to use this flag and skip the remove step. + # Uninstall airflow and providers to keep only the dependencies. In the future when + # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this + # flag and skip the remove step. pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo @@ -1061,7 +1062,7 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \ ARG AIRFLOW_REPO=apache/airflow ARG AIRFLOW_BRANCH=main # Airflow Extras installed -ARG AIRFLOW_EXTRAS="all" +ARG AIRFLOW_EXTRAS="devel-ci" ARG ADDITIONAL_AIRFLOW_EXTRAS="" # Allows to override constraints source ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index ffbe922524..0395481131 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -403,7 +403,7 @@ to get minimal, complete test environment with usual tools and dependencies need | devel-all-dbs | ``pip install -e '.[devel-all-dbs]'`` | Adds all libraries needed to test database providers | +---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ | devel-all | ``pip install -e '.[devel-all]'`` | Everything needed for development including Hadoop, all devel extras, | -| | | all doc extras. Generally: all possible dependencies | +| | | all doc extras. Generally: all possible dependencies except providers | +---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ | devel-ci | ``pip install -e '.[devel-ci]'`` | All dependencies required for CI tests (same as ``devel-all``) | +---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ diff --git a/hatch_build.py b/hatch_build.py index 2d6a4ed861..34a9ef8e27 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -153,12 +153,21 @@ class CustomBuildHook(BuildHookInterface[BuilderConfig]): Any modifications to the build data will be seen by the build target. """ if version == "standard": + all_possible_non_airlfow_dependencies = [] + for extra, deps in self.metadata.core.optional_dependencies.items(): + for dep in deps: + if not dep.startswith("apache-airflow"): + all_possible_non_airlfow_dependencies.append(dep) # remove devel dependencies from optional dependencies for standard packages self.metadata.core._optional_dependencies = { key: value for (key, value) in self.metadata.core.optional_dependencies.items() if not key.startswith("devel") and key not in ["doc", "doc-gen"] } + # This is the special dependency in wheel package that is used to install all possible + # 3rd-party dependencies for airflow for the CI image. It is exposed in the wheel package + # because we want to use for building the image cache from GitHub URL. + self.metadata.core._optional_dependencies["devel-ci"] = all_possible_non_airlfow_dependencies # Replace editable dependencies with provider dependencies for provider packages for dependency_id in DEPENDENCIES.keys(): if DEPENDENCIES[dependency_id]["state"] != "ready": diff --git a/pyproject.toml b/pyproject.toml index a8c4adf269..df9ce1d98c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -491,7 +491,7 @@ cassandra = [ crypto = [ ] druid = [ - "pache-airflow[apache-druid]", + "apache-airflow[apache-druid]", ] gcp = [ "apache-airflow[google]", diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 7a58f2ac15..f8bfabd1e2 100644 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -54,8 +54,9 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz"; common::install_pip_version - # Uninstall airflow to keep only the dependencies. In the future when planned https://github.com/pypa/pip/issues/11440 - # is implemented in pip we might be able to use this flag and skip the remove step. + # Uninstall airflow and providers to keep only the dependencies. In the future when + # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this + # flag and skip the remove step. pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo
