This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 66871a00f3 Add pip caching for faster build (#35026)
66871a00f3 is described below
commit 66871a00f39834b60741fe31a51c11704919eb58
Author: Arthur <[email protected]>
AuthorDate: Tue Oct 31 19:37:21 2023 +0100
Add pip caching for faster build (#35026)
---------
Co-authored-by: Arthur Volant <[email protected]>
Co-authored-by: Jarek Potiuk <[email protected]>
---
Dockerfile | 26 ++++++++++++++++++--------
Dockerfile.ci | 4 ++--
docs/docker-stack/build-arg-ref.rst | 3 +++
docs/docker-stack/build.rst | 12 ++++++++++++
docs/docker-stack/changelog.rst | 4 ++++
scripts/docker/common.sh | 4 ++--
scripts/docker/entrypoint_prod.sh | 2 +-
7 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 1849db6f55..ff50615d43 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -553,9 +553,9 @@ function common::install_pip_version() {
echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
- pip install --disable-pip-version-check --no-cache-dir "pip @
${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
else
- pip install --disable-pip-version-check --no-cache-dir
"pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
@@ -1123,7 +1123,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
>&2 echo " the container starts, so it is only useful for testing
and trying out"
>&2 echo " of adding dependencies."
>&2 echo
- pip install --root-user-action ignore --no-cache-dir
${_PIP_ADDITIONAL_REQUIREMENTS}
+ pip install --root-user-action ignore ${_PIP_ADDITIONAL_REQUIREMENTS}
fi
@@ -1201,7 +1201,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit",
"-o", "nounset", "-o", "n
ARG PYTHON_BASE_IMAGE
ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
+ PIP_CACHE_DIR=/tmp/.cache/pip
ARG DEV_APT_DEPS=""
ARG ADDITIONAL_DEV_APT_DEPS=""
@@ -1386,8 +1387,15 @@ WORKDIR ${AIRFLOW_HOME}
COPY --from=scripts install_from_docker_context_files.sh install_airflow.sh \
install_additional_dependencies.sh /scripts/docker/
-# hadolint ignore=SC2086, SC2010
-RUN if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \
+# Useful for creating a cache id based on the underlying architecture,
preventing the use of cached python packages from
+# an incorrect architecture.
+ARG TARGETARCH
+# Value to be able to easily change cache id and therefore use a bare new cache
+ARG PIP_CACHE_EPOCH="0"
+
+# hadolint ignore=SC2086, SC2010, DL3042
+RUN
--mount=type=cache,id=$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$PIP_CACHE_EPOCH,target=/tmp/.cache/pip,uid=${AIRFLOW_UID}
\
+ if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \
bash /scripts/docker/install_from_docker_context_files.sh; \
fi; \
if ! airflow version 2>/dev/null >/dev/null; then \
@@ -1405,8 +1413,10 @@ RUN if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]];
then \
# In case there is a requirements.txt file in "docker-context-files" it will
be installed
# during the build additionally to whatever has been installed so far. It is
recommended that
# the requirements.txt contains only dependencies with == version specification
-RUN if [[ -f /docker-context-files/requirements.txt ]]; then \
- pip install --no-cache-dir --user -r
/docker-context-files/requirements.txt; \
+# hadolint ignore=DL3042
+RUN
--mount=type=cache,id=additional-requirements-$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$PIP_CACHE_EPOCH,target=/tmp/.cache/pip,uid=${AIRFLOW_UID}
\
+ if [[ -f /docker-context-files/requirements.txt ]]; then \
+ pip install --user -r /docker-context-files/requirements.txt; \
fi
##############################################################################################
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 48301ba232..299d7c8bd0 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -513,9 +513,9 @@ function common::install_pip_version() {
echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
- pip install --disable-pip-version-check --no-cache-dir "pip @
${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
else
- pip install --disable-pip-version-check --no-cache-dir
"pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
diff --git a/docs/docker-stack/build-arg-ref.rst
b/docs/docker-stack/build-arg-ref.rst
index 671ba6240f..aa3315473c 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-arg-ref.rst
@@ -278,3 +278,6 @@ Docker context files.
| |
| This allows to optimize iterations for |
| |
| Image builds and speeds up CI builds. |
+------------------------------------------+------------------------------------------+------------------------------------------+
+| ``PIP_CACHE_EPOCH`` | ``"0"``
| Allow to invalidate cache by passing a |
+| |
| new argument. |
++------------------------------------------+------------------------------------------+------------------------------------------+
diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst
index 6ede683014..5bad72c003 100644
--- a/docs/docker-stack/build.rst
+++ b/docs/docker-stack/build.rst
@@ -972,3 +972,15 @@ The architecture of the images
You can read more details about the images - the context, their parameters and
internal structure in the
`IMAGES.rst <https://github.com/apache/airflow/blob/main/IMAGES.rst>`_
document.
+
+
+Pip packages caching
+....................
+
+To enable faster iteration when building the image locally (especially if you
are testing different combination of
+python packages), pip caching has been enabled. The caching id is based on
four different parameters:
+
+1. ``PYTHON_BASE_IMAGE``: Avoid sharing same cache based on python version and
target os
+2. ``AIRFLOW_PIP_VERSION``
+3. ``TARGETARCH``: Avoid sharing architecture specific cached package
+4. ``PIP_CACHE_EPOCH``: Enable changing cache id by passing
``PIP_CACHE_EPOCH`` as ``--build-arg``
diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst
index d736b4e3ab..aae0194f36 100644
--- a/docs/docker-stack/changelog.rst
+++ b/docs/docker-stack/changelog.rst
@@ -58,6 +58,10 @@ here so that users affected can find the reason for the
changes.
Airflow 2.7
~~~~~~~~~~~
+* 2.7.4
+
+ * PIP caching for local builds has been enabled to speed up local custom
image building
+
* 2.7.3
* Add experimental feature for select type of MySQL Client libraries during
the build custom image via ``INSTALL_MYSQL_CLIENT_TYPE``
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index 701f1242f7..c8e3be7aa3 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -77,9 +77,9 @@ function common::install_pip_version() {
echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
- pip install --disable-pip-version-check --no-cache-dir "pip @
${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
else
- pip install --disable-pip-version-check --no-cache-dir
"pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
diff --git a/scripts/docker/entrypoint_prod.sh
b/scripts/docker/entrypoint_prod.sh
index be2c8c396d..add3ab7ec9 100755
--- a/scripts/docker/entrypoint_prod.sh
+++ b/scripts/docker/entrypoint_prod.sh
@@ -308,7 +308,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
>&2 echo " the container starts, so it is only useful for testing
and trying out"
>&2 echo " of adding dependencies."
>&2 echo
- pip install --root-user-action ignore --no-cache-dir
${_PIP_ADDITIONAL_REQUIREMENTS}
+ pip install --root-user-action ignore ${_PIP_ADDITIONAL_REQUIREMENTS}
fi
