This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4526519a7827fe03d312d731b1e808d9775cdfc4 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Jun 28 17:38:17 2020 +0200 More sensible docker caching strategy for Prod images (#9547) Local caching is now default strategy when building the Production image. You can still change it to pulled - similar to CI builds by providing the right build flag and this is what is used in CI by default. The flags in Breeze are now updated to be more eplanatory and friendly (build-cache-*) and a flag for "disabled" cache option is added as well. Also the Dockerfile and Dockerfile.ci files are not needed any more in the docker context. They used to be needed when we built the Kubernetes image in the container, but since we are now using production image directly - we do not need them any nmore. Combining setting the default strategy to local and removing the Dockerfile from the context has the nice effect that you can iterate much faster on the Production image without triggering rebuilds of half of the docker image as soon as the Dockerfile changes. (cherry picked from commit 6aabd9a86e9582ca762ee143ddce5b6ee1619684) --- .dockerignore | 2 - .github/workflows/ci.yml | 16 ++-- BREEZE.rst | 85 ++++++++++++++++++++-- IMAGES.rst | 52 +++++++++---- breeze | 48 ++++++++++-- breeze-complete | 6 +- ...image_on_ci.sh => ci_prepare_ci_image_on_ci.sh} | 3 - ...age_on_ci.sh => ci_prepare_prod_image_on_ci.sh} | 5 +- scripts/ci/libraries/_build_images.sh | 78 ++++++++++++++++---- scripts/ci/libraries/_initialization.sh | 5 -- 10 files changed, 232 insertions(+), 68 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0a89434..6f89516 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,8 +42,6 @@ !.dockerignore !pytest.ini !CHANGELOG.txt -!Dockerfile.ci -!Dockerfile !LICENSE !MANIFEST.in !NOTICE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67b9e50..bc86961 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Static checks" if: success() run: | @@ -83,7 +83,7 @@ jobs: with: python-version: '3.6' - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Build docs" run: ./scripts/ci/ci_docs.sh @@ -200,7 +200,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh @@ -232,7 +232,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh @@ -262,7 +262,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh @@ -293,7 +293,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh @@ -316,7 +316,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Generate requirements" run: ./scripts/ci/ci_generate_requirements.sh @@ -374,6 +374,6 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Free space" run: ./scripts/ci/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_image_on_ci.sh + run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh - name: "Push CI image ${{ matrix.python-version }}" run: ./scripts/ci/ci_push_ci_image.sh diff --git a/BREEZE.rst b/BREEZE.rst index 441c19b..ee0ff01 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -790,7 +790,9 @@ This is the current syntax for `./breeze <./breeze>`_: Builds docker image (CI or production) without entering the container. You can pass additional options to this command, such as '--force-build-image', - '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour. + '--force-pull-image', '--python', '--build-cache-local' or '-build-cache-pulled' + in order to modify build behaviour. + You can also pass '--production-image' flag to build production image rather than CI image. Flags: @@ -851,9 +853,32 @@ This is the current syntax for `./breeze <./breeze>`_: Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. - -L, --use-local-cache + -L, --build-cache-local Uses local cache to build images. No pulled images will be used, but results of local - builds in the Docker cache are used instead. + builds in the Docker cache are used instead. This will take longer than when the pulled + cache is used for the first time, but subsequent '--build-cache-local' builds will be + faster as they will use mostly the locally build cache. + + This is default strategy used by the Production image builds. + + -U, --build-cache-pulled + Uses images pulled from registry (either DockerHub or GitHub depending on + --github-registry flag) to build images. The pulled images will be used as cache. + Those builds are usually faster than when ''--build-cache-local'' with the exception if + the registry images are not yet updated. The DockerHub images are updated nightly and the + GitHub images are updated after merges to master so it might be that the images are still + outdated vs. the latest version of the Dockerfiles you are using. In this case, the + ''--build-cache-local'' might be faster, especially if you iterate and change the + Dockerfiles yourself. + + This is default strategy used by the CI image builds. + + -X, --build-cache-disabled + Disables cache during docker builds. This is useful if you want to make sure you want to + rebuild everything from scratch. + + This strategy is used by default for both Production and CI images for the scheduled + (nightly) builds in CI. -D, --dockerhub-user DockerHub user used to pull, push and build images. Default: apache. @@ -1199,9 +1224,32 @@ This is the current syntax for `./breeze <./breeze>`_: Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. - -L, --use-local-cache + -L, --build-cache-local Uses local cache to build images. No pulled images will be used, but results of local - builds in the Docker cache are used instead. + builds in the Docker cache are used instead. This will take longer than when the pulled + cache is used for the first time, but subsequent '--build-cache-local' builds will be + faster as they will use mostly the locally build cache. + + This is default strategy used by the Production image builds. + + -U, --build-cache-pulled + Uses images pulled from registry (either DockerHub or GitHub depending on + --github-registry flag) to build images. The pulled images will be used as cache. + Those builds are usually faster than when ''--build-cache-local'' with the exception if + the registry images are not yet updated. The DockerHub images are updated nightly and the + GitHub images are updated after merges to master so it might be that the images are still + outdated vs. the latest version of the Dockerfiles you are using. In this case, the + ''--build-cache-local'' might be faster, especially if you iterate and change the + Dockerfiles yourself. + + This is default strategy used by the CI image builds. + + -X, --build-cache-disabled + Disables cache during docker builds. This is useful if you want to make sure you want to + rebuild everything from scratch. + + This strategy is used by default for both Production and CI images for the scheduled + (nightly) builds in CI. #################################################################################################### @@ -1450,9 +1498,32 @@ This is the current syntax for `./breeze <./breeze>`_: Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. - -L, --use-local-cache + -L, --build-cache-local Uses local cache to build images. No pulled images will be used, but results of local - builds in the Docker cache are used instead. + builds in the Docker cache are used instead. This will take longer than when the pulled + cache is used for the first time, but subsequent '--build-cache-local' builds will be + faster as they will use mostly the locally build cache. + + This is default strategy used by the Production image builds. + + -U, --build-cache-pulled + Uses images pulled from registry (either DockerHub or GitHub depending on + --github-registry flag) to build images. The pulled images will be used as cache. + Those builds are usually faster than when ''--build-cache-local'' with the exception if + the registry images are not yet updated. The DockerHub images are updated nightly and the + GitHub images are updated after merges to master so it might be that the images are still + outdated vs. the latest version of the Dockerfiles you are using. In this case, the + ''--build-cache-local'' might be faster, especially if you iterate and change the + Dockerfiles yourself. + + This is default strategy used by the CI image builds. + + -X, --build-cache-disabled + Disables cache during docker builds. This is useful if you want to make sure you want to + rebuild everything from scratch. + + This strategy is used by default for both Production and CI images for the scheduled + (nightly) builds in CI. **************************************************************************************************** Flags for pulling/pushing Docker images (both CI and production) diff --git a/IMAGES.rst b/IMAGES.rst index 99eaf77..c829176 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -115,36 +115,56 @@ parameter to Breeze: Using cache during builds ========================= -Default mechanism used in Breeze for building images uses - as base - images puled from DockerHub or -GitHub Image Registry. This is in order to speed up local builds and CI builds - instead of 15 minutes +Default mechanism used in Breeze for building CI images uses images pulled from DockerHub or +GitHub Image Registry. This is done to speed up local builds and CI builds - instead of 15 minutes for rebuild of CI images, it takes usually less than 3 minutes when cache is used. For CI builds this is -usually the best strategy - to use default "pull" cache - same for Production Image - it's better to rely -on the "pull" mechanism rather than rebuild the image from the scratch. +usually the best strategy - to use default "pull" cache. This is default strategy when +`<BREEZE.rst>`_ builds are performed. -However when you are iterating on the images and want to rebuild them quickly and often you can provide the -``--use-local-cache`` flag to build commands - this way the standard docker mechanism based on local cache -will be used. The first time you run it, it will take considerably longer time than if you use the -default pull mechanism, but then when you do small, incremental changes to local sources, Dockerfile image -and scripts further rebuilds with --use-local-cache will be considerably faster. +For Production Image - which is far smaller and faster to build, it's better to use local build cache (the +standard mechanism that docker uses. This is the default strategy for production images when +`<BREEZE.rst>`_ builds are performed. The first time you run it, it will take considerably longer time than +if you use the pull mechanism, but then when you do small, incremental changes to local sources, +Dockerfile image= and scripts further rebuilds with local build cache will be considerably faster. + +You can also disable build cache altogether. This is the strategy used by the scheduled builds in CI - they +will always rebuild all the images from scratch. + +You can change the strategy by providing one of the ``--build-cache-local``, ``--build-cache-pulled`` or +even ``--build-cache-disabled`` flags when you run Breeze commands. For example: + +.. code-block:: bash + + ./breeze build-image --python 3.7 --build-cache-local + +Will build the CI image using local build cache (note that it will take quite a long time the first +time you run it). .. code-block:: bash - ./breeze build-image --python 3.7 --production-image --use-local-cache + ./breeze build-image --python 3.7 --production-image --build-cache-pulled + +Will build the production image with pulled images as cache. + + +.. code-block:: bash + + ./breeze build-image --python 3.7 --production-image --build-cache-disabled + +Will build the production image from the scratch. -You can also turn local docker caching by setting DOCKER_CACHE variable to "local" instead of the default -"pulled" and export it to Breeze. +You can also turn local docker caching by setting ``DOCKER_CACHE`` variable to "local", "pulled", +"disabled" and exporting it. .. code-block:: bash export DOCKER_CACHE="local" -You can also - if you really want - disable caching altogether by setting this variable to "no-cache". -This is how "scheduled" builds in our CI are run - those builds take a long time because they -always rebuild everything from scratch. +or .. code-block:: bash - export DOCKER_CACHE="no-cache" + export DOCKER_CACHE="disabled" Choosing image registry diff --git a/breeze b/breeze index cc6dacc..16826bc 100755 --- a/breeze +++ b/breeze @@ -376,7 +376,7 @@ EOF if [[ ${PRODUCTION_IMAGE} == "true" ]]; then cat <<EOF - Use production image. + Production image. Branch name: ${BRANCH_NAME} Docker image: ${AIRFLOW_PROD_IMAGE} @@ -384,7 +384,7 @@ EOF else cat <<EOF - Use CI image. + CI image. Branch name: ${BRANCH_NAME} Docker image: ${AIRFLOW_CI_IMAGE} @@ -675,11 +675,21 @@ function parse_arguments() { export DOCKER_CACHE="no-cache" export FORCE_BUILD_IMAGES="true" shift ;; - -L|--use-local-cache) + -L|--build-cache-local) echo "Use local cache to build images" echo export DOCKER_CACHE="local" shift ;; + -U|--build-cache-pulled) + echo "Use pulled cache to build images" + echo + export DOCKER_CACHE="pulled" + shift ;; + -X|--build-cache-disabled) + echo "Use disabled cache to build images" + echo + export DOCKER_CACHE="disabled" + shift ;; -P|--force-pull-images) echo "Force pulling images before build. Uses pulled images as cache." echo @@ -1052,7 +1062,9 @@ ${CMDNAME} build-image [FLAGS] Builds docker image (CI or production) without entering the container. You can pass additional options to this command, such as '--force-build-image', - '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour. + '--force-pull-image', '--python', '--build-cache-local' or '-build-cache-pulled' + in order to modify build behaviour. + You can also pass '--production-image' flag to build production image rather than CI image. Flags: @@ -1516,9 +1528,33 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. --L, --use-local-cache +-L, --build-cache-local Uses local cache to build images. No pulled images will be used, but results of local - builds in the Docker cache are used instead. + builds in the Docker cache are used instead. This will take longer than when the pulled + cache is used for the first time, but subsequent '--build-cache-local' builds will be + faster as they will use mostly the locally build cache. + + This is default strategy used by the Production image builds. + +-U, --build-cache-pulled + Uses images pulled from registry (either DockerHub or GitHub depending on + --github-registry flag) to build images. The pulled images will be used as cache. + Those builds are usually faster than when ''--build-cache-local'' with the exception if + the registry images are not yet updated. The DockerHub images are updated nightly and the + GitHub images are updated after merges to master so it might be that the images are still + outdated vs. the latest version of the Dockerfiles you are using. In this case, the + ''--build-cache-local'' might be faster, especially if you iterate and change the + Dockerfiles yourself. + + This is default strategy used by the CI image builds. + +-X, --build-cache-disabled + Disables cache during docker builds. This is useful if you want to make sure you want to + rebuild everything from scratch. + + This strategy is used by default for both Production and CI images for the scheduled + (nightly) builds in CI. + " } diff --git a/breeze-complete b/breeze-complete index af9a314..a3cc1eb 100644 --- a/breeze-complete +++ b/breeze-complete @@ -92,7 +92,8 @@ h p: b: i: K: V: l a: t: d: v y n q f -F P I E: C L +F P I E: C +L U X D: R: c g: G: " @@ -101,7 +102,8 @@ help python: backend: integration: kubernetes-mode: kubernetes-version: skip-mounting-local-sources install-airflow-version: install-airflow-reference: db-reset verbose assume-yes assume-no assume-quit forward-credentials -force-build-images force-pull-images production-image extras: force-clean-images use-local-cache +force-build-images force-pull-images production-image extras: force-clean-images +build-cache-local build-cache-pulled build-cache-disabled dockerhub-user: dockerhub-repo: github-registry github-organisation: github-repo: postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: diff --git a/scripts/ci/ci_prepare_image_on_ci.sh b/scripts/ci/ci_prepare_ci_image_on_ci.sh similarity index 91% copy from scripts/ci/ci_prepare_image_on_ci.sh copy to scripts/ci/ci_prepare_ci_image_on_ci.sh index 78cf323..8ced220 100755 --- a/scripts/ci/ci_prepare_image_on_ci.sh +++ b/scripts/ci/ci_prepare_ci_image_on_ci.sh @@ -18,7 +18,4 @@ # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" -export UPGRADE_TO_LATEST_REQUIREMENTS="false" -export SKIP_CI_IMAGE_CHECK="false" - build_ci_image_on_ci diff --git a/scripts/ci/ci_prepare_image_on_ci.sh b/scripts/ci/ci_prepare_prod_image_on_ci.sh similarity index 89% rename from scripts/ci/ci_prepare_image_on_ci.sh rename to scripts/ci/ci_prepare_prod_image_on_ci.sh index 78cf323..066a43b 100755 --- a/scripts/ci/ci_prepare_image_on_ci.sh +++ b/scripts/ci/ci_prepare_prod_image_on_ci.sh @@ -18,7 +18,4 @@ # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" -export UPGRADE_TO_LATEST_REQUIREMENTS="false" -export SKIP_CI_IMAGE_CHECK="false" - -build_ci_image_on_ci +build_prod_image_on_ci diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 0f1d9aa..7195b12 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -316,6 +316,11 @@ function print_build_info() { # Prepares all variables needed by the CI build. Depending on the configuration used (python version # DockerHub user etc. the variables are set so that other functions can use those variables. function prepare_ci_build() { + # We use pulled docker image cache by default for CI images to speed up the builds + export DOCKER_CACHE=${DOCKER_CACHE:="pulled"} + echo + echo "Using ${DOCKER_CACHE} cache strategy for the build." + echo export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" @@ -473,25 +478,39 @@ function rebuild_ci_image_if_needed_and_confirmed() { fi } - -# Builds the CI image in the CI environment. -# Depending on the type of build (push/pr/scheduled) it will either build it incrementally or -# from the scratch without cache (the latter for scheduled builds only) -function build_ci_image_on_ci() { - get_ci_environment - - # In case of CRON jobs we run builds without cache and upgrade to latest requirements +# Determines the strategy to be used for caching based on the type of CI job run. +# In case of CRON jobs we run builds without cache and upgrade to latest requirements +function determine_cache_strategy() { if [[ "${CI_EVENT_TYPE:=}" == "schedule" ]]; then echo echo "Disabling cache for scheduled jobs" echo - export DOCKER_CACHE="no-cache" + export DOCKER_CACHE="disabled" echo - echo "Requirements are upgraded to latest while running Docker build" + echo "Requirements are upgraded to latest for scheduled CI build" echo export UPGRADE_TO_LATEST_REQUIREMENTS="true" + else + echo + echo "Pull cache used for regular CI builds" + echo + export DOCKER_CACHE="pulled" + echo + echo "Requirements are not upgraded to latest ones for regular CI builds" + echo + export UPGRADE_TO_LATEST_REQUIREMENTS="false" fi +} + + +# Builds the CI image in the CI environment. +# Depending on the type of build (push/pr/scheduled) it will either build it incrementally or +# from the scratch without cache (the latter for scheduled builds only) +function build_ci_image_on_ci() { + export SKIP_CI_IMAGE_CHECK="false" + get_ci_environment + determine_cache_strategy prepare_ci_build rm -rf "${BUILD_CACHE_DIR}" @@ -500,12 +519,14 @@ function build_ci_image_on_ci() { rebuild_ci_image_if_needed # Disable force pulling forced above this is needed for the subsequent scripts so that - # They do not try to pull/build images again + # They do not try to pull/build images again. Also skip the image check entirely for + # the rest of the script unset FORCE_PULL_IMAGES unset FORCE_BUILD + export SKIP_CI_IMAGE_CHECK="true" } -# Builds CI image - depending on the caching strategy (pulled, local, no-cache) it +# Builds CI image - depending on the caching strategy (pulled, local, disabled) it # passes the necessary docker build flags via DOCKER_CACHE_CI_DIRECTIVE array # it also passes the right Build args depending on the configuration of the build # selected by Breeze flags or environment variables. @@ -521,7 +542,7 @@ function build_ci_image() { fi pull_ci_image_if_needed - if [[ "${DOCKER_CACHE}" == "no-cache" ]]; then + if [[ "${DOCKER_CACHE}" == "disabled" ]]; then export DOCKER_CACHE_CI_DIRECTIVE=("--no-cache") elif [[ "${DOCKER_CACHE}" == "local" ]]; then export DOCKER_CACHE_CI_DIRECTIVE=() @@ -570,6 +591,11 @@ Docker building ${AIRFLOW_CI_IMAGE}. # Prepares all variables needed by the CI build. Depending on the configuration used (python version # DockerHub user etc. the variables are set so that other functions can use those variables. function prepare_prod_build() { + # We use local docker image cache by default for Production images + export DOCKER_CACHE=${DOCKER_CACHE:="local"} + echo + echo "Using ${DOCKER_CACHE} cache strategy for the build." + echo if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then # When --install-airflow-reference is used then the image is build from github tag EXTRA_DOCKER_PROD_BUILD_FLAGS=( @@ -638,7 +664,29 @@ function prepare_prod_build() { } -# Builds PROD image - depending on the caching strategy (pulled, local, no-cache) it +# Builds the prod image in the CI environment. +# Depending on the type of build (push/pr/scheduled) it will either build it incrementally or +# from the scratch without cache (the latter for scheduled builds only) +function build_prod_image_on_ci() { + get_prod_environment + + determine_cache_strategy + + prepare_prod_build + + rm -rf "${BUILD_CACHE_DIR}" + mkdir -pv "${BUILD_CACHE_DIR}" + + build_prod_image + + # Disable force pulling forced above this is needed for the subsequent scripts so that + # They do not try to pull/build images again + unset FORCE_PULL_IMAGES + unset FORCE_BUILD +} + + +# Builds PROD image - depending on the caching strategy (pulled, local, disabled) it # passes the necessary docker build flags via DOCKER_CACHE_PROD_DIRECTIVE and # DOCKER_CACHE_PROD_BUILD_DIRECTIVE (separate caching options are needed for "build" segment of the image) # it also passes the right Build args depending on the configuration of the build @@ -647,7 +695,7 @@ function build_prod_image() { print_build_info pull_prod_images_if_needed - if [[ "${DOCKER_CACHE}" == "no-cache" ]]; then + if [[ "${DOCKER_CACHE}" == "disabled" ]]; then export DOCKER_CACHE_PROD_DIRECTIVE=("--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}") export DOCKER_CACHE_PROD_BUILD_DIRECTIVE=("--no-cache") elif [[ "${DOCKER_CACHE}" == "local" ]]; then diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 93a6ec5..c0c171a 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -161,11 +161,6 @@ function initialize_common_environment { done fi - # We use pulled docker image cache by default to speed up the builds - # but we can also set different docker caching strategy (for example we can use local cache - # to build the images in case we iterate with the image - export DOCKER_CACHE=${DOCKER_CACHE:="pulled"} - # By default we are not upgrading to latest requirements when building Docker CI image # This will only be done in cron jobs export UPGRADE_TO_LATEST_REQUIREMENTS=${UPGRADE_TO_LATEST_REQUIREMENTS:="false"}
