This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 08d9e0eeac1b6fd2af5db3d9b30b0b40f29cd1e8 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Nov 21 19:21:43 2020 +0100 Fixes unneeded docker-context-files added in CI (#12534) We do not need to add docker-context-files in CI before we run first "cache" PIP installation. Adding it might cause the effect that the cache will always be invalidated in case someone has a file added there before building and pushing the image. This PR fixes the problem by adding docker-context files later in the Dockerfile and changing the constraints location used in the "cache" step to always use the github constraints in this case. Closes #12509 (cherry picked from commit 37548f09acb91edd041565f52051f58610402cb3) --- Dockerfile | 3 ++- Dockerfile.ci | 11 ++++++----- IMAGES.rst | 9 ++++++++- scripts/ci/libraries/_build_images.sh | 35 +++++++++++++++-------------------- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ad3db9..00442bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -176,7 +176,8 @@ RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ fi; \ pip install --user \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" && pip uninstall --yes apache-airflow; \ + --constraint "https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"; \ + && pip uninstall --yes apache-airflow; \ fi ARG AIRFLOW_SOURCES_FROM="." diff --git a/Dockerfile.ci b/Dockerfile.ci index aa426ef..ac51a56 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -262,10 +262,6 @@ ENV AIRFLOW_LOCAL_PIP_WHEELS=${AIRFLOW_LOCAL_PIP_WHEELS} ARG INSTALL_AIRFLOW_VIA_PIP="true" ENV INSTALL_AIRFLOW_VIA_PIP=${INSTALL_AIRFLOW_VIA_PIP} -# If wheel files are found in /docker-context-files during installation -# they are also installed additionally to whatever is installed from Airflow. -COPY docker-context-files /docker-context-files - # In case of CI builds we want to pre-install master version of airflow dependencies so that # We do not have to always reinstall it from the scratch. # This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER. @@ -273,7 +269,8 @@ COPY docker-context-files /docker-context-files RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ - --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \ + --constraint "https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"; \ + && pip uninstall --yes apache-airflow; \ fi @@ -322,6 +319,10 @@ RUN if [[ ${INSTALL_AIRFLOW_VIA_PIP} == "true" ]]; then \ fi; \ fi +# If wheel files are found in /docker-context-files during installation +# they are also installed additionally to whatever is installed from Airflow. +COPY docker-context-files/ /docker-context-files/ + RUN if [[ ${AIRFLOW_LOCAL_PIP_WHEELS} != "true" ]]; then \ if ls /docker-context-files/*.whl 1> /dev/null 2>&1; then \ pip install --no-deps /docker-context-files/*.whl; \ diff --git a/IMAGES.rst b/IMAGES.rst index 8c913db..13ce935 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -399,7 +399,14 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | file has to be in docker context so | | | | it's best to place such file in | | | | one of the folders included in | -| | | dockerignore | +| | | dockerignore . for example in the | +| | | 'docker-context-files'. Note that the | +| | | location does not work for the first | +| | | stage of installation when the | +| | | stage of installation when the | +| | | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` is | +| | | set to true. Default location from | +| | | GitHub is used in this case. | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_LOCAL_PIP_WHEELS`` | ``false`` | If set to true, Airflow and it's | | | | dependencies are installed from locally | diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index fb756ba..5bd2d06 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -97,6 +97,19 @@ function build_images::forget_last_answer() { fi } +function build_images::confirm_via_terminal() { + echo > "${DETECTED_TERMINAL}" + echo > "${DETECTED_TERMINAL}" + echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}" + echo > "${DETECTED_TERMINAL}" + # Make sure to use output of tty rather than stdin/stdout when available - this way confirm + # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks) + # shellcheck disable=SC2094 + "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \ + <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}" + RES=$? +} + # Confirms if hte image should be rebuild and interactively checks it with the user. # In case iit needs to be rebuild. It only ask the user if it determines that the rebuild # is needed and that the rebuild is not already forced. It asks the user using available terminals @@ -144,29 +157,11 @@ function build_images::confirm_image_rebuild() { "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" RES=$? elif [[ ${DETECTED_TERMINAL:=$(tty)} != "not a tty" ]]; then - echo > "${DETECTED_TERMINAL}" - echo > "${DETECTED_TERMINAL}" - echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}" - echo > "${DETECTED_TERMINAL}" - # Make sure to use output of tty rather than stdin/stdout when available - this way confirm - # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks) - # shellcheck disable=SC2094 - "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \ - <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}" - RES=$? export DETECTED_TERMINAL + build_images::confirm_via_terminal elif [[ -c /dev/tty ]]; then export DETECTED_TERMINAL=/dev/tty - # Make sure to use /dev/tty first rather than stdin/stdout when available - this way confirm - # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks) - echo > "${DETECTED_TERMINAL}" - echo > "${DETECTED_TERMINAL}" - echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}" - echo > "${DETECTED_TERMINAL}" - # shellcheck disable=SC2094 - "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \ - <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}" - RES=$? + build_images::confirm_via_terminal else verbosity::print_info verbosity::print_info "No terminal, no stdin - quitting"
