This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-2-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ea900a0f307d2c308750dc2fc71f6c09b7342383 Author: Jarek Potiuk <[email protected]> AuthorDate: Thu Jan 6 04:46:16 2022 +0100 Cleaner output of docker image building scripts (#20679) (cherry picked from commit fb8780013227a20462878b7f9286d083630c0bc2) --- Dockerfile | 5 +---- Dockerfile.ci | 3 +-- IMAGES.rst | 7 +++---- dev/README_RELEASE_PROVIDER_PACKAGES.md | 4 ++-- docs/docker-stack/build-arg-ref.rst | 2 +- scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 2 +- scripts/ci/libraries/_initialization.sh | 2 +- scripts/docker/common.sh | 20 ++++++++++-------- scripts/docker/compile_www_assets.sh | 24 +++++++++++++++++++--- scripts/docker/install_additional_dependencies.sh | 19 ++++++++--------- scripts/docker/install_airflow.sh | 15 +++++++------- ...install_airflow_dependencies_from_branch_tip.sh | 7 ++++--- .../docker/install_from_docker_context_files.sh | 16 ++++++++------- scripts/docker/install_mssql.sh | 6 +++--- scripts/docker/install_mysql.sh | 5 ++--- scripts/docker/install_pip_version.sh | 4 +++- scripts/in_container/_in_container_utils.sh | 4 ++-- scripts/in_container/prod/entrypoint_prod.sh | 2 +- .../in_container/run_prepare_airflow_packages.sh | 2 +- 19 files changed, 84 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index a28de84..bb24aa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ ARG AIRFLOW_UID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.2.4 +ARG AIRFLOW_PIP_VERSION=21.3.1 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"; # By default PIP has progress bar but you can disable it. @@ -186,9 +186,6 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ PATH=${PATH}:/root/.local/bin \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ - # Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image - # from the build image and always in non-editable mode - AIRFLOW_INSTALL_USER_FLAG="--user" \ AIRFLOW_INSTALL_EDITABLE_FLAG="" \ UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} diff --git a/Dockerfile.ci b/Dockerfile.ci index 48962e6..0d956c2 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -223,7 +223,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" ARG INSTALL_FROM_PYPI="true" -ARG AIRFLOW_PIP_VERSION=21.2.4 +ARG AIRFLOW_PIP_VERSION=21.3.1 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -258,7 +258,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ INSTALL_MYSQL_CLIENT="true" \ INSTALL_MSSQL_CLIENT="true" \ AIRFLOW_INSTALLATION_METHOD="." \ - AIRFLOW_INSTALL_USER_FLAG="" \ AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \ diff --git a/IMAGES.rst b/IMAGES.rst index 559ecdb..12f0b91 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -41,14 +41,13 @@ to run Kubernetes tests. See below for the list of arguments that should be prov production image from the local sources. The image is primarily optimised for size of the final image, but also for speed of rebuilds - the -'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing PIP dependencies. +'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing dependencies. It first pre-installs them from the right GitHub branch and only after that final airflow installation is -done from either local sources or remote location (PIP or GitHub repository). +done from either local sources or remote location (PyPI or GitHub repository). You can read more details about building, extending and customizing the PROD image in the `Latest documentation <https://airflow.apache.org/docs/docker-stack/index.html>`_ - CI image -------- @@ -492,7 +491,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined | | | | when installing runtime deps | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index e0b55db..3d928b0 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -611,9 +611,9 @@ provider packages. This is especially helpful when you want to test integrations additional tools. Below is an example Dockerfile, which installs providers for Google/ ```dockerfile -FROM apache/airflow:2.0.0 +FROM apache/airflow:2.2.3 -RUN pip install --upgrade --user apache-airflow-providers-google==2.0.0.rc1 +RUN pip install --user apache-airflow-providers-google==2.2.2.rc1 USER ${AIRFLOW_UID} ``` diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index f142b37..ab75d4a 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. | +------------------------------------------+------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. | +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+---------------------------------------------+ diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index ef920b4..6cf054f 100755 --- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -82,7 +82,7 @@ function create_virtualenv() { . "${virtualenv_path}/bin/activate" - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + pip install "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" local constraints=( --constraint diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index a95ce19..a0e9e30 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -421,7 +421,7 @@ function initialization::initialize_image_build_variables() { export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.4"} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.3.1"} export AIRFLOW_PIP_VERSION # We also pin version of wheel used to get consistent builds diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index d11715e..c3c33f7 100755 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -17,14 +17,11 @@ # under the License. set -euo pipefail -test -v INSTALL_MYSQL_CLIENT -test -v INSTALL_MSSQL_CLIENT -test -v AIRFLOW_INSTALL_USER_FLAG -test -v AIRFLOW_REPO -test -v AIRFLOW_BRANCH -test -v AIRFLOW_PIP_VERSION - -set -x +: "${INSTALL_MYSQL_CLIENT:?Should be true or false}" +: "${INSTALL_MSSQL_CLIENT:?Should be true or false}" +: "${AIRFLOW_REPO:?Should be set}" +: "${AIRFLOW_BRANCH:?Should be set}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION} @@ -42,7 +39,6 @@ function common::override_pip_version_if_needed() { fi } - function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then @@ -60,3 +56,9 @@ function common::get_constraints_location() { AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt" fi } + +function common::show_pip_version_and_location() { + echo "PATH=${PATH}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" +} diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh index 59a7017..5813f1b 100755 --- a/scripts/docker/compile_www_assets.sh +++ b/scripts/docker/compile_www_assets.sh @@ -17,7 +17,6 @@ # under the License. # shellcheck disable=SC2086 set -euo pipefail -set -x # Installs additional dependencies passed as Argument to the Docker build command function compile_www_assets() { @@ -35,8 +34,27 @@ function compile_www_assets() { www_dir="$(python -m site --user-site)/airflow/www" fi pushd ${www_dir} || exit 1 - yarn install --frozen-lockfile --no-cache - yarn run prod + set +e + yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt + local res=$? + if [[ ${res} != 0 ]]; then + >&2 echo + >&2 echo "Error when running yarn install:" + >&2 echo + >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt + exit 1 + fi + yarn run prod 2>/tmp/out-yarn-run.txt + res=$? + if [[ ${res} != 0 ]]; then + >&2 echo + >&2 echo "Error when running yarn install:" + >&2 echo + >&2 cat /tmp/out-yarn-run.txt && rm -f /tmp/out-yarn-run.txt + exit 1 + fi + rm -f /tmp/out-yarn-run.txt + set -e find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}" rm -rf "${www_dir}/node_modules" rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 4f9c05f..3ab4ac7 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -18,11 +18,10 @@ # shellcheck disable=SC2086 set -euo pipefail -test -v UPGRADE_TO_NEWER_DEPENDENCIES -test -v ADDITIONAL_PYTHON_DEPS -test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS -test -v AIRFLOW_INSTALL_USER_FLAG -test -v AIRFLOW_PIP_VERSION +: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" +: "${ADDITIONAL_PYTHON_DEPS:?Should be set}" +: "${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS:?Should be set}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -36,20 +35,19 @@ function install_additional_dependencies() { echo echo Installing additional dependencies while upgrading to newer dependencies echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip check else echo echo Installing additional dependencies upgrading only if needed echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} \ - --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip check fi } @@ -57,5 +55,6 @@ function install_additional_dependencies() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_additional_dependencies diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 61a30c4..e89e09b 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -47,7 +47,7 @@ function install_airflow() { echo Installing all packages with eager upgrade echo # eager upgrade - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + pip install --upgrade --upgrade-strategy eager \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then @@ -59,23 +59,23 @@ function install_airflow() { fi # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip check else \ echo echo Installing all packages with constraints and upgrade if needed echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" # then upgrade if needed without using constraints to account for new limits in setup.py - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ - "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ + "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip check fi @@ -84,5 +84,6 @@ function install_airflow() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 61aaa13..8ebb619 100755 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -39,12 +39,12 @@ function install_airflow_dependencies_from_branch_tip() { fi # Install latest set of dependencies using constraints. In case constraints were upgraded and there # are conflicts, this might fail, but it should be fixed in the following installation steps - pip install ${AIRFLOW_INSTALL_USER_FLAG} \ + pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true echo echo Uninstalling just airflow. Dependencies remain. echo @@ -54,5 +54,6 @@ function install_airflow_dependencies_from_branch_tip() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow_dependencies_from_branch_tip diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index d8ed6bc..0afa9b6 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -34,7 +34,6 @@ function install_airflow_and_providers_from_docker_context_files(){ local pip_flags=( # Don't quote this -- if it is empty we don't want it to create an # empty array element - ${AIRFLOW_INSTALL_USER_FLAG} --find-links="file:///docker-context-files" ) @@ -88,14 +87,14 @@ function install_airflow_and_providers_from_docker_context_files(){ --constraint /tmp/constraints.txt rm /tmp/constraints.txt # make sure correct PIP version is used \ - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install "pip==${AIRFLOW_PIP_VERSION}" # then upgrade if needed without using constraints to account for new limits in setup.py - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} fi # make sure correct PIP version is left installed - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install "pip==${AIRFLOW_PIP_VERSION}" pip check } @@ -104,7 +103,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # without dependencies. This is extremely useful in case you want to install via pip-download # method on air-gaped system where you do not want to download any dependencies from remote hosts # which is a requirement for serious installations -install_all_other_packages_from_docker_context_files() { +function install_all_other_packages_from_docker_context_files() { echo echo Force re-installing all other package from local files without dependencies echo @@ -113,15 +112,18 @@ install_all_other_packages_from_docker_context_files() { reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ grep -v apache_airflow | grep -v apache-airflow || true) if [[ -n "${reinstalling_other_packages}" ]]; then \ - pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install "pip==${AIRFLOW_PIP_VERSION}" fi } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow_and_providers_from_docker_context_files + +common::show_pip_version_and_location install_all_other_packages_from_docker_context_files diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh index b5f8b51..14eb0fe 100755 --- a/scripts/docker/install_mssql.sh +++ b/scripts/docker/install_mssql.sh @@ -15,13 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -set -exuo pipefail +set -euo pipefail function install_mssql_client() { echo echo Installing mssql client echo - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list + curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1 + curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list apt-get update -yqq apt-get upgrade -yqq ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh index 7983eb0..abd4108 100755 --- a/scripts/docker/install_mysql.sh +++ b/scripts/docker/install_mysql.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -set -exuo pipefail +set -euo pipefail declare -a packages MYSQL_VERSION="8.0" @@ -46,14 +46,13 @@ install_mysql_client() { for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) do - gpg --keyserver "${keyserver}" --recv-keys "${key}" && break + gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break done set -e gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - apt-key list > /dev/null 2>&1 echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh index 6e0c3c1..312a954 100755 --- a/scripts/docker/install_pip_version.sh +++ b/scripts/docker/install_pip_version.sh @@ -30,11 +30,13 @@ . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" function install_pip_version() { - pip install --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && mkdir -p /root/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && + mkdir -p ${HOME}/.local/bin } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_pip_version diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 1cf30dd..783ea7d 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -242,7 +242,7 @@ function install_released_airflow_version() { echo rm -rf "${AIRFLOW_SOURCES}"/*.egg-info - pip install --upgrade "apache-airflow==${version}" + pip install "apache-airflow==${version}" } function install_local_airflow_with_eager_upgrade() { @@ -332,7 +332,7 @@ function setup_provider_packages() { function install_supported_pip_version() { group_start "Install supported PIP version ${AIRFLOW_PIP_VERSION}" - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" group_end } diff --git a/scripts/in_container/prod/entrypoint_prod.sh b/scripts/in_container/prod/entrypoint_prod.sh index 2e57773..6f74ce6 100755 --- a/scripts/in_container/prod/entrypoint_prod.sh +++ b/scripts/in_container/prod/entrypoint_prod.sh @@ -290,7 +290,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then >&2 echo " the container starts, so it is onlny useful for testing and trying out" >&2 echo " of adding dependencies." >&2 echo - pip install --no-cache-dir --user ${_PIP_ADDITIONAL_REQUIREMENTS} + pip install --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS} fi diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh index 5148f64..eab50a6 100755 --- a/scripts/in_container/run_prepare_airflow_packages.sh +++ b/scripts/in_container/run_prepare_airflow_packages.sh @@ -34,7 +34,7 @@ function prepare_airflow_packages() { rm -rf -- *egg-info* rm -rf -- build - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" local packages=()
