This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 818c9dfafd178f8d53c6b18a5dc8bb375ce7d4df Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Sep 27 18:56:58 2020 +0200 Conditional MySQL Client installation (#11174) This is the second step of making the Production Docker Image more corporate-environment friendly, by making MySQL client installation optional. Instaling MySQL Client on Debian requires to reach out to oracle deb repositories which might not be approved by security teams when you build the images. Also not everyone needs MySQL client or might want to install their own MySQL client or MariaDB client - from their own repositories. This change makes the installation step separated out to script (with prod/dev installation option). The prod/dev separation is needed because MySQL needs to be installed with dev libraries in the "Build" segment of the image (requiring build essentials etc.) but in "Final" segment of the image only runtime libraries are needed. Part of #11171 Depends on #11173. (cherry picked from commit 044b44125744fc49808db12ab72eed5147aed474) --- .dockerignore | 1 + BREEZE.rst | 12 ++++++ Dockerfile | 71 ++++++++++----------------------- Dockerfile.ci | 25 +----------- breeze | 9 +++++ breeze-complete | 1 + docs/production-deployment.rst | 4 ++ scripts/ci/libraries/_build_images.sh | 2 + scripts/ci/libraries/_initialization.sh | 6 +-- scripts/docker/install_mysql.sh | 58 +++++++++++++++++++++++++++ 10 files changed, 113 insertions(+), 76 deletions(-) diff --git a/.dockerignore b/.dockerignore index 72daed1..7d29561 100644 --- a/.dockerignore +++ b/.dockerignore @@ -37,6 +37,7 @@ # Add those folders to the context so that they are available in the CI container !scripts/in_container +!scripts/docker # Add tests and kubernetes_tests to context. !tests diff --git a/BREEZE.rst b/BREEZE.rst index 4401da2..b998a87 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1222,6 +1222,10 @@ This is the current syntax for `./breeze <./breeze>`_: --disable-pip-cache Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --disable-mysql-client-installation + Disables installation of the mysql client which might be problematic if you are building + image in controlled environment. Only valid for production image. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1672,6 +1676,10 @@ This is the current syntax for `./breeze <./breeze>`_: --disable-pip-cache Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --disable-mysql-client-installation + Disables installation of the mysql client which might be problematic if you are building + image in controlled environment. Only valid for production image. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1985,6 +1993,10 @@ This is the current syntax for `./breeze <./breeze>`_: --disable-pip-cache Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --disable-mysql-client-installation + Disables installation of the mysql client which might be problematic if you are building + image in controlled environment. Only valid for production image. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/Dockerfile b/Dockerfile index 8ad448f..ca7aa7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,29 +122,11 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install MySQL client from Oracle repositories (Debian installs mariadb) -RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \ - && GNUPGHOME="$(mktemp -d)" \ - && export GNUPGHOME \ - && for KEYSERVER in $(shuf -e \ - ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \ - done \ - && gpg --export "${KEY}" | apt-key add - \ - && gpgconf --kill all \ - rm -rf "${GNUPGHOME}"; \ - apt-key list > /dev/null \ - && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - libmysqlclient-dev \ - mysql-client \ - && apt-get autoremove -yqq --purge \ - && apt-get clean && rm -rf /var/lib/apt/lists/* +ARG INSTALL_MYSQL_CLIENT="true" +ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} + +COPY scripts/docker scripts/docker +RUN ./scripts/docker/install_mysql.sh dev ARG AIRFLOW_REPO=apache/airflow ENV AIRFLOW_REPO=${AIRFLOW_REPO} @@ -169,9 +151,12 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} # In case of Production build image segment we want to pre-install master version of airflow # dependencies from github so that we do not have to always reinstall it from the scratch. RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ - pip install --user \ - "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ - --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \ + if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \ + AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \ + fi; \ + pip install --user \ + "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ + --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \ fi ARG AIRFLOW_SOURCES_FROM="." @@ -202,7 +187,11 @@ ENV SLUGIFY_USES_TEXT_UNIDECODE=${SLUGIFY_USES_TEXT_UNIDECODE} WORKDIR /opt/airflow -RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ +# remove mysql from extras if client is not installed +RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \ + AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \ + fi; \ + pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ --constraint "${AIRFLOW_CONSTRAINTS_URL}" && \ if [ -n "${ADDITIONAL_PYTHON_DEPS}" ]; then pip install --user ${ADDITIONAL_PYTHON_DEPS} \ --constraint "${AIRFLOW_CONSTRAINTS_URL}"; fi && \ @@ -307,29 +296,11 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install MySQL client from Oracle repositories (Debian installs mariadb) -RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \ - && GNUPGHOME="$(mktemp -d)" \ - && export GNUPGHOME \ - && for KEYSERVER in $(shuf -e \ - ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \ - done \ - && gpg --export "${KEY}" | apt-key add - \ - && gpgconf --kill all \ - rm -rf "${GNUPGHOME}"; \ - apt-key list > /dev/null \ - && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - libmysqlclient20 \ - mysql-client \ - && apt-get autoremove -yqq --purge \ - && apt-get clean && rm -rf /var/lib/apt/lists/* +ARG INSTALL_MYSQL_CLIENT="true" +ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} + +COPY scripts/docker scripts/docker +RUN ./scripts/docker/install_mysql.sh prod ENV AIRFLOW_UID=${AIRFLOW_UID} ENV AIRFLOW_GID=${AIRFLOW_GID} diff --git a/Dockerfile.ci b/Dockerfile.ci index b100f17..37c405d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -91,29 +91,8 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install MySQL client from Oracle repositories (Debian installs mariadb) -RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \ - && GNUPGHOME="$(mktemp -d)" \ - && export GNUPGHOME \ - && for KEYSERVER in $(shuf -e \ - ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \ - done \ - && gpg --export "${KEY}" | apt-key add - \ - && gpgconf --kill all \ - rm -rf "${GNUPGHOME}"; \ - apt-key list > /dev/null \ - && echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6" | tee -a /etc/apt/sources.list.d/mysql.list \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - libmysqlclient-dev \ - mysql-client \ - && apt-get autoremove -yqq --purge \ - && apt-get clean && rm -rf /var/lib/apt/lists/* +COPY scripts/docker scripts/docker +RUN ./scripts/docker/install_mysql.sh dev RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ diff --git a/breeze b/breeze index bd80d1f..3b802d4 100755 --- a/breeze +++ b/breeze @@ -960,6 +960,11 @@ function breeze::parse_arguments() { echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}" shift 2 ;; + --disable-mysql-client-installation) + export INSTALL_MYSQL_CLIENT="false" + echo "Install MySQL client: ${INSTALL_MYSQL_CLIENT}" + shift + ;; -D | --dockerhub-user) export DOCKERHUB_USER="${2}" echo "Dockerhub user ${DOCKERHUB_USER}" @@ -2136,6 +2141,10 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} --disable-pip-cache Disables GitHub PIP cache during the build. Useful if github is not reachable during build. +--disable-mysql-client-installation + Disables installation of the mysql client which might be problematic if you are building + image in controlled environment. Only valid for production image. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/breeze-complete b/breeze-complete index e281e43..5a09334 100644 --- a/breeze-complete +++ b/breeze-complete @@ -135,6 +135,7 @@ build-cache-local build-cache-pulled build-cache-disabled disable-pip-cache dockerhub-user: dockerhub-repo: github-registry github-repository: github-image-id: postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: +disable-mysql-client-installation load-default-connections load-example-dags " diff --git a/docs/production-deployment.rst b/docs/production-deployment.rst index aedb752..83f455d 100644 --- a/docs/production-deployment.rst +++ b/docs/production-deployment.rst @@ -326,6 +326,10 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | installing in case cassandra extra is | | | | used). | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``INSTALL_MYSQL_CLIENT`` | ``true`` | Whether MySQL client should be installed | +| | | The mysql extra is removed from extras | +| | | if the client is not installed | ++------------------------------------------+------------------------------------------+------------------------------------------+ There are build arguments that determine the installation mechanism of Apache Airflow for the production image. There are three types of build: diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 41856d8..76abff1 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -683,6 +683,7 @@ function build_images::build_prod_images() { "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ + --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -700,6 +701,7 @@ function build_images::build_prod_images() { "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ + --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 7d28a10..7155d29 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -219,9 +219,7 @@ function initialization::initialize_mount_variables() { verbosity::print_info verbosity::print_info "Mounting files folder to Docker" verbosity::print_info - EXTRA_DOCKER_FLAGS+=( - "-v" "${AIRFLOW_SOURCES}/files:/files" - ) + EXTRA_DOCKER_FLAGS+=("-v" "${AIRFLOW_SOURCES}/files:/files") fi EXTRA_DOCKER_FLAGS+=( @@ -325,6 +323,8 @@ function initialization::initialize_image_build_variables() { export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}" # whether pre cached pip packages are used during build export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}" + # by default install mysql client + export INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT:="true"} } # Determine version suffixes used to build backport packages diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh new file mode 100755 index 0000000..1ddc76b --- /dev/null +++ b/scripts/docker/install_mysql.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +declare -a packages + +if [[ "${1}" == "dev" ]]; then + packages=("libmysqlclient-dev" "mysql-client") +elif [[ "${1}" == "prod" ]]; then + packages=("libmysqlclient20" "mysql-client") +else + echo + echo "Specify either prod or dev" + echo +fi + +# Install MySQL Client during the container build +set -euo pipefail + +# Install MySQL client from Oracle repositories (Debian installs mariadb) +# But only if it is not disabled +if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then + KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" + readonly KEY + + GNUPGHOME="$(mktemp -d)" + export GNUPGHOME + set +e + for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) + do + gpg --keyserver "${keyserver}" --recv-keys "${KEY}" && break + done + set -e + gpg --export "${KEY}" | apt-key add - + gpgconf --kill all + rm -rf "${GNUPGHOME}" + apt-key list > /dev/null + echo "deb http://repo.mysql.com/apt/debian/ buster mysql-5.7" | tee -a /etc/apt/sources.list.d/mysql.list + apt-get update + apt-get install --no-install-recommends -y "${packages[@]}" + apt-get autoremove -yqq --purge + apt-get clean && rm -rf /var/lib/apt/lists/* +fi
