This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4b8fbf2f608360ccc86cf539423e5212f5c5778c Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Jun 9 09:33:16 2020 +0200 Improved cloud tool available in the trimmed down CI container (#9167) * Improved cloud tool available in the trimmed down CI container The tools now have shebangs which make them available for python tools. Also /opt/airflow is now mounted from the host Airflow sources which makes it possible for the tools to copy files directly to/from the sources of Airflow. It also contains one small change for Linux users - the files created by docker gcloud are created with root user so in order to fix that the directories mounted from the host are fixed when you exit the tool - their ownership is changed to be owned by the host user (cherry picked from commit de9d3401f9668c703a8f0665769fd17268f607ed) --- BREEZE.rst | 3 ++ Dockerfile.ci | 21 +------- breeze | 1 + scripts/ci/ci_docs.sh | 3 ++ scripts/ci/ci_fix_ownership.sh | 3 ++ scripts/ci/ci_flake8.sh | 6 +++ scripts/ci/ci_mypy.sh | 3 ++ scripts/ci/docker-compose/forward-credentials.yml | 2 + scripts/ci/docker-compose/local-prod.yml | 2 + scripts/ci/docker-compose/local.yml | 2 + scripts/ci/in_container/_in_container_utils.sh | 16 ++++-- scripts/ci/libraries/_initialization.sh | 7 +++ scripts/ci/libraries/_runs.sh | 6 +++ scripts/ci/prepare_tool_scripts.sh | 64 +++++++++++++++++++++++ 14 files changed, 114 insertions(+), 25 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index aec975c..3a1a6a0 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -243,6 +243,9 @@ it is downloaded, it will stay until you remove the downloaded images from your For each of those CLI credentials are taken (automatically) from the credentials you have defined in your ${HOME} directory on host. +Those tools also have host Airflow source directory mounted in /opt/airflow path +so you can directly transfer files to/from your airflow host sources. + Those are currently installed CLIs (they are available as aliases to the docker commands): +-----------------------+----------+-------------------------------------------------+-------------------+ diff --git a/Dockerfile.ci b/Dockerfile.ci index 8035fa2..8051431 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -307,26 +307,7 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ pip install ${ADDITIONAL_PYTHON_DEPS}; \ fi -RUN \ - AWSCLI_IMAGE="amazon/aws-cli:latest" && \ - AZURECLI_IMAGE="mcr.microsoft.com/azure-cli:latest" && \ - GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" && \ - echo "docker run --rm -it -v \${HOST_HOME}/.aws:/root/.aws ${AWSCLI_IMAGE} \"\$@\"" \ - > /usr/bin/aws && \ - echo "docker pull ${AWSCLI_IMAGE}" > /usr/bin/aws-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.azure:/root/.azure ${AZURECLI_IMAGE} \"\$@\"" \ - > /usr/bin/az && \ - echo "docker pull ${AZURECLI_IMAGE}" > /usr/bin/az-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} bq \"\$@\"" \ - > /usr/bin/bq && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/bq-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gcloud \"\$@\"" \ - > /usr/bin/gcloud && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gcloud-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gsutil \"\$@\"" \ - > /usr/bin/gsutil && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gsutil-update && \ - chmod a+x /usr/bin/aws /usr/bin/az /usr/bin/bq /usr/bin/gcloud /usr/bin/gsutil +RUN scripts/ci/prepare_tool_scripts.sh WORKDIR ${AIRFLOW_SOURCES} diff --git a/breeze b/breeze index d9cfd92..005f917 100755 --- a/breeze +++ b/breeze @@ -443,6 +443,7 @@ export POSTGRES_HOST_PORT="${POSTGRES_HOST_PORT}" export POSTGRES_VERSION="${POSTGRES_VERSION}" export MYSQL_HOST_PORT="${MYSQL_HOST_PORT}" export MYSQL_VERSION="${MYSQL_VERSION}" +export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" export MYSQL_ENCODING="${MYSQL_ENCODING}" export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}" export AIRFOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}" diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index e84de9b..417f6a6 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -28,6 +28,9 @@ function run_docs() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build" \ diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/ci_fix_ownership.sh index 775cc58..81f5ac1 100755 --- a/scripts/ci/ci_fix_ownership.sh +++ b/scripts/ci/ci_fix_ownership.sh @@ -33,11 +33,14 @@ export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE} export WEBSERVER_HOST_PORT=28080 HOST_USER_ID="$(id -ur)" HOST_GROUP_ID="$(id -gr)" +HOST_OS="$(uname -s)" export HOST_USER_ID export HOST_GROUP_ID +export HOST_OS docker-compose \ -f "${MY_DIR}/docker-compose/base.yml" \ -f "${MY_DIR}/docker-compose/local.yml" \ + -f "${MY_DIR}/docker-compose/forward-credentials.yml" \ run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index 5a2da56..fac9be4 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -31,6 +31,9 @@ function run_flake8() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ @@ -43,6 +46,9 @@ function run_flake8() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index 8898d30..962d2c0 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -33,6 +33,9 @@ function run_mypy() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ "-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/docker-compose/forward-credentials.yml b/scripts/ci/docker-compose/forward-credentials.yml index 875b1ce..0eb0dbb 100644 --- a/scripts/ci/docker-compose/forward-credentials.yml +++ b/scripts/ci/docker-compose/forward-credentials.yml @@ -22,6 +22,8 @@ services: # Useful for gcloud/aws/kubernetes etc. authorisation to be passed # To inside docker. Use with care - your credentials will be available to # Everything you install in Docker + # If you add it here - also add it to "in_container_fix_ownership" method in + # the _in_container_utils.sh file to make it friendly for Linux users volumes: - ${HOME}/.aws:/root/.aws:cached - ${HOME}/.azure:/root/.azure:cached diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml index ae8317d..a82b4f8 100644 --- a/scripts/ci/docker-compose/local-prod.yml +++ b/scripts/ci/docker-compose/local-prod.yml @@ -40,4 +40,6 @@ services: - HOST_USER_ID - HOST_GROUP_ID - HOST_HOME=${HOME} + - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + - HOST_OS - PYTHONDONTWRITEBYTECODE diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 9603417..3c9e40b 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -60,6 +60,8 @@ services: - HOST_USER_ID - HOST_GROUP_ID - HOST_HOME=${HOME} + - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + - HOST_OS - PYTHONDONTWRITEBYTECODE ports: - "${WEBSERVER_HOST_PORT}:8080" diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 9197a04..5d9fbb6 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -89,13 +89,19 @@ function in_container_cleanup_pycache() { # # Fixes ownership of files generated in container - if they are owned by root, they will be owned by -# The host user. +# The host user. Only needed if the host is Linux - on Mac, ownership of files is automatically +# changed to the Host user via osxfs filesystem # function in_container_fix_ownership() { - set +o pipefail - sudo find "${AIRFLOW_SOURCES}" -print0 -user root \ - | sudo xargs --null chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 - set -o pipefail + if [[ ${HOST_OS:=} == "Linux" ]]; then + set +o pipefail + echo "Fixing ownership of mounted files" + sudo find "${AIRFLOW_SOURCES}" -print0 -user root \ + | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 + sudo find "/root/.aws" "/root/.azure" "/root/.config" "/root/.docker" -print0 -user root \ + | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 + set -o pipefail + fi } function in_container_go_to_airflow_sources() { diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 9f86ba4..c8cd9f7 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -114,6 +114,13 @@ function initialize_common_environment { HOST_GROUP_ID="$(id -gr)" export HOST_GROUP_ID + # Set host OS. This is used to set the ownership properly when exiting + # The container on Linux - all files created inside docker are created with root user + # but they should be restored back to the host user + HOST_OS="$(uname -s)" + export HOST_OS + + # Add the right volume mount for sources, depending which mount strategy is used if [[ ${MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then print_info diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index cd5a44a..76b674d 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -25,6 +25,9 @@ function run_docs() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build" \ @@ -40,6 +43,9 @@ function run_generate_requirements() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --env UPGRADE_WHILE_GENERATING_REQUIREMENTS \ --env PYTHON_MAJOR_MINOR_VERSION \ --env CHECK_REQUIREMENTS_ONLY \ diff --git a/scripts/ci/prepare_tool_scripts.sh b/scripts/ci/prepare_tool_scripts.sh new file mode 100755 index 0000000..7a98c50 --- /dev/null +++ b/scripts/ci/prepare_tool_scripts.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail + +function prepare_tool_script() { + IMAGE="${1}" + VOLUME="${2}" + TOOL="${3}" + COMMAND="${4:-}" + + TARGET_TOOL_PATH="/usr/bin/${TOOL}" + TARGET_TOOL_UPDATE_PATH="/usr/bin/${TOOL}-update" + + cat >"${TARGET_TOOL_PATH}" <<EOF +#!/usr/bin/env bash +docker run --rm -it \ + -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \ + -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \ + -v "\${HOST_AIRFLOW_SOURCES}:/opt/airflow" \ + -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \ + "${IMAGE}" ${COMMAND} "\$@" +RES=\$? +if [[ \${HOST_OS} == "Linux" ]]; then + docker run --rm \ + -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \ + -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \ + -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \ + "\${AIRFLOW_CI_IMAGE}" bash -c \ + "find '/tmp/' '/files/' '/root/${VOLUME}' -user root -print0 | xargs --null chown '\${HOST_USER_ID}.\${HOST_GROUP_ID}' --no-dereference" >/dev/null 2>&1 +fi +exit \${RES} +EOF + + cat >"${TARGET_TOOL_UPDATE_PATH}" <<EOF +#!/usr/bin/env bash +docker pull "${IMAGE}" +EOF + + chmod a+x "${TARGET_TOOL_PATH}" "${TARGET_TOOL_UPDATE_PATH}" +} + +GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + +prepare_tool_script "amazon/aws-cli:latest" ".aws" aws +prepare_tool_script "mcr.microsoft.com/azure-cli:latest" ".azure" az az +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" bq bq +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gcloud gcloud +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gsutil gsutil
