This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6c2c6ba18ce271dae08697e719d5609897ef1008 Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Jun 19 08:58:32 2020 +0200 Fix in-breeze CLI tools to work also on Linux (#9376) Instead of creating the links in the image (which did not work) the links are created now at the entry to the breeze image. The wrappers were not installed via Dockerfile and the ownership fixing did not work on Linux (cherry picked from commit ca8815188755866ae708c968df786c42043656c9) --- Dockerfile.ci | 2 -- scripts/ci/in_container/_in_container_utils.sh | 25 +++++++++++++------ scripts/ci/in_container/entrypoint_ci.sh | 4 +++ scripts/ci/run_cli_tool.sh | 34 ++++++++------------------ 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 651810d..f4ba142 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -307,8 +307,6 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ pip install ${ADDITIONAL_PYTHON_DEPS}; \ fi -RUN source <(bash scripts/ci/run_cli_tool.sh) - WORKDIR ${AIRFLOW_SOURCES} ENV PATH="${HOME}:${PATH}" diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 5d9fbb6..0eb3a8a 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -94,13 +94,24 @@ function in_container_cleanup_pycache() { # function in_container_fix_ownership() { if [[ ${HOST_OS:=} == "Linux" ]]; then - set +o pipefail - echo "Fixing ownership of mounted files" - sudo find "${AIRFLOW_SOURCES}" -print0 -user root \ - | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 - sudo find "/root/.aws" "/root/.azure" "/root/.config" "/root/.docker" -print0 -user root \ - | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 - set -o pipefail + DIRECTORIES_TO_FIX=( + "/tmp" + "/files" + "/root/.aws" + "/root/.azure" + "/root/.config/gcloud" + "/root/.docker" + "${AIRFLOW_SOURCES}" + ) + if [[ ${VERBOSE} == "true" ]]; then + echo "Fixing ownership of mounted files" + fi + sudo find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null \ + | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || + true >/dev/null 2>&1 + if [[ ${VERBOSE} == "true" ]]; then + echo "Fixed ownership of mounted files" + fi fi } diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index 2ea5a14..349b092 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -45,6 +45,10 @@ RUN_TESTS=${RUN_TESTS:="false"} CI=${CI:="false"} INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION:=""}" +# Create links for useful CLI tools +# shellcheck source=scripts/ci/run_cli_tool.sh +source <(bash scripts/ci/run_cli_tool.sh) + if [[ ${AIRFLOW_VERSION} == *1.10* || ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then export RUN_AIRFLOW_1_10="true" else diff --git a/scripts/ci/run_cli_tool.sh b/scripts/ci/run_cli_tool.sh index 8a57c35..3989601 100755 --- a/scripts/ci/run_cli_tool.sh +++ b/scripts/ci/run_cli_tool.sh @@ -45,12 +45,12 @@ if [ ! -L "${BASH_SOURCE[0]}" ] then SCRIPT_PATH=$(readlink -e "${BASH_SOURCE[0]}") # Direct execution - return installation script - >&2 echo "# CLI tool wrappers" - >&2 echo "#" - >&2 echo "# To install, run the following command:" - >&2 echo "# source <(bash ${SCRIPT_PATH@Q})" - >&2 echo "#" - >&2 echo "" + echo "# CLI tool wrappers" + echo "#" + echo "# To install, run the following command:" + echo "# source <(bash ${SCRIPT_PATH@Q})" + echo "#" + echo "" # Print installation script for NAME in "${SUPPORTED_TOOL_NAMES[@]}" do @@ -87,25 +87,20 @@ AWS_CREDENTIALS_DOCKER_ARGS=(-v "${HOST_HOME}/.aws:/root/.aws") AZURE_CREDENTIALS_DOCKER_ARGS=(-v "${HOST_HOME}/.azure:/root/.azure") GOOGLE_CREDENTIALS_DOCKER_ARGS=(-v "${HOST_HOME}/.config/gcloud:/root/.config/gcloud") -DIRECTORIES_TO_FIX=('/tmp/' '/files/') - COMMAND=("${@}") # Configure selected tool case "${TOOL_NAME}" in aws ) COMMON_DOCKER_ARGS+=("${AWS_CREDENTIALS_DOCKER_ARGS[@]}") - DIRECTORIES_TO_FIX+=("/root/.aws") IMAGE_NAME="amazon/aws-cli:latest" ;; az ) COMMON_DOCKER_ARGS+=("${AZURE_CREDENTIALS_DOCKER_ARGS[@]}") - DIRECTORIES_TO_FIX+=("/root/.azure") IMAGE_NAME="mcr.microsoft.com/azure-cli:latest" ;; gcloud | bq | gsutil ) COMMON_DOCKER_ARGS+=("${GOOGLE_CREDENTIALS_DOCKER_ARGS[@]}") - DIRECTORIES_TO_FIX+=("/root/.config/gcloud") IMAGE_NAME="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" COMMAND=("$TOOL_NAME" "${@}") ;; @@ -115,17 +110,11 @@ case "${TOOL_NAME}" in "${AZURE_CREDENTIALS_DOCKER_ARGS[@]}" "${AWS_CREDENTIALS_DOCKER_ARGS[@]}" ) - DIRECTORIES_TO_FIX+=( - "/root/.config/gcloud" - "/root/.aws" - "/root/.azure" - ) IMAGE_NAME="hashicorp/terraform:latest" ;; java ) # TODO: Should we add other credentials? COMMON_DOCKER_ARGS+=("${GOOGLE_CREDENTIALS_DOCKER_ARGS[@]}") - DIRECTORIES_TO_FIX+=("/root/.config/gcloud") IMAGE_NAME="openjdk:8-jre-slim" COMMAND=("/usr/local/openjdk-8/bin/java" "${@}") ;; @@ -150,19 +139,16 @@ if [ -t 0 ] ; then --tty ) fi - +set +e docker run "${TOOL_DOCKER_ARGS[@]}" "${IMAGE_NAME}" "${COMMAND[@]}" RES=$? # Set file permissions to the host user if [[ "${HOST_OS}" == "Linux" ]]; then - FIX_DOCKER_ARGS=(--rm) - FIX_DOCKER_ARGS+=("${COMMON_DOCKER_ARGS[@]}") - FIX_COMMAND=(bash -c - "find ${DIRECTORIES_TO_FIX[@]@Q} -user root -print0 | xargs --null chown '${HOST_USER_ID}.${HOST_GROUP_ID}' --no-dereference") - - docker run "${FIX_DOCKER_ARGS[@]}" "${AIRFLOW_CI_IMAGE}" "${FIX_COMMAND[@]}" >/dev/null 2>&1 + docker run --rm "${COMMON_DOCKER_ARGS[@]}" \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh \ + "${AIRFLOW_CI_IMAGE}" fi exit ${RES}
