This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-0-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f840d1674d3cc3634764fe2ee911332d9872df3a Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Apr 2 07:31:00 2021 +0200 Better handling of docker command (#15080) Not all docker commands are replaced with functions now. Earlier wer replaced all docker commands with a function to be able to capture docker commands used and display it with -v for breeze. This has proven to be harmful as this is an unexpected behaviour for a docker command. This change introduces docker_v command which outputs the command when needed. (cherry picked from commit 535e1a8e692ba28ad8ce9474a66b941af1df4875) --- breeze | 2 +- .../enter_breeze_provider_package_tests.sh | 2 +- scripts/ci/libraries/_build_images.sh | 42 +++++++++++----------- scripts/ci/libraries/_docker_engine_resources.sh | 12 +++---- scripts/ci/libraries/_initialization.sh | 2 ++ scripts/ci/libraries/_kind.sh | 2 +- scripts/ci/libraries/_push_pull_remove_images.sh | 28 +++++++-------- scripts/ci/libraries/_runs.sh | 10 +++--- scripts/ci/libraries/_start_end.sh | 2 +- scripts/ci/libraries/_verbosity.sh | 6 ++-- scripts/ci/libraries/_verify_image.sh | 8 ++--- .../ci_install_and_test_provider_packages.sh | 2 +- scripts/ci/static_checks/bats_tests.sh | 2 +- scripts/ci/static_checks/check_license.sh | 2 +- scripts/ci/static_checks/flake8.sh | 4 +-- .../ci/static_checks/in_container_bats_tests.sh | 4 +-- scripts/ci/static_checks/lint_dockerfile.sh | 4 +-- scripts/ci/static_checks/mypy.sh | 2 +- scripts/ci/static_checks/pylint.sh | 4 +-- scripts/ci/static_checks/refresh_pylint_todo.sh | 2 +- scripts/ci/testing/ci_run_airflow_testing.sh | 2 +- scripts/ci/tools/ci_clear_tmp.sh | 2 +- scripts/ci/tools/ci_fix_ownership.sh | 2 +- scripts/ci/tools/ci_free_space_on_ci.sh | 2 +- 24 files changed, 75 insertions(+), 75 deletions(-) diff --git a/breeze b/breeze index 4df4937..c85a5ac 100755 --- a/breeze +++ b/breeze @@ -609,7 +609,7 @@ if [[ \${VERBOSE} == "true" ]]; then echo echo "Executing script:" echo - echo "${file} \${@}" + echo "${COLOR_CYAN}${file} \${@}${COLOR_RESET}" echo set -x fi diff --git a/dev/provider_packages/enter_breeze_provider_package_tests.sh b/dev/provider_packages/enter_breeze_provider_package_tests.sh index 0e9467f..b67b33f 100755 --- a/dev/provider_packages/enter_breeze_provider_package_tests.sh +++ b/dev/provider_packages/enter_breeze_provider_package_tests.sh @@ -21,7 +21,7 @@ export MOUNT_SELECTED_LOCAL_SOURCES="false" . "$(dirname "${BASH_SOURCE[0]}")/../../scripts/ci/libraries/_script_init.sh" function enter_breeze_with_mapped_sources() { - docker run -it "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run -it "${EXTRA_DOCKER_FLAGS[@]}" \ -v "${AIRFLOW_SOURCES}/setup.py:/airflow_sources/setup.py:cached" \ -v "${AIRFLOW_SOURCES}/setup.cfg:/airflow_sources/setup.cfg:cached" \ -v "${AIRFLOW_SOURCES}/airflow/__init__.py:/airflow_sources/airflow/__init__.py:cached" \ diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 7d7d180..771a06d 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -89,7 +89,7 @@ function build_images::add_build_args_for_remote_install() { # Retrieves version of airflow stored in the production image (used to display the actual # Version we use if it was build from PyPI or GitHub function build_images::get_airflow_version_from_production_image() { - VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"' + docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"' } # Removes the "Forced answer" (yes/no/quit) given previously, unless you specifically want to remember it. @@ -252,7 +252,7 @@ function build_images::confirm_non-empty-docker-context-files() { # We cannot use docker registry APIs as they are available only with authorisation # But this image can be pulled without authentication function build_images::build_ci_image_manifest() { - docker build \ + docker_v build \ --tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \ -f- . <<EOF FROM scratch @@ -270,8 +270,8 @@ function build_images::get_local_build_cache_hash() { set +e # Remove the container just in case - docker rm --force "local-airflow-ci-container" 2>/dev/null >/dev/null - if ! docker inspect "${AIRFLOW_CI_IMAGE}" 2>/dev/null >/dev/null; then + docker_v rm --force "local-airflow-ci-container" 2>/dev/null >/dev/null + if ! docker_v inspect "${AIRFLOW_CI_IMAGE}" 2>/dev/null >/dev/null; then verbosity::print_info verbosity::print_info "Local airflow CI image not available" verbosity::print_info @@ -281,8 +281,8 @@ function build_images::get_local_build_cache_hash() { return fi - docker create --name "local-airflow-ci-container" "${AIRFLOW_CI_IMAGE}" 2>/dev/null - docker cp "local-airflow-ci-container:/build-cache-hash" \ + docker_v create --name "local-airflow-ci-container" "${AIRFLOW_CI_IMAGE}" 2>/dev/null + docker_v cp "local-airflow-ci-container:/build-cache-hash" \ "${LOCAL_IMAGE_BUILD_CACHE_HASH_FILE}" 2>/dev/null || touch "${LOCAL_IMAGE_BUILD_CACHE_HASH_FILE}" set -e @@ -305,7 +305,7 @@ function build_images::get_local_build_cache_hash() { function build_images::get_remote_image_build_cache_hash() { set +e # Pull remote manifest image - if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" 2>/dev/null >/dev/null; then + if ! docker_v pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" 2>/dev/null >/dev/null; then verbosity::print_info verbosity::print_info "Remote docker registry unreachable" verbosity::print_info @@ -317,11 +317,11 @@ function build_images::get_remote_image_build_cache_hash() { set -e rm -f "${REMOTE_IMAGE_CONTAINER_ID_FILE}" # Create container dump out of the manifest image without actually running it - docker create --cidfile "${REMOTE_IMAGE_CONTAINER_ID_FILE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" + docker_v create --cidfile "${REMOTE_IMAGE_CONTAINER_ID_FILE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" # Extract manifest and store it in local file - docker cp "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}"):/build-cache-hash" \ + docker_v cp "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}"):/build-cache-hash" \ "${REMOTE_IMAGE_BUILD_CACHE_HASH_FILE}" - docker rm --force "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}")" + docker_v rm --force "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}")" rm -f "${REMOTE_IMAGE_CONTAINER_ID_FILE}" verbosity::print_info verbosity::print_info "Remote build cache hash: '$(cat "${REMOTE_IMAGE_BUILD_CACHE_HASH_FILE}")'" @@ -490,7 +490,7 @@ function build_image::configure_docker_registry() { verbosity::print_info fi if [[ -n "${token}" ]]; then - echo "${token}" | docker login \ + echo "${token}" | docker_v login \ --username "${GITHUB_USERNAME:-apache}" \ --password-stdin \ "${GITHUB_REGISTRY}" @@ -745,7 +745,7 @@ Docker building ${AIRFLOW_CI_IMAGE}. if [[ -n "${RUNTIME_APT_COMMAND}" ]]; then additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"") fi - docker build \ + docker_v build \ "${EXTRA_DOCKER_CI_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ @@ -781,11 +781,11 @@ Docker building ${AIRFLOW_CI_IMAGE}. set -u if [[ -n "${DEFAULT_CI_IMAGE=}" ]]; then echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${DEFAULT_CI_IMAGE}" - docker tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}" + docker_v tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}" fi if [[ -n "${IMAGE_TAG=}" ]]; then echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${IMAGE_TAG}" - docker tag "${AIRFLOW_CI_IMAGE}" "${IMAGE_TAG}" + docker_v tag "${AIRFLOW_CI_IMAGE}" "${IMAGE_TAG}" fi if [[ -n ${SPIN_PID=} ]]; then kill -HUP "${SPIN_PID}" || true @@ -898,7 +898,7 @@ function build_images::build_prod_images() { if [[ -n "${DEV_APT_COMMAND}" ]]; then additional_dev_args+=("--build-arg" "DEV_APT_COMMAND=\"${DEV_APT_COMMAND}\"") fi - docker build \ + docker_v build \ "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \ --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \ @@ -934,7 +934,7 @@ function build_images::build_prod_images() { if [[ -n "${RUNTIME_APT_COMMAND}" ]]; then additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"") fi - docker build \ + docker_v build \ "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \ --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \ @@ -970,11 +970,11 @@ function build_images::build_prod_images() { set -u if [[ -n "${DEFAULT_PROD_IMAGE:=}" ]]; then echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${DEFAULT_PROD_IMAGE}" - docker tag "${AIRFLOW_PROD_IMAGE}" "${DEFAULT_PROD_IMAGE}" + docker_v tag "${AIRFLOW_PROD_IMAGE}" "${DEFAULT_PROD_IMAGE}" fi if [[ -n "${IMAGE_TAG=}" ]]; then echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${IMAGE_TAG}" - docker tag "${AIRFLOW_PROD_IMAGE}" "${IMAGE_TAG}" + docker_v tag "${AIRFLOW_PROD_IMAGE}" "${IMAGE_TAG}" fi } @@ -1000,7 +1000,7 @@ function build_images::wait_for_image_tag() { while true; do set +e echo "${COLOR_BLUE}Docker pull ${IMAGE_TO_WAIT_FOR} ${COLOR_RESET}" >"${OUTPUT_LOG}" - docker pull "${IMAGE_TO_WAIT_FOR}" >>"${OUTPUT_LOG}" 2>&1 + docker_v pull "${IMAGE_TO_WAIT_FOR}" >>"${OUTPUT_LOG}" 2>&1 set -e local image_hash echo "${COLOR_BLUE} Docker images -q ${IMAGE_TO_WAIT_FOR}${COLOR_RESET}" >>"${OUTPUT_LOG}" @@ -1020,12 +1020,12 @@ function build_images::wait_for_image_tag() { echo echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${IMAGE_NAME}." echo - docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}" + docker_v tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}" for TARGET_TAG in "${@}"; do echo echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${TARGET_TAG}." echo - docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}" + docker_v tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}" done break fi diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh index b5283b3..f5ed3e6 100644 --- a/scripts/ci/libraries/_docker_engine_resources.sh +++ b/scripts/ci/libraries/_docker_engine_resources.sh @@ -22,27 +22,25 @@ function docker_engine_resources::print_overall_stats() { echo "Overall resource statistics" echo docker stats --all --no-stream --no-trunc - docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c "free -h" - df --human || true + docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c "cat /proc/meminfo" + df -h || true } function docker_engine_resources::get_available_memory_in_docker() { - MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \ - 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))') + MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))') echo "${COLOR_BLUE}Memory available for Docker${COLOR_RESET}: $(numfmt --to iec $((MEMORY_AVAILABLE_FOR_DOCKER * 1024 * 1024)))" export MEMORY_AVAILABLE_FOR_DOCKER } function docker_engine_resources::get_available_cpus_in_docker() { - CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \ - 'grep -cE "cpu[0-9]+" </proc/stat') + CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm "debian:buster-slim" grep -cE 'cpu[0-9]+' /proc/stat) echo "${COLOR_BLUE}CPUS available for Docker${COLOR_RESET}: ${CPUS_AVAILABLE_FOR_DOCKER}" export CPUS_AVAILABLE_FOR_DOCKER } function docker_engine_resources::get_available_disk_space_in_docker() { - DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \ + DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c \ 'df / | tail -1 | awk '\''{print $4}'\') echo "${COLOR_BLUE}Disk space available for Docker${COLOR_RESET}: $(numfmt --to iec $((DISK_SPACE_AVAILABLE_FOR_DOCKER * 1024)))" export DISK_SPACE_AVAILABLE_FOR_DOCKER diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 93dabea..cb42693 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -566,11 +566,13 @@ function initialization::set_output_color_variables() { COLOR_RED=$'\e[31m' COLOR_RESET=$'\e[0m' COLOR_YELLOW=$'\e[33m' + COLOR_CYAN=$'\e[36m' export COLOR_BLUE export COLOR_GREEN export COLOR_RED export COLOR_RESET export COLOR_YELLOW + export COLOR_CYAN } # Common environment that is initialized by both Breeze and CI scripts diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index 4fbfee1..a8deaac 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -252,7 +252,7 @@ function kind::check_cluster_ready_for_airflow() { function kind::build_image_for_kubernetes_tests() { start_end::group_start "Build image for kubernetes tests ${AIRFLOW_PROD_IMAGE_KUBERNETES}" cd "${AIRFLOW_SOURCES}" || exit 1 - docker build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF + docker_v build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF FROM ${AIRFLOW_PROD_IMAGE} COPY airflow/example_dags/ \${AIRFLOW_HOME}/dags/ diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 3624a9a..b0723e4 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -25,7 +25,7 @@ function push_pull_remove_images::push_image_with_retries() { set +e echo echo "Trying to push the image ${1}. Number of try: ${try_num}" - docker push "${1}" + docker_v push "${1}" local res=$? set -e if [[ ${res} != "0" ]]; then @@ -61,7 +61,7 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() { echo echo "Pulling the image ${IMAGE_TO_PULL}" echo - docker pull "${IMAGE_TO_PULL}" + docker_v pull "${IMAGE_TO_PULL}" EXIT_VALUE="$?" if [[ ${EXIT_VALUE} != "0" && ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} == "true" ]]; then echo @@ -97,7 +97,7 @@ function push_pull_remove_images::pull_image_github_dockerhub() { set +e if push_pull_remove_images::pull_image_if_not_present_or_forced "${GITHUB_IMAGE}"; then # Tag the image to be the DockerHub one - docker tag "${GITHUB_IMAGE}" "${DOCKERHUB_IMAGE}" + docker_v tag "${GITHUB_IMAGE}" "${DOCKERHUB_IMAGE}" else push_pull_remove_images::pull_image_if_not_present_or_forced "${DOCKERHUB_IMAGE}" fi @@ -109,9 +109,9 @@ function push_pull_remove_images::rebuild_python_base_image() { echo echo "Rebuilding ${AIRFLOW_PYTHON_BASE_IMAGE} from latest ${PYTHON_BASE_IMAGE}" echo - docker pull "${PYTHON_BASE_IMAGE}" + docker_v pull "${PYTHON_BASE_IMAGE}" echo "FROM ${PYTHON_BASE_IMAGE}" | \ - docker build \ + docker_v build \ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}"; \ -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - } @@ -144,7 +144,7 @@ function push_pull_remove_images::pull_base_python_image() { push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PYTHON_BASE_IMAGE}" \ "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}" else - docker pull "${AIRFLOW_PYTHON_BASE_IMAGE}" + docker_v pull "${AIRFLOW_PYTHON_BASE_IMAGE}" fi } @@ -194,7 +194,7 @@ function push_pull_remove_images::pull_prod_images_if_needed() { function push_pull_remove_images::push_ci_images_to_dockerhub() { push_pull_remove_images::push_image_with_retries "${AIRFLOW_PYTHON_BASE_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_IMAGE}" - docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" + docker_v tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" if [[ -n ${DEFAULT_CI_IMAGE=} ]]; then # Only push default image to DockerHub registry if it is defined @@ -214,7 +214,7 @@ function push_pull_remove_images::push_python_image_to_github() { if [[ ${GITHUB_REGISTRY_PUSH_IMAGE_TAG} != "latest" ]]; then PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" fi - docker tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \ + docker_v tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \ "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}" push_pull_remove_images::push_image_with_retries \ "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}" @@ -224,12 +224,12 @@ function push_pull_remove_images::push_python_image_to_github() { function push_pull_remove_images::push_ci_images_to_github() { push_pull_remove_images::push_python_image_to_github AIRFLOW_CI_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" - docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}" + docker_v tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_TAGGED_IMAGE}" if [[ -n ${GITHUB_SHA=} ]]; then # Also push image to GitHub registry with commit SHA AIRFLOW_CI_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${COMMIT_SHA}" - docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}" + docker_v tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_SHA_IMAGE}" fi } @@ -265,17 +265,17 @@ function push_pull_remove_images::push_prod_images_to_dockerhub () { function push_pull_remove_images::push_prod_images_to_github () { push_pull_remove_images::push_python_image_to_github AIRFLOW_PROD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" - docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}" + docker_v tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}" push_pull_remove_images::push_image_with_retries "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" if [[ -n ${COMMIT_SHA=} ]]; then # Also push image to GitHub registry with commit SHA AIRFLOW_PROD_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}" - docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}" + docker_v tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_PROD_SHA_IMAGE}" fi # Also push prod build image AIRFLOW_PROD_BUILD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" - docker tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}" + docker_v tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}" push_pull_remove_images::push_image_with_retries "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}" } @@ -327,7 +327,7 @@ function push_pull_remove_images::check_for_image_in_github_container_registry() local image_to_wait_for="ghcr.io/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}" echo "GitHub Container Registry: checking for ${image_to_wait_for} via docker manifest inspect!" - docker manifest inspect "${image_to_wait_for}" + docker_v manifest inspect "${image_to_wait_for}" local res=$? if [[ ${res} == "0" ]]; then echo "Image: ${image_to_wait_for} found in Container Registry: ${COLOR_GREEN}OK.${COLOR_RESET}" diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index bd80108..45c4c2b 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -19,7 +19,7 @@ # Docker command to build documentation function runs::run_docs() { start_end::group_start "Run build docs" - docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" -t \ -e "GITHUB_ACTIONS=${GITHUB_ACTIONS="false"}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ @@ -30,7 +30,7 @@ function runs::run_docs() { # Docker command to generate constraint files. function runs::run_generate_constraints() { start_end::group_start "Run generate constraints" - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_generate_constraints.sh" @@ -40,7 +40,7 @@ function runs::run_generate_constraints() { # Docker command to prepare airflow packages function runs::run_prepare_airflow_packages() { start_end::group_start "Run prepare airflow packages" - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ -t \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ @@ -53,7 +53,7 @@ function runs::run_prepare_airflow_packages() { # Docker command to prepare provider packages function runs::run_prepare_provider_packages() { # No group here - groups are added internally - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ -t \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ @@ -64,7 +64,7 @@ function runs::run_prepare_provider_packages() { # Docker command to generate release notes for provider packages function runs::run_prepare_provider_documentation() { # No group here - groups are added internally - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ -t \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh index b7fa74f..d2d6a61 100644 --- a/scripts/ci/libraries/_start_end.sh +++ b/scripts/ci/libraries/_start_end.sh @@ -76,7 +76,7 @@ function start_end::dump_container_logs() { echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" echo " Dumping logs from ${container} container" echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" - docker logs "${container}" > "${dump_file}" + docker_v logs "${container}" > "${dump_file}" echo " Container ${container} logs dumped to ${dump_file}" echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" start_end::group_end diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh index 26a077d..dc3ca5a 100644 --- a/scripts/ci/libraries/_verbosity.sh +++ b/scripts/ci/libraries/_verbosity.sh @@ -39,10 +39,10 @@ function verbosity::restore_exit_on_error_status() { # In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be # printed before execution. In case of DRY_RUN_DOCKER flag set to "true" # show the command to execute instead of executing them -function docker { +function docker_v { if [[ ${DRY_RUN_DOCKER} != "false" ]]; then echo - echo "${COLOR_YELLOW}docker" "${@}" "${COLOR_RESET}" + echo "${COLOR_CYAN}docker" "${@}" "${COLOR_RESET}" echo return fi @@ -52,7 +52,7 @@ function docker { ${VERBOSE_COMMANDS:=} != "true" && \ # And when generally printing info is disabled ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then - >&2 echo "docker" "${@}" + >&2 echo "${COLOR_CYAN}docker ${*} ${COLOR_RESET}" fi if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then ${DOCKER_BINARY_PATH} "${@}" >>"${OUTPUT_LOG}" 2>&1 diff --git a/scripts/ci/libraries/_verify_image.sh b/scripts/ci/libraries/_verify_image.sh index f9d2bb6..b0060ac 100644 --- a/scripts/ci/libraries/_verify_image.sh +++ b/scripts/ci/libraries/_verify_image.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. function verify_image::run_command_in_image() { - docker run --rm \ + docker_v run --rm \ -e COLUMNS=180 \ --entrypoint /bin/bash "${DOCKER_IMAGE}" \ -c "${@}" @@ -83,7 +83,7 @@ function verify_image::verify_prod_image_has_airflow_and_providers() { function verify_image::verify_ci_image_dependencies() { start_end::group_start "Checking if Airflow dependencies are non-conflicting in ${DOCKER_IMAGE} image." set +e - docker run --rm --entrypoint /bin/bash "${DOCKER_IMAGE}" -c 'pip check' + docker_v run --rm --entrypoint /bin/bash "${DOCKER_IMAGE}" -c 'pip check' local res=$? if [[ ${res} != "0" ]]; then echo "${COLOR_RED}ERROR: ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}" @@ -212,7 +212,7 @@ function verify_image::verify_prod_image_as_root() { echo "Checking airflow as root" local output local res - output=$(docker run --rm --user 0 "${DOCKER_IMAGE}" "airflow" "info" 2>&1) + output=$(docker_v run --rm --user 0 "${DOCKER_IMAGE}" "airflow" "info" 2>&1) res=$? if [[ ${res} == "0" ]]; then echo "${COLOR_GREEN}OK${COLOR_RESET}" @@ -229,7 +229,7 @@ function verify_image::verify_prod_image_as_root() { tmp_dir="$(mktemp -d)" touch "${tmp_dir}/__init__.py" echo 'print("Awesome")' >> "${tmp_dir}/awesome.py" - output=$(docker run \ + output=$(docker_v run \ --rm \ -e "PYTHONPATH=${tmp_dir}" \ -v "${tmp_dir}:${tmp_dir}" \ diff --git a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh b/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh index 5b29c7e..51575eb 100755 --- a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh +++ b/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh @@ -29,7 +29,7 @@ fi function run_test_package_import_all_classes() { # Groups are added internally - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ -t \ -v "${AIRFLOW_SOURCES}/setup.py:/airflow_sources/setup.py:cached" \ diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh index e54e5b1..eaf9171 100755 --- a/scripts/ci/static_checks/bats_tests.sh +++ b/scripts/ci/static_checks/bats_tests.sh @@ -53,7 +53,7 @@ function run_bats_tests() { # deduplicate FS=" " read -r -a bats_arguments <<< "$(tr ' ' '\n' <<< "${bats_arguments[@]}" | sort -u | tr '\n' ' ' )" if [[ ${#@} == "0" ]]; then - # Run all tests + # Run al tests docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ apache/airflow:bats-2020.09.05-1.2.1 --tap /airflow/tests/bats/ elif [[ ${#bats_arguments} == "0" ]]; then diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh index 8698bc9..d3a8be7 100755 --- a/scripts/ci/static_checks/check_license.sh +++ b/scripts/ci/static_checks/check_license.sh @@ -31,7 +31,7 @@ function run_check_license() { echo "Running license checks. This can take a while." # We mount ALL airflow files for the licence check. We want to check them all! - if ! docker run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \ + if ! docker_v run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \ --user "$(id -ur):$(id -gr)" \ --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ apache/airflow:apache-rat-2020.07.10-0.13 \ diff --git a/scripts/ci/static_checks/flake8.sh b/scripts/ci/static_checks/flake8.sh index 322ab9e..1c5440c 100755 --- a/scripts/ci/static_checks/flake8.sh +++ b/scripts/ci/static_checks/flake8.sh @@ -20,12 +20,12 @@ function run_flake8() { if [[ "${#@}" == "0" ]]; then - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_flake8.sh" else - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_flake8.sh" "${@}" diff --git a/scripts/ci/static_checks/in_container_bats_tests.sh b/scripts/ci/static_checks/in_container_bats_tests.sh index a7c0121..fa4eacd 100644 --- a/scripts/ci/static_checks/in_container_bats_tests.sh +++ b/scripts/ci/static_checks/in_container_bats_tests.sh @@ -20,13 +20,13 @@ function run_in_container_bats_tests() { if [[ "${#@}" == "0" ]]; then - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/opt/bats/bin/bats" \ "-v" "$(pwd):/airflow" \ "${AIRFLOW_CI_IMAGE}" \ --tap "tests/bats/in_container/" else - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/opt/bats/bin/bats" \ "-v" "$(pwd):/airflow" \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/static_checks/lint_dockerfile.sh b/scripts/ci/static_checks/lint_dockerfile.sh index 491662d..38327f7 100755 --- a/scripts/ci/static_checks/lint_dockerfile.sh +++ b/scripts/ci/static_checks/lint_dockerfile.sh @@ -25,7 +25,7 @@ function run_docker_lint() { echo "Running docker lint for all Dockerfiles" echo # shellcheck disable=SC2046 - docker run \ + docker_v run \ -v "$(pwd):/root" \ -w "/root" \ --rm \ @@ -37,7 +37,7 @@ function run_docker_lint() { echo echo "Running docker lint for $*" echo - docker run \ + docker_v run \ -v "$(pwd):/root" \ -w "/root" \ --rm \ diff --git a/scripts/ci/static_checks/mypy.sh b/scripts/ci/static_checks/mypy.sh index a7257a9..7ebbd63 100755 --- a/scripts/ci/static_checks/mypy.sh +++ b/scripts/ci/static_checks/mypy.sh @@ -26,7 +26,7 @@ function run_mypy() { files=("$@") fi - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/static_checks/pylint.sh b/scripts/ci/static_checks/pylint.sh index edadd94..c69498e 100755 --- a/scripts/ci/static_checks/pylint.sh +++ b/scripts/ci/static_checks/pylint.sh @@ -20,12 +20,12 @@ function run_pylint() { if [[ "${#@}" == "0" ]]; then - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_pylint.sh" else - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ --entrypoint "/usr/local/bin/dumb-init" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_pylint.sh" "${@}" diff --git a/scripts/ci/static_checks/refresh_pylint_todo.sh b/scripts/ci/static_checks/refresh_pylint_todo.sh index 05dce88..52474b7 100755 --- a/scripts/ci/static_checks/refresh_pylint_todo.sh +++ b/scripts/ci/static_checks/refresh_pylint_todo.sh @@ -21,7 +21,7 @@ export FORCE_ANSWER_TO_QUESTIONS="quit" . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function refresh_pylint_todo() { - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ "${AIRFLOW_CI_IMAGE}" \ "/opt/airflow/scripts/in_container/refresh_pylint_todo.sh" } diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 0867e3c..af147ad 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -110,7 +110,7 @@ function system_prune_docker() { echo echo "${COLOR_BLUE}System-prune docker${COLOR_RESET}" echo - docker system prune --force --volumes + docker_v system prune --force --volumes echo } diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh index d367967..bef3fa5 100755 --- a/scripts/ci/tools/ci_clear_tmp.sh +++ b/scripts/ci/tools/ci_clear_tmp.sh @@ -27,7 +27,7 @@ sanity_checks::sanitize_mounted_files read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)" -docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ +docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ --rm \ --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh index 2d57d65..56463d2 100755 --- a/scripts/ci/tools/ci_fix_ownership.sh +++ b/scripts/ci/tools/ci_fix_ownership.sh @@ -33,7 +33,7 @@ sanity_checks::sanitize_mounted_files read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)" -docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ +docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ --rm \ --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/tools/ci_free_space_on_ci.sh b/scripts/ci/tools/ci_free_space_on_ci.sh index 4747848..a337545 100755 --- a/scripts/ci/tools/ci_free_space_on_ci.sh +++ b/scripts/ci/tools/ci_free_space_on_ci.sh @@ -26,7 +26,7 @@ echo "${COLOR_BLUE}Cleaning apt${COLOR_RESET}" sudo apt clean echo "${COLOR_BLUE}Pruning docker${COLOR_RESET}" -docker system prune --all --force --volumes +docker_v system prune --all --force --volumes echo "${COLOR_BLUE}Free disk space ${COLOR_RESET}" df -h
