This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-0-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1415dc7a0295ebaf158ce1b823663f5507820659 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Mar 23 01:17:24 2021 +0100 Optimizes image verification steps. (#14780) So far we had matrix of builds that verified images - each image was verified by separate matrix-based job and those verifications were run after all images were alredy available. This step optimizes it. Those steps are run in the same job as "waiting for image", also they run in parallel which will make them a bit faster. This verification is fast and it can be run on any machine in parallel without any problems. (cherry picked from commit c59ab1ddcd0940dd7afaec0cab3a2358e074f5c3) --- .github/workflows/ci.yml | 66 ++-------------------- scripts/ci/images/ci_verify_ci_image.sh | 35 ------------ scripts/ci/images/ci_verify_prod_image.sh | 33 ----------- ....sh => ci_wait_for_and_verify_all_ci_images.sh} | 25 ++++++-- ...h => ci_wait_for_and_verify_all_prod_images.sh} | 24 ++++++-- ...image.sh => ci_wait_for_and_verify_ci_image.sh} | 25 ++++++++ ...age.sh => ci_wait_for_and_verify_prod_image.sh} | 26 ++++++++- scripts/ci/libraries/_parallel.sh | 1 + 8 files changed, 94 insertions(+), 141 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00d82c0..b47022d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,38 +272,9 @@ jobs: # The job will set the output "githubRegistry" - result of auto-detect which registry has # been used by checking where the image can be downloaded from. # - run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh + run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh - verify-ci-images: - timeout-minutes: 20 - name: "Verify CI Image Py${{matrix.python-version}}" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.image-build == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - if: needs.build-info.outputs.waitForImage == 'true' - with: - persist-credentials: false - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: | - needs.build-info.outputs.waitForImage == 'true' - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_verify_ci_image.sh - if: needs.build-info.outputs.waitForImage == 'true' - static-checks: timeout-minutes: 30 name: "Static checks" @@ -948,34 +919,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" env: CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > ${{needs.build-info.outputs.pythonVersionsListAsString}} - run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh - - verify-prod-images: - timeout-minutes: 20 - name: "Verify Prod Image Py${{matrix.python-version}}" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, prod-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - if: needs.build-info.outputs.waitForImage == 'true' - with: - persist-credentials: false - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: needs.build-info.outputs.waitForImage == 'true' - - name: "Prepare PROD Image" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - - name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_verify_prod_image.sh - if: needs.build-info.outputs.waitForImage == 'true' + run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh tests-kubernetes: timeout-minutes: 50 @@ -1068,7 +1012,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - prepare-provider-packages - test-provider-packages-released-airflow - prod-images - - verify-prod-images - docs if: > (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || @@ -1116,7 +1059,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-mysql - tests-kubernetes - ci-images - - verify-ci-images - docs if: > (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || @@ -1204,8 +1146,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" needs: - build-info - constraints - - verify-ci-images - - verify-prod-images + - ci-images + - prod-images - static-checks - static-checks-pylint - tests-sqlite diff --git a/scripts/ci/images/ci_verify_ci_image.sh b/scripts/ci/images/ci_verify_ci_image.sh deleted file mode 100755 index 78a2cd0..0000000 --- a/scripts/ci/images/ci_verify_ci_image.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh" - -function pull_ci_image() { - local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_start "Pulling ${image_name_with_tag} image" - - push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}" - start_end::group_end - -} - - -build_images::prepare_ci_build - -pull_ci_image - -verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}" diff --git a/scripts/ci/images/ci_verify_prod_image.sh b/scripts/ci/images/ci_verify_prod_image.sh deleted file mode 100755 index c492804..0000000 --- a/scripts/ci/images/ci_verify_prod_image.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh" - -function pull_prod_image() { - local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}" - - push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}" - start_end::group_end -} - -build_images::prepare_prod_build - -pull_prod_image - -verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}" diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh similarity index 54% rename from scripts/ci/images/ci_wait_for_all_ci_images.sh rename to scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh index 2451a88..7e09b1c 100755 --- a/scripts/ci/images/ci_wait_for_all_ci_images.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh @@ -15,12 +15,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -euo pipefail + +# We cannot perform full initialization because it will be done later in the "single run" scripts +# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed +LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd) +# shellcheck source=scripts/ci/libraries/_all_libs.sh +source "${LIBRARIES_DIR}/_all_libs.sh" + +initialization::set_output_color_variables + +parallel::make_sure_gnu_parallel_is_installed + echo echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" echo -for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} -do - export PYTHON_MAJOR_MINOR_VERSION - "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh" -done +parallel::initialize_monitoring + +parallel::monitor_progress + +# shellcheck disable=SC2086 +parallel --results "${PARALLEL_MONITORED_DIR}" \ + "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_ci_image.sh" ::: \ + ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh similarity index 54% rename from scripts/ci/images/ci_wait_for_all_prod_images.sh rename to scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh index 626c3ae..2d1da54 100755 --- a/scripts/ci/images/ci_wait_for_all_prod_images.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh @@ -15,13 +15,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -euo pipefail + +# We cannot perform full initialization because it will be done later in the "single run" scripts +# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed +LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd) +# shellcheck source=scripts/ci/libraries/_all_libs.sh +source "${LIBRARIES_DIR}/_all_libs.sh" + +initialization::set_output_color_variables + +parallel::make_sure_gnu_parallel_is_installed echo echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" echo -for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} -do - export PYTHON_MAJOR_MINOR_VERSION - "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh" -done +parallel::initialize_monitoring + +parallel::monitor_progress + +# shellcheck disable=SC2086 +parallel --results "${PARALLEL_MONITORED_DIR}" \ + "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_prod_image.sh" ::: \ + ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} diff --git a/scripts/ci/images/ci_wait_for_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh similarity index 70% rename from scripts/ci/images/ci_wait_for_ci_image.sh rename to scripts/ci/images/ci_wait_for_and_verify_ci_image.sh index 11142f0..e83b0d6 100755 --- a/scripts/ci/images/ci_wait_for_ci_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh @@ -15,9 +15,28 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +if [[ $1 == "" ]]; then + >&2 echo "Requires python MAJOR/MINOR version as first parameter" + exit 1 +fi + +export PYTHON_MAJOR_MINOR_VERSION=$1 +shift + + # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" +function pull_ci_image() { + local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + start_end::group_start "Pulling ${image_name_with_tag} image" + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}" + start_end::group_end + +} + push_pull_remove_images::check_if_github_registry_wait_for_image_enabled build_image::configure_docker_registry @@ -29,4 +48,10 @@ start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear" push_pull_remove_images::wait_for_github_registry_image \ "${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +build_images::prepare_ci_build + +pull_ci_image + +verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}" + start_end::group_end diff --git a/scripts/ci/images/ci_wait_for_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh similarity index 68% rename from scripts/ci/images/ci_wait_for_prod_image.sh rename to scripts/ci/images/ci_wait_for_and_verify_prod_image.sh index ebb6a9b..b4a482a 100755 --- a/scripts/ci/images/ci_wait_for_prod_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh @@ -15,17 +15,41 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +if [[ $1 == "" ]]; then + >&2 echo "Requires python MAJOR/MINOR version as first parameter" + exit 1 +fi + +export PYTHON_MAJOR_MINOR_VERSION=$1 +shift + + # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" +function pull_prod_image() { + local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}" + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}" + start_end::group_end +} + push_pull_remove_images::check_if_github_registry_wait_for_image_enabled build_image::configure_docker_registry export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" + start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE_NAME} image to appear" push_pull_remove_images::wait_for_github_registry_image \ "${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_end + +build_images::prepare_prod_build + +pull_prod_image + +verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}" diff --git a/scripts/ci/libraries/_parallel.sh b/scripts/ci/libraries/_parallel.sh index f8a92ab..09c3121 100644 --- a/scripts/ci/libraries/_parallel.sh +++ b/scripts/ci/libraries/_parallel.sh @@ -71,6 +71,7 @@ function parallel::monitor_loop() { for directory in "${PARALLEL_MONITORED_DIR}"/*/* do parallel_process=$(basename "${directory}") + echo "${COLOR_BLUE}### The last lines for ${parallel_process} process ###${COLOR_RESET}" echo tail -2 "${directory}/stdout" || true
