This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ad8aa653f874141084973b00d1f73b1f1f6e5402 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 2 15:16:24 2020 +0100 Fixes documentation-only selective checks (#12038) There was a problem that documentation-only checks triggered selective checks without docs build (they resulted in basic-checks-only and no images being built. This occured for example in #12025 This PR fixes it by adding image-build and docs-build as two separate outputs. (cherry picked from commit adbf764ade6916b505c3238697bac10f98bfa6eb) --- .github/workflows/build-images-workflow-run.yml | 4 +- .github/workflows/ci.yml | 41 +++++---------- scripts/ci/selective_ci_checks.sh | 69 +++++++++++++++++-------- 3 files changed, 63 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index 6099727..a3277a7 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -201,7 +201,7 @@ jobs: defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} + image-build: ${{ steps.selective-checks.outputs.image-build }} if: > needs.cancel-workflow-runs.outputs.buildImages == 'true' steps: @@ -266,7 +266,7 @@ jobs: image-type: [CI, PROD] fail-fast: true if: > - needs.build-info.outputs.basic-checks-only == 'false' && + needs.build-info.outputs.image-build == 'true' && needs.cancel-workflow-runs.outputs.buildImages == 'true' env: BACKEND: postgres diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5457c0..c217bdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,8 @@ jobs: run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} + image-build: ${{ steps.selective-checks.outputs.image-build }} + docs-build: ${{ steps.selective-checks.outputs.docs-build }} needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} @@ -158,8 +160,7 @@ jobs: name: "Wait for CI images" runs-on: ubuntu-latest needs: [build-info] - if: > - needs.build-info.outputs.basic-checks-only == 'false' + if: needs.build-info.outputs.image-build == 'true' env: BACKEND: sqlite steps: @@ -196,8 +197,7 @@ jobs: env: MOUNT_LOCAL_SOURCES: "true" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: > - needs.build-info.outputs.basic-checks-only == 'false' + if: needs.build-info.outputs.basic-checks-only == 'false' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }} )" uses: actions/checkout@v2 @@ -233,8 +233,7 @@ jobs: SKIP: "build,mypy,flake8,pylint,bats-in-container-tests" MOUNT_LOCAL_SOURCES: "true" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: > - needs.build-info.outputs.basic-checks-only == 'true' + if: needs.build-info.outputs.basic-checks-only == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -266,8 +265,7 @@ jobs: name: "Build docs" runs-on: ubuntu-latest needs: [build-info, ci-images] - if: > - needs.build-info.outputs.basic-checks-only == 'false' + if: needs.build-info.outputs.docs-build == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -348,8 +346,7 @@ jobs: RUN_TESTS: true TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" TEST_TYPE: "" - if: > - needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -401,8 +398,7 @@ jobs: RUN_TESTS: true TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" TEST_TYPE: "" - if: > - needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -451,8 +447,7 @@ jobs: RUN_TESTS: true TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" TEST_TYPE: "" - if: > - needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -506,8 +501,7 @@ jobs: TEST_TYPE: "" NUM_RUNS: 10 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: > - needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -588,8 +582,7 @@ jobs: env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: > - needs.build-info.outputs.basic-checks-only == 'false' + if: needs.build-info.outputs.image-build == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -633,8 +626,7 @@ jobs: KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}" KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - if: > - needs.build-info.outputs.run-kubernetes-tests == 'true' + if: needs.build-info.outputs.run-kubernetes-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -744,7 +736,6 @@ jobs: - ci-images - docs if: > - needs.build-info.outputs.basic-checks-only == 'false' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) && github.event_name != 'schedule' strategy: @@ -780,9 +771,7 @@ jobs: - ci-images env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - if: > - needs.build-info.outputs.basic-checks-only == 'false' && - (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : merge commit ${{ github.merge_commit_sha }})" uses: actions/checkout@v2 @@ -815,9 +804,7 @@ jobs: - tests-mysql - tests-postgres - tests-kubernetes - if: > - needs.build-info.outputs.basic-checks-only == 'false' && - (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 10b74df..0faa541 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -139,7 +139,6 @@ function get_changed_files() { readonly CHANGED_FILES } - function run_tests() { initialization::ga_output run-tests "${@}" } @@ -160,11 +159,18 @@ function needs_python_scans() { initialization::ga_output needs-python-scans "${@}" } - function set_test_types() { initialization::ga_output test-types "${@}" } +function set_docs_build() { + initialization::ga_output docs-build "${@}" +} + +function set_image_build() { + initialization::ga_output image-build "${@}" +} + function set_basic_checks_only() { initialization::ga_output basic-checks-only "${@}" } @@ -181,6 +187,8 @@ function set_outputs_run_everything_and_exit() { run_kubernetes_tests "true" set_test_types "${ALL_TESTS}" set_basic_checks_only "false" + set_docs_build "true" + set_image_build "true" exit } @@ -189,9 +197,10 @@ function set_outputs_run_all_tests() { run_kubernetes_tests "true" set_test_types "${ALL_TESTS}" set_basic_checks_only "false" + set_image_build "true" } -function set_output_skip_all_tests_and_exit() { +function set_output_skip_all_tests_and_docs_and_exit() { needs_api_tests "false" needs_helm_tests "false" needs_javascript_scans "false" @@ -200,6 +209,22 @@ function set_output_skip_all_tests_and_exit() { run_kubernetes_tests "false" set_test_types "" set_basic_checks_only "true" + set_docs_build "false" + set_image_build "false" + exit +} + +function set_output_skip_tests_but_build_images_and_exit() { + needs_api_tests "false" + needs_helm_tests "false" + needs_javascript_scans "false" + needs_python_scans "false" + run_tests "false" + run_kubernetes_tests "false" + set_test_types "" + set_basic_checks_only "true" + set_docs_build "true" + set_image_build "true" exit } @@ -292,14 +317,15 @@ function check_if_docs_should_be_generated() { echo "None of the docs changed" else image_build_needed="true" + docs_build_needed="true" fi } AIRFLOW_SOURCES_TRIGGERING_TESTS=( - "^airflow" - "^chart" - "^tests" - "^kubernetes_tests" + "^airflow" + "^chart" + "^tests" + "^kubernetes_tests" ) readonly AIRFLOW_SOURCES_TRIGGERING_TESTS @@ -308,8 +334,13 @@ function check_if_tests_are_needed_at_all() { show_changed_files if [[ $(count_changed_files) == "0" ]]; then - echo "None of the important files changed, Skipping tests" - set_output_skip_all_tests_and_exit + if [[ ${image_build_needed} == "true" ]]; then + echo "No tests needed, Skipping tests but building images." + set_output_skip_tests_but_build_images_and_exit + else + echo "None of the important files changed, Skipping tests" + set_output_skip_all_tests_and_docs_and_exit + fi else image_build_needed="true" tests_needed="true" @@ -398,6 +429,7 @@ readonly FULL_TESTS_NEEDED output_all_basic_variables image_build_needed="false" +docs_build_needed="false" tests_needed="false" kubernetes_tests_needed="false" @@ -405,27 +437,20 @@ get_changed_files "${1}" run_all_tests_if_environment_files_changed check_if_docs_should_be_generated check_if_helm_tests_should_be_run -check_if_tests_are_needed_at_all check_if_javascript_security_scans_should_be_run check_if_python_security_scans_should_be_run +check_if_tests_are_needed_at_all get_count_all_files get_count_kubernetes_files calculate_test_types_to_run +set_image_build "${image_build_needed}" if [[ ${image_build_needed} == "true" ]]; then set_basic_checks_only "false" else set_basic_checks_only "true" fi - -if [[ ${tests_needed} == "true" ]]; then - run_tests "true" -else - run_tests "false" -fi - -if [[ ${kubernetes_tests_needed} == "true" ]]; then - run_kubernetes_tests "true" -else - run_kubernetes_tests "false" -fi +set_basic_checks_only "${image_build_needed}" +set_docs_build "${docs_build_needed}" +run_tests "${tests_needed}" +run_kubernetes_tests "${kubernetes_tests_needed}"
