This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5bb69e5dd1da6e667f3bad7841ba1edc6ff4331a Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Oct 12 00:28:11 2020 +0200 Selective tests - depends on files changed in the commit. (#11417) This is final step of implementing #10507 - selective tests. Depending on files changed by the incoming commit, only subset of the tests are exucted. The conditions below are evaluated in the sequence specified below as well: * In case of "push" and "schedule" type of events, all tests are executed. * If no important files and folders changed - no tests are executed. This is a typical case for doc-only changes. * If any of the environment files (Dockerfile/setup.py etc.) all tests are executed. * If no "core/other" files are changed, only the relevant types of tests are executed: * API - if any of the API files/tests changed * CLI - if any of the CLI files/tests changed * WWW - if any of the WWW files/tests changed * Providers - if any of the Providers files/tests changed * Integration Heisentests, Quarantined, Postgres and MySQL runs are always run unless all tests are skipped like in case of doc-only changes. * If "Kubernetes" related files/tests are changed, the "Kubernetes" tests with Kind are run. Note that those tests are run separately using Host environment and those tests are stored in "kubernetes_tests" folder. * If some of the core/other files change, all tests are run. This is calculated by substracting all the files count calculated above from the total count of important files. Fixes: #10507 (cherry picked from commit 369bbf0427b84eb1fcd293622a1c165a69ad53c5) --- .github/workflows/ci.yml | 121 +++---- scripts/ci/selective_tests.sh | 350 +++++++++++++++++++++ .../ci/tools/ci_check_if_tests_should_be_run.sh | 56 ---- scripts/ci/tools/ci_count_changed_files.sh | 63 ---- 4 files changed, 385 insertions(+), 205 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c7df5c..d27d84c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,6 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}" GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - TEST_TYPES: '["Core", "Integration", "Heisentests"]' # You can switch between building the image in "Build Images" workflow or building them in CI workflow # Separately for each job. @@ -70,25 +69,27 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} - pythonVersions: ${{ steps.versions.outputs.python-versions }} - defaultPythonVersion: ${{ steps.versions.outputs.default-python-version }} - kubernetesVersions: ${{ steps.versions.outputs.kubernetes-versions }} - defaultKubernetesVersion: ${{ steps.versions.outputs.default-kubernetes-version }} - kubernetesModes: ${{ steps.versions.outputs.kubernetes-modes }} - defaultKubernetesMode: ${{ steps.versions.outputs.default-kubernetes-mode }} - postgresVersions: ${{ steps.versions.outputs.postgres-versions }} - defaultPostgresVersion: ${{ steps.versions.outputs.default-postgres-version }} - mysqlVersions: ${{ steps.versions.outputs.mysql-versions }} - defaultMySQLVersion: ${{ steps.versions.outputs.default-mysql-version }} - helmVersions: ${{ steps.versions.outputs.helm-versions }} - defaultHelmVersion: ${{ steps.versions.outputs.default-helm-version }} - kindVersions: ${{ steps.versions.outputs.kind-versions }} - defaultKindVersion: ${{ steps.versions.outputs.default-kind-version }} - testTypes: ${{ steps.versions.outputs.test-types }} - postgresExclude: ${{ steps.versions.outputs.postgres-exclude }} - mysqlExclude: ${{ steps.versions.outputs.mysql-exclude }} - sqliteExclude: ${{ steps.versions.outputs.sqlite-exclude }} - kubernetesExclude: ${{ steps.versions.outputs.kubernetes-exclude }} + pythonVersions: ${{ steps.selective-tests.outputs.python-versions }} + defaultPythonVersion: ${{ steps.selective-tests.outputs.default-python-version }} + kubernetesVersions: ${{ steps.selective-tests.outputs.kubernetes-versions }} + defaultKubernetesVersion: ${{ steps.selective-tests.outputs.default-kubernetes-version }} + kubernetesModes: ${{ steps.selective-tests.outputs.kubernetes-modes }} + defaultKubernetesMode: ${{ steps.selective-tests.outputs.default-kubernetes-mode }} + postgresVersions: ${{ steps.selective-tests.outputs.postgres-versions }} + defaultPostgresVersion: ${{ steps.selective-tests.outputs.default-postgres-version }} + mysqlVersions: ${{ steps.selective-tests.outputs.mysql-versions }} + defaultMySQLVersion: ${{ steps.selective-tests.outputs.default-mysql-version }} + helmVersions: ${{ steps.selective-tests.outputs.helm-versions }} + defaultHelmVersion: ${{ steps.selective-tests.outputs.default-helm-version }} + kindVersions: ${{ steps.selective-tests.outputs.kind-versions }} + defaultKindVersion: ${{ steps.selective-tests.outputs.default-kind-version }} + testTypes: ${{ steps.selective-tests.outputs.test-types }} + postgresExclude: ${{ steps.selective-tests.outputs.postgres-exclude }} + mysqlExclude: ${{ steps.selective-tests.outputs.mysql-exclude }} + sqliteExclude: ${{ steps.selective-tests.outputs.sqlite-exclude }} + kubernetesExclude: ${{ steps.selective-tests.outputs.kubernetes-exclude }} + run-tests: ${{ steps.selective-tests.outputs.run-tests }} + run-kubernetes-tests: ${{ steps.selective-tests.outputs.run-kubernetes-tests }} if: github.repository == 'apache/airflow' || github.event_name != 'schedule' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -109,62 +110,9 @@ jobs: else echo "::set-output name=wait-for-image::false" fi - - name: Set versions - id: versions - run: | - . ./scripts/ci/libraries/_script_init.sh - - initialization::ga_output python-versions \ - "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" - initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" - - initialization::ga_output kubernetes-versions \ - "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")" - initialization::ga_output default-kubernetes-version "${KUBERNETES_VERSION}" - - initialization::ga_output kubernetes-modes \ - "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")" - initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}" - - initialization::ga_output postgres-versions \ - "$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")" - initialization::ga_output default-postgres-version "${POSTGRES_VERSION}" - - initialization::ga_output mysql-versions \ - "$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")" - initialization::ga_output default-mysql-version "${MYSQL_VERSION}" - - initialization::ga_output kind-versions \ - "$(initialization::parameters_to_json "${CURRENT_KIND_VERSIONS[@]}")" - initialization::ga_output default-kind-version "${KIND_VERSION}" - - initialization::ga_output helm-versions \ - "$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")" - initialization::ga_output default-helm-version "${HELM_VERSION}" - - initialization::ga_output test-types "${TEST_TYPES}" - - initialization::ga_output postgres-exclude '[{ "python-version": "3.6" }]' - - initialization::ga_output mysql-exclude '[{ "python-version": "3.7" }]' - - initialization::ga_output sqlite-exclude '[{ "python-version": "3.8" }]' - - initialization::ga_output kubernetes-exclude '[]' - - trigger-tests: - timeout-minutes: 5 - name: "Trigger tests" - runs-on: ubuntu-latest - outputs: - run-tests: ${{ steps.trigger-tests.outputs.run-tests }} - if: github.repository == 'apache/airflow' || github.event_name != 'schedule' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - - name: "Check if tests should be run" - run: ./scripts/ci/tools/ci_check_if_tests_should_be_run.sh - id: trigger-tests + - name: Selective tests + id: selective-tests + run: ./scripts/ci/selective_tests.sh helm-tests: timeout-minutes: 5 @@ -267,7 +215,7 @@ jobs: timeout-minutes: 60 name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [build-info, trigger-tests, ci-images] + needs: [build-info, ci-images] strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} @@ -288,7 +236,7 @@ jobs: RUN_TESTS: true TEST_TYPE: ${{ matrix.test-type }} if: > - (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') && + needs.build-info.outputs.run-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -320,7 +268,7 @@ jobs: timeout-minutes: 60 name: "${{matrix.test-type}}:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [build-info, trigger-tests, ci-images] + needs: [build-info, ci-images] strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} @@ -341,7 +289,7 @@ jobs: RUN_TESTS: true TEST_TYPE: ${{ matrix.test-type }} if: > - (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') && + needs.build-info.outputs.run-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -372,7 +320,7 @@ jobs: timeout-minutes: 60 name: "${{matrix.test-type}}:Sqlite Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [build-info, trigger-tests, ci-images] + needs: [build-info, ci-images] strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} @@ -384,7 +332,8 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} RUN_TESTS: true TEST_TYPE: ${{ matrix.test-type }} - if: (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') && + if: > + needs.build-info.outputs.run-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -416,7 +365,7 @@ jobs: name: "Quarantined tests" runs-on: ubuntu-latest continue-on-error: true - needs: [build-info, trigger-tests, ci-images] + needs: [build-info, ci-images] strategy: matrix: include: @@ -433,7 +382,7 @@ jobs: NUM_RUNS: 10 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: > - (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') && + needs.build-info.outputs.run-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -533,7 +482,7 @@ jobs: timeout-minutes: 50 name: "K8s: ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}}" runs-on: ubuntu-latest - needs: [build-info, trigger-tests, prod-images] + needs: [build-info, prod-images] strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} @@ -553,7 +502,7 @@ jobs: KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" if: > - (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') && + needs.build-info.outputs.run-kubernetes-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/scripts/ci/selective_tests.sh b/scripts/ci/selective_tests.sh new file mode 100755 index 0000000..897dc78 --- /dev/null +++ b/scripts/ci/selective_tests.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. ./scripts/ci/libraries/_script_init.sh + +declare -a pattern_array + +function output_all_basic_variables() { + initialization::ga_output python-versions \ + "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" + initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" + + initialization::ga_output kubernetes-versions \ + "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")" + initialization::ga_output default-kubernetes-version "${KUBERNETES_VERSION}" + + initialization::ga_output kubernetes-modes \ + "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")" + initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}" + + initialization::ga_output postgres-versions \ + "$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")" + initialization::ga_output default-postgres-version "${POSTGRES_VERSION}" + + initialization::ga_output mysql-versions \ + "$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")" + initialization::ga_output default-mysql-version "${MYSQL_VERSION}" + + initialization::ga_output kind-versions \ + "$(initialization::parameters_to_json "${CURRENT_KIND_VERSIONS[@]}")" + initialization::ga_output default-kind-version "${KIND_VERSION}" + + initialization::ga_output helm-versions \ + "$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")" + initialization::ga_output default-helm-version "${HELM_VERSION}" + + initialization::ga_output postgres-exclude '[{ "python-version": "3.6" }]' + + initialization::ga_output mysql-exclude '[{ "python-version": "3.7" }]' + + initialization::ga_output sqlite-exclude '[{ "python-version": "3.8" }]' + + initialization::ga_output kubernetes-exclude '[]' +} + +function set_outputs_run_all_tests() { + initialization::ga_output run-tests "true" + initialization::ga_output run-kubernetes-tests "true" + initialization::ga_output test-types \ + '["Core", "Other", "API", "CLI", "Providers", "WWW", "Integration", "Heisentests"]' +} + +function set_output_skip_all_tests() { + initialization::ga_output run-tests "false" + initialization::ga_output run-kubernetes-tests "false" + initialization::ga_output test-types '[]' +} + +function initialize_git_repo() { + git remote add target "https://github.com/${CI_TARGET_REPO}"; + git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 + echo + echo "My commit SHA: ${COMMIT_SHA}" + echo + echo + echo "Retrieved changed files from ${COMMIT_SHA} comparing to ${CI_TARGET_BRANCH} in ${CI_TARGET_REPO}" + echo + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${COMMIT_SHA}" "${CI_TARGET_BRANCH}" || true) + echo + echo "Changed files:" + echo + echo "${CHANGED_FILES}" + echo + readonly CHANGED_FILES +} + +# Converts array of patterns into single | pattern string +# pattern_array - array storing regexp patterns +# Outputs - pattern string +function get_regexp_from_patterns() { + local test_triggering_regexp="" + local separator="" + local pattern + for pattern in "${pattern_array[@]}"; do + test_triggering_regexp="${test_triggering_regexp}${separator}${pattern}" + separator="|" + done + echo "${test_triggering_regexp}" +} + +# Shows changed files in the commit vs. the target. +# Input: +# pattern_array - array storing regexp patterns +function show_changed_files() { + local the_regexp + the_regexp=$(get_regexp_from_patterns) + echo + echo "Changed files matching the ${the_regexp} pattern:" + echo + echo "${CHANGED_FILES}" | grep -E "${the_regexp}" || true + echo +} + +# Counts changed files in the commit vs. the target +# Input: +# pattern_array - array storing regexp patterns +# Output: +# Count of changed files matching the patterns +function count_changed_files() { + count_changed_files=$(echo "${CHANGED_FILES}" | grep -c -E "$(get_regexp_from_patterns)" || true) + echo "${count_changed_files}" +} + +function run_all_tests_when_push_or_schedule() { + if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "schedule" ]]; then + echo + echo "Always run all tests in case of push/schedule events" + echo + set_outputs_run_all_tests + exit + fi +} + +function check_if_tests_should_be_run_at_all() { + TEST_TRIGGERING_PATTERNS=( + "^airflow" + "^.github/workflows/" + "^Dockerfile" + "^scripts" + "^chart" + "^setup.py" + "^tests" + "^kubernetes_tests" + ) + readonly TEST_TRIGGERING_PATTERNS + + pattern_array=("${TEST_TRIGGERING_PATTERNS[@]}") + show_changed_files "$(get_regexp_from_patterns)" + + if [[ $(count_changed_files) == "0" ]]; then + echo "None of the important files changed, Skipping tests" + set_output_skip_all_tests + exit + else + initialization::ga_output run-tests "true" + fi +} + +function check_if_environment_files_changed() { + ENVIRONMENT_TRIGGERING_PATTERNS=( + "^.github/workflows/" + "^Dockerfile" + "^scripts" + "^setup.py" + ) + readonly ENVIRONMENT_TRIGGERING_PATTERNS + + pattern_array=("${ENVIRONMENT_TRIGGERING_PATTERNS[@]}") + show_changed_files "$(get_regexp_from_patterns)" + + if [[ $(count_changed_files) != "0" ]]; then + echo "Important environment files changed. Running all tests" + set_outputs_run_all_tests + exit + fi +} + +function get_count_all_files() { + echo + echo "Count All files" + echo + ALL_TRIGGERING_PATTERNS=( + "^airflow" + "^chart" + "^tests" + "^kubernetes_tests" + ) + readonly ALL_TRIGGERING_PATTERNS + pattern_array=("${ALL_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_ALL_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_ALL_CHANGED_FILES}" + readonly COUNT_ALL_CHANGED_FILES +} + +function get_count_api_files() { + echo + echo "Count API files" + echo + API_TRIGGERING_PATTERNS=( + "^airflow/api" + "^airflow/api_connexion" + "^tests/api" + "^tests/api_connexion" + ) + readonly API_TRIGGERING_PATTERNS + pattern_array=("${API_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_API_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_API_CHANGED_FILES}" + readonly COUNT_API_CHANGED_FILES +} + +function get_count_cli_files() { + echo + echo "Count CLI files" + echo + CLI_TRIGGERING_PATTERNS=( + "^airflow/cli" + "^tests/cli" + ) + readonly CLI_TRIGGERING_PATTERNS + pattern_array=("${CLI_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_CLI_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_CLI_CHANGED_FILES}" + readonly COUNT_CLI_CHANGED_FILES +} + +function get_count_providers_files() { + echo + echo "Count Providers files" + echo + PROVIDERS_TRIGGERING_PATTERNS=( + "^airflow/providers" + "^tests/providers" + ) + readonly PROVIDERS_TRIGGERING_PATTERNS + pattern_array=("${PROVIDERS_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_PROVIDERS_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_PROVIDERS_CHANGED_FILES}" + readonly COUNT_PROVIDERS_CHANGED_FILES +} + +function get_count_www_files() { + echo + echo "Count WWW files" + echo + WWW_TRIGGERING_PATTERNS=( + "^airflow/www" + "^tests/www" + ) + readonly WWW_TRIGGERING_PATTERNS + pattern_array=("${WWW_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_WWW_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_WWW_CHANGED_FILES}" + readonly COUNT_WWW_CHANGED_FILES +} + +function get_count_kubernetes_files() { + echo + echo "Count Kubernetes files" + echo + KUBERNETES_TRIGGERING_PATTERNS=( + "^airflow/kubernetes" + "^chart" + "^tests/kubernetes_tests" + ) + readonly KUBERNETES_TRIGGERING_PATTERNS + pattern_array=("${KUBERNETES_TRIGGERING_PATTERNS[@]}") + show_changed_files + COUNT_KUBERNETES_CHANGED_FILES=$(count_changed_files) + echo "Files count: ${COUNT_KUBERNETES_CHANGED_FILES}" + readonly COUNT_KUBERNETES_CHANGED_FILES +} + +function calculate_test_types_to_run() { + echo + echo "Count Core/Other files" + echo + COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES - COUNT_WWW_CHANGED_FILES - \ + COUNT_PROVIDERS_CHANGED_FILES - COUNT_CLI_CHANGED_FILES - COUNT_API_CHANGED_FILES - \ + COUNT_KUBERNETES_CHANGED_FILES)) + + readonly COUNT_CORE_OTHER_CHANGED_FILES + echo + echo "Files count: ${COUNT_CORE_OTHER_CHANGED_FILES}" + echo + if [[ ${COUNT_CORE_OTHER_CHANGED_FILES} -gt 0 ]]; then + # Running all tests because some core or other files changed + echo + echo "Looks like ${COUNT_CORE_OTHER_CHANGED_FILES} files changed in the core/other area and" + echo "We have to run all tests. This will take longer than usual" + echo + set_outputs_run_all_tests + else + if [[ ${COUNT_KUBERNETES_CHANGED_FILES} != "0" ]]; then + initialization::ga_output run-kubernetes-tests "true" + else + initialization::ga_output run-kubernetes-tests "false" + fi + initialization::ga_output run-tests "true" + SELECTED_TESTS="" + if [[ ${COUNT_API_CHANGED_FILES} != "0" ]]; then + echo + echo "Adding API to selected files as ${COUNT_API_CHANGED_FILES} API files changed" + echo + SELECTED_TESTS="${SELECTED_TESTS}, \"API\"" + fi + if [[ ${COUNT_CLI_CHANGED_FILES} != "0" ]]; then + echo + echo "Adding CLI to selected files as ${COUNT_CLI_CHANGED_FILES} CLI files changed" + echo + SELECTED_TESTS="${SELECTED_TESTS}, \"CLI\"" + fi + if [[ ${COUNT_PROVIDERS_CHANGED_FILES} != "0" ]]; then + echo + echo "Adding Providers to selected files as ${COUNT_PROVIDERS_CHANGED_FILES} Provider files changed" + echo + SELECTED_TESTS="${SELECTED_TESTS}, \"Providers\"" + fi + if [[ ${COUNT_WWW_CHANGED_FILES} != "0" ]]; then + echo + echo "Adding WWW to selected files as ${COUNT_WWW_CHANGED_FILES} WWW files changed" + echo + SELECTED_TESTS="${SELECTED_TESTS}, \"WWW\"" + fi + initialization::ga_output test-types "[ \"Integration\", \"Heisentests\" ${SELECTED_TESTS} ]" + fi +} + +output_all_basic_variables +run_all_tests_when_push_or_schedule +initialize_git_repo +check_if_tests_should_be_run_at_all +check_if_environment_files_changed +get_count_all_files +get_count_api_files +get_count_cli_files +get_count_providers_files +get_count_www_files +get_count_kubernetes_files +calculate_test_types_to_run diff --git a/scripts/ci/tools/ci_check_if_tests_should_be_run.sh b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh deleted file mode 100755 index 5408060..0000000 --- a/scripts/ci/tools/ci_check_if_tests_should_be_run.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -CHANGED_FILES_PATTERNS=( - "^airflow" - "^.github/workflows/" - "^Dockerfile" - "^scripts" - "^chart" - "^setup.py" - "^tests" - "^kubernetes_tests" -) -readonly CHANGED_FILES_PATTERNS - -changed_files_regexp="" - -separator="" -for PATTERN in "${CHANGED_FILES_PATTERNS[@]}" -do - changed_files_regexp="${changed_files_regexp}${separator}${PATTERN}" - separator="|" -done - -echo -echo "GitHub SHA: ${COMMIT_SHA}" -echo - -set +e -"${SCRIPTS_CI_DIR}/tools/ci_count_changed_files.sh" "${changed_files_regexp}" -count_changed_files=$? -set -e - -if [[ ${count_changed_files} == "0" ]]; then - echo "::set-output name=run-tests::false" -else - echo "::set-output name=run-tests::true" -fi diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh deleted file mode 100755 index 786d8c4..0000000 --- a/scripts/ci/tools/ci_count_changed_files.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Returns number of files matching the pattern changed in revision specified -# Versus the tip of the target branch -# Parameters -# $1: Revision to compare -# $2: Pattern to match -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -if [[ ${CI_EVENT_TYPE} == "push" ]]; then - echo - echo "Always run all tests on push" - echo - exit 1 -fi - -git remote add target "https://github.com/${CI_TARGET_REPO}"; - -git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 - -echo -echo "Retrieve changed files from ${COMMIT_SHA} comparing to ${CI_TARGET_BRANCH}" -echo - -changed_files=$(git diff-tree --no-commit-id --name-only -r "${COMMIT_SHA}" "${CI_TARGET_BRANCH}" || true) - -echo -echo "Changed files:" -echo -echo "${changed_files}" -echo - -echo -echo "Changed files matching the ${1} pattern" -echo -echo "${changed_files}" | grep -E "${1}" || true -echo - -echo -echo "Count changed files matching the ${1} pattern" -echo -count_changed_files=$(echo "${changed_files}" | grep -c -E "${1}" || true) -echo "${count_changed_files}" -echo - -exit "${count_changed_files}"
