This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d33f6d0bbbfbc4b383b5eaea0773a0c21d7e4057 Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Jul 17 10:30:56 2020 +0200 Reorganizing of CI tests (#9654) * we come back to idea of having one CI workflow * cancel and openapi are incorporated into that CI workflow * cancel retrieves workflow id automatically (works for forks) * static checks are now merged into one job * less dependencies between jobs so that waiting is minimised * better name for check if tests should be run * separated out script for tests should be run check (cherry picked from commit 496ed6f1b279dc5ca560dec0044c8373531565f4) --- .github/workflows/ci.yml | 102 ++++++++++++--------- .../get_workflow_id.sh} | 18 ++-- scripts/ci/{ => kubernetes}/ci_run_helm_testing.sh | 2 +- scripts/ci/libraries/_initialization.sh | 2 +- ...files.sh => ci_check_if_tests_should_be_run.sh} | 69 +++++++------- scripts/ci/tools/ci_count_changed_files.sh | 19 +++- 6 files changed, 119 insertions(+), 93 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a849d58..604fa0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,30 @@ env: jobs: + cancel-previous-workflow-run: + timeout-minutes: 60 + name: "Cancel previous workflow run" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Get ci workflow id + run: "scripts/ci/cancel/get_workflow_id.sh" + env: + WORKFLOW: ci + GITHUB_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repositoru }} + - name: Cancel workflow ${{ github.workflow }} + uses: styfle/[email protected] + with: + workflow_id: ${{ env.WORKFLOW_ID }} + access_token: ${{ github.token }} + static-checks: timeout-minutes: 60 name: "Checks" runs-on: ubuntu-latest + needs: + - cancel-previous-workflow-run env: MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true" CI_JOB_TYPE: "Static checks" @@ -69,11 +89,12 @@ jobs: python -m pip install pre-commit \ --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt ./scripts/ci/static_checks/ci_run_static_checks.sh - docs: timeout-minutes: 60 - name: Build docs + name: "Build docs" runs-on: ubuntu-latest + needs: + - cancel-previous-workflow-run env: CI_JOB_TYPE: "Documentation" steps: @@ -86,42 +107,25 @@ jobs: - name: "Build docs" run: ./scripts/ci/docs/ci_docs.sh - build-prod-image: - timeout-minutes: 60 - name: "Build prod image Py${{ matrix.python-version }}" - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8] - env: - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - CI_JOB_TYPE: "Prod image" - steps: - - uses: actions/checkout@master - - name: "Build PROD image ${{ matrix.python-version }}" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - trigger-tests: - timeout-minutes: 10 - name: "Count changed important files" + timeout-minutes: 5 + name: "Checks if tests should be run" runs-on: ubuntu-latest + needs: + - cancel-previous-workflow-run outputs: - count: ${{ steps.trigger-tests.outputs.count }} + run-tests: ${{ steps.trigger-tests.outputs.run-tests }} steps: - uses: actions/checkout@master - - name: "Get count of changed python files" - run: | - set +e - ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \ - '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests' - echo "::set-output name=count::$?" + - name: "Check if tests should be run" + run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh" id: trigger-tests tests-kubernetes: timeout-minutes: 80 name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}} ${{matrix.kubernetes-version}}" runs-on: ubuntu-latest - needs: [static-checks, trigger-tests] + needs: [trigger-tests] strategy: matrix: python-version: [3.6, 3.7] @@ -147,8 +151,7 @@ jobs: KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}" KIND_VERSION: "${{ matrix.kind-version }}" HELM_VERSION: "${{ matrix.helm-version }}" - # For pull requests only run tests when important files changed - if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request' + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@master - uses: actions/setup-python@v1 @@ -186,7 +189,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') timeout-minutes: 80 name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [static-checks, trigger-tests] + needs: [trigger-tests] strategy: matrix: python-version: [2.7, 3.5, 3.6, 3.7, 3.8] @@ -200,8 +203,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') RUN_TESTS: "true" CI_JOB_TYPE: "Tests" TEST_TYPE: ${{ matrix.test-type }} - # For pull requests only run tests when important files changed - if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request' + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@master - uses: actions/setup-python@v1 @@ -218,7 +220,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') timeout-minutes: 80 name: "${{matrix.test-type}}:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [static-checks, trigger-tests] + needs: [trigger-tests] strategy: matrix: python-version: [2.7, 3.5, 3.6, 3.7, 3.8] @@ -232,8 +234,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') RUN_TESTS: "true" CI_JOB_TYPE: "Tests" TEST_TYPE: ${{ matrix.test-type }} - # For pull requests only run tests when important files changed - if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request' + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@master - uses: actions/setup-python@v1 @@ -250,7 +251,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') timeout-minutes: 80 name: "${{matrix.test-type}}:Sqlite Py${{matrix.python-version}}" runs-on: ubuntu-latest - needs: [static-checks, trigger-tests] + needs: [trigger-tests] strategy: matrix: python-version: [2.7, 3.5, 3.6, 3.7, 3.8] @@ -262,8 +263,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') TEST_TYPE: ${{ matrix.test-type }} RUN_TESTS: "true" CI_JOB_TYPE: "Tests" - # For pull requests only run tests when python files changed - if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request' + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@master - uses: actions/setup-python@v1 @@ -281,7 +281,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}" runs-on: ubuntu-latest continue-on-error: true - needs: [static-checks, trigger-tests] + needs: [trigger-tests] strategy: matrix: python-version: [3.6] @@ -295,8 +295,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') RUN_TESTS: "true" CI_JOB_TYPE: "Tests" TEST_TYPE: ${{ matrix.test-type }} - # For pull requests only run tests when important files changed - if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request' + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@master - uses: actions/setup-python@v1 @@ -313,12 +312,14 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') timeout-minutes: 5 name: "Checks: Helm tests" runs-on: ubuntu-latest + needs: + - cancel-previous-workflow-run env: CI_JOB_TYPE: "Tests" steps: - uses: actions/checkout@master - name: "Helm Tests" - run: ./scripts/ci/ci_run_helm_testing.sh + run: ./scripts/ci/kubernetes/ci_run_helm_testing.sh requirements: timeout-minutes: 80 @@ -328,6 +329,8 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') matrix: python-version: [2.7, 3.5, 3.6, 3.7, 3.8] fail-fast: false + needs: + - cancel-previous-workflow-run env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} CHECK_REQUIREMENTS_ONLY: true @@ -343,6 +346,21 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Generate requirements" run: ./scripts/ci/requirements/ci_generate_requirements.sh + build-prod-image: + timeout-minutes: 60 + name: "Build prod image Py${{ matrix.python-version }}" + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + CI_JOB_TYPE: "Prod image" + steps: + - uses: actions/checkout@master + - name: "Build PROD image ${{ matrix.python-version }}" + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh + push-prod-images-to-github-cache: timeout-minutes: 80 name: "Push PROD images" diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/cancel/get_workflow_id.sh similarity index 67% copy from scripts/ci/ci_run_helm_testing.sh copy to scripts/ci/cancel/get_workflow_id.sh index 0a267d4..4fa6187 100755 --- a/scripts/ci/ci_run_helm_testing.sh +++ b/scripts/ci/cancel/get_workflow_id.sh @@ -15,14 +15,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -echo "Running helm tests" - -CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/" - -echo "Chart directory is $CHART_DIR" - -docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \ - --entrypoint /bin/sh \ - aneeshkj/helm-unittest \ - -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ." +set -euo pipefail +echo "Getting workflow id for ${WORKFLOW}. Github Repo: ${GITHUB_REPOSITORY}" +URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW}.yml"; +echo "Calling URL: ${URL}" +WORKFLOW_ID=$(curl "Authorization: token ${GITHUB_TOKEN}" "${URL}" | jq '.id') +echo "Workflow id for ${WORKFLOW}: ${WORKFLOW_ID}" +echo "::set-env name=WORKFLOW_ID::${WORKFLOW_ID}" diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/kubernetes/ci_run_helm_testing.sh similarity index 98% rename from scripts/ci/ci_run_helm_testing.sh rename to scripts/ci/kubernetes/ci_run_helm_testing.sh index 0a267d4..d48e298 100755 --- a/scripts/ci/ci_run_helm_testing.sh +++ b/scripts/ci/kubernetes/ci_run_helm_testing.sh @@ -18,7 +18,7 @@ echo "Running helm tests" -CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/" +CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../../chart/" echo "Chart directory is $CHART_DIR" diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index d0b14a7..e759b03 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -280,7 +280,7 @@ function get_environment_for_builds_on_ci() { fi elif [[ ${GITHUB_ACTIONS:=} == "true" ]]; then export CI_TARGET_REPO="${GITHUB_REPOSITORY}" - export CI_TARGET_BRANCH="${GITHUB_BASE_REF}" + export CI_TARGET_BRANCH="${GITHUB_BASE_REF:=${CI_TARGET_BRANCH}}" export CI_BUILD_ID="${GITHUB_RUN_ID}" export CI_JOB_ID="${GITHUB_JOB}" if [[ ${GITHUB_EVENT_NAME:=} == "pull_request" ]]; then diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh similarity index 53% copy from scripts/ci/tools/ci_count_changed_files.sh copy to scripts/ci/tools/ci_check_if_tests_should_be_run.sh index e3d2c9e..5a51048 100755 --- a/scripts/ci/tools/ci_count_changed_files.sh +++ b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh @@ -16,40 +16,41 @@ # specific language governing permissions and limitations # under the License. -# Returns number of files matching the pattern changed in revision specified -# Versus the tip of the target branch -# Parameters -# $1: Revision to compare -# $2: Pattern to match - # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -get_environment_for_builds_on_ci - -git remote add target "https://github.com/${CI_TARGET_REPO}"; - -git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 - -CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}" "${CI_TARGET_BRANCH}" || true) - -echo -echo "Changed files:" -echo -echo "${CHANGED_FILES}" -echo - -echo -echo "Changed files matching the ${2} pattern" -echo -echo "${CHANGED_FILES}" | grep -E "${2}" || true -echo - -echo -echo "Count changed files matching the ${2} pattern" -echo -COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true) -echo "${COUNT_CHANGED_FILES}" -echo - -exit "${COUNT_CHANGED_FILES}" +CHANGED_FILES_PATTERNS=( + "^airflow" + "^.github/workflows/" + "^Dockerfile" + "^scripts" + "^chart" + "^setup.py" + "^requirements" + "^tests" + "^kubernetes_tests" +) + +CHANGED_FILES_REGEXP="" + +SEPARATOR="" +for PATTERN in "${CHANGED_FILES_PATTERNS[@]}" +do + CHANGED_FILES_REGEXP="${CHANGED_FILES_REGEXP}${SEPARATOR}${PATTERN}" + SEPARATOR="|" +done + +echo +echo "GitHub SHA: ${GITHUB_SHA}" +echo + +set +e +"${SCRIPTS_CI_DIR}/tools/ci_count_changed_files.sh" "${CHANGED_FILES_REGEXP}" +COUNT_CHANGED_FILES=$? +set -e + +if [[ ${COUNT_CHANGED_FILES} == "0" ]]; then + echo "::set-output name=run-tests::false" +else + echo "::set-output name=run-tests::true" +fi diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh index e3d2c9e..ef16eb1 100755 --- a/scripts/ci/tools/ci_count_changed_files.sh +++ b/scripts/ci/tools/ci_count_changed_files.sh @@ -27,11 +27,22 @@ get_environment_for_builds_on_ci +if [[ ${CI_EVENT_TYPE} == "push" ]]; then + echo + echo "Always run all tests on push" + echo + exit 1 +fi + git remote add target "https://github.com/${CI_TARGET_REPO}"; git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 -CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}" "${CI_TARGET_BRANCH}" || true) +echo +echo "Retrieve changed files from ${GITHUB_SHA} comparing to ${CI_TARGET_BRANCH}" +echo + +CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${GITHUB_SHA}" "${CI_TARGET_BRANCH}" || true) echo echo "Changed files:" @@ -40,13 +51,13 @@ echo "${CHANGED_FILES}" echo echo -echo "Changed files matching the ${2} pattern" +echo "Changed files matching the ${1} pattern" echo -echo "${CHANGED_FILES}" | grep -E "${2}" || true +echo "${CHANGED_FILES}" | grep -E "${1}" || true echo echo -echo "Count changed files matching the ${2} pattern" +echo "Count changed files matching the ${1} pattern" echo COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true) echo "${COUNT_CHANGED_FILES}"
