This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 496ed6f Reorganizing of CI tests (#9654)
496ed6f is described below
commit 496ed6f1b279dc5ca560dec0044c8373531565f4
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Jul 17 10:30:56 2020 +0200
Reorganizing of CI tests (#9654)
* we come back to idea of having one CI workflow
* cancel and openapi are incorporated into that CI workflow
* cancel retrieves workflow id automatically (works for forks)
* static checks are now merged into one job
* less dependencies between jobs so that waiting is minimised
* better name for check if tests should be run
* separated out script for tests should be run check
---
.github/workflows/ci.yml | 155 +++++++++++----------
.github/workflows/openapi.yml | 35 -----
.../ci/cancel/get_workflow_id.sh | 22 ++-
scripts/ci/{ => kubernetes}/ci_run_helm_testing.sh | 2 +-
scripts/ci/libraries/_initialization.sh | 2 +-
...files.sh => ci_check_if_tests_should_be_run.sh} | 69 ++++-----
scripts/ci/tools/ci_count_changed_files.sh | 19 ++-
7 files changed, 141 insertions(+), 163 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0df0c66..ce5b0ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,44 +42,31 @@ env:
jobs:
- static-checks-1:
+ cancel-previous-workflow-run:
timeout-minutes: 60
- name: "Checks: some checks"
+ name: "Cancel previous workflow run"
runs-on: ubuntu-latest
- env:
- MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
- CI_JOB_TYPE: "Static checks"
steps:
- uses: actions/checkout@master
- - uses: actions/setup-python@v1
- with:
- python-version: '3.x'
- - name: Cache pre-commit env
- uses: actions/cache@v2
+ - name: Get ci workflow id
+ run: "scripts/ci/cancel/get_workflow_id.sh"
env:
- cache-name: cache-pre-commit-v3
+ WORKFLOW: ci
+ GITHUB_TOKEN: ${{ github.token }}
+ GITHUB_REPOSITORY: ${{ github.repositoru }}
+ - name: Cancel workflow ${{ github.workflow }}
+ uses: styfle/[email protected]
with:
- path: ~/.cache/pre-commit
- key: ${{ env.cache-name }}-${{ github.job }}-${{
hashFiles('.pre-commit-config.yaml') }}
- - name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- - name: "Build CI image"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Static checks"
- if: success()
- env:
- PYTHON_VERSION: 3.6
- run: |
- python -m pip install pre-commit \
- --constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
- ./scripts/ci/static_checks/ci_run_static_checks.sh pylint-tests mypy
yamllint flake8
+ workflow_id: ${{ env.WORKFLOW_ID }}
+ access_token: ${{ github.token }}
- static-checks-2:
+ static-checks:
timeout-minutes: 60
- name: "Checks: all other"
+ name: "Static checks"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
- SKIP: pylint-tests,mypy,yamllint,flake8
MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
CI_JOB_TYPE: "Static checks"
steps:
@@ -103,11 +90,12 @@ jobs:
python -m pip install pre-commit \
--constraint
requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
./scripts/ci/static_checks/ci_run_static_checks.sh
-
docs:
timeout-minutes: 60
- name: Build docs
+ name: "Build docs"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
CI_JOB_TYPE: "Documentation"
steps:
@@ -117,33 +105,18 @@ jobs:
- name: "Build docs"
run: ./scripts/ci/docs/ci_docs.sh
- build-prod-image:
- timeout-minutes: 60
- name: "Build prod image Py${{ matrix.python-version }}"
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: [3.6, 3.7, 3.8]
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- CI_JOB_TYPE: "Prod image"
- steps:
- - uses: actions/checkout@master
- - name: "Build PROD image ${{ matrix.python-version }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
-
prepare-backport-packages:
timeout-minutes: 60
name: "Backport packages"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
INSTALL_AIRFLOW_VERSION: "1.10.10"
CI_JOB_TYPE: "Prepare & test packages"
PYTHON_MAJOR_MINOR_VERSION: 3.6
steps:
- uses: actions/checkout@master
- with:
- fetch-depth: 0
- uses: actions/setup-python@v1
with:
python-version: '3.x'
@@ -152,30 +125,27 @@ jobs:
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare & test backport packages"
- run: |
-
./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
+ run:
"./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh"
trigger-tests:
- timeout-minutes: 10
- name: "Count changed important files"
+ timeout-minutes: 5
+ name: "Checks if tests should be run"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
outputs:
- count: ${{ steps.trigger-tests.outputs.count }}
+ run-tests: ${{ steps.trigger-tests.outputs.run-tests }}
steps:
- uses: actions/checkout@master
- - name: "Get count of changed python files"
- run: |
- set +e
- ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
-
'^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
- echo "::set-output name=count::$?"
+ - name: "Check if tests should be run"
+ run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh"
id: trigger-tests
tests-kubernetes:
timeout-minutes: 80
name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}}
${{matrix.kubernetes-version}}"
runs-on: ubuntu-latest
- needs: [static-checks-1, static-checks-2, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6, 3.7]
@@ -201,8 +171,7 @@ jobs:
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
KIND_VERSION: "${{ matrix.kind-version }}"
HELM_VERSION: "${{ matrix.helm-version }}"
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name !=
'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name
!= 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -240,7 +209,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name:
"${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks-1, static-checks-2, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6, 3.7]
@@ -254,8 +223,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name !=
'pull_request'
+ if: needs.trigger-tests.outputs.count == 'true' || github.event_name !=
'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -272,7 +240,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name: "${{matrix.test-type}}:MySQL${{matrix.mysql-version}},
Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks-1, static-checks-2, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.7, 3.8]
@@ -286,8 +254,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name !=
'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name
!= 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -304,7 +271,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name: "${{matrix.test-type}}:Sqlite Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks-1, static-checks-2, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6, 3.8]
@@ -316,8 +283,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
TEST_TYPE: ${{ matrix.test-type }}
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
- # For pull requests only run tests when python files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name !=
'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name
!= 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -335,7 +301,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
name:
"${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
runs-on: ubuntu-latest
continue-on-error: true
- needs: [static-checks-1, static-checks-2, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6]
@@ -349,8 +315,7 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name !=
'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name
!= 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -367,12 +332,14 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 5
name: "Checks: Helm tests"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
CI_JOB_TYPE: "Tests"
steps:
- uses: actions/checkout@master
- name: "Helm Tests"
- run: ./scripts/ci/ci_run_helm_testing.sh
+ run: ./scripts/ci/kubernetes/ci_run_helm_testing.sh
requirements:
timeout-minutes: 80
@@ -382,6 +349,8 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
matrix:
python-version: [3.6, 3.7, 3.8]
fail-fast: false
+ needs:
+ - cancel-previous-workflow-run
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
CHECK_REQUIREMENTS_ONLY: true
@@ -397,6 +366,23 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
- name: "Generate requirements"
run: ./scripts/ci/requirements/ci_generate_requirements.sh
+ build-prod-image:
+ timeout-minutes: 60
+ name: "Build prod image Py${{ matrix.python-version }}"
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7, 3.8]
+ needs:
+ - cancel-previous-workflow-run
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ CI_JOB_TYPE: "Prod image"
+ steps:
+ - uses: actions/checkout@master
+ - name: "Build PROD image ${{ matrix.python-version }}"
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+
push-prod-images-to-github-cache:
timeout-minutes: 80
name: "Push PROD images"
@@ -486,3 +472,24 @@ ${{
hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
tags: true
force: true
branch: master
+
+ test-openapi-client-generation:
+ name: "Test OpenAPI client generation"
+ runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
+ steps:
+ - uses: actions/checkout@master
+ - name: Get workflow id
+ run: "scripts/ci/cancel/get_workflow_id.sh"
+ env:
+ WORKFLOW: openapi
+ GITHUB_TOKEN: ${{ github.token }}
+ GITHUB_REPOSITORY: ${{ github.repositoru }}
+ - name: Cancel workflow ${{ github.workflow }}
+ uses: styfle/[email protected]
+ with:
+ workflow_id: ${{ env.WORKFLOW_ID }}
+ access_token: ${{ github.token }}
+ - name: "Generate client codegen diff"
+ run: ./scripts/ci/openapi/client_codegen_diff.sh
diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml
deleted file mode 100644
index 915c40e..0000000
--- a/.github/workflows/openapi.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: OpenAPI Build
-on:
- push:
- branches: ['master']
- pull_request:
- branches: ['master']
-
-jobs:
- test-openapi-client-generation:
- name: "Test OpenAPI client generation"
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@master
- with:
- fetch-depth: 0
- - name: "Generate client codegen diff"
- run: ./scripts/ci/openapi/client_codegen_diff.sh
diff --git a/.github/workflows/cancel.yml b/scripts/ci/cancel/get_workflow_id.sh
old mode 100644
new mode 100755
similarity index 65%
rename from .github/workflows/cancel.yml
rename to scripts/ci/cancel/get_workflow_id.sh
index e831a28..4fa6187
--- a/.github/workflows/cancel.yml
+++ b/scripts/ci/cancel/get_workflow_id.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,17 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
----
-name: Cancel
-on: [push, pull_request]
-jobs:
- cancel:
- name: 'Cancel Previous Runs'
- runs-on: ubuntu-latest
- timeout-minutes: 3
- steps:
- - uses: styfle/[email protected]
- with:
- # Fetch via
https://api.github.com/repos/apache/airflow/actions/workflows/ci.yml
- workflow_id: 1029499
- access_token: ${{ github.token }}
+set -euo pipefail
+echo "Getting workflow id for ${WORKFLOW}. Github Repo: ${GITHUB_REPOSITORY}"
+URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW}.yml";
+echo "Calling URL: ${URL}"
+WORKFLOW_ID=$(curl "Authorization: token ${GITHUB_TOKEN}" "${URL}" | jq '.id')
+echo "Workflow id for ${WORKFLOW}: ${WORKFLOW_ID}"
+echo "::set-env name=WORKFLOW_ID::${WORKFLOW_ID}"
diff --git a/scripts/ci/ci_run_helm_testing.sh
b/scripts/ci/kubernetes/ci_run_helm_testing.sh
similarity index 98%
rename from scripts/ci/ci_run_helm_testing.sh
rename to scripts/ci/kubernetes/ci_run_helm_testing.sh
index 0a267d4..d48e298 100755
--- a/scripts/ci/ci_run_helm_testing.sh
+++ b/scripts/ci/kubernetes/ci_run_helm_testing.sh
@@ -18,7 +18,7 @@
echo "Running helm tests"
-CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/"
+CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../../chart/"
echo "Chart directory is $CHART_DIR"
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index 6ee8930..8fc96a9 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -272,7 +272,7 @@ function get_environment_for_builds_on_ci() {
fi
elif [[ ${GITHUB_ACTIONS:=} == "true" ]]; then
export CI_TARGET_REPO="${GITHUB_REPOSITORY}"
- export CI_TARGET_BRANCH="${GITHUB_BASE_REF}"
+ export CI_TARGET_BRANCH="${GITHUB_BASE_REF:=${CI_TARGET_BRANCH}}"
export CI_BUILD_ID="${GITHUB_RUN_ID}"
export CI_JOB_ID="${GITHUB_JOB}"
if [[ ${GITHUB_EVENT_NAME:=} == "pull_request" ]]; then
diff --git a/scripts/ci/tools/ci_count_changed_files.sh
b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh
similarity index 53%
copy from scripts/ci/tools/ci_count_changed_files.sh
copy to scripts/ci/tools/ci_check_if_tests_should_be_run.sh
index e3d2c9e..5a51048 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh
@@ -16,40 +16,41 @@
# specific language governing permissions and limitations
# under the License.
-# Returns number of files matching the pattern changed in revision specified
-# Versus the tip of the target branch
-# Parameters
-# $1: Revision to compare
-# $2: Pattern to match
-
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-get_environment_for_builds_on_ci
-
-git remote add target "https://github.com/${CI_TARGET_REPO}";
-
-git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
-
-CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}"
"${CI_TARGET_BRANCH}" || true)
-
-echo
-echo "Changed files:"
-echo
-echo "${CHANGED_FILES}"
-echo
-
-echo
-echo "Changed files matching the ${2} pattern"
-echo
-echo "${CHANGED_FILES}" | grep -E "${2}" || true
-echo
-
-echo
-echo "Count changed files matching the ${2} pattern"
-echo
-COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true)
-echo "${COUNT_CHANGED_FILES}"
-echo
-
-exit "${COUNT_CHANGED_FILES}"
+CHANGED_FILES_PATTERNS=(
+ "^airflow"
+ "^.github/workflows/"
+ "^Dockerfile"
+ "^scripts"
+ "^chart"
+ "^setup.py"
+ "^requirements"
+ "^tests"
+ "^kubernetes_tests"
+)
+
+CHANGED_FILES_REGEXP=""
+
+SEPARATOR=""
+for PATTERN in "${CHANGED_FILES_PATTERNS[@]}"
+do
+ CHANGED_FILES_REGEXP="${CHANGED_FILES_REGEXP}${SEPARATOR}${PATTERN}"
+ SEPARATOR="|"
+done
+
+echo
+echo "GitHub SHA: ${GITHUB_SHA}"
+echo
+
+set +e
+"${SCRIPTS_CI_DIR}/tools/ci_count_changed_files.sh" "${CHANGED_FILES_REGEXP}"
+COUNT_CHANGED_FILES=$?
+set -e
+
+if [[ ${COUNT_CHANGED_FILES} == "0" ]]; then
+ echo "::set-output name=run-tests::false"
+else
+ echo "::set-output name=run-tests::true"
+fi
diff --git a/scripts/ci/tools/ci_count_changed_files.sh
b/scripts/ci/tools/ci_count_changed_files.sh
index e3d2c9e..ef16eb1 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_count_changed_files.sh
@@ -27,11 +27,22 @@
get_environment_for_builds_on_ci
+if [[ ${CI_EVENT_TYPE} == "push" ]]; then
+ echo
+ echo "Always run all tests on push"
+ echo
+ exit 1
+fi
+
git remote add target "https://github.com/${CI_TARGET_REPO}";
git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
-CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}"
"${CI_TARGET_BRANCH}" || true)
+echo
+echo "Retrieve changed files from ${GITHUB_SHA} comparing to
${CI_TARGET_BRANCH}"
+echo
+
+CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${GITHUB_SHA}"
"${CI_TARGET_BRANCH}" || true)
echo
echo "Changed files:"
@@ -40,13 +51,13 @@ echo "${CHANGED_FILES}"
echo
echo
-echo "Changed files matching the ${2} pattern"
+echo "Changed files matching the ${1} pattern"
echo
-echo "${CHANGED_FILES}" | grep -E "${2}" || true
+echo "${CHANGED_FILES}" | grep -E "${1}" || true
echo
echo
-echo "Count changed files matching the ${2} pattern"
+echo "Count changed files matching the ${1} pattern"
echo
COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true)
echo "${COUNT_CHANGED_FILES}"
