This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cf497c03b58d3ca67d0dffde6ee487564770eadc Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Jul 3 17:57:22 2021 +0200 Removes coverage from kubernetes tests (#16794) The coverage generated by parallel runs of K8S tests cause often failures of tests because temporary .coverage files were generated in airflow sources. However the coverage of those tests was actually wrong - it did not check the coverage of Airflow code (it is running inside K8S in scheduler/workers/webserver pods). This is quite a bit complex task captured in #16793 but for now we should simply disable the coverage for those tests. (cherry picked from commit 5399f9124a4e75c7bb89e47c267d89b5280060ad) --- .github/workflows/ci.yml | 65 ++++++++++++++++++++++-- scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 9 ++-- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b2735c..57ab690 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -975,15 +975,72 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: failure() with: name: > - kind-logs- + kind-logs-${{matrix.executor}} path: /tmp/kind_logs_* retention-days: 7 - - name: "Upload artifact for coverage" + + tests-helm-executor-upgrade: + timeout-minutes: 50 + name: Helm Chart Executor Upgrade + runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + needs: [build-info, prod-images] + + env: + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + BACKEND: postgres + RUN_TESTS: "true" + RUNTIME: "kubernetes" + KUBERNETES_MODE: "image" + EXECUTOR: "KubernetesExecutor" + KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" + HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" + GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > + ${{needs.build-info.outputs.pythonVersionsListAsString}} + CURRENT_KUBERNETES_VERSIONS_AS_STRING: > + ${{needs.build-info.outputs.kubernetesVersionsListAsString}} + if: needs.build-info.outputs.run-kubernetes-tests == 'true' + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + - name: "Setup python" + uses: actions/setup-python@v2 + with: + python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + - name: "Free space" + run: ./scripts/ci/tools/ci_free_space_on_ci.sh + - name: "Get all PROD images" + run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh + - name: "Cache virtualenv for kubernetes testing" + uses: actions/cache@v2 + with: + path: ".build/.kubernetes_venv" + key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ + -${{needs.build-info.outputs.kubernetesVersionsListAsString}} + -${{needs.build-info.outputs.pythonVersionsListAsString}} + -${{ hashFiles('setup.py','setup.cfg') }}" + restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ + -${{needs.build-info.outputs.kubernetesVersionsListAsString}} + -${{needs.build-info.outputs.pythonVersionsListAsString}}" + - name: "Cache bin folder with tools for kubernetes testing" + uses: actions/cache@v2 + with: + path: ".build/kubernetes-bin" + key: "kubernetes-binaries + -${{ needs.build-info.outputs.defaultKindVersion }}\ + -${{ needs.build-info.outputs.defaultHelmVersion }}" + restore-keys: "kubernetes-binaries" + - name: "Kubernetes Helm Chart Executor Upgrade Tests" + run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh + - name: "Upload KinD logs" uses: actions/upload-artifact@v2 + if: failure() with: name: > - coverage-k8s- - path: "./files/coverage*.xml" + kind-logs-KubernetesExecutor + path: /tmp/kind_logs_* retention-days: 7 push-prod-images-to-github-registry: diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index 487b16b..6cab719 100755 --- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -18,6 +18,8 @@ # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" +: "${EXECUTOR:?You must set EXECUTOR to one of 'KubernetesExecutor', 'CeleryExecutor', 'CeleryKubernetesExecutor' }" + kind::make_sure_kubernetes_tools_are_installed kind::get_kind_cluster_name @@ -60,9 +62,6 @@ function parse_tests_to_run() { "--verbosity=1" "--strict-markers" "--durations=100" - "--cov=airflow/" - "--cov-config=.coveragerc" - "--cov-report=xml:files/coverage=${KIND_CLUSTER_NAME}.xml" "--color=yes" "--maxfail=50" "--pythonwarnings=ignore::DeprecationWarning" @@ -76,7 +75,7 @@ function create_virtualenv() { HOST_PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') readonly HOST_PYTHON_VERSION - local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}" + local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}_${EXECUTOR}" mkdir -pv "${BUILD_CACHE_DIR}/.kubernetes_venv/" if [[ ! -d ${virtualenv_path} ]]; then @@ -90,7 +89,7 @@ function create_virtualenv() { pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" - pip install pytest freezegun pytest-cov \ + pip install pytest freezegun \ --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${HOST_PYTHON_VERSION}.txt"; pip install -e ".[cncf.kubernetes,postgres]" \
