This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 81dd5c88f0e6ad9437305b9fd21d48027fffe846 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Jul 19 22:30:42 2022 +0200 Speed up Kubernetes upgrade tests at least 2x (#25159) We do not need to run upgrade kubernetes tests for all python versions and all kubernetes versions. It is enough to run them for the minimum and maximum versions. We already run all k8s tests in main for all executors, so they "generally" work, and it is very, very unlikely that if it works for Python 3.7, and 3.10, it will not work for 3.8 or 3.9 (similarly for K8S version). This will speed up the build at least 2 times. (cherry picked from commit 907e720b9e312e9da1e645973a747ddbd3ba69f5) --- .github/workflows/ci.yml | 23 +++++++++++++--------- .../src/airflow_breeze/utils/selective_checks.py | 16 +++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd056e4238..c459b7001d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,8 +134,13 @@ jobs: all-python-versions-list-as-string: >- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + min-max-python-versions-as-string: >- + ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }} kubernetes-versions-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} + default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} + min-max-kubernetes-versions-as-string: >- + ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }} postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} @@ -1524,7 +1529,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-helm-executor-upgrade: timeout-minutes: 150 - name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} + name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-prod-images] env: @@ -1536,10 +1541,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" EXECUTOR: "KubernetesExecutor" KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}" HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}" - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.python-versions-list-as-string}} - CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >- + ${{needs.build-info.outputs.min-max-python-versions-as-string}} + CURRENT_KUBERNETES_VERSIONS_AS_STRING: >- + ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} if: > needs.build-info.outputs.run-kubernetes-tests == 'true' && needs.build-info.outputs.default-branch == 'main' @@ -1569,12 +1574,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" with: path: ".build/.kubernetes_venv" key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\ - -${{needs.build-info.outputs.kubernetes-versions-list-as-string}} - -${{needs.build-info.outputs.python-versions-list-as-string}} + -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} + -${{needs.build-info.outputs.min-max-python-versions-as-string}} -${{ hashFiles('setup.py','setup.cfg') }}" restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\ - -${{needs.build-info.outputs.kubernetes-versions-list-as-string}} - -${{needs.build-info.outputs.python-versions-list-as-string}}" + -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \ + -${{needs.build-info.outputs.min-max-python-versions-as-string}}" - name: "Cache bin folder with tools for kubernetes testing" uses: actions/cache@v3 with: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 642bb75043..af5b7a154d 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -330,6 +330,14 @@ class SelectiveChecks: def python_versions_list_as_string(self) -> str: return " ".join(self.python_versions) + @cached_property + def min_max_python_versions_as_string(self) -> str: + return " ".join( + [CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[0], CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[-1]] + if self._full_tests_needed + else [DEFAULT_PYTHON_MAJOR_MINOR_VERSION] + ) + @cached_property def all_python_versions(self) -> list[str]: return ( @@ -386,6 +394,14 @@ class SelectiveChecks: def kubernetes_versions(self) -> list[str]: return CURRENT_KUBERNETES_VERSIONS if self._full_tests_needed else [DEFAULT_KUBERNETES_VERSION] + @cached_property + def min_max_kubernetes_versions_as_string(self) -> str: + return " ".join( + [CURRENT_KUBERNETES_VERSIONS[0], CURRENT_KUBERNETES_VERSIONS[-1]] + if self._full_tests_needed + else [DEFAULT_KUBERNETES_VERSION] + ) + @cached_property def kubernetes_versions_list_as_string(self) -> str: return " ".join(self.kubernetes_versions)
