This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch switch-cache-building-to-public-macos-runners in repository https://gitbox.apache.org/repos/asf/airflow.git
commit af971f08943451ef8150db8b851a50f9b73d3883 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Mar 30 23:19:27 2024 +0100 Switch building images for ARM to public GitHub macos-14 runners GitHub recently added a possibility for OSS projects to run the macos-14 runners for ARM building. We are switching to build cache images using those runners. --- .github/workflows/additional-ci-image-checks.yml | 6 +++++- .github/workflows/ci.yml | 5 ++++- .github/workflows/finalize-tests.yml | 6 +++++- .github/workflows/push-image-cache.yml | 6 ------ dev/breeze/src/airflow_breeze/global_constants.py | 1 + dev/breeze/src/airflow_breeze/utils/selective_checks.py | 14 +++++++++++--- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 2331ceba9c..877de552ff 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy description: "The array of labels (in json form) determining default runner used for the build." required: true type: string + macos-runs-on-as-string: + description: "The array of labels (in json form) determining macos runners." + required: true + type: string public-runs-on-as-string: description: "The array of labels (in json form) determining public runners." required: true @@ -96,7 +100,7 @@ jobs: packages: write secrets: inherit with: - runs-on-as-string: ${{ inputs.public-runs-on-as-string }} + runs-on-as-string: ${{ inputs.macos-runs-on-as-string }} cache-type: "Early" include-prod-images: "false" push-latest-images: "false" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 361fbd4674..6436c17e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,8 +103,9 @@ jobs: helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} default-runs-on-as-string: ${{ steps.selective-checks.outputs.default-runs-on-as-string }} - self-hosted-runs-on-as-string: ${{ steps.selective-checks.outputs.self-hosted-runs-on-as-string }} + macos-runs-on-as-string: ${{ steps.selective-checks.outputs.macos-runs-on-as-string }} public-runs-on-as-string: ${{ steps.selective-checks.outputs.public-runs-on-as-string }} + self-hosted-runs-on-as-string: ${{ steps.selective-checks.outputs.self-hosted-runs-on-as-string }} is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} @@ -239,6 +240,7 @@ jobs: if: needs.build-info.outputs.canary-run == 'true' with: default-runs-on-as-string: ${{ needs.build-info.outputs.default-runs-on-as-string }} + macos-runs-on-as-string: ${{ needs.build-info.outputs.macos-runs-on-as-string }} public-runs-on-as-string: ${{ needs.build-info.outputs.public-runs-on-as-string }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -609,6 +611,7 @@ jobs: uses: ./.github/workflows/finalize-tests.yml with: public-runs-on-as-string: ${{ needs.build-info.outputs.public-runs-on-as-string }} + macos-runs-on-as-string: ${{ needs.build-info.outputs.macos-runs-on-as-string }} self-hosted-runs-on-as-string: ${{ needs.build-info.outputs.self-hosted-runs-on-as-string }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index aa8e34bf48..c000a4f0b8 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy description: "The array of labels (in json form) determining public runners." required: true type: string + macos-runs-on-as-string: + description: "The array of labels (in json form) determining macos runners." + required: true + type: string self-hosted-runs-on-as-string: description: "The array of labels (in json form) determining public runners." required: true @@ -158,7 +162,7 @@ jobs: packages: write secrets: inherit with: - runs-on-as-string: ${{ inputs.self-hosted-runs-on-as-string }} + runs-on-as-string: ${{ inputs.macos-runs-on-as-string }} cache-type: "Regular ARM" include-prod-images: "true" push-latest-images: "true" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 94cd9eb9d1..3b0b5dce10 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -178,9 +178,6 @@ jobs: with: name: prod-packages path: ./docker-context-files - - name: "Start ARM instance" - run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: inputs.platform == 'linux/arm64' - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} ${{ inputs.platform }}" @@ -189,9 +186,6 @@ jobs: --prepare-buildx-cache --platform "${{ inputs.platform }}" --install-packages-from-context --airflow-constraints-mode constraints-source-providers --python ${{ matrix.python }} - - name: "Stop ARM instance" - run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && inputs.platform == 'linux/arm64' # We only push "AMD" images as it is really only needed for any kind of automated builds in CI # and currently there is not an easy way to make multi-platform image from two separate builds # and we can do it after we stopped the ARM instance as it is not needed anymore diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 745913e038..71267bd47d 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -30,6 +30,7 @@ from typing import Iterable from airflow_breeze.utils.host_info_utils import Architecture from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +RUNS_ON_MACOS_RUNNER = '["macos-14"]' RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]' # we should get more sophisticated logic here in the future, but for now we just check if # we use self airflow, vm-based, amd hosted runner as a default diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 2b318d6f43..bf382b6c5f 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -43,6 +43,7 @@ from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, HELM_VERSION, KIND_VERSION, + RUNS_ON_MACOS_RUNNER, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, GithubEvents, @@ -1133,6 +1134,10 @@ class SelectiveChecks: def public_runs_on_as_string(self) -> str: return RUNS_ON_PUBLIC_RUNNER + @cached_property + def macos_runs_on_as_string(self) -> str: + return RUNS_ON_MACOS_RUNNER + @cached_property def is_self_hosted_runner(self) -> bool: """ @@ -1172,7 +1177,7 @@ class SelectiveChecks: or "x64" == label.lower() or "asf-runner" == label or ("ubuntu" in label and "arm" not in label.lower()) - for label in json.loads(self.public_runs_on_as_string) + for label in json.loads(self.default_runs_on_as_string) ] ) @@ -1187,8 +1192,11 @@ class SelectiveChecks: """ return any( [ - "arm" == label.lower() or "arm64" == label.lower() or "asf-arm" == label - for label in json.loads(self.public_runs_on_as_string) + "arm" == label.lower() + or "arm64" == label.lower() + or "asf-arm" == label + or "macos-14" == label + for label in json.loads(self.default_runs_on_as_string) ] )
