This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch restore-arm-builds in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8bef337dcfe6599911e28df1b3fca32e8a7c9fb4 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun May 4 23:56:27 2025 +0200 Simplify runs-on conditions to only differentiate AMD/ARM For now - when we do not have self-hosted runners yet, we want to simplify the selective check rules to only differentiate the two options we have now: * AMD runners (public) * ARM runners (also bublic) The selective checks now return two options: * runs-on-amd * runs-on-arm That should simplify choices and allow to bring back some of the disabled ARM jobs, while open up for the future possibility of enabling self-hosted runners. --- .github/workflows/additional-ci-image-checks.yml | 66 ++-- .github/workflows/additional-prod-image-tests.yml | 12 +- .github/workflows/airflow-distributions-tests.yml | 11 +- .github/workflows/basic-tests.yml | 14 +- .github/workflows/ci-image-build.yml | 10 +- .github/workflows/ci-image-checks.yml | 18 +- .github/workflows/ci.yml | 74 ++--- .github/workflows/finalize-tests.yml | 62 ++-- .github/workflows/generate-constraints.yml | 6 +- .github/workflows/helm-tests.yml | 12 +- .github/workflows/integration-system-tests.yml | 8 +- .github/workflows/k8s-tests.yml | 6 +- .github/workflows/prod-image-build.yml | 8 +- .github/workflows/prod-image-extra-checks.yml | 8 +- .github/workflows/publish-docs-to-s3.yml | 60 +++- .github/workflows/push-image-cache.yml | 12 +- .github/workflows/release_dockerhub_image.yml | 4 +- .github/workflows/run-unit-tests.yml | 6 +- .github/workflows/special-tests.yml | 26 +- .github/workflows/test-providers.yml | 8 +- dev/breeze/doc/ci/04_selective_checks.md | 15 +- .../src/airflow_breeze/commands/ci_commands.py | 9 +- dev/breeze/src/airflow_breeze/global_constants.py | 12 +- .../src/airflow_breeze/utils/selective_checks.py | 137 +------- dev/breeze/tests/test_pr_info.py | 158 +++------ dev/breeze/tests/test_selective_checks.py | 366 --------------------- 26 files changed, 290 insertions(+), 838 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 86740494b64..664aa746a06 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,16 +20,12 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining AMD public runners." required: true type: string - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." - required: true - type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runs-on-arm: + description: "The array of labels (in json form) determining ARM public runners." required: true type: string python-versions: @@ -103,8 +99,8 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ inputs.runs-on-amd }} + runs-on-arm: ${{ inputs.runs-on-arm }} cache-type: "Early" include-prod-images: "false" push-latest-images: "false" @@ -124,7 +120,7 @@ jobs: check-that-image-builds-quickly: timeout-minutes: 11 name: Check that image builds quickly - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} @@ -156,28 +152,28 @@ jobs: - name: "Check that image builds quickly" run: breeze shell --max-time 600 --platform "linux/amd64" -# # This is only a check if ARM images are successfully building when committer runs PR from -# # Apache repository. This is needed in case you want to fix failing cache job in "canary" run -# # There is no point in running this one in "canary" run, because the above step is doing the -# # same build anyway. -# build-ci-arm-images: -# name: Build CI ARM images -# uses: ./.github/workflows/ci-image-build.yml -# permissions: -# contents: read -# packages: write -# with: -# platform: "linux/arm64" -# push-image: "false" -# upload-image-artifact: "true" -# upload-mount-cache-artifact: ${{ inputs.canary-run }} -# runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} -# runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} -# python-versions: ${{ inputs.python-versions }} -# branch: ${{ inputs.branch }} -# constraints-branch: ${{ inputs.constraints-branch }} -# use-uv: ${{ inputs.use-uv }} -# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} -# docker-cache: ${{ inputs.docker-cache }} -# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} + # This is only a check if ARM images are successfully building when committer runs PR from + # Apache repository. This is needed in case you want to fix failing cache job in "canary" run + # There is no point in running this one in "canary" run, because the above step is doing the + # same build anyway. + build-ci-arm-images: + name: Build CI ARM images + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + packages: write + with: + runs-on-amd: ${{ inputs.runs-on-amd }} + runs-on-arm: ${{ inputs.runs-on-arm }} + platform: "linux/arm64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: ${{ inputs.canary-run }} + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + constraints-branch: ${{ inputs.constraints-branch }} + use-uv: ${{ inputs.use-uv }} + upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ inputs.docker-cache }} + disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} # diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 56badcd10ad..863f1ba7775 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,8 +20,8 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining AMD public runners." required: true type: string default-branch: @@ -63,7 +63,7 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-amd: ${{ inputs.runs-on-amd }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -78,7 +78,7 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-amd: ${{ inputs.runs-on-amd }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -92,7 +92,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of PROD image building" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -130,7 +130,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index c7071c5f34d..59fe65be132 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -21,6 +21,10 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: # Static inputs defined to choose which distribution to test to run + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." + required: true + type: string distribution-name: description: "The name of the distribution to test" required: true @@ -33,11 +37,6 @@ on: # yamllint disable-line rule:truthy description: "distribution test type" # eg task-sdk-tests required: true type: string - # Environment inputs - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." - required: true - type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -60,7 +59,7 @@ jobs: distributions-tests: timeout-minutes: 80 name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 7efc3deedc1..00d13151ff4 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,7 +20,7 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: + runs-on-amd: description: "The array of labels (in json form) determining public runners." required: true type: string @@ -66,7 +66,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} steps: - name: "Cleanup repo" shell: bash @@ -87,7 +87,7 @@ jobs: tests-ui: timeout-minutes: 15 name: React UI tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} if: inputs.run-ui-tests == 'true' steps: - name: "Cleanup repo" @@ -155,7 +155,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" steps: @@ -183,7 +183,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} needs: install-pre-commit if: inputs.basic-checks-only == 'true' steps: @@ -236,7 +236,7 @@ jobs: upgrade-check: timeout-minutes: 45 name: "Upgrade checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -306,7 +306,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1df2e26c090..5c217466429 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,12 +20,12 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runs-on-arm: + description: "The array of labels (in json form) determining public ARM runners." required: true type: string target-commit-sha: @@ -110,7 +110,7 @@ jobs: # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" # instead of an array of strings. # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-amd) || fromJSON(inputs.runs-on-arm) }} env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 93b8fd1ece2..0a091b7b98c 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -20,12 +20,8 @@ name: CI Image Checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." - required: true - type: string - runs-on-as-json-docs-build: - description: "The array of labels (in json form) determining the labels used for docs build." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string needs-mypy: @@ -117,7 +113,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache (only canary runs)" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.basic-checks-only == 'false' @@ -161,7 +157,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -201,7 +197,7 @@ jobs: mypy: timeout-minutes: 45 name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} needs: install-pre-commit if: inputs.needs-mypy == 'true' strategy: @@ -245,7 +241,7 @@ jobs: build-docs: timeout-minutes: 150 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} if: inputs.docs-build == 'true' strategy: fail-fast: false @@ -386,7 +382,7 @@ jobs: test-python-api-client: timeout-minutes: 60 name: "Test Python API client" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} if: inputs.needs-api-codegen == 'true' env: BACKEND: "postgres" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79434589238..d1c614450a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,12 +83,6 @@ jobs: include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} individual-providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} - is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} - is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} - is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} - is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} - is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} - is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} kubernetes-combos-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} @@ -123,11 +117,8 @@ jobs: run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} - runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} - runs-on-as-json-docs-build: ${{ steps.selective-checks.outputs.runs-on-as-json-docs-build }} - runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted-asf: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }} - runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ steps.selective-checks.outputs.runs-on-amd }} + runs-on-arm: ${{ steps.selective-checks.outputs.runs-on-arm }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} @@ -182,7 +173,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} @@ -203,17 +194,17 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} + runs-on-arm: ${{ needs.build-info.outputs.runs-on-arm }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} if: needs.build-info.outputs.ci-image-build == 'true' @@ -228,9 +219,8 @@ jobs: id-token: write if: needs.build-info.outputs.canary-run == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} + runs-on-arm: ${{ needs.build-info.outputs.runs-on-arm }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} @@ -251,7 +241,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.ci-image-build == 'true' with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # generate no providers constraints only in canary builds - they take quite some time to generate # they are not needed for regular builds, they are only needed to update constraints in canaries @@ -267,8 +257,7 @@ jobs: id-token: write contents: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -304,7 +293,7 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-tests == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -326,8 +315,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -344,7 +332,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -371,7 +359,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -398,7 +386,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -425,7 +413,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -453,7 +441,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -482,7 +470,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -512,7 +500,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "sqlite" test-name: "" test-name-separator: "" @@ -540,7 +528,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} backend: "sqlite" test-name: "" test-name-separator: "" @@ -574,7 +562,7 @@ jobs: needs.build-info.outputs.full-tests-needed == 'true') with: default-branch: ${{ needs.build-info.outputs.default-branch }} - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} core-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > @@ -599,7 +587,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} run-system-tests: ${{ needs.build-info.outputs.run-tests }} @@ -622,7 +610,7 @@ jobs: if: > needs.build-info.outputs.run-tests == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -650,7 +638,7 @@ jobs: packages: read if: needs.build-info.outputs.run-tests == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -679,7 +667,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} build-type: "Regular" platform: "linux/amd64" push-image: "false" @@ -700,7 +688,7 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -720,7 +708,7 @@ jobs: packages: read with: platform: "linux/amd64" - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -738,7 +726,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -759,7 +747,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -806,8 +794,8 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} + runs-on-arm: ${{ needs.build-info.outputs.runs-on-arm }} python-versions: ${{ needs.build-info.outputs.python-versions }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 12d3906de4c..2c7ecb50367 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,12 +20,12 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runs-on-arm: + description: "The array of labels (in json form) determining public ARM runners." required: true type: string python-versions: @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -142,8 +142,8 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} + runs-on-amd: ${{ inputs.runs-on-amd }} + runs-on-arm: ${{ inputs.runs-on-arm }} cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" @@ -159,34 +159,34 @@ jobs: inputs.canary-run == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - # push-buildx-cache-to-github-registry-arm: - # name: Push Regular ARM Image Cache - # needs: [update-constraints] - # uses: ./.github/workflows/push-image-cache.yml - # permissions: - # contents: read - # packages: write - # with: - # runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - # runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} - # cache-type: "Regular ARM" - # include-prod-images: "true" - # push-latest-images: "true" - # platform: "linux/arm64" - # python-versions: ${{ inputs.python-versions }} - # branch: ${{ inputs.branch }} - # constraints-branch: ${{ inputs.constraints-branch }} - # use-uv: "true" - # include-success-outputs: ${{ inputs.include-success-outputs }} - # docker-cache: ${{ inputs.docker-cache }} - # if: > - # inputs.canary-run == 'true' && - # (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + push-buildx-cache-to-github-registry-arm: + name: Push Regular ARM Image Cache + needs: [update-constraints] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + packages: write + with: + runs-on-amd: ${{ inputs.runs-on-amd }} + runs-on-arm: ${{ inputs.runs-on-arm }} + cache-type: "Regular ARM" + include-prod-images: "true" + push-latest-images: "true" + platform: "linux/arm64" + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + constraints-branch: ${{ inputs.constraints-branch }} + use-uv: "true" + include-success-outputs: ${{ inputs.include-success-outputs }} + docker-cache: ${{ inputs.docker-cache }} + if: > + inputs.canary-run == 'true' && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') summarize-warnings: timeout-minutes: 15 name: "Summarize warnings" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 46cb9f250bc..91c18694081 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,8 +20,8 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string python-versions-list-as-string: @@ -46,7 +46,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1b4aa19cbe5..12913ebbf84 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,12 +20,8 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." - required: true - type: string - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string helm-test-packages: @@ -46,7 +42,7 @@ jobs: tests-helm: timeout-minutes: 80 name: "Unit tests Helm: ${{ matrix.helm-test-package }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: @@ -85,7 +81,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" steps: diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index fc3159223e8..a60433601c5 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -20,7 +20,7 @@ name: Integration and system tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: + runs-on-amd: description: "The array of labels (in json form) determining public runners." required: true type: string @@ -71,7 +71,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: @@ -120,7 +120,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true' name: "Integration: providers ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: @@ -168,7 +168,7 @@ jobs: timeout-minutes: 30 if: inputs.run-system-tests == 'true' name: "System Tests" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} env: BACKEND: "postgres" BACKEND_VERSION: ${{ inputs.default-postgres-version }}" diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 2632408b813..4eefb022828 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -24,8 +24,8 @@ on: # yamllint disable-line rule:truthy description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string python-versions-list-as-string: @@ -54,7 +54,7 @@ jobs: tests-kubernetes: timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 844b0d46e7d..54602894804 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,8 +20,8 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining default runner for AMD build." required: true type: string build-type: @@ -114,7 +114,7 @@ jobs: build-prod-packages: name: "Build Airflow and provider distributions" timeout-minutes: 10 - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -186,7 +186,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 80 name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} needs: - build-prod-packages env: diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 356900c9222..c6959f9604d 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,8 +20,8 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string python-versions: @@ -62,7 +62,7 @@ jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-amd: ${{ inputs.runs-on-amd }} build-type: "MySQL Client" upload-image-artifact: "false" upload-package-artifact: "false" @@ -83,7 +83,7 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-amd: ${{ inputs.runs-on-amd }} build-type: "pip" upload-image-artifact: "false" upload-package-artifact: "false" diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index f538eb5e662..28a5020ab45 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -44,17 +44,73 @@ env: permissions: contents: read jobs: + build-info: + timeout-minutes: 10 + name: "Build Info" + runs-on: ["ubuntu-24.04"] + outputs: + runs-on-amd: ${{ steps.selective-checks.outputs.runs-on-amd }} + runs-on-arm: ${{ steps.selective-checks.outputs.runs-on-arm }} + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + VERBOSE: true + REF: ${{ inputs.ref }} + EXCLUDE_DOCS: ${{ inputs.exclude-docs }} + DESTINATION_LOCATION: ${{ inputs.destination-location }} + DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + if: contains(fromJSON('[ + "ashb", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2" + ]'), github.event.sender.login) + steps: + - name: "Input parameters summary" + shell: bash + run: | + echo "Input parameters summary" + echo "=========================" + echo "Ref: '${AIRFLOW_VERSION}'" + echo "Exclude docs: '${EXCLUDE_DOCS}'" + echo "Destination location: '${DESTINATION_LOCATION}'" + echo "Docs list as string: '${DOCS_LIST_AS_STRING}'" + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "true" + - name: Selective checks + id: selective-checks + env: + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + build-ci-images: name: Build CI images uses: ./.github/workflows/ci-image-build.yml + needs: [build-info] permissions: contents: read # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: '["ubuntu-22.04"]' - runs-on-as-json-self-hosted: '["ubuntu-22.04"]' + runs-on-amd: ${{ needs.build-info.outputs.runs-on-amd }} + runs-on-arm: ${{ needs.build-info.outputs.runs-on-arm }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 7db441c3cc7..9ba02c6aab0 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,12 +20,12 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runs-on-arm: + description: "The array of labels (in json form) determining public ARM runners." required: true type: string cache-type: @@ -87,7 +87,7 @@ jobs: # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" # instead of an array of strings. # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-amd) || fromJSON(inputs.runs-on-arm) }} permissions: contents: read packages: write @@ -162,7 +162,7 @@ jobs: # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" # instead of an array of strings. # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-amd) || fromJSON(inputs.runs-on-arm) }} permissions: contents: read packages: write diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index a82738d9973..e6da818cec1 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -48,6 +48,8 @@ jobs: platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} skipLatest: ${{ steps.selective-checks.outputs.skipLatest }} + runs-on-amd: ${{ steps.selective-checks.outputs.runs-on-amd }} + runs-on-arm: ${{ steps.selective-checks.outputs.runs-on-arm }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true @@ -109,7 +111,7 @@ jobs: timeout-minutes: 50 # yamllint disable rule:line-length name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }}" - runs-on: ${{ matrix.platform == 'linux/amd64' && fromJSON('["ubuntu-22.04"]') || fromJSON('["ubuntu-22.04-arm"]') }} + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(needs.build-info.outputs.runs-on-amd) || fromJSON(needs.build-info.outputs.runs-on-arm) }} needs: [build-info] strategy: fail-fast: false diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 939fa2b6341..9a46ceff2fa 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,8 +20,8 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string test-group: @@ -125,7 +125,7 @@ jobs: ${{ inputs.test-scope }}-${{ inputs.test-group }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ ${{ matrix.python-version}}:${{ matrix.test-types.description }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index d47907f923f..3d87e07ac7e 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,8 +20,8 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string default-branch: @@ -90,7 +90,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -113,7 +113,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -136,7 +136,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -160,7 +160,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -185,7 +185,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -209,7 +209,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -233,7 +233,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} test-name: "Postgres" test-scope: "Quarantined" test-group: "core" @@ -256,7 +256,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} test-name: "Postgres" test-scope: "Quarantined" test-group: "providers" @@ -280,7 +280,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} test-name: "Postgres" test-scope: "ARM collection" test-group: "core" @@ -304,7 +304,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} test-name: "Postgres" test-scope: "ARM collection" test-group: "providers" @@ -329,7 +329,7 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runs-on-amd: ${{ inputs.runs-on-amd }} test-name: "SystemTest" test-scope: "System" test-group: "core" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index 9775df78061..e642b5abc8e 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -20,8 +20,8 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runs-on-amd: + description: "The array of labels (in json form) determining public AMD runners." required: true type: string canary-run: @@ -68,7 +68,7 @@ jobs: prepare-install-verify-provider-distributions: timeout-minutes: 80 name: "Providers ${{ matrix.package-format }} tests" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: @@ -163,7 +163,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.compat.test-types.description }} - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runs-on-amd) }} strategy: fail-fast: false matrix: diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index c450342f7d1..b5e845643a8 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -196,14 +196,8 @@ GitHub Actions to pass the list of parameters to a command to execute | helm-version | Which Helm version to use for tests | v3.15.3 | | | include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | | individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | -| is-airflow-runner | Whether runner used is an airflow or infrastructure runner (true if airflow/false if infrastructure) | false | | -| is-amd-runner | Whether runner used is an AMD one | true | | -| is-arm-runner | Whether runner used is an ARM one | false | | | is-committer-build | Whether the build is triggered by a committer | false | | -| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | | is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | -| is-self-hosted-runner | Whether the runner is self-hosted | false | | -| is-vm-runner | Whether the runner uses VM to run | true | | | kind-version | Which Kind version to use for tests | v0.24.0 | | | kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.10-v1.28.13 | * | | kubernetes-versions | All Kubernetes versions to use for tests as JSON array | \['v1.25.2'\] | | @@ -234,11 +228,8 @@ GitHub Actions to pass the list of parameters to a command to execute | run-tests | Whether unit tests should be run ("true"/"false") | true | | | run-ui-tests | Whether UI tests should be run ("true"/"false") | true | | | run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | -| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | \["ubuntu-22.04"\] | | -| runs-on-as-json-docs-build | List of labels assigned for runners for that build for ddcs build (as string) | \["ubuntu-22.04"\] | | -| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | \["self-hosted", "Linux", "X64"\] | | -| runs-on-as-json-self-hosted-asf | List of labels assigned for runners for that build for ASF self hosted runners | \["self-hosted", "Linux", "X64"\] | | -| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | \["ubuntu-22.04"\] | | +| runs-on-amd | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | | +| runs-on-arm | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | | | selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | | skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | | skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | @@ -329,8 +320,6 @@ This table summarizes the labels you can use on PRs to control the selective che | latest versions only | *-versions-*, *-versions-* | If set, the number of Python, Kubernetes, DB versions will be limited to the latest ones. | | non committer build | is-committer-build | If set, the scripts used for images are used from target branch for committers. | | upgrade to newer dependencies | upgrade-to-newer-dependencies | If set to true (default false) then dependencies in the CI image build are upgraded. | -| use public runners | runs-on-as-json-public | Force using public runners as default runners. | -| use self-hosted runners | runs-on-as-json-default | Force using self-hosted runners as default runners. | ----- diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index 18e76153b95..273143a2728 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -41,8 +41,7 @@ from airflow_breeze.commands.common_options import ( ) from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - RUNS_ON_PUBLIC_RUNNER, - RUNS_ON_SELF_HOSTED_RUNNER, + RUNS_ON_PUBLIC_RUNNER_AMD, GithubEvents, github_events, ) @@ -307,10 +306,8 @@ class WorkflowInfo(NamedTuple): for label in self.pull_request_labels: if "use public runners" in label: get_console().print("[info]Force running on public runners") - return RUNS_ON_PUBLIC_RUNNER - if not os.environ.get("AIRFLOW_SELF_HOSTED_RUNNER"): - return RUNS_ON_PUBLIC_RUNNER - return RUNS_ON_SELF_HOSTED_RUNNER + return RUNS_ON_PUBLIC_RUNNER_AMD + return RUNS_ON_PUBLIC_RUNNER_AMD def is_canary_run(self) -> str: if ( diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index d0c4e2e6fa6..c7cb48558cb 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -34,16 +34,8 @@ from airflow_breeze.utils.path_utils import ( AIRFLOW_ROOT_PATH, ) -RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]' -# we should get more sophisticated logic here in the future, but for now we just check if -# we use self airflow, vm-based, amd hosted runner as a default -# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners -RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]' -RUNS_ON_SELF_HOSTED_ASF_RUNNER = '["self-hosted", "asf-runner"]' -# TODO: when we have it properly set-up with labels we should change it to -# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]' -# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]' -SELF_HOSTED_RUNNERS_CPU_COUNT = 8 +RUNS_ON_PUBLIC_RUNNER_AMD = '["ubuntu-22.04"]' +RUNS_ON_PUBLIC_RUNNER_ARM = '["ubuntu-22.04-arm"]' ANSWER = "" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index a11bc50dcb2..9a6b35531a9 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -46,9 +46,8 @@ from airflow_breeze.global_constants import ( KIND_VERSION, NUMBER_OF_LOW_DEP_SLICES, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, - RUNS_ON_PUBLIC_RUNNER, - RUNS_ON_SELF_HOSTED_ASF_RUNNER, - RUNS_ON_SELF_HOSTED_RUNNER, + RUNS_ON_PUBLIC_RUNNER_AMD, + RUNS_ON_PUBLIC_RUNNER_ARM, TESTABLE_CORE_INTEGRATIONS, TESTABLE_PROVIDERS_INTEGRATIONS, GithubEvents, @@ -86,7 +85,6 @@ LOG_WITHOUT_MOCK_IN_TESTS_EXCEPTION_LABEL = "log exception" NON_COMMITTER_BUILD_LABEL = "non committer build" UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies" USE_PUBLIC_RUNNERS_LABEL = "use public runners" -USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners" ALL_CI_SELECTIVE_TEST_TYPES = "API Always CLI Core Other Serialization" @@ -1261,135 +1259,12 @@ class SelectiveChecks: return " ".join(sorted(affected_providers)) @cached_property - def runs_on_as_json_default(self) -> str: - if self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY: - if self._is_canary_run(): - return RUNS_ON_SELF_HOSTED_RUNNER - if self._pr_labels and USE_PUBLIC_RUNNERS_LABEL in self._pr_labels: - # Forced public runners - return RUNS_ON_PUBLIC_RUNNER - actor = self._github_actor - if self._github_event in (GithubEvents.PULL_REQUEST, GithubEvents.PULL_REQUEST_TARGET): - try: - actor = self._github_context_dict["event"]["pull_request"]["user"]["login"] - get_console().print( - f"[warning]The actor: {actor} retrieved from GITHUB_CONTEXT's" - f" event.pull_request.user.login[/]" - ) - except Exception as e: - get_console().print(f"[warning]Exception when reading user login: {e}[/]") - get_console().print( - f"[info]Could not find the actor from pull request, " - f"falling back to the actor who triggered the PR: {actor}[/]" - ) - if ( - actor not in COMMITTERS - and self._pr_labels - and USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels - ): - get_console().print( - f"[error]The PR has `{USE_SELF_HOSTED_RUNNERS_LABEL}` label, but " - f"{actor} is not a committer. This is not going to work.[/]" - ) - sys.exit(1) - if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels: - # Forced self-hosted runners - return RUNS_ON_SELF_HOSTED_RUNNER - return RUNS_ON_PUBLIC_RUNNER - return RUNS_ON_PUBLIC_RUNNER - - @cached_property - def runs_on_as_json_self_hosted(self) -> str: - return RUNS_ON_SELF_HOSTED_RUNNER + def runs_on_amd(self) -> str: + return RUNS_ON_PUBLIC_RUNNER_AMD @cached_property - def runs_on_as_json_self_hosted_asf(self) -> str: - return RUNS_ON_SELF_HOSTED_ASF_RUNNER - - @cached_property - def runs_on_as_json_docs_build(self) -> str: - # We used to run docs build on self-hosted runners because they had more space, but - # It turned out that public runners have a lot of space in /mnt folder that we can utilise - # but in the future we might want to switch back to self-hosted runners so we have this - # separate property to determine that and place to implement different logic if needed - return RUNS_ON_PUBLIC_RUNNER - - @cached_property - def runs_on_as_json_public(self) -> str: - return RUNS_ON_PUBLIC_RUNNER - - @cached_property - def is_self_hosted_runner(self) -> bool: - """ - True if the job has runs_on labels indicating It should run on "self-hosted" runner. - - All self-hosted runners have "self-hosted" label. - """ - return "self-hosted" in json.loads(self.runs_on_as_json_default) - - @cached_property - def is_airflow_runner(self) -> bool: - """ - True if the job has runs_on labels indicating It should run on Airflow managed runner. - - All Airflow team-managed runners will have "airflow-runner" label. - """ - # TODO: when we have it properly set-up with labels we should just check for - # "airflow-runner" presence in runs_on - runs_on_array = json.loads(self.runs_on_as_json_default) - return "Linux" in runs_on_array and "X64" in runs_on_array and "self-hosted" in runs_on_array - - @cached_property - def is_amd_runner(self) -> bool: - """ - True if the job has runs_on labels indicating AMD architecture. - - Matching amd label, asf-runner, and any ubuntu that does not contain arm - The last case is just in case - currently there are no public runners that have ARM - instances, but they can add them in the future. It might be that for compatibility - they will just add arm in the runner name - because currently GitHub users use just - one label "ubuntu-*" for all their work and depend on them being AMD ones. - """ - return any( - [ - label.lower() == "amd" - or label.lower() == "amd64" - or label.lower() == "x64" - or label == "asf-runner" - or ("ubuntu" in label and "arm" not in label.lower()) - for label in json.loads(self.runs_on_as_json_public) - ] - ) - - @cached_property - def is_arm_runner(self) -> bool: - """ - True if the job has runs_on labels indicating ARM architecture. - - Matches any label containing arm - including ASF-specific "asf-arm" label. - - # See https://cwiki.apache.org/confluence/pages/viewpage.action?spaceKey=INFRA&title=ASF+Infra+provided+self-hosted+runners - """ - return any( - [ - label.lower() == "arm" or label.lower() == "arm64" or label == "asf-arm" - for label in json.loads(self.runs_on_as_json_public) - ] - ) - - @cached_property - def is_vm_runner(self) -> bool: - """Whether the runner is VM runner (managed by airflow).""" - # TODO: when we have it properly set-up with labels we should just check for - # "airflow-runner" presence in runs_on - return self.is_airflow_runner - - @cached_property - def is_k8s_runner(self) -> bool: - """Whether the runner is K8s runner (managed by airflow).""" - # TODO: when we have it properly set-up with labels we should just check for - # "k8s-runner" presence in runs_on - return False + def runs_on_arm(self) -> str: + return RUNS_ON_PUBLIC_RUNNER_ARM @cached_property def has_migrations(self) -> bool: diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py index 1c874e5c4db..2a2f814c7dd 100644 --- a/dev/breeze/tests/test_pr_info.py +++ b/dev/breeze/tests/test_pr_info.py @@ -16,9 +16,7 @@ # under the License. from __future__ import annotations -import os from pathlib import Path -from unittest import mock from airflow_breeze.commands.ci_commands import workflow_info @@ -26,123 +24,57 @@ TEST_PR_INFO_DIR = Path(__file__).parent / "test_pr_info_files" def test_pr_info(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}): - json_string = (TEST_PR_INFO_DIR / "pr_github_context.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == [ - "area:providers", - "area:dev-tools", - "area:logging", - "kind:documentation", - ] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "test/airflow" - assert wi.event_name == "pull_request" - assert wi.pr_number == 26004 - assert wi.get_runs_on() == '["ubuntu-22.04"]' - assert wi.is_canary_run() == "false" - assert wi.run_coverage() == "false" + json_string = (TEST_PR_INFO_DIR / "pr_github_context.json").read_text() + wi = workflow_info(json_string) + assert wi.pull_request_labels == [ + "area:providers", + "area:dev-tools", + "area:logging", + "kind:documentation", + ] + assert wi.target_repo == "apache/airflow" + assert wi.head_repo == "test/airflow" + assert wi.event_name == "pull_request" + assert wi.pr_number == 26004 + assert wi.get_runs_on() == '["ubuntu-22.04"]' + assert wi.is_canary_run() == "false" + assert wi.run_coverage() == "false" def test_push_info(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}): - json_string = (TEST_PR_INFO_DIR / "push_github_context.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == [] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "push" - assert wi.pr_number is None - assert wi.get_runs_on() == '["ubuntu-22.04"]' - assert wi.is_canary_run() == "true" - assert wi.run_coverage() == "true" + json_string = (TEST_PR_INFO_DIR / "push_github_context.json").read_text() + wi = workflow_info(json_string) + assert wi.pull_request_labels == [] + assert wi.target_repo == "apache/airflow" + assert wi.head_repo == "apache/airflow" + assert wi.event_name == "push" + assert wi.pr_number is None + assert wi.get_runs_on() == '["ubuntu-22.04"]' + assert wi.is_canary_run() == "true" + assert wi.run_coverage() == "true" def test_schedule(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}): - json_string = (TEST_PR_INFO_DIR / "schedule_github_context.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == [] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "schedule" - assert wi.pr_number is None - assert wi.get_runs_on() == '["ubuntu-22.04"]' - assert wi.is_canary_run() == "true" - assert wi.run_coverage() == "false" - - -def test_runs_on_self_hosted(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}): - json_string = (TEST_PR_INFO_DIR / "simple_pr.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == ["another"] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "pull_request" - assert wi.pr_number == 1234 - # TODO: fix it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' - assert wi.is_canary_run() == "false" - assert wi.run_coverage() == "false" - - -def test_runs_on_forced_public_runner(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}): - json_string = (TEST_PR_INFO_DIR / "self_hosted_forced_pr.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == ["use public runners", "another"] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "pull_request" - assert wi.pr_number == 1234 - assert wi.get_runs_on() == '["ubuntu-22.04"]' - assert wi.is_canary_run() == "false" - assert wi.run_coverage() == "false" + json_string = (TEST_PR_INFO_DIR / "schedule_github_context.json").read_text() + wi = workflow_info(json_string) + assert wi.pull_request_labels == [] + assert wi.target_repo == "apache/airflow" + assert wi.head_repo == "apache/airflow" + assert wi.event_name == "schedule" + assert wi.pr_number is None + assert wi.get_runs_on() == '["ubuntu-22.04"]' + assert wi.is_canary_run() == "true" + assert wi.run_coverage() == "false" def test_runs_on_simple_pr_other_repo(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}): - json_string = (TEST_PR_INFO_DIR / "simple_pr_different_repo.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == ["another"] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "test/airflow" - assert wi.event_name == "pull_request" - assert wi.pr_number == 1234 - assert wi.get_runs_on() == '["ubuntu-22.04"]' - assert wi.is_canary_run() == "false" - assert wi.run_coverage() == "false" - - -def test_runs_on_push_other_branch(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}): - json_string = (TEST_PR_INFO_DIR / "push_other_branch.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == [] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "push" - assert wi.pr_number is None - # TODO: revert it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' - assert wi.is_canary_run() == "false" - assert wi.run_coverage() == "false" - - -def test_runs_on_push_v_test_branch(): - with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}): - json_string = (TEST_PR_INFO_DIR / "push_v_test_branch.json").read_text() - wi = workflow_info(json_string) - assert wi.pull_request_labels == [] - assert wi.target_repo == "apache/airflow" - assert wi.head_repo == "apache/airflow" - assert wi.event_name == "push" - assert wi.pr_number is None - # TODO: revert it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' - assert wi.is_canary_run() == "true" - assert wi.run_coverage() == "false" + json_string = (TEST_PR_INFO_DIR / "simple_pr_different_repo.json").read_text() + wi = workflow_info(json_string) + assert wi.pull_request_labels == ["another"] + assert wi.target_repo == "apache/airflow" + assert wi.head_repo == "test/airflow" + assert wi.event_name == "pull_request" + assert wi.pr_number == 1234 + assert wi.get_runs_on() == '["ubuntu-22.04"]' + assert wi.is_canary_run() == "false" + assert wi.run_coverage() == "false" diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 9369e320d5c..82b9725c71f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -28,7 +28,6 @@ from rich.console import Console from airflow_breeze.global_constants import ( ALLOWED_KUBERNETES_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, - COMMITTERS, DEFAULT_KUBERNETES_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, NUMBER_OF_LOW_DEP_SLICES, @@ -2422,302 +2421,6 @@ def test_provider_yaml_or_pyproject_toml_changes_trigger_ci_build( assert_outputs_are_printed(expected_outputs, str(stderr)) [email protected]( - ( - "github_event, github_actor, github_repository, pr_labels, " - "github_context_dict, runs_on_as_json_default, runs_on_as_docs_build, is_self_hosted_runner, " - "is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner, exception" - ), - [ - pytest.param( - GithubEvents.PUSH, - "user", - "apache/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - # "true", - # "true", - "true", - "false", - # TODO: revert it when we fix self-hosted runners - "false", - # "true", - "false", - False, - id="Push event", - ), - pytest.param( - GithubEvents.PUSH, - "user", - "private/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Push event for private repo", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - "user", - "apache/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - "user", - "private/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request private repo", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - COMMITTERS[0], - "apache/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - COMMITTERS[0], - "apache/airflow", - (), - dict(event=dict(pull_request=dict(user=dict(login="user")))), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request committer pr non-committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - COMMITTERS[0], - "private/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request private repo committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST_TARGET, - "user", - "apache/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request target", - ), - pytest.param( - GithubEvents.PULL_REQUEST_TARGET, - "user", - "private/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request target private repo", - ), - pytest.param( - GithubEvents.PULL_REQUEST_TARGET, - COMMITTERS[0], - "apache/airflow", - [], - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request target committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - COMMITTERS[0], - "apache/airflow", - (), - dict(event=dict(pull_request=dict(user=dict(login="user")))), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request target committer pr non-committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST_TARGET, - COMMITTERS[0], - "private/airflow", - (), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request targe private repo committer", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - "user", - "apache/airflow", - ("use self-hosted runners",), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - True, - id="Pull request by non committer with 'use self-hosted runners' label.", - ), - pytest.param( - GithubEvents.PULL_REQUEST, - COMMITTERS[0], - "apache/airflow", - ("use public runners",), - dict(), - '["ubuntu-22.04"]', - '["ubuntu-22.04"]', - "false", - "false", - "true", - "false", - "false", - "false", - False, - id="Pull request by committer with 'use public runners' label.", - ), - ], -) -def test_runs_on( - github_event: GithubEvents, - github_actor: str, - github_repository: str, - pr_labels: tuple[str, ...], - github_context_dict: dict[str, Any], - runs_on_as_json_default: str, - runs_on_as_docs_build: str, - is_self_hosted_runner: str, - is_airflow_runner: str, - is_amd_runner: str, - is_arm_runner: str, - is_vm_runner: str, - is_k8s_runner: str, - exception: bool, -): - def get_output() -> str: - return str( - SelectiveChecks( - files=(), - commit_ref="", - github_repository=github_repository, - github_event=github_event, - github_actor=github_actor, - github_context_dict=github_context_dict, - pr_labels=pr_labels, - default_branch="main", - ) - ) - - if exception: - with pytest.raises(SystemExit): - get_output() - else: - stderr = get_output() - assert_outputs_are_printed({"runs-on-as-json-default": runs_on_as_json_default}, str(stderr)) - assert_outputs_are_printed({"runs-on-as-json-docs-build": runs_on_as_docs_build}, str(stderr)) - assert_outputs_are_printed({"is-self-hosted-runner": is_self_hosted_runner}, str(stderr)) - assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner}, str(stderr)) - assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr)) - assert_outputs_are_printed({"is-arm-runner": is_arm_runner}, str(stderr)) - assert_outputs_are_printed({"is-vm-runner": is_vm_runner}, str(stderr)) - assert_outputs_are_printed({"is-k8s-runner": is_k8s_runner}, str(stderr)) - - @pytest.mark.parametrize( "files, has_migrations", [ @@ -2857,75 +2560,6 @@ def test_mypy_matches( assert_outputs_are_printed(expected_outputs, str(stderr)) [email protected]( - "files, expected_outputs, github_actor, pr_labels", - [ - pytest.param( - ("README.md",), - { - "is-committer-build": "false", - "runs-on-as-json-default": '["ubuntu-22.04"]', - }, - "", - (), - id="Regular pr", - ), - pytest.param( - ("README.md",), - { - "is-committer-build": "true", - "runs-on-as-json-default": '["ubuntu-22.04"]', - }, - "potiuk", - (), - id="Committer regular PR", - ), - pytest.param( - ("README.md",), - { - "is-committer-build": "false", - "runs-on-as-json-default": '["ubuntu-22.04"]', - }, - "potiuk", - ("non committer build",), - id="Committer regular PR - forcing non-committer build", - ), - pytest.param( - ("README.md",), - { - "docker-cache": "disabled", - "disable-airflow-repo-cache": "true", - }, - "potiuk", - ("disable image cache",), - id="Disabled cache", - ), - pytest.param( - ("README.md",), - { - "docker-cache": "registry", - "disable-airflow-repo-cache": "false", - }, - "potiuk", - (), - id="Standard cache", - ), - ], -) -def test_pr_labels( - files: tuple[str, ...], expected_outputs: dict[str, str], github_actor: str, pr_labels: tuple[str, ...] -): - stderr = SelectiveChecks( - files=files, - commit_ref=NEUTRAL_COMMIT, - default_branch="main", - github_actor=github_actor, - github_event=GithubEvents.PULL_REQUEST, - pr_labels=pr_labels, - ) - assert_outputs_are_printed(expected_outputs, str(stderr)) - - @pytest.mark.parametrize( "files, pr_labels, github_event", [
