This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8c1b105a82 Switch to new labelling scheme allowing for ASF self-hosted
runners (#35109)
8c1b105a82 is described below
commit 8c1b105a82b87e83ebbd149b3d6585669655fc17
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Oct 23 10:01:54 2023 +0200
Switch to new labelling scheme allowing for ASF self-hosted runners (#35109)
We have been using a labelling scheme where just "self-hosted" label
was used for our CI jobs to indicate that the job should run on
the "airflow" provided self-hosted runners. Since the ASF infra
now provides self-hosted runners managed by them when we want to
make use of it, we should change the scheme used by us to allow
more fine-grained distinction between the runners.
This will also be helpful when we switch to new runners based on k8s,
where we want to run them side-by-side with the old VM-based runners, it
will allow us to selectively choose which self-hosted runners we want
to use (and possibly dynamically swap between those).
---
.github/workflows/build-images.yml | 24 ++-
.github/workflows/ci.yml | 114 +++++++------
.github/workflows/codeql-analysis.yml | 4 +-
.github/workflows/recheck-old-bug-report.yml | 2 +-
.github/workflows/release_dockerhub_image.yml | 10 +-
.github/workflows/stale.yml | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 8 +-
.../src/airflow_breeze/utils/selective_checks.py | 73 +++++++++
dev/breeze/tests/test_pr_info.py | 16 +-
dev/breeze/tests/test_selective_checks.py | 178 ++++++++++++++++++---
10 files changed, 332 insertions(+), 99 deletions(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index d8fe6af8a0..e57dca99fc 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -66,7 +66,13 @@ jobs:
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
- runs-on: ${{ steps.selective-checks.outputs.runs-on }}
+ runs-on: ${{steps.selective-checks.outputs.runs-on}}
+ is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
+ is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
+ is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
+ is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
+ is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
+ is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
target-commit-sha:
"${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
github.sha
@@ -183,7 +189,7 @@ jobs:
timeout-minutes: 80
name: >
Build CI images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: |
needs.build-info.outputs.image-build == 'true' &&
@@ -192,7 +198,7 @@ jobs:
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
+ RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
@@ -256,7 +262,7 @@ jobs:
name: >
Build PROD images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images]
if: |
needs.build-info.outputs.image-build == 'true' &&
@@ -265,7 +271,7 @@ jobs:
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
+ RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
@@ -312,18 +318,20 @@ jobs:
build-ci-images-arm:
timeout-minutes: 50
name: "Build ARM CI images
${{needs.build-info.outputs.all-python-versions-list-as-string}}"
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-prod-images]
+ # We can change the job to run on ASF ARM runners and do not start our
instance once we enable ASF runners
if: |
needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
- needs.build-info.outputs.runs-on == 'self-hosted' &&
+ needs.build-info.outputs.is-self-hosted-runner == 'true' &&
+ needs.build-info.outputs.is-airflow-runner == 'true' &&
github.repository == 'apache/airflow'
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
+ RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 412bc15512..f769520820 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,7 +114,13 @@ jobs:
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
- runs-on: ${{ steps.selective-checks.outputs.runs-on }}
+ runs-on: ${{steps.selective-checks.outputs.runs-on}}
+ is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
+ is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
+ is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
+ is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
+ is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
+ is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -192,7 +198,7 @@ jobs:
push-early-buildx-cache-to-github-registry:
timeout-minutes: 50
name: "Push Early Image Cache"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
strategy:
@@ -264,7 +270,7 @@ jobs:
check-that-image-builds-quickly:
timeout-minutes: 5
name: "Check that image builds quickly"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- push-early-buildx-cache-to-github-registry
@@ -295,7 +301,7 @@ jobs:
name: >-
${{needs.build-info.outputs.build-job-description}} CI images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
@@ -333,7 +339,7 @@ jobs:
name: >
Preview constraints
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images]
env:
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
@@ -390,7 +396,7 @@ jobs:
name: >
${{needs.build-info.outputs.build-job-description}} PROD images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images, generate-constraints]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
@@ -428,7 +434,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
steps:
- name: Cleanup repo
@@ -453,7 +459,7 @@ jobs:
tests-www:
timeout-minutes: 10
name: React WWW tests
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.run-www-tests == 'true'
steps:
@@ -481,7 +487,7 @@ jobs:
test-openapi-client-generation:
timeout-minutes: 10
name: "Test OpenAPI client generation"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.needs-api-codegen == 'true'
steps:
@@ -498,7 +504,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of production image building"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.image-build == 'true'
steps:
@@ -524,7 +530,7 @@ jobs:
test-git-clone-on-windows:
timeout-minutes: 5
name: "Test git clone on Windows"
- runs-on: windows-latest
+ runs-on: "windows-latest"
needs: [build-info]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -532,7 +538,6 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- if: needs.build-info.outputs.runs-on != 'self-hosted'
wait-for-ci-images:
timeout-minutes: 120
@@ -567,7 +572,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -612,7 +617,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -666,7 +671,7 @@ jobs:
docs:
timeout-minutes: 90
name: "Build docs"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
@@ -732,7 +737,7 @@ jobs:
prepare-test-provider-packages-wheel:
timeout-minutes: 80
name: "Provider packages wheel (verify)"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -787,7 +792,7 @@ jobs:
provider-airflow-compatibility-check:
timeout-minutes: 80
name: "Providers Airflow 2.5 compatibility check"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -854,7 +859,7 @@ jobs:
prepare-install-provider-packages-sdist:
timeout-minutes: 80
name: "Provider packages sdist (install)"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -904,9 +909,11 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: >
+ needs.build-info.outputs.is-self-hosted-runner == 'true'
+ && needs.build-info.outputs.is-airflow-runner == 'false'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
@@ -939,7 +946,7 @@ jobs:
tests-helm:
timeout-minutes: 80
name: "Unit tests Helm: ${{matrix.helm-test-package}}"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
@@ -983,7 +990,7 @@ jobs:
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
@@ -1039,7 +1046,7 @@ jobs:
LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1088,7 +1095,7 @@ jobs:
MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1137,7 +1144,7 @@ jobs:
InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1185,7 +1192,7 @@ jobs:
name: >
MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
@@ -1238,7 +1245,7 @@ jobs:
name: >
MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
@@ -1264,7 +1271,8 @@ jobs:
PARALLELISM: "${{needs.build-info.outputs.mssql-parallelism}}"
if: >
needs.build-info.outputs.run-tests == 'true' &&
- (needs.build-info.outputs.runs-on == 'self-hosted' ||
+ (needs.build-info.outputs.is-self-hosted-runner == 'true' &&
+ needs.build-info.outputs.is-airflow-runner == 'true' ||
needs.build-info.outputs.full-tests-needed == 'true' ||
needs.build-info.outputs.has-migrations == 'true')
steps:
@@ -1297,7 +1305,7 @@ jobs:
timeout-minutes: 130
name: >
Sqlite Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
@@ -1345,7 +1353,7 @@ jobs:
tests-integration-postgres:
timeout-minutes: 130
name: Integration Tests Postgres
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1375,35 +1383,35 @@ jobs:
run: |
breeze testing integration-tests --integration cassandra
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: mongo"
run: |
breeze testing integration-tests --integration mongo
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: pinot"
run: |
breeze testing integration-tests --integration pinot
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: celery"
run: |
breeze testing integration-tests --integration celery
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: trino, kerberos"
run: |
breeze testing integration-tests --integration trino --integration
kerberos
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: Kafka"
run: |
breeze testing integration-tests --integration kafka
breeze down
- if: needs.build-info.outputs.runs-on != 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: all-testable"
run: breeze testing integration-tests --integration all-testable
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration Postgres"
uses: ./.github/actions/post_tests_success
@@ -1417,7 +1425,7 @@ jobs:
tests-integration-mysql:
timeout-minutes: 130
name: Integration Tests MySQL
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1437,32 +1445,32 @@ jobs:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Prepare breeze & CI image:
${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Integration Tests MySQL: all-testable"
run: breeze testing integration-tests --integration all-testable
- if: needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration MySQL"
uses: ./.github/actions/post_tests_success
- if: success() && needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests failure: Integration MySQL"
uses: ./.github/actions/post_tests_failure
- if: failure() && needs.build-info.outputs.runs-on == 'self-hosted'
+ if: failure()
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
continue-on-error: true
needs: [build-info, wait-for-ci-images]
env:
@@ -1473,7 +1481,7 @@ jobs:
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID:
"quarantined-${{needs.build-info.outputs.default-python-version}}"
COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
- if: needs.build-info.outputs.run-tests == 'true' &&
needs.build-info.outputs.runs-on == 'self-hosted'
+ if: needs.build-info.outputs.run-tests == 'true' &&
needs.build-info.outputs.is-airflow-runner == 'true'
steps:
- name: Cleanup repo
shell: bash
@@ -1526,7 +1534,7 @@ jobs:
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- tests-postgres
@@ -1560,7 +1568,7 @@ jobs:
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for & verify PROD images"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
@@ -1602,7 +1610,7 @@ jobs:
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Test docker-compose quick start"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
@@ -1632,7 +1640,7 @@ jobs:
name: "\
Helm: ${{matrix.executor}} - ${{matrix.use-standard-naming}} - \
${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
strategy:
matrix:
@@ -1704,7 +1712,7 @@ jobs:
packages: write
timeout-minutes: 80
name: "Update constraints"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- docs
@@ -1767,7 +1775,7 @@ jobs:
push-buildx-cache-to-github-registry:
timeout-minutes: 50
name: "Push Image Cache"
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- generate-constraints
@@ -1861,7 +1869,7 @@ jobs:
name: >
Build CI ARM images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: "${{needs.build-info.outputs.runs-on}}"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- wait-for-ci-images
diff --git a/.github/workflows/codeql-analysis.yml
b/.github/workflows/codeql-analysis.yml
index 656ab5d7c5..b71283c02a 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -33,7 +33,7 @@ concurrency:
jobs:
selective-checks:
name: Selective checks
- runs-on: ubuntu-22.04
+ runs-on: "ubuntu-22.04"
outputs:
needs-python-scans: ${{
steps.selective-checks.outputs.needs-python-scans }}
needs-javascript-scans: ${{
steps.selective-checks.outputs.needs-javascript-scans }}
@@ -54,7 +54,7 @@ jobs:
analyze:
name: Analyze
- runs-on: ubuntu-22.04
+ runs-on: "ubuntu-22.04"
needs: [selective-checks]
strategy:
fail-fast: false
diff --git a/.github/workflows/recheck-old-bug-report.yml
b/.github/workflows/recheck-old-bug-report.yml
index 093e5df4ef..7c3adcd0a6 100644
--- a/.github/workflows/recheck-old-bug-report.yml
+++ b/.github/workflows/recheck-old-bug-report.yml
@@ -26,7 +26,7 @@ permissions:
issues: write
jobs:
recheck-old-bug-report:
- runs-on: ubuntu-22.04
+ runs-on: "ubuntu-22.04"
steps:
- uses: actions/stale@v8
with:
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 1114126f62..9c25aaea0b 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -39,7 +39,9 @@ jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-22.04' }}
+ # TODO: when we have it properly set-up with labels we should check for
+ # "airflow-runner" presence in runs_on
+ runs-on: ["self-hosted", "Linux", "X64"]
outputs:
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
allPythonVersions: ${{
steps.selective-checks.outputs.all-python-versions }}
@@ -67,14 +69,16 @@ jobs:
release-images:
timeout-minutes: 120
name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-22.04' }}
+ # TODO: when we have it properly set-up with labels we should check for
+ # "airflow-runner" presence in runs_on
+ runs-on: ["self-hosted", "Linux", "X64"]
needs: [build-info]
strategy:
fail-fast: false
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions)
}}
env:
- RUNS_ON: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-22.04' }}
+ RUNS_ON: '["self-hosted", "Linux", "X64"]'
if: contains(fromJSON('[
"ashb",
"eladkal",
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 92e31cc685..fb680826aa 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -27,7 +27,7 @@ permissions:
issues: write
jobs:
stale:
- runs-on: ubuntu-22.04
+ runs-on: "ubuntu-22.04"
steps:
- uses: actions/stale@v8
with:
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 00d7e1c99f..094376b04b 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -28,8 +28,12 @@ from pathlib import Path
from airflow_breeze.utils.host_info_utils import Architecture
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
PROVIDER_DEPENDENCIES_JSON_FILE_PATH
-RUNS_ON_PUBLIC_RUNNER = "ubuntu-22.04"
-RUNS_ON_SELF_HOSTED_RUNNER = "self-hosted"
+RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
+# we should get more sophisticated logic here in the future, but for now we
just check if
+# we use self airflow, vm-based, amd hosted runner as a default
+# TODO: when we have it properly set-up with labels we should change it to
+# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner",
"X64"]'
+RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
SELF_HOSTED_RUNNERS_CPU_COUNT = 8
ANSWER = ""
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index f2843ada5a..3e1001ee8e 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -801,6 +801,79 @@ class SelectiveChecks:
return RUNS_ON_SELF_HOSTED_RUNNER
return RUNS_ON_PUBLIC_RUNNER
+ @cached_property
+ def is_self_hosted_runner(self) -> bool:
+ """
+ True if the job has runs_on labels indicating It should run on
"self-hosted" runner.
+
+ All self-hosted runners have "self-hosted" label.
+ """
+ return "self-hosted" in json.loads(self.runs_on)
+
+ @cached_property
+ def is_airflow_runner(self) -> bool:
+ """
+ True if the job has runs_on labels indicating It should run on Airflow
managed runner.
+
+ All Airflow team-managed runners will have "airflow-runner" label.
+ """
+ # TODO: when we have it properly set-up with labels we should just
check for
+ # "airflow-runner" presence in runs_on
+ runs_on_array = json.loads(self.runs_on)
+ return "Linux" in runs_on_array and "X64" in runs_on_array and
"self-hosted" in runs_on_array
+
+ @cached_property
+ def is_amd_runner(self) -> bool:
+ """
+ True if the job has runs_on labels indicating AMD architecture.
+
+ Matching amd label, asf-runner, and any ubuntu that does not contain
arm
+ The last case is just in case - currently there are no public runners
that have ARM
+ instances, but they can add them in the future. It might be that for
compatibility
+ they will just add arm in the runner name - because currently GitHub
users use just
+ one label "ubuntu-*" for all their work and depend on them being AMD
ones.
+ """
+ return any(
+ [
+ "amd" == label.lower()
+ or "amd64" == label.lower()
+ or "x64" == label.lower()
+ or "asf-runner" == label
+ or ("ubuntu" in label and "arm" not in label.lower())
+ for label in json.loads(self.runs_on)
+ ]
+ )
+
+ @cached_property
+ def is_arm_runner(self) -> bool:
+ """
+ True if the job has runs_on labels indicating ARM architecture.
+
+ Matches any label containing arm - including ASF-specific "asf-arm"
label.
+
+ # See
https://cwiki.apache.org/confluence/pages/viewpage.action?spaceKey=INFRA&title=ASF+Infra+provided+self-hosted+runners
+ """
+ return any(
+ [
+ "arm" == label.lower() or "arm64" == label.lower() or
"asf-arm" == label
+ for label in json.loads(self.runs_on)
+ ]
+ )
+
+ @cached_property
+ def is_vm_runner(self) -> bool:
+ """Whether the runner is VM runner (managed by airflow)."""
+ # TODO: when we have it properly set-up with labels we should just
check for
+ # "airflow-runner" presence in runs_on
+ return self.is_airflow_runner
+
+ @cached_property
+ def is_k8s_runner(self) -> bool:
+ """Whether the runner is K8s runner (managed by airflow)."""
+ # TODO: when we have it properly set-up with labels we should just
check for
+ # "k8s-runner" presence in runs_on
+ return False
+
@cached_property
def mssql_parallelism(self) -> int:
# Limit parallelism for MSSQL to 1 for public runners due to race
conditions generated there
diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py
index 7a8d910fe9..3513cb5670 100644
--- a/dev/breeze/tests/test_pr_info.py
+++ b/dev/breeze/tests/test_pr_info.py
@@ -39,7 +39,7 @@ def test_pr_info():
assert wi.head_repo == "test/airflow"
assert wi.event_name == "pull_request"
assert wi.pr_number == 26004
- assert wi.get_runs_on() == "ubuntu-22.04"
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -53,7 +53,7 @@ def test_push_info():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "push"
assert wi.pr_number is None
- assert wi.get_runs_on() == "ubuntu-22.04"
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
assert wi.is_canary_run() == "true"
assert wi.run_coverage() == "true"
@@ -67,7 +67,7 @@ def test_schedule():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "schedule"
assert wi.pr_number is None
- assert wi.get_runs_on() == "ubuntu-22.04"
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -81,7 +81,7 @@ def test_runs_on_self_hosted():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "pull_request"
assert wi.pr_number == 1234
- assert wi.get_runs_on() == "self-hosted"
+ assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -95,7 +95,7 @@ def test_runs_on_forced_public_runner():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "pull_request"
assert wi.pr_number == 1234
- assert wi.get_runs_on() == "ubuntu-22.04"
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -109,7 +109,7 @@ def test_runs_on_simple_pr_other_repo():
assert wi.head_repo == "test/airflow"
assert wi.event_name == "pull_request"
assert wi.pr_number == 1234
- assert wi.get_runs_on() == "ubuntu-22.04"
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -123,7 +123,7 @@ def test_runs_on_push_other_branch():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "push"
assert wi.pr_number is None
- assert wi.get_runs_on() == "self-hosted"
+ assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -137,6 +137,6 @@ def test_runs_on_push_v_test_branch():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "push"
assert wi.pr_number is None
- assert wi.get_runs_on() == "self-hosted"
+ assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
assert wi.is_canary_run() == "true"
assert wi.run_coverage() == "false"
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index eb2eec345b..c33767982c 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -17,9 +17,11 @@
from __future__ import annotations
import re
+from functools import lru_cache
from typing import Any
import pytest
+from rich.console import Console
from airflow_breeze.global_constants import COMMITTERS, GithubEvents
from airflow_breeze.utils.selective_checks import SelectiveChecks
@@ -35,17 +37,51 @@ def escape_ansi_colors(line):
return ANSI_COLORS_MATCHER.sub("", line)
+# Can be replaced with cache when we move to Python 3.9 (when 3.8 is EOL)
+@lru_cache(maxsize=None)
+def get_rich_console() -> Console:
+ return Console(color_system="truecolor", force_terminal=True)
+
+
+def print_in_color(s: Any = ""):
+ get_rich_console().print(s)
+
+
def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
escaped_stderr = escape_ansi_colors(stderr)
- for name, value in expected_outputs.items():
- if value is None:
- search_string = rf"^{re.escape(name)}="
- if re.search(search_string, escaped_stderr, re.MULTILINE):
- raise AssertionError(f"The {name} output should not be in
{escaped_stderr}")
+ received_output_as_dict = dict(line.split("=", 1) for line in
escaped_stderr.splitlines() if "=" in line)
+ for expected_key, expected_value in expected_outputs.items():
+ if expected_value is None:
+ if expected_key in received_output_as_dict:
+ print_in_color(f"\n[red]ERROR: The '{expected_key}' should not
be present in:[/]")
+ print_in_color(received_output_as_dict)
+ print_in_color("\n")
+ assert expected_key is not None
+
else:
- search_string = rf"^{re.escape(name)}={re.escape(value)}$"
- if not re.search(search_string, escaped_stderr, re.MULTILINE):
- raise AssertionError(f"Expected {name}={value} not found in
{escaped_stderr}")
+ received_value = received_output_as_dict.get(expected_key)
+ if received_value != expected_value:
+ if received_value is not None:
+ print_in_color(f"\n[red]ERROR: The key '{expected_key}'
has unexpected value:")
+ print_in_color(received_value)
+ print_in_color("Expected value:\n")
+ print_in_color(expected_value)
+ print_in_color("\nOutput received:")
+ print_in_color(received_output_as_dict)
+ print_in_color()
+ assert expected_value == received_value
+ else:
+ print_in_color(
+ f"\n[red]ERROR: The key '{expected_key}' missing but "
+ f"it is expected. Expected value:"
+ )
+ print_in_color(expected_value)
+ print_in_color("\nOutput received:")
+ print_in_color(received_output_as_dict)
+ print_in_color()
+ print_in_color(received_output_as_dict)
+ print_in_color()
+ assert received_value is not None
@pytest.mark.parametrize(
@@ -1107,20 +1143,54 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
@pytest.mark.parametrize(
- "github_event, github_actor, github_repository, pr_labels,
github_context_dict, runs_on",
+ "github_event, github_actor, github_repository, pr_labels,
github_context_dict, "
+ "runs_on, "
+ "is_self_hosted_runner, is_airflow_runner, is_amd_runner, is_arm_runner,
is_vm_runner, is_k8s_runner",
[
- pytest.param(GithubEvents.PUSH, "user", "apache/airflow", [], dict(),
"self-hosted", id="Push event"),
+ pytest.param(
+ GithubEvents.PUSH,
+ "user",
+ "apache/airflow",
+ [],
+ dict(),
+ '["self-hosted", "Linux", "X64"]',
+ "true",
+ "true",
+ "true",
+ "false",
+ "true",
+ "false",
+ id="Push event",
+ ),
pytest.param(
GithubEvents.PUSH,
"user",
"private/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Push event for private repo",
),
pytest.param(
- GithubEvents.PULL_REQUEST, "user", "apache/airflow", [], dict(),
"ubuntu-22.04", id="Pull request"
+ GithubEvents.PULL_REQUEST,
+ "user",
+ "apache/airflow",
+ [],
+ dict(),
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
+ id="Pull request",
),
pytest.param(
GithubEvents.PULL_REQUEST,
@@ -1128,7 +1198,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"private/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request private repo",
),
pytest.param(
@@ -1137,7 +1213,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(),
- "self-hosted",
+ '["self-hosted", "Linux", "X64"]',
+ "true",
+ "true",
+ "true",
+ "false",
+ "true",
+ "false",
id="Pull request committer",
),
pytest.param(
@@ -1146,7 +1228,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(event=dict(pull_request=dict(user=dict(login="user")))),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request committer pr non-committer",
),
pytest.param(
@@ -1155,7 +1243,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"private/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request private repo committer",
),
pytest.param(
@@ -1164,7 +1258,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request target",
),
pytest.param(
@@ -1173,7 +1273,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"private/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request target private repo",
),
pytest.param(
@@ -1182,7 +1288,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(),
- "self-hosted",
+ '["self-hosted", "Linux", "X64"]',
+ "true",
+ "true",
+ "true",
+ "false",
+ "true",
+ "false",
id="Pull request target committer",
),
pytest.param(
@@ -1191,7 +1303,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(event=dict(pull_request=dict(user=dict(login="user")))),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request target committer pr non-committer",
),
pytest.param(
@@ -1200,7 +1318,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"private/airflow",
[],
dict(),
- "ubuntu-22.04",
+ '["ubuntu-22.04"]',
+ "false",
+ "false",
+ "true",
+ "false",
+ "false",
+ "false",
id="Pull request targe private repo committer",
),
],
@@ -1212,6 +1336,12 @@ def test_runs_on(
pr_labels: list[str],
github_context_dict: dict[str, Any],
runs_on: str,
+ is_self_hosted_runner: str,
+ is_airflow_runner: str,
+ is_amd_runner: str,
+ is_arm_runner: str,
+ is_vm_runner: str,
+ is_k8s_runner: str,
):
stderr = SelectiveChecks(
files=(),
@@ -1224,6 +1354,12 @@ def test_runs_on(
default_branch="main",
)
assert_outputs_are_printed({"runs-on": runs_on}, str(stderr))
+ assert_outputs_are_printed({"is-self-hosted-runner":
is_self_hosted_runner}, str(stderr))
+ assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner},
str(stderr))
+ assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr))
+ assert_outputs_are_printed({"is-arm-runner": is_arm_runner}, str(stderr))
+ assert_outputs_are_printed({"is-vm-runner": is_vm_runner}, str(stderr))
+ assert_outputs_are_printed({"is-k8s-runner": is_k8s_runner}, str(stderr))
@pytest.mark.parametrize(
