This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new bd8ca5c382c [v3-0-test] Simplify runs-on conditions to only
differentiate AMD/ARM (#50181) (#50196)
bd8ca5c382c is described below
commit bd8ca5c382c1e03da3745fa1575576447785f3aa
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon May 5 12:11:56 2025 +0200
[v3-0-test] Simplify runs-on conditions to only differentiate AMD/ARM
(#50181) (#50196)
* Simplify runs-on conditions to only differentiate AMD/ARM
For now - when we do not have self-hosted runners yet, we want to
simplify the selective check rules to only differentiate the two
options we have now:
* AMD runners (public)
* ARM runners (also bublic)
The selective checks now return two options:
* amd-runners
* arm-runners
That should simplify choices and allow to bring back some of the
disabled ARM jobs, while open up for the future possibility of
enabling self-hosted runners. It should also allow us to finally
run ARM tests in parallel to AMD tests and turn off experimental
status of the ARM image.
Fixes: #41935
* Update .github/workflows/publish-docs-to-s3.yml
---------
(cherry picked from commit 3b80419e28a6ea95246b546460ffba78d09f3255)
Co-authored-by: GPK <[email protected]>
---
.github/workflows/additional-ci-image-checks.yml | 35 +-
.github/workflows/additional-prod-image-tests.yml | 12 +-
.github/workflows/airflow-distributions-tests.yml | 11 +-
.github/workflows/basic-tests.yml | 14 +-
.github/workflows/ci-image-build.yml | 10 +-
.github/workflows/ci-image-checks.yml | 18 +-
.github/workflows/ci.yml | 74 ++---
.github/workflows/finalize-tests.yml | 62 ++--
.github/workflows/generate-constraints.yml | 6 +-
.github/workflows/helm-tests.yml | 12 +-
.github/workflows/integration-system-tests.yml | 8 +-
.github/workflows/k8s-tests.yml | 6 +-
.github/workflows/prod-image-build.yml | 8 +-
.github/workflows/prod-image-extra-checks.yml | 8 +-
.github/workflows/publish-docs-to-s3.yml | 260 +++++++++++++++
.github/workflows/push-image-cache.yml | 12 +-
.github/workflows/release_dockerhub_image.yml | 4 +-
.github/workflows/run-unit-tests.yml | 6 +-
.github/workflows/special-tests.yml | 26 +-
.github/workflows/test-providers.yml | 8 +-
dev/breeze/doc/ci/04_selective_checks.md | 15 +-
.../src/airflow_breeze/commands/ci_commands.py | 9 +-
dev/breeze/src/airflow_breeze/global_constants.py | 12 +-
.../src/airflow_breeze/utils/selective_checks.py | 137 +-------
dev/breeze/tests/test_pr_info.py | 158 +++------
dev/breeze/tests/test_selective_checks.py | 366 ---------------------
26 files changed, 476 insertions(+), 821 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 86740494b64..6ef786e8a86 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -20,16 +20,12 @@ name: Additional CI image checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining AMD
public runners."
required: true
type: string
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
- required: true
- type: string
- runs-on-as-json-self-hosted:
- description: "The array of labels (in json form) determining
self-hosted runners."
+ arm-runners:
+ description: "The array of labels (in json form) determining ARM
public runners."
required: true
type: string
python-versions:
@@ -103,8 +99,8 @@ jobs:
# from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
packages: write
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ inputs.amd-runners }}
+ arm-runners: ${{ inputs.arm-runners }}
cache-type: "Early"
include-prod-images: "false"
push-latest-images: "false"
@@ -124,7 +120,7 @@ jobs:
check-that-image-builds-quickly:
timeout-minutes: 11
name: Check that image builds quickly
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
@@ -156,10 +152,10 @@ jobs:
- name: "Check that image builds quickly"
run: breeze shell --max-time 600 --platform "linux/amd64"
-# # This is only a check if ARM images are successfully building when
committer runs PR from
-# # Apache repository. This is needed in case you want to fix failing cache
job in "canary" run
-# # There is no point in running this one in "canary" run, because the above
step is doing the
-# # same build anyway.
+# This is only a check if ARM images are successfully building when committer
runs PR from
+# Apache repository. This is needed in case you want to fix failing cache job
in "canary" run
+# There is no point in running this one in "canary" run, because the above
step is doing the
+# same build anyway.
# build-ci-arm-images:
# name: Build CI ARM images
# uses: ./.github/workflows/ci-image-build.yml
@@ -167,17 +163,16 @@ jobs:
# contents: read
# packages: write
# with:
+# amd-runners: ${{ inputs.amd-runners }}
+# arm-runners: ${{ inputs.arm-runners }}
# platform: "linux/arm64"
# push-image: "false"
# upload-image-artifact: "true"
-# upload-mount-cache-artifact: ${{ inputs.canary-run }}
-# runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-# runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+# upload-mount-cache-artifact: ${{ inputs.canary-run }}
# python-versions: ${{ inputs.python-versions }}
# branch: ${{ inputs.branch }}
# constraints-branch: ${{ inputs.constraints-branch }}
# use-uv: ${{ inputs.use-uv }}
# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
# docker-cache: ${{ inputs.docker-cache }}
-# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
-#
+# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 56badcd10ad..3834efad76e 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -20,8 +20,8 @@ name: Additional PROD image tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining AMD
public runners."
required: true
type: string
default-branch:
@@ -63,7 +63,7 @@ jobs:
name: PROD image extra checks (main)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+ amd-runners: ${{ inputs.amd-runners }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
@@ -78,7 +78,7 @@ jobs:
name: PROD image extra checks (release)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+ amd-runners: ${{ inputs.amd-runners }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
@@ -92,7 +92,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of PROD image building"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -130,7 +130,7 @@ jobs:
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker Compose quick start with PROD image verifying"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/airflow-distributions-tests.yml
b/.github/workflows/airflow-distributions-tests.yml
index c7071c5f34d..77b0e76462a 100644
--- a/.github/workflows/airflow-distributions-tests.yml
+++ b/.github/workflows/airflow-distributions-tests.yml
@@ -21,6 +21,10 @@ on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
# Static inputs defined to choose which distribution to test to run
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
+ required: true
+ type: string
distribution-name:
description: "The name of the distribution to test"
required: true
@@ -33,11 +37,6 @@ on: # yamllint disable-line rule:truthy
description: "distribution test type" # eg task-sdk-tests
required: true
type: string
- # Environment inputs
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
- required: true
- type: string
default-python-version:
description: "Which version of python should be used by default"
required: true
@@ -60,7 +59,7 @@ jobs:
distributions-tests:
timeout-minutes: 80
name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 7efc3deedc1..cdd619c9f4c 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -20,7 +20,7 @@ name: Basic tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
+ amd-runners:
description: "The array of labels (in json form) determining public
runners."
required: true
type: string
@@ -66,7 +66,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -87,7 +87,7 @@ jobs:
tests-ui:
timeout-minutes: 15
name: React UI tests
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
if: inputs.run-ui-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -155,7 +155,7 @@ jobs:
install-pre-commit:
timeout-minutes: 5
name: "Install pre-commit for cache"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
steps:
@@ -183,7 +183,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
needs: install-pre-commit
if: inputs.basic-checks-only == 'true'
steps:
@@ -236,7 +236,7 @@ jobs:
upgrade-check:
timeout-minutes: 45
name: "Upgrade checks"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
needs: install-pre-commit
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -306,7 +306,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 1df2e26c090..9c032fbff37 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -20,12 +20,12 @@ name: Build CI images
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
- runs-on-as-json-self-hosted:
- description: "The array of labels (in json form) determining
self-hosted runners."
+ arm-runners:
+ description: "The array of labels (in json form) determining public
ARM runners."
required: true
type: string
target-commit-sha:
@@ -110,7 +110,7 @@ jobs:
# adding space before (with >) apparently turns the `runs-on` processed
line into a string "Array"
# instead of an array of strings.
# yamllint disable-line rule:line-length
- runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
+ runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }}
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 76c850132ea..3dca5a75cf7 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -20,12 +20,8 @@ name: CI Image Checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
- required: true
- type: string
- runs-on-as-json-docs-build:
- description: "The array of labels (in json form) determining the
labels used for docs build."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
needs-mypy:
@@ -117,7 +113,7 @@ jobs:
install-pre-commit:
timeout-minutes: 5
name: "Install pre-commit for cache (only canary runs)"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
if: inputs.basic-checks-only == 'false'
@@ -161,7 +157,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
needs: install-pre-commit
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -201,7 +197,7 @@ jobs:
mypy:
timeout-minutes: 45
name: "MyPy checks"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
needs: install-pre-commit
if: inputs.needs-mypy == 'true'
strategy:
@@ -245,7 +241,7 @@ jobs:
build-docs:
timeout-minutes: 150
name: "Build documentation"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
if: inputs.docs-build == 'true'
strategy:
fail-fast: false
@@ -387,7 +383,7 @@ jobs:
test-python-api-client:
timeout-minutes: 60
name: "Test Python API client"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
if: inputs.needs-api-codegen == 'true'
env:
BACKEND: "postgres"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 879e16df373..b94e5d8ee39 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -83,12 +83,6 @@ jobs:
include-success-outputs: ${{
steps.selective-checks.outputs.include-success-outputs }}
individual-providers-test-types-list-as-strings-in-json: >-
${{
steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json
}}
- is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
- is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
- is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
- is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
- is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
- is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos
}}
kubernetes-combos-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
@@ -123,11 +117,8 @@ jobs:
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
- runs-on-as-json-default: ${{
steps.selective-checks.outputs.runs-on-as-json-default }}
- runs-on-as-json-docs-build: ${{
steps.selective-checks.outputs.runs-on-as-json-docs-build }}
- runs-on-as-json-public: ${{
steps.selective-checks.outputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted-asf: ${{
steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }}
- runs-on-as-json-self-hosted: ${{
steps.selective-checks.outputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ steps.selective-checks.outputs.amd-runners }}
+ arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
selected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.selected-providers-list-as-string }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
@@ -182,7 +173,7 @@ jobs:
needs: [build-info]
uses: ./.github/workflows/basic-tests.yml
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}}
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
@@ -203,17 +194,17 @@ jobs:
# from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
packages: write
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted: ${{
needs.build-info.outputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
+ arm-runners: ${{ needs.build-info.outputs.arm-runners }}
platform: "linux/amd64"
push-image: "false"
upload-image-artifact: "true"
upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
+ constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{
needs.build-info.outputs.disable-airflow-repo-cache }}
if: needs.build-info.outputs.ci-image-build == 'true'
@@ -228,9 +219,8 @@ jobs:
id-token: write
if: needs.build-info.outputs.canary-run == 'true'
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted: ${{
needs.build-info.outputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
+ arm-runners: ${{ needs.build-info.outputs.arm-runners }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
@@ -251,7 +241,7 @@ jobs:
uses: ./.github/workflows/generate-constraints.yml
if: needs.build-info.outputs.ci-image-build == 'true'
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
# generate no providers constraints only in canary builds - they take
quite some time to generate
# they are not needed for regular builds, they are only needed to update
constraints in canaries
@@ -267,8 +257,7 @@ jobs:
id-token: write
contents: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
- runs-on-as-json-docs-build: ${{
needs.build-info.outputs.runs-on-as-json-docs-build }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
@@ -303,7 +292,7 @@ jobs:
needs.build-info.outputs.skip-providers-tests != 'true' &&
needs.build-info.outputs.latest-versions-only != 'true'
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
@@ -325,8 +314,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -343,7 +331,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
@@ -370,7 +358,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
@@ -397,7 +385,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
@@ -424,7 +412,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
@@ -452,7 +440,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
@@ -481,7 +469,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
@@ -511,7 +499,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
@@ -539,7 +527,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
@@ -573,7 +561,7 @@ jobs:
needs.build-info.outputs.full-tests-needed == 'true')
with:
default-branch: ${{ needs.build-info.outputs.default-branch }}
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
core-test-types-list-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
providers-test-types-list-as-strings-in-json: >
@@ -598,7 +586,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
testable-core-integrations: ${{
needs.build-info.outputs.testable-core-integrations }}
testable-providers-integrations: ${{
needs.build-info.outputs.testable-providers-integrations }}
run-system-tests: ${{ needs.build-info.outputs.run-tests }}
@@ -621,7 +609,7 @@ jobs:
if: >
needs.build-info.outputs.run-tests == 'true'
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
test-scope: "All"
@@ -649,7 +637,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-tests == 'true'
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
test-scope: "All"
@@ -678,7 +666,7 @@ jobs:
# from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
packages: write
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
build-type: "Regular"
platform: "linux/amd64"
push-image: "false"
@@ -699,7 +687,7 @@ jobs:
needs: [build-info, build-prod-images, generate-constraints]
uses: ./.github/workflows/additional-prod-image-tests.yml
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
@@ -719,7 +707,7 @@ jobs:
packages: read
with:
platform: "linux/amd64"
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -737,7 +725,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -758,7 +746,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -806,8 +794,8 @@ jobs:
- tests-with-lowest-direct-resolution-providers
uses: ./.github/workflows/finalize-tests.yml
with:
- runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted: ${{
needs.build-info.outputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
+ arm-runners: ${{ needs.build-info.outputs.arm-runners }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index 12d3906de4c..ae1d77e0741 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -20,12 +20,12 @@ name: Finalize tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
- runs-on-as-json-self-hosted:
- description: "The array of labels (in json form) determining
self-hosted runners."
+ arm-runners:
+ description: "The array of labels (in json form) determining public
ARM runners."
required: true
type: string
python-versions:
@@ -80,7 +80,7 @@ permissions:
contents: read
jobs:
update-constraints:
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
timeout-minutes: 80
name: "Update constraints"
permissions:
@@ -142,8 +142,8 @@ jobs:
# from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
packages: write
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
- runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+ amd-runners: ${{ inputs.amd-runners }}
+ arm-runners: ${{ inputs.arm-runners }}
cache-type: "Regular AMD"
include-prod-images: "true"
push-latest-images: "true"
@@ -159,34 +159,34 @@ jobs:
inputs.canary-run == 'true' &&
(github.event_name == 'schedule' || github.event_name ==
'workflow_dispatch')
- # push-buildx-cache-to-github-registry-arm:
- # name: Push Regular ARM Image Cache
- # needs: [update-constraints]
- # uses: ./.github/workflows/push-image-cache.yml
- # permissions:
- # contents: read
- # packages: write
- # with:
- # runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
- # runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
- # cache-type: "Regular ARM"
- # include-prod-images: "true"
- # push-latest-images: "true"
- # platform: "linux/arm64"
- # python-versions: ${{ inputs.python-versions }}
- # branch: ${{ inputs.branch }}
- # constraints-branch: ${{ inputs.constraints-branch }}
- # use-uv: "true"
- # include-success-outputs: ${{ inputs.include-success-outputs }}
- # docker-cache: ${{ inputs.docker-cache }}
- # if: >
- # inputs.canary-run == 'true' &&
- # (github.event_name == 'schedule' || github.event_name ==
'workflow_dispatch')
+ # push-buildx-cache-to-github-registry-arm:
+ # name: Push Regular ARM Image Cache
+ # needs: [update-constraints]
+ # uses: ./.github/workflows/push-image-cache.yml
+ # permissions:
+ # contents: read
+ # packages: write
+ # with:
+ # amd-runners: ${{ inputs.amd-runners }}
+ # arm-runners: ${{ inputs.arm-runners }}
+ # cache-type: "Regular ARM"
+ # include-prod-images: "true"
+ # push-latest-images: "true"
+ # platform: "linux/arm64"
+ # python-versions: ${{ inputs.python-versions }}
+ # branch: ${{ inputs.branch }}
+ # constraints-branch: ${{ inputs.constraints-branch }}
+ # use-uv: "true"
+ # include-success-outputs: ${{ inputs.include-success-outputs }}
+ # docker-cache: ${{ inputs.docker-cache }}
+ # if: >
+ # inputs.canary-run == 'true' &&
+ # (github.event_name == 'schedule' || github.event_name ==
'workflow_dispatch')
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
steps:
- name: "Cleanup repo"
shell: bash
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 46cb9f250bc..91b56942454 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -20,8 +20,8 @@ name: Generate constraints
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
python-versions-list-as-string:
@@ -46,7 +46,7 @@ jobs:
contents: read
timeout-minutes: 70
name: Generate constraints ${{ inputs.python-versions-list-as-string }}
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
DEBUG_RESOURCES: ${{ inputs.debug-resources }}
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 1b4aa19cbe5..21c7f01a528 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -20,12 +20,8 @@ name: Helm tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
- required: true
- type: string
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
helm-test-packages:
@@ -46,7 +42,7 @@ jobs:
tests-helm:
timeout-minutes: 80
name: "Unit tests Helm: ${{ matrix.helm-test-package }}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
@@ -85,7 +81,7 @@ jobs:
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
steps:
diff --git a/.github/workflows/integration-system-tests.yml
b/.github/workflows/integration-system-tests.yml
index fc3159223e8..9c3480432b5 100644
--- a/.github/workflows/integration-system-tests.yml
+++ b/.github/workflows/integration-system-tests.yml
@@ -20,7 +20,7 @@ name: Integration and system tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
+ amd-runners:
description: "The array of labels (in json form) determining public
runners."
required: true
type: string
@@ -71,7 +71,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-core-integrations != '[]'
name: "Integration core ${{ matrix.integration }}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
@@ -120,7 +120,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-providers-integrations != '[]' &&
inputs.skip-providers-tests != 'true'
name: "Integration: providers ${{ matrix.integration }}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
@@ -168,7 +168,7 @@ jobs:
timeout-minutes: 30
if: inputs.run-system-tests == 'true'
name: "System Tests"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
env:
BACKEND: "postgres"
BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
index 2632408b813..43b30e40a97 100644
--- a/.github/workflows/k8s-tests.yml
+++ b/.github/workflows/k8s-tests.yml
@@ -24,8 +24,8 @@ on: # yamllint disable-line rule:truthy
description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
required: true
type: string
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
python-versions-list-as-string:
@@ -54,7 +54,7 @@ jobs:
tests-kubernetes:
timeout-minutes: 60
name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo
}}-${{ matrix.use-standard-naming }}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 844b0d46e7d..bf4ead70add 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -20,8 +20,8 @@ name: Build PROD images
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining default
runner for AMD build."
required: true
type: string
build-type:
@@ -114,7 +114,7 @@ jobs:
build-prod-packages:
name: "Build Airflow and provider distributions"
timeout-minutes: 10
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
if: inputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -186,7 +186,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) ||
fromJSON('[""]') }}
timeout-minutes: 80
name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version
}}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
needs:
- build-prod-packages
env:
diff --git a/.github/workflows/prod-image-extra-checks.yml
b/.github/workflows/prod-image-extra-checks.yml
index 356900c9222..640d924d873 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -20,8 +20,8 @@ name: PROD images extra checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
python-versions:
@@ -62,7 +62,7 @@ jobs:
myssql-client-image:
uses: ./.github/workflows/prod-image-build.yml
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+ amd-runners: ${{ inputs.amd-runners }}
build-type: "MySQL Client"
upload-image-artifact: "false"
upload-package-artifact: "false"
@@ -83,7 +83,7 @@ jobs:
pip-image:
uses: ./.github/workflows/prod-image-build.yml
with:
- runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+ amd-runners: ${{ inputs.amd-runners }}
build-type: "pip"
upload-image-artifact: "false"
upload-package-artifact: "false"
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
new file mode 100644
index 00000000000..71cd4e2d290
--- /dev/null
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -0,0 +1,260 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Publish Docs to S3
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: "The branch or tag to checkout for the docs publishing"
+ required: true
+ type: string
+ exclude-docs:
+ description: "Comma separated list of docs to exclude"
+ required: false
+ default: "NO_DOCS"
+ type: string
+ destination-location:
+ description: "The destination location in S3"
+ required: false
+ default: "s3://staging-docs-airflow-apache-org/docs"
+ type: string
+ docs-list-as-string:
+ description: "Space separated list of docs to build"
+ required: false
+ default: ""
+ type: string
+env:
+ AIRFLOW_ROOT_PATH: "/home/runner/work/temp-airflow-repo-reference" #
checkout dir for referenced tag
+permissions:
+ contents: read
+jobs:
+ build-info:
+ timeout-minutes: 10
+ name: "Build Info"
+ runs-on: ["ubuntu-24.04"]
+ outputs:
+ amd-runners: ${{ steps.selective-checks.outputs.amd-runners }}
+ arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
+ env:
+ GITHUB_CONTEXT: ${{ toJson(github) }}
+ VERBOSE: true
+ REF: ${{ inputs.ref }}
+ EXCLUDE_DOCS: ${{ inputs.exclude-docs }}
+ DESTINATION_LOCATION: ${{ inputs.destination-location }}
+ DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
+ if: contains(fromJSON('[
+ "ashb",
+ "eladkal",
+ "ephraimbuddy",
+ "jedcunningham",
+ "kaxil",
+ "pierrejeambrun",
+ "potiuk",
+ "utkarsharma2"
+ ]'), github.event.sender.login)
+ steps:
+ - name: "Input parameters summary"
+ shell: bash
+ run: |
+ echo "Input parameters summary"
+ echo "========================="
+ echo "Ref: '${REF}'"
+ echo "Exclude docs: '${EXCLUDE_DOCS}'"
+ echo "Destination location: '${DESTINATION_LOCATION}'"
+ echo "Docs list as string: '${DOCS_LIST_AS_STRING}'"
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: "Cleanup docker"
+ run: ./scripts/ci/cleanup_docker.sh
+ - name: "Install uv"
+ run: curl -LsSf https://astral.sh/uv/install.sh | sh
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ use-uv: "true"
+ - name: Selective checks
+ id: selective-checks
+ env:
+ VERBOSE: "false"
+ run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
+
+ build-ci-images:
+ name: Build CI images
+ uses: ./.github/workflows/ci-image-build.yml
+ needs: [build-info]
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
+ packages: write
+ with:
+ amd-runners: ${{ needs.build-info.outputs.amd-runners }}
+ arm-runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/amd64"
+ push-image: "false"
+ upload-image-artifact: "true"
+ upload-mount-cache-artifact: false
+ python-versions: "['3.9']"
+ branch: ${{ inputs.ref }}
+ use-uv: true
+ upgrade-to-newer-dependencies: false
+ constraints-branch: "constraints-main"
+ docker-cache: registry
+ disable-airflow-repo-cache: false
+
+ build-docs:
+ needs: [build-ci-images]
+ timeout-minutes: 150
+ name: "Build documentation"
+ runs-on: ubuntu-latest
+ env:
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_USERNAME: ${{ github.actor }}
+ INCLUDE_NOT_READY_PROVIDERS: "true"
+ INCLUDE_SUCCESS_OUTPUTS: false
+ PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
+ VERBOSE: "true"
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} "
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: "Checkout from ${{ inputs.ref }} to build docs"
+ run: |
+ git clone https://github.com/apache/airflow.git
"${AIRFLOW_ROOT_PATH}"
+ cd "${AIRFLOW_ROOT_PATH}" && git checkout ${REF}
+ env:
+ REF: ${{ inputs.ref }}
+ - name: "Prepare breeze & CI image: 3.9"
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ platform: "linux/amd64"
+ python: 3.9
+ use-uv: true
+ - name: "Building docs with --docs-only flag"
+ env:
+ DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
+ run: >
+ breeze build-docs ${DOCS_LIST_AS_STRING} --docs-only
+ - name: "Upload build docs"
+ uses: actions/upload-artifact@v4
+ with:
+ name: airflow-docs
+ path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build
+ retention-days: '7'
+ if-no-files-found: 'error'
+ overwrite: 'true'
+
+ publish-docs-to-s3:
+ needs: [build-docs]
+ name: "Publish documentation to S3"
+ permissions:
+ id-token: write
+ contents: read
+ runs-on: ubuntu-latest
+ env:
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_USERNAME: ${{ github.actor }}
+ INCLUDE_NOT_READY_PROVIDERS: "true"
+ INCLUDE_SUCCESS_OUTPUTS: false
+ PYTHON_MAJOR_MINOR_VERSION: 3.9
+ VERBOSE: "true"
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} "
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: "Cleanup docker"
+ run: ./scripts/ci/cleanup_docker.sh
+ - name: "Checkout ${{ inputs.ref }}"
+ run: |
+ git clone https://github.com/apache/airflow.git
"${AIRFLOW_ROOT_PATH}"
+ cd "${AIRFLOW_ROOT_PATH}" && git checkout ${REF}
+ env:
+ REF: ${{ inputs.ref }}
+ - name: "Download docs prepared as artifacts"
+ uses: actions/download-artifact@v4
+ with:
+ name: airflow-docs
+ path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build
+ - name: Check disk space available
+ run: df -h
+ # Here we will create temp airflow-site dir to publish
+ # docs and for back-references
+ - name: Create /mnt/airflow-site directory
+ run: |
+ sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}"
/mnt/airflow-site
+ echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/" >> "$GITHUB_ENV"
+ - name: "Prepare breeze & CI image: 3.9"
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ platform: "linux/amd64"
+ python: 3.9
+ use-uv: true
+ - name: "Publish docs to tmp directory"
+ env:
+ DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
+ run: >
+ breeze release-management publish-docs --override-versioned
--run-in-parallel
+ ${DOCS_LIST_AS_STRING}
+ - name: Check disk space available
+ run: df -h
+ - name: "Generate back references for providers"
+ run: breeze release-management add-back-references all-providers
+ - name: "Generate back references for apache-airflow"
+ run: breeze release-management add-back-references apache-airflow
+ - name: "Generate back references for docker-stack"
+ run: breeze release-management add-back-references docker-stack
+ - name: "Generate back references for helm-chart"
+ run: breeze release-management add-back-references helm-chart
+ - name: Install AWS CLI v2
+ run: |
+ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"; -o
/tmp/awscliv2.zip
+ unzip -q /tmp/awscliv2.zip -d /tmp
+ rm /tmp/awscliv2.zip
+ sudo /tmp/aws/install --update
+ rm -rf /tmp/aws/
+ - name: Configure AWS credentials
+ uses:
aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
# v4.0.1
+ with:
+ aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+ aws-region: us-east-2
+ - name: "Syncing docs to S3"
+ env:
+ DESTINATION_LOCATION: "${{ inputs.destination-location }}"
+ SOURCE_DIR_PATH: "/mnt/airflow-site/docs-archive/"
+ EXCLUDE_DOCS: "${{ inputs.exclude-docs }}"
+ run: |
+ breeze release-management publish-docs-to-s3 --source-dir-path
${SOURCE_DIR_PATH} \
+ --destination-location ${DESTINATION_LOCATION} --stable-versions \
+ --exclude-docs ${EXCLUDE_DOCS}
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 7db441c3cc7..2dc539956d7 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -20,12 +20,12 @@ name: Push image cache
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public
runners."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
- runs-on-as-json-self-hosted:
- description: "The array of labels (in json form) determining
self-hosted runners."
+ arm-runners:
+ description: "The array of labels (in json form) determining public
ARM runners."
required: true
type: string
cache-type:
@@ -87,7 +87,7 @@ jobs:
# adding space before (with >) apparently turns the `runs-on` processed
line into a string "Array"
# instead of an array of strings.
# yamllint disable-line rule:line-length
- runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
+ runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }}
permissions:
contents: read
packages: write
@@ -162,7 +162,7 @@ jobs:
# adding space before (with >) apparently turns the `runs-on` processed
line into a string "Array"
# instead of an array of strings.
# yamllint disable-line rule:line-length
- runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.runs-on-as-json-public) ||
fromJSON(inputs.runs-on-as-json-self-hosted) }}
+ runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }}
permissions:
contents: read
packages: write
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index a82738d9973..9ed8590e32d 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -48,6 +48,8 @@ jobs:
platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }}
airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }}
skipLatest: ${{ steps.selective-checks.outputs.skipLatest }}
+ amd-runners: ${{ steps.selective-checks.outputs.amd-runners }}
+ arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
VERBOSE: true
@@ -109,7 +111,7 @@ jobs:
timeout-minutes: 50
# yamllint disable rule:line-length
name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}"
- runs-on: ${{ matrix.platform == 'linux/amd64' &&
fromJSON('["ubuntu-22.04"]') || fromJSON('["ubuntu-22.04-arm"]') }}
+ runs-on: ${{ (inputs.platform == 'linux/amd64') &&
fromJSON(needs.build-info.outputs.amd-runners) ||
fromJSON(needs.build-info.outputs.arm-runners) }}
needs: [build-info]
strategy:
fail-fast: false
diff --git a/.github/workflows/run-unit-tests.yml
b/.github/workflows/run-unit-tests.yml
index 939fa2b6341..cdfb7b58f0d 100644
--- a/.github/workflows/run-unit-tests.yml
+++ b/.github/workflows/run-unit-tests.yml
@@ -20,8 +20,8 @@ name: Unit tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
test-group:
@@ -125,7 +125,7 @@ jobs:
${{ inputs.test-scope }}-${{ inputs.test-group }}:\
${{ inputs.test-name }}${{ inputs.test-name-separator }}${{
matrix.backend-version }}:\
${{ matrix.python-version}}:${{ matrix.test-types.description }}"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index d47907f923f..7f20ce11111 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -20,8 +20,8 @@ name: Special tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
default-branch:
@@ -90,7 +90,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
test-scope: "DB"
@@ -113,7 +113,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
test-scope: "DB"
@@ -136,7 +136,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
test-scope: "All"
@@ -160,7 +160,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
test-scope: "All"
@@ -185,7 +185,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
test-scope: "All"
@@ -209,7 +209,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
test-scope: "All"
@@ -233,7 +233,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
test-name: "Postgres"
test-scope: "Quarantined"
test-group: "core"
@@ -256,7 +256,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
test-name: "Postgres"
test-scope: "Quarantined"
test-group: "providers"
@@ -280,7 +280,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
test-name: "Postgres"
test-scope: "ARM collection"
test-group: "core"
@@ -304,7 +304,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
test-name: "Postgres"
test-scope: "ARM collection"
test-group: "providers"
@@ -329,7 +329,7 @@ jobs:
contents: read
packages: read
with:
- runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+ amd-runners: ${{ inputs.amd-runners }}
test-name: "SystemTest"
test-scope: "System"
test-group: "core"
diff --git a/.github/workflows/test-providers.yml
b/.github/workflows/test-providers.yml
index 9775df78061..0fbeaaee052 100644
--- a/.github/workflows/test-providers.yml
+++ b/.github/workflows/test-providers.yml
@@ -20,8 +20,8 @@ name: Provider tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ amd-runners:
+ description: "The array of labels (in json form) determining public
AMD runners."
required: true
type: string
canary-run:
@@ -68,7 +68,7 @@ jobs:
prepare-install-verify-provider-distributions:
timeout-minutes: 80
name: "Providers ${{ matrix.package-format }} tests"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
@@ -163,7 +163,7 @@ jobs:
timeout-minutes: 80
# yamllint disable rule:line-length
name: Compat ${{ matrix.compat.airflow-version }}:P${{
matrix.compat.python-version }}:${{ matrix.compat.test-types.description }}
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ runs-on: ${{ fromJSON(inputs.amd-runners) }}
strategy:
fail-fast: false
matrix:
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index c450342f7d1..615bee819b7 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -196,14 +196,8 @@ GitHub Actions to pass the list of parameters to a command
to execute
| helm-version | Which Helm version
to use for tests
| v3.15.3 | |
| include-success-outputs | Whether to include
outputs of successful parallel tests ("true"/"false")
| false | |
| individual-providers-test-types-list-as-strings-in-json | Which test types
should be run for unit tests for providers (individually listed)
| Providers[\amazon\] Providers\[google\] | * |
-| is-airflow-runner | Whether runner
used is an airflow or infrastructure runner (true if airflow/false if
infrastructure) | false | |
-| is-amd-runner | Whether runner
used is an AMD one
| true | |
-| is-arm-runner | Whether runner
used is an ARM one
| false | |
| is-committer-build | Whether the build
is triggered by a committer
| false | |
-| is-k8s-runner | Whether the build
runs on our k8s infrastructure
| false | |
| is-legacy-ui-api-labeled | Whether the PR is
labeled as legacy UI/API
| false | |
-| is-self-hosted-runner | Whether the runner
is self-hosted
| false | |
-| is-vm-runner | Whether the runner
uses VM to run
| true | |
| kind-version | Which Kind version
to use for tests
| v0.24.0 | |
| kubernetes-combos-list-as-string | All combinations
of Python version and Kubernetes version to use for tests as space-separated
string | 3.9-v1.25.2 3.10-v1.28.13 | * |
| kubernetes-versions | All Kubernetes
versions to use for tests as JSON array
| \['v1.25.2'\] | |
@@ -234,11 +228,8 @@ GitHub Actions to pass the list of parameters to a command
to execute
| run-tests | Whether unit tests
should be run ("true"/"false")
| true | |
| run-ui-tests | Whether UI tests
should be run ("true"/"false")
| true | |
| run-www-tests | Whether Legacy WWW
tests should be run ("true"/"false")
| true | |
-| runs-on-as-json-default | List of labels
assigned for runners for that build for default runs for that build (as string)
| \["ubuntu-22.04"\] | |
-| runs-on-as-json-docs-build | List of labels
assigned for runners for that build for ddcs build (as string)
| \["ubuntu-22.04"\] | |
-| runs-on-as-json-self-hosted | List of labels
assigned for runners for that build for self hosted runners
| \["self-hosted", "Linux", "X64"\] | |
-| runs-on-as-json-self-hosted-asf | List of labels
assigned for runners for that build for ASF self hosted runners
| \["self-hosted", "Linux", "X64"\] | |
-| runs-on-as-json-public | List of labels
assigned for runners for that build for public runners
| \["ubuntu-22.04"\] | |
+| amd-runners | List of labels
assigned for runners for that build for public AMD runners
| \["ubuntu-22.04"\] | |
+| arm-runners | List of labels
assigned for runners for that build for public ARM runners
| \["ubuntu-22.04-arm"\] | |
| selected-providers-list-as-string | List of providers
affected when they are selectively affected.
| airbyte http | * |
| skip-pre-commits | Which pre-commits
should be skipped during the static-checks run
| flynt,identity | |
| skip-providers-tests | When provider
tests should be skipped (on non-main branch or when no provider changes
detected) | true | |
@@ -329,8 +320,6 @@ This table summarizes the labels you can use on PRs to
control the selective che
| latest versions only | *-versions-*, *-versions-* | If
set, the number of Python, Kubernetes, DB versions will be limited to the
latest ones. |
| non committer build | is-committer-build | If
set, the scripts used for images are used from target branch for committers.
|
| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set
to true (default false) then dependencies in the CI image build are upgraded.
|
-| use public runners | runs-on-as-json-public | Force
using public runners as default runners.
|
-| use self-hosted runners | runs-on-as-json-default | Force
using self-hosted runners as default runners.
|
-----
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
index 18e76153b95..a5453794a31 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
@@ -41,8 +41,7 @@ from airflow_breeze.commands.common_options import (
)
from airflow_breeze.global_constants import (
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- RUNS_ON_PUBLIC_RUNNER,
- RUNS_ON_SELF_HOSTED_RUNNER,
+ PUBLIC_AMD_RUNNERS,
GithubEvents,
github_events,
)
@@ -307,10 +306,8 @@ class WorkflowInfo(NamedTuple):
for label in self.pull_request_labels:
if "use public runners" in label:
get_console().print("[info]Force running on public runners")
- return RUNS_ON_PUBLIC_RUNNER
- if not os.environ.get("AIRFLOW_SELF_HOSTED_RUNNER"):
- return RUNS_ON_PUBLIC_RUNNER
- return RUNS_ON_SELF_HOSTED_RUNNER
+ return PUBLIC_AMD_RUNNERS
+ return PUBLIC_AMD_RUNNERS
def is_canary_run(self) -> str:
if (
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 2bd56e49eef..fcb9b4f70ed 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -34,16 +34,8 @@ from airflow_breeze.utils.path_utils import (
AIRFLOW_ROOT_PATH,
)
-RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
-# we should get more sophisticated logic here in the future, but for now we
just check if
-# we use self airflow, vm-based, amd hosted runner as a default
-# TODO: temporarily we need to switch to public runners to avoid issues with
self-hosted runners
-RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]'
-RUNS_ON_SELF_HOSTED_ASF_RUNNER = '["self-hosted", "asf-runner"]'
-# TODO: when we have it properly set-up with labels we should change it to
-# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner",
"X64"]'
-# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
-SELF_HOSTED_RUNNERS_CPU_COUNT = 8
+PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]'
+PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]'
ANSWER = ""
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 52c14bcadc6..64836539432 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -46,9 +46,8 @@ from airflow_breeze.global_constants import (
KIND_VERSION,
NUMBER_OF_LOW_DEP_SLICES,
PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
- RUNS_ON_PUBLIC_RUNNER,
- RUNS_ON_SELF_HOSTED_ASF_RUNNER,
- RUNS_ON_SELF_HOSTED_RUNNER,
+ PUBLIC_AMD_RUNNERS,
+ PUBLIC_ARM_RUNNERS,
TESTABLE_CORE_INTEGRATIONS,
TESTABLE_PROVIDERS_INTEGRATIONS,
GithubEvents,
@@ -86,7 +85,6 @@ LOG_WITHOUT_MOCK_IN_TESTS_EXCEPTION_LABEL = "log exception"
NON_COMMITTER_BUILD_LABEL = "non committer build"
UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies"
USE_PUBLIC_RUNNERS_LABEL = "use public runners"
-USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners"
ALL_CI_SELECTIVE_TEST_TYPES = "API Always CLI Core Other Serialization"
@@ -1257,135 +1255,12 @@ class SelectiveChecks:
return " ".join(sorted(affected_providers))
@cached_property
- def runs_on_as_json_default(self) -> str:
- if self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY:
- if self._is_canary_run():
- return RUNS_ON_SELF_HOSTED_RUNNER
- if self._pr_labels and USE_PUBLIC_RUNNERS_LABEL in self._pr_labels:
- # Forced public runners
- return RUNS_ON_PUBLIC_RUNNER
- actor = self._github_actor
- if self._github_event in (GithubEvents.PULL_REQUEST,
GithubEvents.PULL_REQUEST_TARGET):
- try:
- actor =
self._github_context_dict["event"]["pull_request"]["user"]["login"]
- get_console().print(
- f"[warning]The actor: {actor} retrieved from
GITHUB_CONTEXT's"
- f" event.pull_request.user.login[/]"
- )
- except Exception as e:
- get_console().print(f"[warning]Exception when reading user
login: {e}[/]")
- get_console().print(
- f"[info]Could not find the actor from pull request, "
- f"falling back to the actor who triggered the PR:
{actor}[/]"
- )
- if (
- actor not in COMMITTERS
- and self._pr_labels
- and USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels
- ):
- get_console().print(
- f"[error]The PR has `{USE_SELF_HOSTED_RUNNERS_LABEL}`
label, but "
- f"{actor} is not a committer. This is not going to
work.[/]"
- )
- sys.exit(1)
- if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels:
- # Forced self-hosted runners
- return RUNS_ON_SELF_HOSTED_RUNNER
- return RUNS_ON_PUBLIC_RUNNER
- return RUNS_ON_PUBLIC_RUNNER
-
- @cached_property
- def runs_on_as_json_self_hosted(self) -> str:
- return RUNS_ON_SELF_HOSTED_RUNNER
+ def amd_runners(self) -> str:
+ return PUBLIC_AMD_RUNNERS
@cached_property
- def runs_on_as_json_self_hosted_asf(self) -> str:
- return RUNS_ON_SELF_HOSTED_ASF_RUNNER
-
- @cached_property
- def runs_on_as_json_docs_build(self) -> str:
- # We used to run docs build on self-hosted runners because they had
more space, but
- # It turned out that public runners have a lot of space in /mnt folder
that we can utilise
- # but in the future we might want to switch back to self-hosted
runners so we have this
- # separate property to determine that and place to implement different
logic if needed
- return RUNS_ON_PUBLIC_RUNNER
-
- @cached_property
- def runs_on_as_json_public(self) -> str:
- return RUNS_ON_PUBLIC_RUNNER
-
- @cached_property
- def is_self_hosted_runner(self) -> bool:
- """
- True if the job has runs_on labels indicating It should run on
"self-hosted" runner.
-
- All self-hosted runners have "self-hosted" label.
- """
- return "self-hosted" in json.loads(self.runs_on_as_json_default)
-
- @cached_property
- def is_airflow_runner(self) -> bool:
- """
- True if the job has runs_on labels indicating It should run on Airflow
managed runner.
-
- All Airflow team-managed runners will have "airflow-runner" label.
- """
- # TODO: when we have it properly set-up with labels we should just
check for
- # "airflow-runner" presence in runs_on
- runs_on_array = json.loads(self.runs_on_as_json_default)
- return "Linux" in runs_on_array and "X64" in runs_on_array and
"self-hosted" in runs_on_array
-
- @cached_property
- def is_amd_runner(self) -> bool:
- """
- True if the job has runs_on labels indicating AMD architecture.
-
- Matching amd label, asf-runner, and any ubuntu that does not contain
arm
- The last case is just in case - currently there are no public runners
that have ARM
- instances, but they can add them in the future. It might be that for
compatibility
- they will just add arm in the runner name - because currently GitHub
users use just
- one label "ubuntu-*" for all their work and depend on them being AMD
ones.
- """
- return any(
- [
- label.lower() == "amd"
- or label.lower() == "amd64"
- or label.lower() == "x64"
- or label == "asf-runner"
- or ("ubuntu" in label and "arm" not in label.lower())
- for label in json.loads(self.runs_on_as_json_public)
- ]
- )
-
- @cached_property
- def is_arm_runner(self) -> bool:
- """
- True if the job has runs_on labels indicating ARM architecture.
-
- Matches any label containing arm - including ASF-specific "asf-arm"
label.
-
- # See
https://cwiki.apache.org/confluence/pages/viewpage.action?spaceKey=INFRA&title=ASF+Infra+provided+self-hosted+runners
- """
- return any(
- [
- label.lower() == "arm" or label.lower() == "arm64" or label ==
"asf-arm"
- for label in json.loads(self.runs_on_as_json_public)
- ]
- )
-
- @cached_property
- def is_vm_runner(self) -> bool:
- """Whether the runner is VM runner (managed by airflow)."""
- # TODO: when we have it properly set-up with labels we should just
check for
- # "airflow-runner" presence in runs_on
- return self.is_airflow_runner
-
- @cached_property
- def is_k8s_runner(self) -> bool:
- """Whether the runner is K8s runner (managed by airflow)."""
- # TODO: when we have it properly set-up with labels we should just
check for
- # "k8s-runner" presence in runs_on
- return False
+ def arm_runners(self) -> str:
+ return PUBLIC_ARM_RUNNERS
@cached_property
def has_migrations(self) -> bool:
diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py
index 1c874e5c4db..2a2f814c7dd 100644
--- a/dev/breeze/tests/test_pr_info.py
+++ b/dev/breeze/tests/test_pr_info.py
@@ -16,9 +16,7 @@
# under the License.
from __future__ import annotations
-import os
from pathlib import Path
-from unittest import mock
from airflow_breeze.commands.ci_commands import workflow_info
@@ -26,123 +24,57 @@ TEST_PR_INFO_DIR = Path(__file__).parent /
"test_pr_info_files"
def test_pr_info():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}):
- json_string = (TEST_PR_INFO_DIR / "pr_github_context.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == [
- "area:providers",
- "area:dev-tools",
- "area:logging",
- "kind:documentation",
- ]
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "test/airflow"
- assert wi.event_name == "pull_request"
- assert wi.pr_number == 26004
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- assert wi.is_canary_run() == "false"
- assert wi.run_coverage() == "false"
+ json_string = (TEST_PR_INFO_DIR / "pr_github_context.json").read_text()
+ wi = workflow_info(json_string)
+ assert wi.pull_request_labels == [
+ "area:providers",
+ "area:dev-tools",
+ "area:logging",
+ "kind:documentation",
+ ]
+ assert wi.target_repo == "apache/airflow"
+ assert wi.head_repo == "test/airflow"
+ assert wi.event_name == "pull_request"
+ assert wi.pr_number == 26004
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
+ assert wi.is_canary_run() == "false"
+ assert wi.run_coverage() == "false"
def test_push_info():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}):
- json_string = (TEST_PR_INFO_DIR /
"push_github_context.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == []
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "push"
- assert wi.pr_number is None
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- assert wi.is_canary_run() == "true"
- assert wi.run_coverage() == "true"
+ json_string = (TEST_PR_INFO_DIR / "push_github_context.json").read_text()
+ wi = workflow_info(json_string)
+ assert wi.pull_request_labels == []
+ assert wi.target_repo == "apache/airflow"
+ assert wi.head_repo == "apache/airflow"
+ assert wi.event_name == "push"
+ assert wi.pr_number is None
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
+ assert wi.is_canary_run() == "true"
+ assert wi.run_coverage() == "true"
def test_schedule():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}):
- json_string = (TEST_PR_INFO_DIR /
"schedule_github_context.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == []
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "schedule"
- assert wi.pr_number is None
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- assert wi.is_canary_run() == "true"
- assert wi.run_coverage() == "false"
-
-
-def test_runs_on_self_hosted():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}):
- json_string = (TEST_PR_INFO_DIR / "simple_pr.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == ["another"]
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "pull_request"
- assert wi.pr_number == 1234
- # TODO: fix it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
- assert wi.is_canary_run() == "false"
- assert wi.run_coverage() == "false"
-
-
-def test_runs_on_forced_public_runner():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}):
- json_string = (TEST_PR_INFO_DIR /
"self_hosted_forced_pr.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == ["use public runners", "another"]
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "pull_request"
- assert wi.pr_number == 1234
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- assert wi.is_canary_run() == "false"
- assert wi.run_coverage() == "false"
+ json_string = (TEST_PR_INFO_DIR /
"schedule_github_context.json").read_text()
+ wi = workflow_info(json_string)
+ assert wi.pull_request_labels == []
+ assert wi.target_repo == "apache/airflow"
+ assert wi.head_repo == "apache/airflow"
+ assert wi.event_name == "schedule"
+ assert wi.pr_number is None
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
+ assert wi.is_canary_run() == "true"
+ assert wi.run_coverage() == "false"
def test_runs_on_simple_pr_other_repo():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": ""}):
- json_string = (TEST_PR_INFO_DIR /
"simple_pr_different_repo.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == ["another"]
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "test/airflow"
- assert wi.event_name == "pull_request"
- assert wi.pr_number == 1234
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- assert wi.is_canary_run() == "false"
- assert wi.run_coverage() == "false"
-
-
-def test_runs_on_push_other_branch():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}):
- json_string = (TEST_PR_INFO_DIR / "push_other_branch.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == []
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "push"
- assert wi.pr_number is None
- # TODO: revert it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
- assert wi.is_canary_run() == "false"
- assert wi.run_coverage() == "false"
-
-
-def test_runs_on_push_v_test_branch():
- with mock.patch.dict(os.environ, {"AIRFLOW_SELF_HOSTED_RUNNER": "true"}):
- json_string = (TEST_PR_INFO_DIR /
"push_v_test_branch.json").read_text()
- wi = workflow_info(json_string)
- assert wi.pull_request_labels == []
- assert wi.target_repo == "apache/airflow"
- assert wi.head_repo == "apache/airflow"
- assert wi.event_name == "push"
- assert wi.pr_number is None
- # TODO: revert it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
- assert wi.is_canary_run() == "true"
- assert wi.run_coverage() == "false"
+ json_string = (TEST_PR_INFO_DIR /
"simple_pr_different_repo.json").read_text()
+ wi = workflow_info(json_string)
+ assert wi.pull_request_labels == ["another"]
+ assert wi.target_repo == "apache/airflow"
+ assert wi.head_repo == "test/airflow"
+ assert wi.event_name == "pull_request"
+ assert wi.pr_number == 1234
+ assert wi.get_runs_on() == '["ubuntu-22.04"]'
+ assert wi.is_canary_run() == "false"
+ assert wi.run_coverage() == "false"
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index b49ab34c00c..11997e9fdfd 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -28,7 +28,6 @@ from rich.console import Console
from airflow_breeze.global_constants import (
ALLOWED_KUBERNETES_VERSIONS,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
- COMMITTERS,
DEFAULT_KUBERNETES_VERSION,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
NUMBER_OF_LOW_DEP_SLICES,
@@ -2422,302 +2421,6 @@ def
test_provider_yaml_or_pyproject_toml_changes_trigger_ci_build(
assert_outputs_are_printed(expected_outputs, str(stderr))
[email protected](
- (
- "github_event, github_actor, github_repository, pr_labels, "
- "github_context_dict, runs_on_as_json_default, runs_on_as_docs_build,
is_self_hosted_runner, "
- "is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner,
is_k8s_runner, exception"
- ),
- [
- pytest.param(
- GithubEvents.PUSH,
- "user",
- "apache/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- # "true",
- # "true",
- "true",
- "false",
- # TODO: revert it when we fix self-hosted runners
- "false",
- # "true",
- "false",
- False,
- id="Push event",
- ),
- pytest.param(
- GithubEvents.PUSH,
- "user",
- "private/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Push event for private repo",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- "user",
- "apache/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- "user",
- "private/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request private repo",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- COMMITTERS[0],
- "apache/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- COMMITTERS[0],
- "apache/airflow",
- (),
- dict(event=dict(pull_request=dict(user=dict(login="user")))),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request committer pr non-committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- COMMITTERS[0],
- "private/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request private repo committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST_TARGET,
- "user",
- "apache/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request target",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST_TARGET,
- "user",
- "private/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request target private repo",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST_TARGET,
- COMMITTERS[0],
- "apache/airflow",
- [],
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request target committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- COMMITTERS[0],
- "apache/airflow",
- (),
- dict(event=dict(pull_request=dict(user=dict(login="user")))),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request target committer pr non-committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST_TARGET,
- COMMITTERS[0],
- "private/airflow",
- (),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request targe private repo committer",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- "user",
- "apache/airflow",
- ("use self-hosted runners",),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- True,
- id="Pull request by non committer with 'use self-hosted runners'
label.",
- ),
- pytest.param(
- GithubEvents.PULL_REQUEST,
- COMMITTERS[0],
- "apache/airflow",
- ("use public runners",),
- dict(),
- '["ubuntu-22.04"]',
- '["ubuntu-22.04"]',
- "false",
- "false",
- "true",
- "false",
- "false",
- "false",
- False,
- id="Pull request by committer with 'use public runners' label.",
- ),
- ],
-)
-def test_runs_on(
- github_event: GithubEvents,
- github_actor: str,
- github_repository: str,
- pr_labels: tuple[str, ...],
- github_context_dict: dict[str, Any],
- runs_on_as_json_default: str,
- runs_on_as_docs_build: str,
- is_self_hosted_runner: str,
- is_airflow_runner: str,
- is_amd_runner: str,
- is_arm_runner: str,
- is_vm_runner: str,
- is_k8s_runner: str,
- exception: bool,
-):
- def get_output() -> str:
- return str(
- SelectiveChecks(
- files=(),
- commit_ref="",
- github_repository=github_repository,
- github_event=github_event,
- github_actor=github_actor,
- github_context_dict=github_context_dict,
- pr_labels=pr_labels,
- default_branch="main",
- )
- )
-
- if exception:
- with pytest.raises(SystemExit):
- get_output()
- else:
- stderr = get_output()
- assert_outputs_are_printed({"runs-on-as-json-default":
runs_on_as_json_default}, str(stderr))
- assert_outputs_are_printed({"runs-on-as-json-docs-build":
runs_on_as_docs_build}, str(stderr))
- assert_outputs_are_printed({"is-self-hosted-runner":
is_self_hosted_runner}, str(stderr))
- assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner},
str(stderr))
- assert_outputs_are_printed({"is-amd-runner": is_amd_runner},
str(stderr))
- assert_outputs_are_printed({"is-arm-runner": is_arm_runner},
str(stderr))
- assert_outputs_are_printed({"is-vm-runner": is_vm_runner}, str(stderr))
- assert_outputs_are_printed({"is-k8s-runner": is_k8s_runner},
str(stderr))
-
-
@pytest.mark.parametrize(
"files, has_migrations",
[
@@ -2857,75 +2560,6 @@ def test_mypy_matches(
assert_outputs_are_printed(expected_outputs, str(stderr))
[email protected](
- "files, expected_outputs, github_actor, pr_labels",
- [
- pytest.param(
- ("README.md",),
- {
- "is-committer-build": "false",
- "runs-on-as-json-default": '["ubuntu-22.04"]',
- },
- "",
- (),
- id="Regular pr",
- ),
- pytest.param(
- ("README.md",),
- {
- "is-committer-build": "true",
- "runs-on-as-json-default": '["ubuntu-22.04"]',
- },
- "potiuk",
- (),
- id="Committer regular PR",
- ),
- pytest.param(
- ("README.md",),
- {
- "is-committer-build": "false",
- "runs-on-as-json-default": '["ubuntu-22.04"]',
- },
- "potiuk",
- ("non committer build",),
- id="Committer regular PR - forcing non-committer build",
- ),
- pytest.param(
- ("README.md",),
- {
- "docker-cache": "disabled",
- "disable-airflow-repo-cache": "true",
- },
- "potiuk",
- ("disable image cache",),
- id="Disabled cache",
- ),
- pytest.param(
- ("README.md",),
- {
- "docker-cache": "registry",
- "disable-airflow-repo-cache": "false",
- },
- "potiuk",
- (),
- id="Standard cache",
- ),
- ],
-)
-def test_pr_labels(
- files: tuple[str, ...], expected_outputs: dict[str, str], github_actor:
str, pr_labels: tuple[str, ...]
-):
- stderr = SelectiveChecks(
- files=files,
- commit_ref=NEUTRAL_COMMIT,
- default_branch="main",
- github_actor=github_actor,
- github_event=GithubEvents.PULL_REQUEST,
- pr_labels=pr_labels,
- )
- assert_outputs_are_printed(expected_outputs, str(stderr))
-
-
@pytest.mark.parametrize(
"files, pr_labels, github_event",
[
