This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b5bc253fc2d Run maintainer and canary CI on self-hosted RunsOn runners
(#74889)
b5bc253fc2d is described below
commit b5bc253fc2d594ccedb7da6104f2b03d26485651
Author: Shahar Epstein <[email protected]>
AuthorDate: Sun Oct 25 10:48:19 2026 +0200
Run maintainer and canary CI on self-hosted RunsOn runners (#74889)
This PR implements AIP-118 for integrating self-hosted runners in Apache
Airflow's CI, utilizing the project's AWS instance and runs-on.com's runners.
---
.github/actions/breeze/action.yml | 2 +
.../actions/prepare_breeze_and_image/action.yml | 5 +-
.github/actions/prepare_single_ci_image/action.yml | 5 +-
.../actions/runs-on-cost-summary/action.yml | 32 +-
.github/actions/stash-restore/action.yml | 137 +++++++
.github/actions/stash-save/action.yml | 129 +++++++
.github/workflows/additional-ci-image-checks.yml | 2 +-
.github/workflows/additional-prod-image-tests.yml | 30 +-
.github/workflows/basic-tests.yml | 26 +-
.github/workflows/ci-amd.yml | 61 +--
.github/workflows/ci-arm.yml | 63 ++--
.github/workflows/ci-image-build.yml | 36 +-
.github/workflows/ci-image-checks.yml | 8 +-
.github/workflows/finalize-tests.yml | 6 +-
.github/workflows/generate-constraints.yml | 4 +-
.github/workflows/helm-tests.yml | 4 +-
.github/workflows/integration-system-tests.yml | 6 +-
.github/workflows/k8s-tests.yml | 4 +-
.github/workflows/kustomize-overlays-tests.yml | 2 +-
.github/workflows/prod-image-build.yml | 6 +-
.github/workflows/push-image-cache.yml | 4 +-
.github/workflows/special-tests.yml | 22 +-
.github/workflows/test-providers.yml | 4 +-
.github/workflows/ui-e2e-tests.yml | 2 +-
README.md | 2 +
dev/breeze/doc/ci/04_selective_checks.md | 90 ++++-
dev/breeze/doc/ci/06_debugging.md | 6 +-
dev/breeze/src/airflow_breeze/global_constants.py | 8 +
.../src/airflow_breeze/utils/selective_checks.py | 116 ++++++
dev/breeze/tests/test_selective_checks.py | 411 +++++++++++++++++++++
scripts/ci/make_mnt_writeable.sh | 21 +-
scripts/ci/move_docker_to_mnt.sh | 7 +-
scripts/ci/prek/check_ci_workflows_in_sync.py | 5 +-
33 files changed, 1096 insertions(+), 170 deletions(-)
diff --git a/.github/actions/breeze/action.yml
b/.github/actions/breeze/action.yml
index cb49d1d6d57..9fe5b551096 100644
--- a/.github/actions/breeze/action.yml
+++ b/.github/actions/breeze/action.yml
@@ -29,6 +29,8 @@ outputs:
runs:
using: "composite"
steps:
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Setup python"
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #
v6.2.0
with:
diff --git a/.github/actions/prepare_breeze_and_image/action.yml
b/.github/actions/prepare_breeze_and_image/action.yml
index 6c91e28bff8..263db0346d4 100644
--- a/.github/actions/prepare_breeze_and_image/action.yml
+++ b/.github/actions/prepare_breeze_and_image/action.yml
@@ -82,12 +82,13 @@ runs:
- name: >
Restore ${{ inputs.image-type }} docker image ${{ inputs.platform
}}:${{ inputs.python }}
${{ inputs.image-stash-ref != '' && format('built for ref {0}',
inputs.image-stash-ref) || '' }}
- # yamllint disable-line rule:line-length
- uses:
apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0
# restore/v1.0.0
+ uses: ./.github/actions/stash-restore
with:
key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform
}}-${{ inputs.python }}\
${{ inputs.image-stash-ref != '' && format('-{0}',
inputs.image-stash-ref) || '' }}"
path: "/mnt/"
+ # Must stay true: runs on RunsOn (canaries and release-branch pushes)
save these image
+ # stashes as S3 digests only.
only-current-branch: 'true'
fail-on-download: 'true'
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{
inputs.python }}"
diff --git a/.github/actions/prepare_single_ci_image/action.yml
b/.github/actions/prepare_single_ci_image/action.yml
index d70012c80f7..8a734ef1102 100644
--- a/.github/actions/prepare_single_ci_image/action.yml
+++ b/.github/actions/prepare_single_ci_image/action.yml
@@ -36,11 +36,12 @@ runs:
using: "composite"
steps:
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python
}}"
- # yamllint disable-line rule:line-length
- uses:
apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0
# restore/v1.0.0
+ uses: ./.github/actions/stash-restore
with:
key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}
path: "/mnt/"
+ # Must stay true: runs on RunsOn (canaries and release-branch pushes)
save these image
+ # stashes as S3 digests only.
only-current-branch: 'true'
fail-on-download: 'true'
if: contains(inputs.python-versions-list-as-string, inputs.python)
diff --git a/scripts/ci/make_mnt_writeable.sh
b/.github/actions/runs-on-cost-summary/action.yml
old mode 100755
new mode 100644
similarity index 58%
copy from scripts/ci/make_mnt_writeable.sh
copy to .github/actions/runs-on-cost-summary/action.yml
index ddabf84b6b7..fd0d2007f10
--- a/scripts/ci/make_mnt_writeable.sh
+++ b/.github/actions/runs-on-cost-summary/action.yml
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,20 +14,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-function make_mnt_writeable {
- set -x
- echo "Investigating node disks"
- lsblk
- sudo blkid
- echo "Checking free space!"
- df -H
- echo "Cleaning /mnt just in case it is not empty"
- sudo mkdir -p /mnt
- sudo rm -rf /mnt/*
- echo "Checking free space!"
- df -H
- echo "Making sure that /mnt is writeable"
- sudo chown -R "${USER}" /mnt
-}
-
-make_mnt_writeable
+#
+---
+# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary.
Used by every job
+# that can run on RunsOn: through the breeze action, or directly by jobs that
do not install Breeze.
+name: 'RunsOn cost summary'
+description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op
elsewhere.'
+runs:
+ using: 'composite'
+ steps:
+ - name: "RunsOn cost summary"
+ if: runner.environment == 'self-hosted'
+ uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
+ with:
+ show_costs: summary
diff --git a/.github/actions/stash-restore/action.yml
b/.github/actions/stash-restore/action.yml
new file mode 100644
index 00000000000..ed9535c1722
--- /dev/null
+++ b/.github/actions/stash-restore/action.yml
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On
RunsOn (the only
+# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's
in-region S3 cache
+# bucket instead of leaving AWS as GitHub artifacts.
+#
+# Every RunsOn job can write anywhere in that bucket, so the bucket is not
trusted: objects are
+# named by their sha256, and the digest itself travels as a small ASF stash
artifact, which
+# GitHub scopes to the run / branch exactly like a regular stash. A blob that
does not match the
+# digest the ASF restore found for this run, branch and repository is never
extracted. The bucket's
+# lifecycle rule (S3CacheExpirationInDays) must outlive the longest
retention-days of a digest
+# (6 days today), or a digest hit finds no blob.
+name: 'Restore stash'
+description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn
runners.'
+inputs:
+ key:
+ description: 'Stash key.'
+ required: true
+ path:
+ description: >
+ Directory to restore into (GitHub-hosted only; the S3 tarball restores
entries to the
+ paths they were saved from, and the restore then fails if any of them is
outside this directory).
+ required: false
+ default: ${{ github.workspace }}
+ only-current-branch:
+ description: 'Do not fall back to the base branch stash.'
+ required: false
+ default: 'false'
+ fail-on-download:
+ description: 'Fail when a stash was found but could not be restored.'
+ required: false
+ default: 'false'
+outputs:
+ stash-hit:
+ description: "'true' if a stash was restored, 'false' otherwise."
+ value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit
}}
+runs:
+ using: 'composite'
+ steps:
+ - name: "Restore stash from GitHub artifacts"
+ id: artifact
+ if: runner.environment != 'self-hosted'
+ # yamllint disable-line rule:line-length
+ uses:
apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3
# restore/v1.0.1
+ with:
+ key: ${{ inputs.key }}
+ path: ${{ inputs.path }}
+ only-current-branch: ${{ inputs.only-current-branch }}
+ fail-on-download: ${{ inputs.fail-on-download }}
+ - name: "Restore stash digest from GitHub artifacts"
+ id: digest
+ if: runner.environment == 'self-hosted'
+ # yamllint disable-line rule:line-length
+ uses:
apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3
# restore/v1.0.1
+ with:
+ key: ${{ inputs.key }}-s3
+ path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}
+ only-current-branch: ${{ inputs.only-current-branch }}
+ fail-on-download: ${{ inputs.fail-on-download }}
+ - name: "Restore stash from S3"
+ id: s3
+ if: runner.environment == 'self-hosted'
+ shell: bash
+ env:
+ STASH_KEY: ${{ inputs.key }}
+ STASH_PATH: ${{ inputs.path }}
+ DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }}
+ DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key
}}/sha256
+ FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }}
+ run: |
+ set -uo pipefail
+ payload="${RUNNER_TEMP}/stash-restore-$$.tar"
+ entries="${RUNNER_TEMP}/stash-restore-$$.entries"
+ miss() {
+ rm -f "${payload}" "${entries}"
+ echo "stash-hit=false" >> "${GITHUB_OUTPUT}"
+ if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true"
]]; then
+ echo "::error::$1"
+ exit 1
+ fi
+ echo "$1"
+ exit 0
+ }
+ if [[ "${DIGEST_HIT}" != "true" ]]; then
+ miss "No stash '${STASH_KEY}'"
+ fi
+ digest="$(cat "${DIGEST_FILE}")"
+ if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
+ miss "Malformed digest for stash '${STASH_KEY}'"
+ fi
+ bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
+ export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a
RunsOn runner}"
+ object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"
+ if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then
+ miss "Could not download ${object} for stash '${STASH_KEY}'"
+ fi
+ if ! echo "${digest} ${payload}" | sha256sum --check --status; then
+ miss "${object} does not match the digest recorded for stash
'${STASH_KEY}'"
+ fi
+ # GNU tar detects the compression stash-save chose.
+ if ! tar -xPvf "${payload}" > "${entries}"; then
+ miss "Could not extract ${object} for stash '${STASH_KEY}'"
+ fi
+ rm -f "${payload}"
+ # The S3 tarball ignores `path`: fail loudly if a caller saves and
restores different
+ # directories, which would work on GitHub-hosted runners only.
+ restore_dir="$(realpath -ms "${STASH_PATH}")"
+ restore_dir="${restore_dir%/}"
+ outside=()
+ while IFS= read -r entry; do
+ [[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}"
+ [[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}")
+ done < "${entries}"
+ rm -f "${entries}"
+ if [[ ${#outside[@]} -gt 0 ]]; then
+ printf '%s\n' "${outside[@]}"
+ echo "::error::Stash '${STASH_KEY}' restored the entries above
outside of ${restore_dir}/"
+ exit 1
+ fi
+ echo "stash-hit=true" >> "${GITHUB_OUTPUT}"
+ echo "Restored stash '${STASH_KEY}' from ${object}"
diff --git a/.github/actions/stash-save/action.yml
b/.github/actions/stash-save/action.yml
new file mode 100644
index 00000000000..ecfd2409438
--- /dev/null
+++ b/.github/actions/stash-save/action.yml
@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn
the tarball goes
+# to the stack's S3 cache bucket under its sha256 and only the digest is saved
as an ASF stash
+# artifact. See stash-restore for why.
+name: 'Save stash'
+description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.'
+inputs:
+ key:
+ description: 'Stash key.'
+ required: true
+ path:
+ description: 'File, directory or glob to stash.'
+ required: true
+ if-no-files-found:
+ description: "'error' fails when path matches nothing, 'warn' or 'ignore'
do not."
+ required: false
+ default: 'warn'
+ retention-days:
+ description: >
+ Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's
+ S3CacheExpirationInDays lifecycle rule expires the tarball itself.
+ required: false
+ default: '5'
+ compression-level:
+ description: "Zlib compression level (0-9). On RunsOn, '0' stores the
tarball uncompressed."
+ required: false
+ default: '6'
+runs:
+ using: 'composite'
+ steps:
+ - name: "Save stash to GitHub artifacts"
+ if: runner.environment != 'self-hosted'
+ uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ with:
+ key: ${{ inputs.key }}
+ path: ${{ inputs.path }}
+ if-no-files-found: ${{ inputs.if-no-files-found }}
+ retention-days: ${{ inputs.retention-days }}
+ compression-level: ${{ inputs.compression-level }}
+ - name: "Save stash to S3"
+ id: s3
+ if: runner.environment == 'self-hosted'
+ shell: bash
+ env:
+ STASH_KEY: ${{ inputs.key }}
+ STASH_PATH: ${{ inputs.path }}
+ DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
+ IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }}
+ COMPRESSION_LEVEL: ${{ inputs.compression-level }}
+ run: |
+ set -euo pipefail
+ bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
+ export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a
RunsOn runner}"
+ shopt -s nullglob
+ # shellcheck disable=SC2206
+ candidates=( ${STASH_PATH} )
+ existing=()
+ for p in "${candidates[@]}"; do
+ if [[ -e "${p}" ]]; then
+ existing+=("${p}")
+ fi
+ done
+ if [[ ${#existing[@]} -eq 0 ]]; then
+ if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then
+ echo "::error::No files found for path '${STASH_PATH}'"
+ exit 1
+ fi
+ if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then
+ echo "::warning::No files found for path '${STASH_PATH}', skipping
save"
+ fi
+ exit 0
+ fi
+ # The tarball is streamed to S3 while being hashed: a multi-GB image
never touches the
+ # runner's disk. Its key is only known at the end, so it lands under a
temporary key and
+ # is copied server-side to its digest key.
+ prefix="s3://${bucket}/cache/airflow-stash"
+ upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar"
+ digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo"
+ digest_file="${RUNNER_TEMP}/stash-save-$$.sha256"
+ trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT
+ mkfifo "${digest_fifo}"
+ sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" &
+ digest_pid=$!
+ compression=()
+ if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then
+ compression=(-z)
+ fi
+ tar -cP "${compression[@]}" -f - "${existing[@]}" \
+ | tee "${digest_fifo}" \
+ | aws s3 cp --only-show-errors - "${upload}"
+ wait "${digest_pid}"
+ digest="$(cat "${digest_file}")"
+ if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
+ echo "::error::Could not hash the stash '${STASH_KEY}'"
+ exit 1
+ fi
+ object="${prefix}/sha256/${digest}.tar"
+ aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}"
+ # Leaving the upload behind only costs storage until the bucket's
lifecycle rule expires it.
+ aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not
remove ${upload}"
+ mkdir -p "${DIGEST_DIR}"
+ echo "${digest}" > "${DIGEST_DIR}/sha256"
+ echo "saved=true" >> "${GITHUB_OUTPUT}"
+ echo "Saved ${existing[*]} to ${object}"
+ - name: "Save stash digest to GitHub artifacts"
+ if: runner.environment == 'self-hosted' && steps.s3.outputs.saved ==
'true'
+ uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ with:
+ key: ${{ inputs.key }}-s3
+ path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
+ if-no-files-found: error
+ retention-days: ${{ inputs.retention-days }}
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index de26e37fbfc..7d60cf6ef41 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -62,7 +62,7 @@ jobs:
# layer that builds Python from source, which takes 11-18 minutes for
Python 3.11+ on hosted runners.
timeout-minutes: 35
name: Check that image builds quickly
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index dc5da4b211e..c850b783deb 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -152,7 +152,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of PROD image building"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -189,7 +189,7 @@ jobs:
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker Compose quick start with PROD image verifying"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -220,7 +220,7 @@ jobs:
task-sdk-integration-tests:
timeout-minutes: 60
name: "Task SDK integration tests with PROD image"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -254,7 +254,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Regular e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -264,7 +264,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Remote logging e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -276,7 +276,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Elasticsearch remote logging e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -288,7 +288,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenSearch remote logging e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -300,7 +300,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "XCom object storage backend e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -311,7 +311,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Event driven e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -323,7 +323,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Java SDK e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -335,7 +335,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Go SDK e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -347,7 +347,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "TypeScript SDK e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -359,7 +359,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenLineage e2e test"
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -377,7 +377,7 @@ jobs:
compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }}
uses: ./.github/workflows/openlineage-e2e-compat-tests.yml
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -423,7 +423,7 @@ jobs:
airflow-ctl-integration-tests:
timeout-minutes: 60
name: "Airflow CTL integration tests with PROD image"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 4212f4bd872..88740c02c46 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -87,7 +87,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -105,7 +105,7 @@ jobs:
run-breeze-integration-tests:
timeout-minutes: 120
name: Breeze integration tests
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-breeze-integration-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -136,13 +136,15 @@ jobs:
# Without a pin uv picks the runner's system Python, which can be older
than Airflow supports.
UV_PYTHON: "${{ inputs.default-python-version }}"
SHARED_DISTRIBUTIONS: ${{
join(fromJSON(inputs.shared-distributions-as-json), ' ') }}
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
fetch-depth: 1
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
@@ -171,7 +173,7 @@ jobs:
tests-scripts:
timeout-minutes: 10
name: Scripts tests
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-scripts-tests == 'true'
env:
# Without a pin uv picks the runner's system Python, which can be older
than Airflow supports.
@@ -182,6 +184,8 @@ jobs:
with:
fetch-depth: 1
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
@@ -200,7 +204,7 @@ jobs:
tests-ui:
timeout-minutes: 15
name: React UI tests
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.run-ui-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -210,6 +214,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 #
v6.1.0
with:
@@ -237,7 +243,7 @@ jobs:
check-translation-completness:
timeout-minutes: 15
name: "Check translation completeness"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
@@ -254,7 +260,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -327,7 +333,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -381,7 +387,7 @@ jobs:
test-airflow-standalone:
timeout-minutes: 30
name: "Test Airflow standalone commands"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
AIRFLOW_HOME: ~/airflow
FORCE_COLOR: 1
@@ -390,6 +396,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 92f1903a1b7..05fa7900c0f 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -20,8 +20,9 @@
# `release-branch-ci.yml`. Companion to `ci-arm.yml`, which runs the same
pipeline on ARM as the
# scheduled canary. The two workflows are physical copies of each other
# (GitHub Actions has no cross-file YAML include), differing only in their
-# trigger surface, the hardcoded `runner-type` / `platform` values in
-# `build-info`'s outputs, and the architecture-specific print-platform job.
+# trigger surface, the hardcoded `platform` value in `build-info`'s outputs
+# (and the `PLATFORM` selective checks pick the runner type for), and the
+# architecture-specific print-platform job.
# Edit both files together when changing pipeline shape.
name: Tests (AMD)
on: # yamllint disable-line rule:truthy
@@ -305,7 +306,7 @@ jobs:
latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
- # platform / runner-type are hardcoded per workflow file (this is the
+ # platform is hardcoded per workflow file (this is the
# AMD copy). The matching ARM copy lives in ci-arm.yml.
platform: "linux/amd64"
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
@@ -347,7 +348,7 @@ jobs:
run-task-sdk-integration-tests: ${{
steps.selective-checks.outputs.run-task-sdk-integration-tests }}
run-breeze-integration-tests: ${{
steps.selective-checks.outputs.run-breeze-integration-tests }}
run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests
}}
- runner-type: '["ubuntu-22.04"]'
+ runner-type: ${{ steps.selective-checks.outputs.runner-type }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }}
@@ -429,6 +430,8 @@ jobs:
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
+ # Repository variable: "true" moves every run, pushes included, back
to public runners.
+ USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
PLATFORM: "linux/amd64"
SELECTIVE_TESTS_ON_PUSH: "true"
PUSH_BASE_COMMIT: ${{
needs.check-run-needed.outputs.push-base-commit }}
@@ -566,7 +569,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -607,7 +610,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -729,7 +732,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -758,7 +761,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -787,7 +790,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -817,7 +820,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -846,7 +849,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -877,7 +880,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -908,7 +911,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -938,7 +941,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -1024,7 +1027,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1054,7 +1057,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1165,7 +1168,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1181,7 +1184,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1197,6 +1200,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
# keep this in sync with go.mod in go-sdk/
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e #
v7.0.0
@@ -1235,7 +1240,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1245,6 +1250,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 #
v6.1.0
with:
@@ -1269,7 +1276,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32,
128) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1285,6 +1292,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: Setup Java
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad #
v6.1.0
with:
@@ -1299,7 +1308,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1331,7 +1340,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1368,7 +1377,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1389,7 +1398,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1669,7 +1678,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -1679,6 +1688,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index 180182647a6..1bde91c7aca 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -20,9 +20,10 @@
# `release-branch-ci.yml` — never on PRs. Companion to `ci-amd.yml`, which
runs the
# same pipeline on AMD for every PR. The two workflows are physical
# copies of each other (GitHub Actions has no cross-file YAML include),
-# differing only in their trigger surface, the hardcoded `runner-type` /
-# `platform` values in `build-info`'s outputs, and the architecture-specific
-# print-platform job. Edit both files together when changing pipeline shape.
+# differing only in their trigger surface, the hardcoded `platform` value in
+# `build-info`'s outputs (and the `PLATFORM` selective checks pick the runner
+# type for), and the architecture-specific print-platform job. Edit both files
+# together when changing pipeline shape.
name: Tests (ARM)
on: # yamllint disable-line rule:truthy
schedule:
@@ -187,7 +188,7 @@ jobs:
latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
- # platform / runner-type are hardcoded per workflow file (this is the
+ # platform is hardcoded per workflow file (this is the
# ARM copy). The matching AMD copy lives in ci-amd.yml.
platform: "linux/arm64"
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
@@ -229,7 +230,7 @@ jobs:
run-task-sdk-integration-tests: ${{
steps.selective-checks.outputs.run-task-sdk-integration-tests }}
run-breeze-integration-tests: ${{
steps.selective-checks.outputs.run-breeze-integration-tests }}
run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests
}}
- runner-type: '["ubuntu-22.04-arm"]'
+ runner-type: ${{ steps.selective-checks.outputs.runner-type }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }}
@@ -311,6 +312,8 @@ jobs:
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
+ # Repository variable: "true" moves every run, pushes included, back
to public runners.
+ USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
PLATFORM: "linux/arm64"
SELECTIVE_TESTS_ON_PUSH: "true"
PUSH_BASE_COMMIT: ${{
needs.check-run-needed.outputs.push-base-commit }}
@@ -448,7 +451,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -489,7 +492,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -611,7 +614,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -640,7 +643,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -669,7 +672,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -699,7 +702,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -728,7 +731,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -759,7 +762,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -790,7 +793,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -820,7 +823,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -906,7 +909,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -936,7 +939,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1047,7 +1050,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1063,7 +1066,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1079,6 +1082,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
# keep this in sync with go.mod in go-sdk/
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e #
v7.0.0
@@ -1117,7 +1122,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1127,6 +1132,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 #
v6.1.0
with:
@@ -1151,7 +1158,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32,
128) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1167,6 +1174,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: Setup Java
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad #
v6.1.0
with:
@@ -1181,7 +1190,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1213,7 +1222,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1250,7 +1259,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1271,7 +1280,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ needs.build-info.outputs.runner-type }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1550,7 +1559,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
- runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -1560,6 +1569,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 315256420fd..23f8e1f724c 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -128,7 +128,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 110
name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
@@ -147,24 +147,38 @@ jobs:
with:
ref: ${{ inputs.checkout-ref }}
persist-credentials: false
+ # A local action is read from the workspace, and `checkout-ref` may be a
release ref cut
+ # before the stash actions existed, so they are loaded from the
workflow's own version. The
+ # runner-preparing scripts too: an older `make_mnt_writeable.sh` empties
/mnt, which on
+ # RunsOn holds the runner's own disks.
+ - name: "Checkout the workflow version for its actions and runner
scripts"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ persist-credentials: false
+ path: workflow-version
+ sparse-checkout: |
+ .github/actions
+ scripts/ci
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
- name: "Make /mnt writeable"
- run: ./scripts/ci/make_mnt_writeable.sh
+ run: ./workflow-version/scripts/ci/make_mnt_writeable.sh
- name: "Move docker to /mnt"
- run: ./scripts/ci/move_docker_to_mnt.sh
+ run: ./workflow-version/scripts/ci/move_docker_to_mnt.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
# A ref is not a commit: a branch documented twice moves between the two
publishes, and
# an image built before it moved is a cache, not an answer. The commit
each stashed image
# is saved with tells the two apart, and it is stashed separately so
that deciding costs
# a few bytes rather than the multi-gigabyte image the decision may make
unnecessary.
+ # `only-current-branch` must stay true on the image restores below: runs
on RunsOn (canaries
+ # and release-branch pushes) save these image stashes as S3 digests
only, so a fallback to the
+ # base branch would find nothing on public runners anyway.
- name: >
Restore the commit the CI image stashed for ref ${{
inputs.image-stash-ref }}
was built from
- # yamllint disable-line rule:line-length
- uses:
apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3
# restore/v1.0.1
+ uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-commit-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -197,8 +211,7 @@ jobs:
- name: >
Restore CI docker image built for ref ${{ inputs.image-stash-ref }}
${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- # yamllint disable-line rule:line-length
- uses:
apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3
# restore/v1.0.1
+ uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -244,8 +257,7 @@ jobs:
--cache-file
/tmp/ci-cache-mount-save-v4-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: steps.restore-cache-mount.outputs.stash-hit == 'true'
- name: "Restore CI docker image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
- # yamllint disable-line rule:line-length
- uses:
apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3
# restore/v1.0.1
+ uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
path: "/mnt/"
@@ -336,7 +348,7 @@ jobs:
shell: bash
if: inputs.upload-image-artifact == 'true' &&
steps.stashed-image.outputs.reusable != 'true'
- name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
- uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ uses: ./workflow-version/.github/actions/stash-save
with:
key: ci-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION
}}.tar"
@@ -350,7 +362,7 @@ jobs:
# what a later publish of that same ref reads - hence a retention that
outlives the shared
# one, the same ref being rebuilt three days apart at the very least.
- name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref
}}"
- uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ uses: ./workflow-version/.github/actions/stash-save
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -364,7 +376,7 @@ jobs:
# Saved last and with the image's retention, so that finding this commit
is enough to
# know the image it describes is there to be restored.
- name: "Stash the commit the CI image for ref ${{
inputs.image-stash-ref }} was built from"
- uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ uses: ./workflow-version/.github/actions/stash-save
with:
key: "ci-image-commit-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index affe6ebdb8d..1a1c06c6804 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -119,7 +119,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies
}}"
@@ -175,7 +175,7 @@ jobs:
build-docs:
timeout-minutes: 150
name: "Build documentation"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
if: inputs.docs-build == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -334,7 +334,7 @@ jobs:
id-token: write
contents: read
needs: build-docs
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -466,7 +466,7 @@ jobs:
test-python-api-client:
timeout-minutes: 60
name: "Test Python API client"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-api-codegen == 'true'
env:
BACKEND: "postgres"
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index 043aa8f0ebf..b664e49cf06 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -80,7 +80,7 @@ permissions:
contents: read
jobs:
update-constraints:
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
timeout-minutes: 80
name: "Update constraints"
permissions:
@@ -103,6 +103,8 @@ jobs:
with:
# Needed to perform push action
persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
@@ -124,7 +126,7 @@ jobs:
# Constraint commits are now handled by the dedicated
update-constraints-on-push workflow
# triggered when uv.lock changes are pushed to main/test branches.
dependency-upgrade-summary:
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
needs: [update-constraints]
if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform ==
'linux/amd64'
name: "Deps ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 9846e44ce71..58312ea63c4 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -85,7 +85,7 @@ jobs:
contents: read
timeout-minutes: 70
name: Generate constraints for ${{ matrix.python-version }} on ${{
inputs.platform }}
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -114,6 +114,8 @@ jobs:
platform: ${{ inputs.platform }}
python: ${{ matrix.python-version }}
use-uv: ${{ inputs.use-uv }}
+ # Runs the checkout-ref's own make_mnt_writeable.sh, which on old
refs empties /mnt and
+ # would kill a RunsOn runner: every caller passing checkout-ref pins
public runners.
make-mnt-writeable-and-cleanup: true
- name: "Install prek"
uses: ./.github/actions/install-prek
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 7e779afcc2a..2cbe0603a43 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -52,7 +52,7 @@ jobs:
name: >-
Unit tests Helm: ${{ matrix.helm-test-package }}
(K8S ${{ matrix.kubernetes-version }})
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -98,7 +98,7 @@ jobs:
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/integration-system-tests.yml
b/.github/workflows/integration-system-tests.yml
index 202d70cd9ef..efc8d578b5c 100644
--- a/.github/workflows/integration-system-tests.yml
+++ b/.github/workflows/integration-system-tests.yml
@@ -75,7 +75,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-core-integrations != '[]'
name: "Integration core ${{ matrix.integration }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -126,7 +126,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-providers-integrations != '[]' &&
inputs.skip-providers-tests != 'true'
name: "Integration: providers ${{ matrix.integration }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -176,7 +176,7 @@ jobs:
timeout-minutes: 30
if: inputs.run-system-tests == 'true'
name: "System Tests"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
BACKEND: "postgres"
BACKEND_VERSION: ${{ inputs.default-postgres-version }}
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
index d295b9d713d..39b0277e110 100644
--- a/.github/workflows/k8s-tests.yml
+++ b/.github/workflows/k8s-tests.yml
@@ -62,7 +62,7 @@ jobs:
tests-kubernetes:
timeout-minutes: 60
name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo
}}-${{ matrix.use-standard-naming }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -149,7 +149,7 @@ jobs:
tests-kubernetes-lang-sdk:
timeout-minutes: 60
name: "K8S Lang-SDK:${{ inputs.lang-sdk-kubernetes-combo }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
DEBUG_RESOURCES: ${{ inputs.debug-resources }}
INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
diff --git a/.github/workflows/kustomize-overlays-tests.yml
b/.github/workflows/kustomize-overlays-tests.yml
index 1cca2b85000..5fef9c1ac8a 100644
--- a/.github/workflows/kustomize-overlays-tests.yml
+++ b/.github/workflows/kustomize-overlays-tests.yml
@@ -53,7 +53,7 @@ jobs:
smoke-test-overlay:
timeout-minutes: 40
name: "Overlay smoke test: ${{ matrix.overlay }}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
strategy:
fail-fast: false
max-parallel: 4
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index f136cd1ce61..31eaabace18 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -119,7 +119,7 @@ jobs:
build-prod-packages:
name: "Build Airflow and provider distributions"
timeout-minutes: 10
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -199,7 +199,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 80
name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version
}}"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
needs:
- build-prod-packages
env:
@@ -289,7 +289,7 @@ jobs:
breeze prod-image save --platform "${PLATFORM}" --image-file-dir
"/mnt"
if: inputs.upload-image-artifact == 'true'
- name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
- uses:
apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9
# save/v1.0.2
+ uses: ./.github/actions/stash-save
with:
key: prod-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION
}}.tar"
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 4d04deb09da..3dc4693e0df 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -79,7 +79,7 @@ on: # yamllint disable-line rule:truthy
jobs:
push-ci-image-cache:
name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
permissions:
contents: read
packages: write
@@ -149,7 +149,7 @@ jobs:
push-prod-image-cache:
name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
permissions:
contents: read
packages: write
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index 6ac4d60d1ba..ff56fbb9df5 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -94,7 +94,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -119,7 +119,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -144,7 +144,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -169,7 +169,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -194,7 +194,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -220,7 +220,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -246,7 +246,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -272,7 +272,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -298,7 +298,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -323,7 +323,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -348,7 +348,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ inputs.runners }}
+ runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
test-name: "SystemTest"
test-scope: "System"
diff --git a/.github/workflows/test-providers.yml
b/.github/workflows/test-providers.yml
index e7b3575b3f1..406ebde52fa 100644
--- a/.github/workflows/test-providers.yml
+++ b/.github/workflows/test-providers.yml
@@ -72,7 +72,7 @@ jobs:
prepare-install-verify-provider-distributions:
timeout-minutes: 80
name: "Providers ${{ matrix.package-format }} tests"
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
matrix:
@@ -178,7 +178,7 @@ jobs:
timeout-minutes: 80
# yamllint disable rule:line-length
name: Compat ${{ matrix.compat.airflow-version }}:P${{
matrix.compat.python-version }}:${{ matrix.test-types.description }}
- runs-on: ${{ fromJSON(inputs.runners) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/ui-e2e-tests.yml
b/.github/workflows/ui-e2e-tests.yml
index 8b8ad21e4bf..1ee5e1e6a67 100644
--- a/.github/workflows/ui-e2e-tests.yml
+++ b/.github/workflows/ui-e2e-tests.yml
@@ -88,7 +88,7 @@ jobs:
test-ui-e2e-tests:
timeout-minutes: 90
name: ${{ inputs.workflow-name || 'UI E2E Tests' }}
- runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }}
+ runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0], 8,
32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11'
}}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/README.md b/README.md
index 8e7f8f70b41..e1a994222e9 100644
--- a/README.md
+++ b/README.md
@@ -603,3 +603,5 @@ The CI infrastructure for Apache Airflow has been sponsored
by:
<a href="https://astronomer.io";><img
src="https://assets2.astronomer.io/logos/logoForLIGHTbackground.png";
alt="astronomer.io" width="250px"></a>
<a href="https://aws.amazon.com/opensource/";><img
src="https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/[email protected]?raw=true";
alt="AWS OpenSource" width="130px"></a>
+
+The self-hosted CI runners on AWS are managed by
[RunsOn](https://runs-on.com), which is free for open-source projects.
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 2356d61c3ea..d15ff81dd5e 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -33,7 +33,7 @@
- [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks)
- [Suspended providers](#suspended-providers)
- [Selective check outputs](#selective-check-outputs)
- - [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs)
+ - [Self-hosted (RunsOn) vs. public
runners](#self-hosted-runson-vs-public-runners)
- [Changing behaviours of the CI runs by setting
labels](#changing-behaviours-of-the-ci-runs-by-setting-labels)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -689,6 +689,7 @@ GitHub Actions to pass the list of parameters to a command
to execute
| run-www-tests | Whether Legacy WWW
tests should be run ("true"/"false")
| true | |
| amd-runners | List of labels
assigned for runners for that build for public AMD runners
| \["ubuntu-22.04"\] | |
| arm-runners | List of labels
assigned for runners for that build for public ARM runners
| \["ubuntu-22.04-arm"\] | |
+| runner-type | Runner labels for
the CI jobs on this platform: public, or a RunsOn template jobs fill with
cpu/ram | \["ubuntu-22.04"\] | |
| selected-providers-list-as-string | List of providers
affected when they are selectively affected.
| airbyte http | * |
| skip-prek-hooks | Which prek hooks
should be skipped during the static-checks run
| flynt,identity | |
| skip-providers-tests | When provider
tests should be skipped (on non-main branch or when no provider changes
detected) | true | |
@@ -698,27 +699,82 @@ GitHub Actions to pass the list of parameters to a
command to execute
| upgrade-to-newer-dependencies | Whether the image
build should attempt to upgrade all dependencies (true/false or commit hash)
| false | |
-## Committer vs. Non-committer PRs
-
-There is a difference in how the CI jobs are run for committer and
non-committer PRs from forks.
-The main reason is security; we do not want to run untrusted code on our
infrastructure for self-hosted runners.
-
-Currently there is no difference because we are not using `self-hosted`
runners (until we implement `Action
-Runner Controller` but most of the jobs, committer builds will use
"Self-hosted" runners by default,
-while non-committer builds will use "Public" runners. For committers, this can
be overridden by setting the
-`use public runners` label in the PR.
+## Self-hosted (RunsOn) vs. public runners
+
+The `Tests (AMD)` / `Tests (ARM)` jobs run either on GitHub-hosted ("public")
runners or on self-hosted
+[RunsOn](https://runs-on.com) EC2 runners in the Airflow AWS account.
Selective checks route runs of
+code from `apache/airflow` itself there; this is a routing preference, not
what keeps fork code off
+RunsOn (see below). The `runner-type` output is decided as follows, first
match wins:
+
+1. Runs in any repository other than `apache/airflow` (forks running their own
CI) use public runners.
+2. Any run uses public runners while the repository variable
`USE_PUBLIC_RUNNERS` is `true`: the
+ switch for moving `main` off RunsOn without a PR, e.g. when the stack is
unavailable.
+3. PRs with the `use public runners` label use public runners.
+4. Any `push`, `schedule` or `workflow_dispatch` in `apache/airflow` (canary
runs and the release
+ branch pushes, all triggered by people with write access) uses RunsOn.
+5. Other events than `pull_request` use public runners.
+6. PRs with the `use self-hosted runners` label use RunsOn, if a committer's
(`COMMITTERS` in
+ `global_constants.py`) latest review of the PR's head commit is an approval
and no committer's
+ request for changes stands. The label outlives the commit it was added for,
so on its own it would
+ vouch for every later push: a new push sends the PR back to public runners
until the head commit is
+ approved again, and so does a dismissed approval if no other committer's
approval of the head commit
+ remains. A committer's request for changes keeps the PR on public runners
across pushes, as it blocks
+ a merge on GitHub, until that committer reviews again or the review is
dismissed. The label must
+ also still be on the PR: a re-run reuses the labels of its original event,
so they are checked
+ against the PR's current ones. If the PR or its reviews cannot be read, the
label is ignored as well.
+7. PRs with the `non committer build` label use public runners.
+8. PRs opened by a bot (Dependabot and the like), or from a `dependabot/`
branch, use public runners:
+ they carry code nobody has reviewed yet.
+9. PRs from a branch of `apache/airflow` use RunsOn; PRs from forks use public
runners.
+
+Labels are read from the event that started the run, and the CI workflows do
not run on `labeled`. After
+adding or removing a label, push a commit or close and reopen the PR;
re-running an earlier run reuses
+its original labels, and "Re-run failed jobs" also reuses that run's runner
decision. A push voids the
+approval `use self-hosted runners` needs, so for that label approve the head
commit and then close and
+reopen the PR.
+
+The RunsOn value is a label template such as
+`["runs-on=<run_id>/family=.../image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]`.
Every job
+picks its size class with `format(fromJSON(inputs.runners)[0], <cpu>, <ram>)`
(4/16, 8/32 or 32/128),
+or, for a single-job reusable workflow, its caller passes `format(<runners>,
<cpu>, <ram>)`;
+on a public label the same call returns the label unchanged. Steps that only
make sense on RunsOn are
+gated on `runner.environment == 'self-hosted'`:
+
+* The EC2 cost summary of each job.
+* The image stashes (`.github/actions/stash-restore` / `stash-save`). On
RunsOn the image tarballs go to
+ the stack's S3 cache bucket, named by their sha256, and only the digest is
stashed as a GitHub artifact.
+ Any RunsOn job can write to the bucket, so a tarball is used only if it
matches the digest stashed by
+ the same run, branch and repository (or, for a PR, its base branch). The
smaller caches (mount cache, prek, docs inventory) use
+ the ASF stash action on both kinds of runners, so PRs on public runners keep
the ones canary runs save;
+ the price is that every RunsOn canary uploads the mount cache from AWS to
GitHub.
+
+The decision above is not a security boundary. Under the `pull_request` event
the code that
+evaluates it is the PR's own: a fork PR runs its own workflow files and breeze
sources, and can name
+RunsOn labels in `runs-on:` directly. RunsOn launches a runner for any job of
a repository its GitHub
+App is installed on, so nothing in this repository can prevent that. Keeping
untrusted code away from
+anything that matters has to be enforced outside the repository, by:
+
+* The repository's Actions setting "Approval for running fork pull request
workflows from
+ contributors", which must be "Require approval for all external
contributors". That still exempts
+ members of the `apache` GitHub organization (ASF committers of any project
who linked their GitHub
+ account) and repository collaborators, but every exempt person is
identifiable.
+* The RunsOn stack, which must be dedicated to this repository and give the
runners an instance role
+ that reaches nothing beyond the stack's own cache bucket (whose contents are
never trusted, see the
+ stash actions), no EBS permissions, no SSH, network isolation between
runners and a maximum run
+ time. RunsOn has no cap on instance sizes or counts and its budget only
alerts, so the AWS account's
+ EC2 vCPU quotas are the cost ceiling.
Pushes in a contributor fork are never committer builds, whoever pushes,
because forks have no
self-hosted runners.
## Changing behaviours of the CI runs by setting labels
-Also, currently for most of the jobs, committer builds by default use
"Self-hosted" runners, while
-non-committer builds use "Public" runners. For committers, this can be
overridden by setting the
-`use public runners` label in the PR. In the future when we might also switch
committers to public runners.
-Committers will be able to use `use self-hosted runners` label in the PR to
force using self-hosted runners.
-The `use public runners` label will still be available for committers and they
will be able to set it for
-builds that also have `canary` label set to also switch the `canary` builds to
public runners.
+PRs from `apache/airflow` branches run on self-hosted RunsOn runners and PRs
from forks run on public
+runners (see above). Set `use public runners` to move a PR to public runners;
a committer can set
+`use self-hosted runners` and approve the PR's head commit to move a fork PR
to RunsOn, provided no
+committer's request for changes stands, then close and reopen the PR (a push
would need a fresh
+approval; a committer should do the reopen, since an external author's reopen
still needs a maintainer
+to approve the workflow run).
If you are testing CI workflow changes and want to test it for more complete
matrix combinations generated by
the jobs - you can set `all versions` label in the PR. This will run the PRs
with the same combinations
@@ -726,7 +782,7 @@ of versions as the `canary` main build. Using `all
versions` is automatically se
change in `pyproject.toml`.
If you are running an `apache` PR, you can also set `canary` label for such PR
and in this case, all the
-`canary` properties of build will be used: `self-hosted` runners, `full tests
needed` mode, `all versions`
+`canary` properties of build will be used: `full tests needed` mode, `all
versions`
as well as all canary-specific jobs will run there. You can modify this
behaviour of the `canary` run by
applying `use public runners`, and `default versions only` labels to the PR as
well which will still run
a `canary` equivalent build but with public runners an default Python/K8S
versions only - respectively.
diff --git a/dev/breeze/doc/ci/06_debugging.md
b/dev/breeze/doc/ci/06_debugging.md
index c87c2b3f5b0..dfe9b968931 100644
--- a/dev/breeze/doc/ci/06_debugging.md
+++ b/dev/breeze/doc/ci/06_debugging.md
@@ -44,12 +44,16 @@ in some cases, you need to run the PR as coming from the
"apache"
repository rather than from your fork.
You can also apply the label later and rebase the PR or close/reopen
-the PR to apply the label to the PR.
+the PR to apply the label to the PR. For `use self-hosted runners`, a
+committer approves the head commit (with no committer's request for
+changes standing) and closes/reopens the PR: a rebase needs a fresh
+approval.
| Action to perform
| Label to set | PR from "apache" repo |
|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|:---------------------:|
| Run the build with all combinations of all<br>python, backends, kubernetes
etc on PR, <br>and run all types of tests for all test<br>groups.
| full tests needed | |
| Force to use public runners for the build
| use public runners | |
+| Run a PR from a fork on the self-hosted RunsOn<br>runners (needs a
committer's approval of its head commit<br>and no committer's request for
changes) | use self-hosted runners | |
| Debug resources used during the build for <br>parallel jobs
| debug ci resources | |
| Force running PR on latest versions of<br>python, backends, kubernetes etc.
when you<br>want to save resources and test only latest<br>versions
| latest versions only | |
| Force running PR on minimal (default) <br>versions of python, backends,
kubernetes etc.<br>in order to save resources and run tests only<br>for minimum
versions | default versions only | |
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index d7fec43e8bd..15a98856e5e 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -38,6 +38,14 @@ from airflow_breeze.utils.path_utils import (
PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]'
PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]'
+# RunsOn (runs-on.com) EC2 runner label templates. Jobs fill in their size
class with
+# `format(<label>, cpu, ram)` in the workflow; the same call leaves the public
labels unchanged.
+RUNS_ON_AMD_RUNNER_TEMPLATE = (
+
"runs-on={run_id}/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={{0}}/ram={{1}}"
+)
+RUNS_ON_ARM_RUNNER_TEMPLATE = (
+
"runs-on={run_id}/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={{0}}/ram={{1}}"
+)
# Platform the tests of a CI run execute on.
CI_AMD_PLATFORM = "linux/amd64"
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 7a598fda8be..2d420e8ef90 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -55,6 +55,8 @@ from airflow_breeze.global_constants import (
PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
PUBLIC_AMD_RUNNERS,
PUBLIC_ARM_RUNNERS,
+ RUNS_ON_AMD_RUNNER_TEMPLATE,
+ RUNS_ON_ARM_RUNNER_TEMPLATE,
TESTABLE_CORE_INTEGRATIONS,
TESTABLE_PROVIDERS_INTEGRATION_OWNERS,
TESTABLE_PROVIDERS_INTEGRATIONS,
@@ -95,6 +97,7 @@ LATEST_VERSIONS_ONLY_LABEL = "latest versions only"
NON_COMMITTER_BUILD_LABEL = "non committer build"
UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies"
USE_PUBLIC_RUNNERS_LABEL = "use public runners"
+USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners"
ALLOW_PROVIDER_DEPENDENCY_BUMP_LABEL = "allow provider dependency bump"
SKIP_COMMON_COMPAT_CHECK_LABEL = "skip common compat check"
AREA_E2E_TESTS_LABEL = "area:e2e-tests"
@@ -2179,6 +2182,119 @@ class SelectiveChecks:
def arm_runners(self) -> str:
return PUBLIC_ARM_RUNNERS
+ @cached_property
+ def runner_type(self) -> str:
+ """Runner labels for the jobs of a CI run on this run's platform.
+
+ The RunsOn label is a template the jobs fill in with their size class.
+ """
+ # RunsOn needs the run id in the label; without one (e.g. a local run)
the jobs would never start.
+ run_id = self._github_context_dict.get("run_id")
+ if not run_id or not self._use_runs_on_runners():
+ return PUBLIC_ARM_RUNNERS if self._platform == CI_ARM_PLATFORM
else PUBLIC_AMD_RUNNERS
+ template = (
+ RUNS_ON_ARM_RUNNER_TEMPLATE if self._platform == CI_ARM_PLATFORM
else RUNS_ON_AMD_RUNNER_TEMPLATE
+ )
+ return json.dumps([template.format(run_id=run_id)])
+
+ def _use_runs_on_runners(self) -> bool:
+ """Whether this run should use the self-hosted RunsOn runners.
+
+ This only routes runs; it does not keep fork code off RunsOn. A
``pull_request`` run uses
+ the PR's own workflow files and breeze, so a fork can request a RunsOn
label directly -
+ that restriction has to be enforced by the RunsOn stack and the
repository's Actions
+ settings, not here. Routed there: pushes, schedules and dispatches,
and PRs from
+ apache/airflow branches except bots' (e.g. Dependabot's). A fork PR
goes there while it
+ carries the ``use self-hosted runners`` label, a committer has
approved its head commit and
+ no committer's request for changes is outstanding.
+ The repository variable ``USE_PUBLIC_RUNNERS=true`` moves every run
back to public runners.
+ """
+ if self._github_repository != APACHE_AIRFLOW_GITHUB_REPOSITORY:
+ return False
+ if os.environ.get("USE_PUBLIC_RUNNERS", "").casefold() == "true":
+ return False
+ if USE_PUBLIC_RUNNERS_LABEL in self._pr_labels:
+ return False
+ if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE,
GithubEvents.WORKFLOW_DISPATCH]:
+ return True
+ if self._github_event != GithubEvents.PULL_REQUEST:
+ return False
+ if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels and
self._is_self_hosted_label_approved():
+ return True
+ if NON_COMMITTER_BUILD_LABEL in self._pr_labels:
+ return False
+ pull_request = self._github_context_dict.get("event",
{}).get("pull_request", {})
+ # The PR author, not github.actor: a committer rebasing a Dependabot
PR becomes the actor.
+ # The branch too: anyone can open their own PR from a Dependabot
branch.
+ if (pull_request.get("user") or {}).get("type") == "Bot" or (
+ pull_request.get("head", {}).get("ref",
"").startswith("dependabot/")
+ ):
+ return False
+ # A PR from a deleted fork has no head repo.
+ head_repo = (pull_request.get("head", {}).get("repo") or
{}).get("full_name")
+ return head_repo == APACHE_AIRFLOW_GITHUB_REPOSITORY
+
+ def _is_self_hosted_label_approved(self) -> bool:
+ """Whether the PR still has the label, a committer approved its head
commit and no committer's veto stands.
+
+ A re-run reuses the labels of its original event, so a later removal
only shows in the PR
+ itself. The label outlives the commit it was added for, so on its own
it would vouch for
+ every later push. A request for changes stays in force across pushes,
as on GitHub, until
+ that committer reviews again or it is dismissed. Any failure to tell
counts as "no".
+ """
+ # Lazy: breeze keeps its top-level imports minimal
(check-breeze-top-dependencies-limited).
+ import requests
+
+ pull_request = self._github_context_dict.get("event",
{}).get("pull_request", {})
+ head_sha = pull_request.get("head", {}).get("sha")
+ if not head_sha:
+ return False
+ pull_request_url = (
+
f"https://api.github.com/repos/{self._github_repository}/pulls/{pull_request.get('number')}"
+ )
+ headers = {"Accept": "application/vnd.github+json"}
+ if token := os.environ.get("GITHUB_TOKEN"):
+ headers["Authorization"] = f"Bearer {token}"
+ # Each reviewer's latest verdict, on the head commit and on any
commit; a dismissed review is
+ # listed as DISMISSED in place, and comment-only reviews change
nothing.
+ latest_on_head: dict[str, str] = {}
+ latest_overall: dict[str, str] = {}
+ try:
+ response = requests.get(pull_request_url, headers=headers,
timeout=30)
+ response.raise_for_status()
+ if USE_SELF_HOSTED_RUNNERS_LABEL not in {label["name"] for label
in response.json()["labels"]}:
+ console_print(
+ f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r}
label: the PR no longer has it"
+ )
+ return False
+ url: str | None = f"{pull_request_url}/reviews?per_page=100"
+ while url:
+ response = requests.get(url, headers=headers, timeout=30)
+ response.raise_for_status()
+ for review in response.json():
+ if review.get("state") not in {"APPROVED",
"CHANGES_REQUESTED", "DISMISSED"}:
+ continue
+ login = ((review.get("user") or {}).get("login") or
"").casefold()
+ latest_overall[login] = review["state"]
+ if review.get("commit_id") == head_sha:
+ latest_on_head[login] = review["state"]
+ url = response.links.get("next", {}).get("url")
+ # AttributeError / KeyError / TypeError: a 200 whose body is not the
expected PR or list of reviews.
+ except (requests.RequestException, AttributeError, KeyError,
TypeError) as e:
+ console_print(f"[warning]Could not read the labels and reviews of
the PR: {e}")
+ return False
+ committers = {committer.casefold() for committer in COMMITTERS}
+ approved = any(latest_on_head.get(committer) == "APPROVED" for
committer in committers)
+ vetoed = any(latest_overall.get(committer) == "CHANGES_REQUESTED" for
committer in committers)
+ if approved and not vetoed:
+ return True
+ console_print(
+ f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r} label: "
+ f"the head commit {head_sha} is not approved by a committer, or a
committer's request for "
+ "changes stands"
+ )
+ return False
+
@cached_property
def has_migrations(self) -> bool:
return any([file.startswith("airflow-core/src/airflow/migrations/")
for file in self._files])
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 0e19fe1f6a5..90ee9560f0d 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -23,6 +23,7 @@ from typing import Any
from unittest.mock import Mock, patch
import pytest
+import requests
from rich.console import Console
from airflow_breeze.global_constants import (
@@ -30,6 +31,7 @@ from airflow_breeze.global_constants import (
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
CI_AMD_PLATFORM,
CI_ARM_PLATFORM,
+ COMMITTERS,
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS,
DEFAULT_KUBERNETES_VERSION,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
@@ -4905,3 +4907,412 @@ def
test_providers_with_test_side_effects_are_shared_process_providers():
for provider in PROVIDERS_WITH_PROCESS_GLOBAL_TEST_SIDE_EFFECTS:
assert provider in get_available_distributions()
assert provider not in LONG_RUNNING_TEST_PROVIDERS
+
+
+RUNS_ON_AMD =
'["runs-on=42/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]'
+RUNS_ON_ARM =
'["runs-on=42/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={0}/ram={1}"]'
+
+
+HEAD_SHA = "a" * 40
+
+
[email protected](autouse=True)
+def _no_use_public_runners_variable(monkeypatch):
+ monkeypatch.delenv("USE_PUBLIC_RUNNERS", raising=False)
+
+
+def _pull_request_context(head_repo: str | None, author_type: str = "User") ->
dict[str, Any]:
+ head = {"repo": {"full_name": head_repo} if head_repo else None, "sha":
HEAD_SHA}
+ return {
+ "run_id": 42,
+ "event": {"pull_request": {"number": 7, "head": head, "user": {"type":
author_type}}},
+ }
+
+
[email protected](
+ ("github_event", "github_repository", "github_context_dict", "pr_labels",
"platform", "expected"),
+ [
+ pytest.param(
+ GithubEvents.PUSH,
+ "apache/airflow",
+ {"run_id": 42},
+ (),
+ CI_AMD_PLATFORM,
+ RUNS_ON_AMD,
+ id="canary push uses RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.SCHEDULE,
+ "apache/airflow",
+ {"run_id": 42},
+ (),
+ CI_ARM_PLATFORM,
+ RUNS_ON_ARM,
+ id="scheduled ARM canary uses ARM RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.WORKFLOW_DISPATCH,
+ "apache/airflow",
+ {"run_id": 42},
+ (),
+ CI_AMD_PLATFORM,
+ RUNS_ON_AMD,
+ id="workflow_dispatch uses RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("apache/airflow", author_type="Bot"),
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="bot PR from an apache/airflow branch uses public runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ {
+ "run_id": 42,
+ "event": {
+ "pull_request": {
+ "head": {"ref": "dependabot/uv/foo", "repo":
{"full_name": "apache/airflow"}},
+ "user": {"type": "User"},
+ }
+ },
+ },
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="user PR from a Dependabot branch uses public runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("apache/airflow", author_type="Bot"),
+ ("use self-hosted runners", "non committer build"),
+ CI_AMD_PLATFORM,
+ RUNS_ON_AMD,
+ id="committer's self-hosted label wins over bot author and non
committer build",
+ ),
+ pytest.param(
+ GithubEvents.PUSH,
+ "apache/airflow",
+ {},
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="push without a run id stays on public runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("someone/airflow"),
+ ("use self-hosted runners", "use public runners"),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="public runners label wins over the self-hosted label",
+ ),
+ pytest.param(
+ GithubEvents.PUSH,
+ "someone/airflow",
+ {"run_id": 42},
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="push in a fork repository stays on public runners",
+ ),
+ pytest.param(
+ GithubEvents.SCHEDULE,
+ "someone/airflow",
+ {"run_id": 42},
+ (),
+ CI_ARM_PLATFORM,
+ '["ubuntu-22.04-arm"]',
+ id="schedule in a fork repository stays on public ARM runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("apache/airflow"),
+ (),
+ CI_AMD_PLATFORM,
+ RUNS_ON_AMD,
+ id="PR from an apache/airflow branch uses RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("someone/airflow"),
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="PR from a fork uses public runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context(None),
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="PR from a deleted fork uses public runners",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("someone/airflow"),
+ ("use self-hosted runners",),
+ CI_AMD_PLATFORM,
+ RUNS_ON_AMD,
+ id="PR from a fork with the self-hosted label uses RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("someone/airflow"),
+ ("canary", "full tests needed"),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="canary label does not route a fork PR to RunsOn",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("apache/airflow"),
+ ("use public runners",),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="public runners label opts an apache/airflow PR out",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST,
+ "apache/airflow",
+ _pull_request_context("apache/airflow"),
+ ("non committer build",),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="non committer build label treats an apache/airflow PR as a
fork PR",
+ ),
+ pytest.param(
+ GithubEvents.PULL_REQUEST_TARGET,
+ "apache/airflow",
+ _pull_request_context("someone/airflow"),
+ (),
+ CI_AMD_PLATFORM,
+ '["ubuntu-22.04"]',
+ id="pull_request_target uses public runners",
+ ),
+ ],
+)
[email protected](SelectiveChecks, "_is_self_hosted_label_approved",
autospec=True, return_value=True)
+def test_runner_type(
+ _mock_is_self_hosted_label_approved,
+ github_event: GithubEvents,
+ github_repository: str,
+ github_context_dict: dict[str, Any],
+ pr_labels: tuple[str, ...],
+ platform: str,
+ expected: str,
+):
+ selective_checks = SelectiveChecks(
+ github_event=github_event,
+ github_repository=github_repository,
+ github_context_dict=github_context_dict,
+ pr_labels=pr_labels,
+ platform=platform,
+ )
+ assert selective_checks.runner_type == expected
+
+
[email protected](
+ ("value", "expected"),
+ [
+ pytest.param("true", '["ubuntu-22.04"]', id="true moves a push to
public runners"),
+ pytest.param("True", '["ubuntu-22.04"]', id="case does not matter"),
+ pytest.param("false", RUNS_ON_AMD, id="false keeps RunsOn"),
+ pytest.param("", RUNS_ON_AMD, id="an unset variable keeps RunsOn"),
+ ],
+)
+def test_use_public_runners_variable(monkeypatch, value: str, expected: str):
+ monkeypatch.setenv("USE_PUBLIC_RUNNERS", value)
+ selective_checks = SelectiveChecks(
+ github_event=GithubEvents.PUSH, github_repository="apache/airflow",
github_context_dict={"run_id": 42}
+ )
+ assert selective_checks.runner_type == expected
+
+
[email protected](SelectiveChecks, "_is_self_hosted_label_approved",
autospec=True, return_value=True)
+def test_use_public_runners_variable_beats_the_self_hosted_label(
+ _mock_is_self_hosted_label_approved, monkeypatch
+):
+ monkeypatch.setenv("USE_PUBLIC_RUNNERS", "true")
+ selective_checks = SelectiveChecks(
+ github_event=GithubEvents.PULL_REQUEST,
+ github_repository="apache/airflow",
+ github_context_dict=_pull_request_context("apache/airflow"),
+ pr_labels=("use self-hosted runners",),
+ )
+ assert selective_checks.runner_type == '["ubuntu-22.04"]'
+
+
+def _review(login: str, state: str, commit_id: str = HEAD_SHA) -> dict[str,
Any]:
+ return {"user": {"login": login}, "state": state, "commit_id": commit_id}
+
+
[email protected](
+ ("pages", "expected"),
+ [
+ pytest.param([[_review(COMMITTERS[0], "APPROVED")]], True,
id="approved by a committer"),
+ pytest.param(
+ [[_review(COMMITTERS[0].swapcase(), "APPROVED")]],
+ True,
+ id="approved by a committer whose login differs in case",
+ ),
+ pytest.param([[_review("a-triager", "APPROVED")]], False, id="approved
by a non-committer"),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED", commit_id="b" * 40)]],
+ False,
+ id="approved an earlier commit only",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0],
"CHANGES_REQUESTED")]],
+ False,
+ id="changes requested after approving",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "CHANGES_REQUESTED"),
_review(COMMITTERS[0], "APPROVED")]],
+ True,
+ id="approved after requesting changes",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0],
"DISMISSED")]],
+ False,
+ id="approval dismissed",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0],
"COMMENTED")]],
+ True,
+ id="a later comment keeps the approval",
+ ),
+ pytest.param(
+ [[_review("a-triager", "APPROVED")], [_review(COMMITTERS[0],
"APPROVED")]],
+ True,
+ id="approved on a later page",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[1],
"CHANGES_REQUESTED")]],
+ False,
+ id="another committer's request for changes vetoes an approval",
+ ),
+ pytest.param(
+ [[_review(COMMITTERS[0], "APPROVED"), _review("a-triager",
"CHANGES_REQUESTED")]],
+ True,
+ id="a non-committer's request for changes does not veto",
+ ),
+ pytest.param(
+ [
+ [
+ _review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b"
* 40),
+ _review(COMMITTERS[0], "APPROVED"),
+ ]
+ ],
+ False,
+ id="a committer's request for changes on an earlier commit still
vetoes",
+ ),
+ pytest.param(
+ [
+ [
+ _review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b"
* 40),
+ _review(COMMITTERS[1], "APPROVED", commit_id="b" * 40),
+ _review(COMMITTERS[0], "APPROVED"),
+ ]
+ ],
+ True,
+ id="a request for changes withdrawn by its committer's later
approval does not veto",
+ ),
+ pytest.param([[]], False, id="no review"),
+ pytest.param([{"message": "Not Found"}], False, id="error object
instead of reviews"),
+ ],
+)
+@patch("requests.get", autospec=True)
+def test_is_self_hosted_label_approved(mock_get, pages: list[list[dict[str,
Any]]], expected: bool):
+ pull_request = Mock(spec=requests.Response)
+ pull_request.json.return_value = {"labels": [{"name": "use self-hosted
runners"}]}
+ responses = [pull_request]
+ for index, page in enumerate(pages):
+ response = Mock(spec=requests.Response)
+ response.json.return_value = page
+ response.links = {"next": {"url": f"page-{index + 1}"}} if index + 1 <
len(pages) else {}
+ responses.append(response)
+ mock_get.side_effect = responses
+ selective_checks = SelectiveChecks(
+ github_context_dict=_pull_request_context("someone/airflow"),
+ pr_labels=("use self-hosted runners",),
+ )
+ assert selective_checks._is_self_hosted_label_approved() is expected
+ assert [call.args[0] for call in mock_get.call_args_list[:2]] == [
+ "https://api.github.com/repos/apache/airflow/pulls/7";,
+
"https://api.github.com/repos/apache/airflow/pulls/7/reviews?per_page=100";,
+ ]
+
+
[email protected](
+ "pull_request_body",
+ [
+ pytest.param({"labels": [{"name": "full tests needed"}]}, id="label
removed since the event"),
+ pytest.param({"message": "Not Found"}, id="error object instead of the
PR"),
+ ],
+)
+@patch("requests.get", autospec=True)
+def test_is_self_hosted_label_approved_needs_the_current_label(mock_get,
pull_request_body: dict[str, Any]):
+ # A re-run reuses the labels of its original event, so the label is still
in pr_labels.
+ response = Mock(spec=requests.Response)
+ response.json.return_value = pull_request_body
+ mock_get.return_value = response
+ selective_checks = SelectiveChecks(
+ github_context_dict=_pull_request_context("someone/airflow"),
+ pr_labels=("use self-hosted runners",),
+ )
+ assert selective_checks._is_self_hosted_label_approved() is False
+ mock_get.assert_called_once()
+
+
+@patch("requests.get", autospec=True)
+def test_is_self_hosted_label_approved_without_head_sha(mock_get):
+ # A review whose commit_id is missing must not match a missing head sha.
+ response = Mock(spec=requests.Response)
+ response.json.return_value = [{"user": {"login": COMMITTERS[0]}, "state":
"APPROVED"}]
+ response.links = {}
+ mock_get.return_value = response
+ context = _pull_request_context("someone/airflow")
+ del context["event"]["pull_request"]["head"]["sha"]
+ selective_checks = SelectiveChecks(github_context_dict=context,
pr_labels=("use self-hosted runners",))
+ assert selective_checks._is_self_hosted_label_approved() is False
+ mock_get.assert_not_called()
+
+
[email protected](
+ ("head_repo", "expected"),
+ [
+ pytest.param("someone/airflow", '["ubuntu-22.04"]', id="fork PR falls
back to public runners"),
+ pytest.param("apache/airflow", RUNS_ON_AMD, id="apache/airflow PR
stays on RunsOn"),
+ ],
+)
[email protected](
+ "failure",
+ [
+ pytest.param(requests.ConnectionError("down"), id="connection error"),
+ pytest.param(requests.HTTPError("403"), id="non-2xx response"),
+ ],
+)
+@patch("requests.get", autospec=True)
+def test_self_hosted_label_ignored_when_reviews_cannot_be_read(
+ mock_get, failure: Exception, head_repo: str, expected: str
+):
+ mock_get.side_effect = failure
+ selective_checks = SelectiveChecks(
+ github_event=GithubEvents.PULL_REQUEST,
+ github_context_dict=_pull_request_context(head_repo),
+ pr_labels=("use self-hosted runners",),
+ )
+ assert selective_checks.runner_type == expected
diff --git a/scripts/ci/make_mnt_writeable.sh b/scripts/ci/make_mnt_writeable.sh
index ddabf84b6b7..ed1e4d044ce 100755
--- a/scripts/ci/make_mnt_writeable.sh
+++ b/scripts/ci/make_mnt_writeable.sh
@@ -22,13 +22,26 @@ function make_mnt_writeable {
sudo blkid
echo "Checking free space!"
df -H
- echo "Cleaning /mnt just in case it is not empty"
+ echo "Cleaning /mnt just in case it is not empty and making sure that /mnt
is writeable"
sudo mkdir -p /mnt
- sudo rm -rf /mnt/*
+ if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then
+ # RunsOn mounts an instance's local NVMe disks under /mnt and keeps
docker, /home/runner and
+ # /tmp on them, so emptying or re-owning such a mount would break the
running job.
+ local entry
+ for entry in /mnt/*; do
+ if mountpoint -q "${entry}"; then
+ echo "Keeping mount point ${entry}"
+ else
+ sudo rm -rf --one-file-system "${entry}"
+ fi
+ done
+ sudo chown "${USER}" /mnt
+ else
+ sudo rm -rf /mnt/*
+ sudo chown -R "${USER}" /mnt
+ fi
echo "Checking free space!"
df -H
- echo "Making sure that /mnt is writeable"
- sudo chown -R "${USER}" /mnt
}
make_mnt_writeable
diff --git a/scripts/ci/move_docker_to_mnt.sh b/scripts/ci/move_docker_to_mnt.sh
index e3d69ef60be..4fa5e4a0711 100755
--- a/scripts/ci/move_docker_to_mnt.sh
+++ b/scripts/ci/move_docker_to_mnt.sh
@@ -29,7 +29,12 @@ function cleanup_runner {
# Hence we only move docker to /mnt on x86_64 where we have a separate
/mnt mount
# If we get short on disk space on arm64 as well we need to revisit this
logic
# and make the idle nvme being used as well.
- if uname -i|grep -q x86_64; then
+ if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then
+ # RunsOn keeps docker either on the instance's local NVMe disks, which
may also back
+ # /home/runner and /tmp, or - on instance types without them - on the
same EBS root
+ # volume /mnt is on. Moving it gains nothing and could delete the
running job's files.
+ echo "Self-hosted runner - keeping /var/lib/docker where it is"
+ elif uname -i|grep -q x86_64; then
local target_docker_volume_location="/mnt/var-lib-docker"
# This is faster than docker prune
echo "Stopping docker"
diff --git a/scripts/ci/prek/check_ci_workflows_in_sync.py
b/scripts/ci/prek/check_ci_workflows_in_sync.py
index 134df167f1d..321ca94194c 100755
--- a/scripts/ci/prek/check_ci_workflows_in_sync.py
+++ b/scripts/ci/prek/check_ci_workflows_in_sync.py
@@ -37,8 +37,8 @@ Documented divergences:
its ``gate`` job runs selective tests only for pushes to the branches
in CANONICAL_PUSH_BRANCH_FILTERS (fork_ci_gate.py).
4. ``concurrency.group`` prefix — ``ci-arm-`` vs ``ci-amd-``
-5. ``build-info`` outputs ``platform`` and ``runner-type`` — hardcoded per
- architecture (and the surrounding comment naming the "ARM/AMD copy")
+5. ``build-info`` output ``platform`` — hardcoded per architecture (and the
+ surrounding comment naming the "ARM/AMD copy")
6. ``PLATFORM`` env on the ``selective-checks`` step — the platform this
workflow's tests run on.
7. ``print-platform`` job — ``name:`` and the architecture echoed to
@@ -96,7 +96,6 @@ LINE_RULES: list[tuple[str, str]] = [
" ci-PLACEHOLDER-${{ github.event.pull_request.number || github.ref
}}${{",
),
(r'^ platform: "linux/(?:arm64|amd64)"$', ' platform:
"linux/PLACEHOLDER"'),
- (r"^ runner-type: '\[\"ubuntu-22\.04(?:-arm)?\"\]'$", "
runner-type: 'PLACEHOLDER'"),
(r'^ PLATFORM: "linux/(?:arm64|amd64)"$', ' PLATFORM:
"linux/PLACEHOLDER"'),
(
r"^ # (?:ARM|AMD) copy\)\. The matching (?:AMD|ARM) copy lives in
ci-(?:amd|arm)\.yml\.$",
![https://assets2.astronomer.io/logos/logoForLIGHTbackground.png"](https://assets2.astronomer.io/logos/logoForLIGHTbackground.png"){kind=link}
![https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/[email protected]?raw=true"](https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/aws-cloud-alt_light...@4x.png?raw=true"){kind=link}