This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d17f1213630 Widen the Spot pool for RunsOn runners: more families,
capacity-optimized, size ranges (#75822)
d17f1213630 is described below
commit d17f1213630755ad17b84433e63fb041c0bf263c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Oct 27 22:43:27 2026 +0100
Widen the Spot pool for RunsOn runners: more families, capacity-optimized,
size ranges (#75822)
* Let RunsOn x64 runners use more instance families for Spot
Generated-by: Claude Opus 5
* Take RunsOn Spot runners from capacity-optimized pools
Generated-by: Claude Opus 5
* Let RunsOn runners take Spot from up to twice the requested size
Generated-by: Claude Opus 5
---
.github/workflows/additional-ci-image-checks.yml | 2 +-
.github/workflows/additional-prod-image-tests.yml | 30 ++++++++--------
.github/workflows/basic-tests.yml | 18 +++++-----
.github/workflows/ci-amd.yml | 42 +++++++++++------------
.github/workflows/ci-arm.yml | 42 +++++++++++------------
.github/workflows/ci-image-build.yml | 2 +-
.github/workflows/ci-image-checks.yml | 8 ++---
.github/workflows/finalize-tests.yml | 4 +--
.github/workflows/generate-constraints.yml | 2 +-
.github/workflows/helm-tests.yml | 4 +--
.github/workflows/integration-system-tests.yml | 6 ++--
.github/workflows/k8s-tests.yml | 4 +--
.github/workflows/kustomize-overlays-tests.yml | 2 +-
.github/workflows/prod-image-build.yml | 4 +--
.github/workflows/push-image-cache.yml | 4 +--
.github/workflows/special-tests.yml | 22 ++++++------
.github/workflows/test-providers.yml | 4 +--
.github/workflows/ui-e2e-tests.yml | 2 +-
dev/breeze/doc/ci/04_selective_checks.md | 14 +++++---
dev/breeze/src/airflow_breeze/global_constants.py | 13 ++++---
dev/breeze/tests/test_selective_checks.py | 14 +++++---
21 files changed, 129 insertions(+), 114 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 7d60cf6ef41..e1dbdd2a4e8 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -62,7 +62,7 @@ jobs:
# layer that builds Python from source, which takes 11-18 minutes for
Python 3.11+ on hosted runners.
timeout-minutes: 35
name: Check that image builds quickly
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index c21fe129f89..a0109fb65a6 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -163,7 +163,7 @@ jobs:
timeout-minutes: 60
name: "Test examples of PROD image building"
if: inputs.run-generic-prod-image-tests == 'true'
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -201,7 +201,7 @@ jobs:
timeout-minutes: 60
name: "Docker Compose quick start with PROD image verifying"
if: inputs.run-generic-prod-image-tests == 'true'
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -232,7 +232,7 @@ jobs:
task-sdk-integration-tests:
timeout-minutes: 60
name: "Task SDK integration tests with PROD image"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -267,7 +267,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Regular e2e test"
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -277,7 +277,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Remote logging e2e test"
- runners: ${{ format(inputs.runners, 4, 16) }}
+ runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -291,7 +291,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Elasticsearch remote logging e2e test"
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -305,7 +305,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenSearch remote logging e2e test"
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -320,7 +320,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "XCom object storage backend e2e test"
- runners: ${{ format(inputs.runners, 4, 16) }}
+ runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -331,7 +331,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Event driven e2e test"
- runners: ${{ format(inputs.runners, 32, 128) }}
+ runners: ${{ format(inputs.runners, '32+64', '128+256') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -345,7 +345,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Java SDK e2e test"
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -359,7 +359,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Go SDK e2e test"
- runners: ${{ format(inputs.runners, 32, 128) }}
+ runners: ${{ format(inputs.runners, '32+64', '128+256') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -373,7 +373,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "TypeScript SDK e2e test"
- runners: ${{ format(inputs.runners, 32, 128) }}
+ runners: ${{ format(inputs.runners, '32+64', '128+256') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -387,7 +387,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenLineage e2e test"
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -407,7 +407,7 @@ jobs:
compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }}
uses: ./.github/workflows/openlineage-e2e-compat-tests.yml
with:
- runners: ${{ format(inputs.runners, 8, 32) }}
+ runners: ${{ format(inputs.runners, '8+16', '32+64') }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -453,7 +453,7 @@ jobs:
airflow-ctl-integration-tests:
timeout-minutes: 60
name: "Airflow CTL integration tests with PROD image"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 9d3a9046e56..d1b84c5a156 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -98,7 +98,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -116,7 +116,7 @@ jobs:
run-breeze-integration-tests:
timeout-minutes: 120
name: Breeze integration tests
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
if: inputs.run-breeze-integration-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -148,7 +148,7 @@ jobs:
# Without a pin uv picks the runner's system Python, which can be older
than Airflow supports.
UV_PYTHON: "${{ inputs.default-python-version }}"
SHARED_DISTRIBUTIONS: ${{
join(fromJSON(inputs.shared-distributions-as-json), ' ') }}
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
@@ -185,7 +185,7 @@ jobs:
tests-scripts:
timeout-minutes: 10
name: Scripts tests
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
if: inputs.run-scripts-tests == 'true'
env:
# Without a pin uv picks the runner's system Python, which can be older
than Airflow supports.
@@ -216,7 +216,7 @@ jobs:
tests-ui:
timeout-minutes: 15
name: React UI tests
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
if: inputs.run-ui-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -255,7 +255,7 @@ jobs:
check-translation-completness:
timeout-minutes: 15
name: "Check translation completeness"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
@@ -272,7 +272,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -358,7 +358,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -413,7 +413,7 @@ jobs:
timeout-minutes: 30
name: "Test Airflow standalone commands"
if: inputs.run-generic-basic-tests == 'true'
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
env:
AIRFLOW_HOME: ~/airflow
FORCE_COLOR: 1
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index f709ee36fe2..75c3ff8edc4 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -605,7 +605,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'4+8', '16+32') }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -646,7 +646,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -768,7 +768,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -797,7 +797,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -826,7 +826,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -856,7 +856,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -885,7 +885,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -916,7 +916,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -947,7 +947,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -977,7 +977,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -1064,7 +1064,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1094,7 +1094,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1206,7 +1206,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1222,7 +1222,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1291,7 +1291,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1327,7 +1327,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32,
128) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'32+64', '128+256') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1359,7 +1359,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 30
permissions:
contents: read
@@ -1391,7 +1391,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'4+8', '16+32') }}
timeout-minutes: 30
permissions:
contents: read
@@ -1428,7 +1428,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '4+8',
'16+32') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1449,7 +1449,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '8+16',
'32+64') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1772,7 +1772,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index ba3a3733c0c..341e1e02160 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -486,7 +486,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'4+8', '16+32') }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -527,7 +527,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
@@ -649,7 +649,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -678,7 +678,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -707,7 +707,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -737,7 +737,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -766,7 +766,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -797,7 +797,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -828,7 +828,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -858,7 +858,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -945,7 +945,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, 8,
'32+64') }}
+ runners: ${{ format(needs.build-info.outputs.in-memory-runner-type,
'8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -975,7 +975,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1087,7 +1087,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '32+64',
'128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1103,7 +1103,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1172,7 +1172,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1208,7 +1208,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32,
128) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'32+64', '128+256') }}
timeout-minutes: 15
permissions:
contents: read
@@ -1240,7 +1240,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
timeout-minutes: 30
permissions:
contents: read
@@ -1272,7 +1272,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4,
16) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'4+8', '16+32') }}
timeout-minutes: 30
permissions:
contents: read
@@ -1309,7 +1309,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '4+8',
'16+32') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1330,7 +1330,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
+ runners: ${{ format(needs.build-info.outputs.runner-type, '8+16',
'32+64') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1651,7 +1651,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
- runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 23f8e1f724c..b16296c8a14 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -128,7 +128,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 110
name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index ce2e8c41b33..4208aee7b9b 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -124,7 +124,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies
}}"
@@ -192,7 +192,7 @@ jobs:
build-docs:
timeout-minutes: 150
name: "Build documentation"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
if: inputs.docs-build == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -351,7 +351,7 @@ jobs:
id-token: write
contents: read
needs: build-docs
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -483,7 +483,7 @@ jobs:
test-python-api-client:
timeout-minutes: 60
name: "Test Python API client"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
if: inputs.run-api-codegen == 'true'
env:
BACKEND: "postgres"
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index c0a04fafe5f..9715a8cfc61 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -80,7 +80,7 @@ permissions:
contents: read
jobs:
update-constraints:
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
timeout-minutes: 80
name: "Update constraints"
permissions:
@@ -128,7 +128,7 @@ jobs:
# Constraint commits are now handled by the dedicated
update-constraints-on-push workflow
# triggered when uv.lock changes are pushed to main/test branches.
dependency-upgrade-summary:
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
needs: [update-constraints]
if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform ==
'linux/amd64'
name: "Deps ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 662fb0bfa63..17a460aaa68 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -85,7 +85,7 @@ jobs:
contents: read
timeout-minutes: 70
name: Generate constraints for ${{ matrix.python-version }} on ${{
inputs.platform }}
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
strategy:
fail-fast: false
max-parallel: 20
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 0d0facfa4aa..cf140b98c62 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -52,7 +52,7 @@ jobs:
name: >-
Unit tests Helm: ${{ matrix.helm-test-package }}
(K8S ${{ matrix.kubernetes-version }})
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
strategy:
fail-fast: false
max-parallel: 20
@@ -98,7 +98,7 @@ jobs:
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/integration-system-tests.yml
b/.github/workflows/integration-system-tests.yml
index efc8d578b5c..8b24223a55f 100644
--- a/.github/workflows/integration-system-tests.yml
+++ b/.github/workflows/integration-system-tests.yml
@@ -75,7 +75,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-core-integrations != '[]'
name: "Integration core ${{ matrix.integration }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }}
strategy:
fail-fast: false
max-parallel: 20
@@ -126,7 +126,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-providers-integrations != '[]' &&
inputs.skip-providers-tests != 'true'
name: "Integration: providers ${{ matrix.integration }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
strategy:
fail-fast: false
max-parallel: 20
@@ -176,7 +176,7 @@ jobs:
timeout-minutes: 30
if: inputs.run-system-tests == 'true'
name: "System Tests"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
env:
BACKEND: "postgres"
BACKEND_VERSION: ${{ inputs.default-postgres-version }}
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
index 39b0277e110..73d0ae9840f 100644
--- a/.github/workflows/k8s-tests.yml
+++ b/.github/workflows/k8s-tests.yml
@@ -62,7 +62,7 @@ jobs:
tests-kubernetes:
timeout-minutes: 60
name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo
}}-${{ matrix.use-standard-naming }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
strategy:
fail-fast: false
max-parallel: 20
@@ -149,7 +149,7 @@ jobs:
tests-kubernetes-lang-sdk:
timeout-minutes: 60
name: "K8S Lang-SDK:${{ inputs.lang-sdk-kubernetes-combo }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
env:
DEBUG_RESOURCES: ${{ inputs.debug-resources }}
INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
diff --git a/.github/workflows/kustomize-overlays-tests.yml
b/.github/workflows/kustomize-overlays-tests.yml
index 5fef9c1ac8a..5f9be6f52fb 100644
--- a/.github/workflows/kustomize-overlays-tests.yml
+++ b/.github/workflows/kustomize-overlays-tests.yml
@@ -53,7 +53,7 @@ jobs:
smoke-test-overlay:
timeout-minutes: 40
name: "Overlay smoke test: ${{ matrix.overlay }}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
strategy:
fail-fast: false
max-parallel: 4
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 8825103d328..b2ce6b25f7d 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -119,7 +119,7 @@ jobs:
build-prod-packages:
name: "Build Airflow and provider distributions"
timeout-minutes: 10
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
if: inputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -199,7 +199,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 80
name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version
}}"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
needs:
- build-prod-packages
env:
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 3dc4693e0df..8fdb06f95ce 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -79,7 +79,7 @@ on: # yamllint disable-line rule:truthy
jobs:
push-ci-image-cache:
name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
permissions:
contents: read
packages: write
@@ -149,7 +149,7 @@ jobs:
push-prod-image-cache:
name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
permissions:
contents: read
packages: write
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index 4343e17b2e7..09658246df9 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -98,7 +98,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -123,7 +123,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -148,7 +148,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -173,7 +173,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -198,7 +198,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -224,7 +224,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -250,7 +250,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -276,7 +276,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.in-memory-runners, 8, '32+64') }}
+ runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -302,7 +302,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.runners, 4, 16) }}
+ runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -327,7 +327,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.runners, 4, 16) }}
+ runners: ${{ format(inputs.runners, '4+8', '16+32') }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -352,7 +352,7 @@ jobs:
contents: read
packages: read
with:
- runners: ${{ format(inputs.runners, 32, 128) }}
+ runners: ${{ format(inputs.runners, '32+64', '128+256') }}
platform: ${{ inputs.platform }}
test-name: "SystemTest"
test-scope: "System"
diff --git a/.github/workflows/test-providers.yml
b/.github/workflows/test-providers.yml
index 2e99506d276..ccf9dc12304 100644
--- a/.github/workflows/test-providers.yml
+++ b/.github/workflows/test-providers.yml
@@ -72,7 +72,7 @@ jobs:
prepare-install-verify-provider-distributions:
timeout-minutes: 80
name: "Providers ${{ matrix.package-format }} tests"
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
strategy:
fail-fast: false
matrix:
@@ -178,7 +178,7 @@ jobs:
timeout-minutes: 80
# yamllint disable rule:line-length
name: Compat ${{ matrix.compat.airflow-version }}:P${{
matrix.compat.python-version }}:${{ matrix.test-types.description }}
- runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/ui-e2e-tests.yml
b/.github/workflows/ui-e2e-tests.yml
index 1ee5e1e6a67..22c24913a1d 100644
--- a/.github/workflows/ui-e2e-tests.yml
+++ b/.github/workflows/ui-e2e-tests.yml
@@ -88,7 +88,7 @@ jobs:
test-ui-e2e-tests:
timeout-minutes: 90
name: ${{ inputs.workflow-name || 'UI E2E Tests' }}
- runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0], 8,
32) }}
+ runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0],
'8+16', '32+64') }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11'
}}"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 12f36559984..4bf5fffa284 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -762,9 +762,13 @@ approval `use self-hosted runners` needs, so for that
label approve the head com
reopen the PR.
The RunsOn value is a label template such as
-`["runs-on=<run_id>/family=.../image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]`.
Every job
-picks its size class with `format(fromJSON(inputs.runners)[0], <cpu>, <ram>)`
(4/16, 8/32 or 32/128),
-or, for a single-job reusable workflow, its caller passes `format(<runners>,
<cpu>, <ram>)`;
+`["runs-on=<run_id>/family=.../image=ubuntu22-full-x64/volume=150gb/spot=co/cpu={0}/ram={1}"]`.
Every job
+picks its size class with `format(fromJSON(inputs.runners)[0], <cpu>, <ram>)`
(`'4+8', '16+32'`,
+`'8+16', '32+64'` or `'32+64', '128+256'`), or, for a single-job reusable
workflow, its caller passes
+`format(<runners>, <cpu>, <ram>)`. Each class is a range up to twice its size,
so Spot can come from
+the larger instance types too; xdist follows the host's CPUs, so a larger
runner runs more test workers.
+`spot=co` takes Spot from the pools with the most capacity regardless of
price, which is cheaper than
+the on-demand launch and whole-job rerun a capacity miss or interruption costs;
on a public label the same call returns the label unchanged. Steps that only
make sense on RunsOn are
gated on `runner.environment == 'self-hosted'`:
@@ -781,8 +785,8 @@ jobs whose databases wait on disk syncs: the Postgres,
MySQL and Sqlite test job
core tests and the SQLAlchemy, Boto and Pendulum special tests. Its label adds
[`extras=tmpfs`](https://runs-on.com/docs/runners/capabilities/yolo-mode/),
which puts
`/var/lib/docker`, `/tmp` and `/home/runner` on a RAM disk, and also accepts
the memory-optimized
-r-family. The jobs ask for 8 CPUs and 32 or 64 GB (`ram=32+64`), as the RAM
disk is capped at the
-runner's memory before RunsOn v3.5.0. On public runners the output equals
`runner-type`, and so it does
+r-family. The jobs ask for 8 to 16 CPUs and 32 to 128 GB
(`cpu=8+16/ram=32+128`), as the RAM disk
+is capped at the runner's memory before RunsOn v3.5.0. On public runners the
output equals `runner-type`, and so it does
on RunsOn while the repository variable `USE_IN_MEMORY_DOCKER` is `false`: the
switch for moving docker
back to disk without a PR.
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 336ada83fec..efa1c3425b0 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -40,21 +40,26 @@ PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]'
PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]'
# RunsOn (runs-on.com) EC2 runner label templates. Jobs fill in their size
class with
# `format(<label>, cpu, ram)` in the workflow; the same call leaves the public
labels unchanged.
+# `spot=co` (capacity-optimized) takes Spot from the pools least likely to run
out or be reclaimed
+# instead of RunsOn's default price-capacity-optimized: a capacity miss or an
interruption sends the
+# runner to on-demand (and reruns the whole job), which costs more than the
higher Spot price.
RUNS_ON_AMD_RUNNER_TEMPLATE = (
-
"runs-on={run_id}/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={{0}}/ram={{1}}"
+
"runs-on={run_id}/family=m8a+m8i+m7a+m7i+m6a+m6i+m5a+m5/image=ubuntu22-full-x64/volume=150gb/spot=co"
+ "/cpu={{0}}/ram={{1}}"
)
RUNS_ON_ARM_RUNNER_TEMPLATE = (
-
"runs-on={run_id}/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={{0}}/ram={{1}}"
+
"runs-on={run_id}/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/spot=co/cpu={{0}}/ram={{1}}"
)
# The same runners with docker, /tmp and /home/runner on a RAM disk (RunsOn
`extras=tmpfs`), for jobs whose
# databases wait on disk syncs. The RAM disk is capped at 100% of RAM before
RunsOn v3.5.0, so these jobs
# ask for memory headroom and also accept the memory-optimized r-family.
RUNS_ON_AMD_IN_MEMORY_RUNNER_TEMPLATE = (
-
"runs-on={run_id}/family=m7a+m7i+m6a+m6i+r7a+r7i+r6a+r6i/image=ubuntu22-full-x64/volume=150gb"
+
"runs-on={run_id}/family=m8a+m8i+m7a+m7i+m6a+m6i+m5a+m5+r8i+r7a+r7i+r6a+r6i"
+ "/image=ubuntu22-full-x64/volume=150gb/spot=co"
"/cpu={{0}}/ram={{1}}/extras=tmpfs"
)
RUNS_ON_ARM_IN_MEMORY_RUNNER_TEMPLATE = (
-
"runs-on={run_id}/family=m8g+m7g+m6g+r8g+r7g+r6g/image=ubuntu22-full-arm64/volume=150gb"
+
"runs-on={run_id}/family=m8g+m7g+m6g+r8g+r7g+r6g/image=ubuntu22-full-arm64/volume=150gb/spot=co"
"/cpu={{0}}/ram={{1}}/extras=tmpfs"
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index f2e8e05e298..ddce03204ad 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -4910,14 +4910,20 @@ def
test_providers_with_test_side_effects_are_shared_process_providers():
assert provider not in LONG_RUNNING_TEST_PROVIDERS
-RUNS_ON_AMD =
'["runs-on=42/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]'
-RUNS_ON_ARM =
'["runs-on=42/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={0}/ram={1}"]'
+RUNS_ON_AMD = (
+
'["runs-on=42/family=m8a+m8i+m7a+m7i+m6a+m6i+m5a+m5/image=ubuntu22-full-x64/volume=150gb/spot=co'
+ '/cpu={0}/ram={1}"]'
+)
+RUNS_ON_ARM = (
+
'["runs-on=42/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/spot=co/cpu={0}/ram={1}"]'
+)
RUNS_ON_AMD_IN_MEMORY = (
-
'["runs-on=42/family=m7a+m7i+m6a+m6i+r7a+r7i+r6a+r6i/image=ubuntu22-full-x64/volume=150gb'
+ '["runs-on=42/family=m8a+m8i+m7a+m7i+m6a+m6i+m5a+m5+r8i+r7a+r7i+r6a+r6i'
+ "/image=ubuntu22-full-x64/volume=150gb/spot=co"
'/cpu={0}/ram={1}/extras=tmpfs"]'
)
RUNS_ON_ARM_IN_MEMORY = (
-
'["runs-on=42/family=m8g+m7g+m6g+r8g+r7g+r6g/image=ubuntu22-full-arm64/volume=150gb'
+
'["runs-on=42/family=m8g+m7g+m6g+r8g+r7g+r6g/image=ubuntu22-full-arm64/volume=150gb/spot=co'
'/cpu={0}/ram={1}/extras=tmpfs"]'
)
