This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new beae4cd54f0 [v3-4-test] Run docs publishing and provider registry
workflows on self-hosted runners (#76168) (#76176)
beae4cd54f0 is described below
commit beae4cd54f026acd9ec8b38c2c0ff46fdb5fed97
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Oct 31 22:04:21 2026 +0100
[v3-4-test] Run docs publishing and provider registry workflows on
self-hosted runners (#76168) (#76176)
Publishing docs and the provider registry builds a full CI image on public
runners, which are slower and queue behind the rest of the project's jobs.
These workflows now pick their runners the same way CI does, so they use
RunsOn in apache/airflow and keep USE_PUBLIC_RUNNERS as the switch back.
The registry build checks out the ref being published, and older refs ship a
make_mnt_writeable.sh that empties /mnt - on RunsOn that is the runner's own
disks - so the runner-preparation scripts now come from the workflow's
version.
(cherry picked from commit c30f6c9ea0f20257a8cd8cee2b8135e3288c815a)
Generated-by: Claude Opus 5
Co-authored-by: Jarek Potiuk <[email protected]>
---
.github/workflows/publish-docs-to-s3.yml | 52 ++++++++++++++++++++++++++------
.github/workflows/registry-backfill.yml | 42 +++++++++++++++++++++++---
.github/workflows/registry-build.yml | 41 +++++++++++++++++++++++--
3 files changed, 117 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index d69319ad315..833aef3810d 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -72,6 +72,38 @@ on: # yamllint disable-line rule:truthy
permissions:
contents: read
jobs:
+ # Picks the runner the way CI does: RunsOn in apache/airflow, public runners
when the repository
+ # variable USE_PUBLIC_RUNNERS is "true". A job of its own because build-info
checks out the ref
+ # being published, whose breeze may predate RunsOn; this one runs the
workflow's own breeze.
+ runner-info:
+ timeout-minutes: 10
+ name: "Runner info"
+ runs-on: ["ubuntu-slim"]
+ permissions:
+ contents: read
+ outputs:
+ runner-type: ${{ steps.selective-checks.outputs.runner-type }}
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ free-space: "false"
+ - name: "Selective checks"
+ id: selective-checks
+ env:
+ COMMIT_REF: "${{ github.sha }}"
+ VERBOSE: "false"
+ # Only what the runner choice reads: the full context would carry
the dispatch inputs,
+ # and this job runs before build-info's committer allowlist.
+ GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{
github.ref_name }}"}'
+ PLATFORM: "linux/amd64"
+ USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
+ run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
build-info:
timeout-minutes: 10
name: "Build Info"
@@ -282,10 +314,10 @@ jobs:
echo "publish-dag-schema=${PUBLISH_DAG}" >> ${GITHUB_OUTPUT}
build-docs:
- needs: [build-info]
+ needs: [build-info, runner-info]
timeout-minutes: 150
name: "Build documentation"
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'8+16', '32+64') }}
permissions:
contents: read
packages: read
@@ -473,10 +505,10 @@ jobs:
overwrite: 'true'
build-java-sdk-docs:
- needs: [build-info]
+ needs: [build-info, runner-info]
timeout-minutes: 30
name: "Build Java SDK Javadoc"
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'4+8', '16+32') }}
permissions:
contents: read
packages: read
@@ -539,10 +571,10 @@ jobs:
overwrite: 'true'
build-ts-sdk-docs:
- needs: [build-info]
+ needs: [build-info, runner-info]
timeout-minutes: 30
name: "Build TypeScript SDK API reference"
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'4+8', '16+32') }}
permissions:
contents: read
packages: read
@@ -604,7 +636,7 @@ jobs:
overwrite: 'true'
publish-docs-to-s3:
- needs: [build-docs, build-java-sdk-docs, build-ts-sdk-docs, build-info,
update-registry]
+ needs: [build-docs, build-java-sdk-docs, build-ts-sdk-docs, build-info,
runner-info, update-registry]
# `update-registry` is skipped for non-provider distributions, and a job
whose dependency was
# skipped is skipped in turn, so its result is checked rather than implied
- which means the
# two that are not conditional have to be asserted here as well.
@@ -626,7 +658,7 @@ jobs:
id-token: write
contents: read
packages: write
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'4+8', '16+32') }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -775,7 +807,7 @@ jobs:
--destination-location "${SCHEMAS_DESTINATION}" "${args[@]}"
update-packages-metadata:
- needs: [build-info, publish-docs-to-s3]
+ needs: [build-info, runner-info, publish-docs-to-s3]
# Keyed on the sync step rather than the job, so a schema publish failing
after a complete sync
# still gets its docs into the metadata, while a partial sync is not
advertised.
if: ${{ !cancelled() && needs.publish-docs-to-s3.outputs.docs-synced ==
'true' }}
@@ -789,7 +821,7 @@ jobs:
cancel-in-progress: false
permissions:
contents: read
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'2+4', '8+16') }}
steps:
- name: "Cleanup repo"
shell: bash
diff --git a/.github/workflows/registry-backfill.yml
b/.github/workflows/registry-backfill.yml
index 6dd2967594a..58085f9ae61 100644
--- a/.github/workflows/registry-backfill.yml
+++ b/.github/workflows/registry-backfill.yml
@@ -42,7 +42,39 @@ permissions:
packages: read
jobs:
+ # Picks the runner the way CI does: RunsOn in apache/airflow, public runners
when the repository
+ # variable USE_PUBLIC_RUNNERS is "true".
+ runner-info:
+ timeout-minutes: 10
+ name: "Runner info"
+ runs-on: ["ubuntu-slim"]
+ permissions:
+ contents: read
+ outputs:
+ runner-type: ${{ steps.selective-checks.outputs.runner-type }}
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ free-space: "false"
+ - name: "Selective checks"
+ id: selective-checks
+ env:
+ COMMIT_REF: "${{ github.sha }}"
+ VERBOSE: "false"
+ # Only what the runner choice reads: the full context would carry
the dispatch inputs,
+ # and this job runs before the committer allowlist of the image
build.
+ GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{
github.ref_name }}"}'
+ PLATFORM: "linux/amd64"
+ USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
+ run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
build-ci-image:
+ needs: [runner-info]
name: "Build CI image"
uses: ./.github/workflows/ci-image-build.yml
permissions:
@@ -65,7 +97,7 @@ jobs:
"vincbeck"
]'), github.event.sender.login)
with:
- runners: '["ubuntu-22.04"]'
+ runners: ${{ needs.runner-info.outputs.runner-type }}
platform: "linux/amd64"
push-image: "false"
upload-image-artifact: "true"
@@ -115,8 +147,8 @@ jobs:
>> "${GITHUB_OUTPUT}"
backfill:
- needs: [prepare, build-ci-image]
- runs-on: ubuntu-latest
+ needs: [prepare, runner-info, build-ci-image]
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'4+8', '16+32') }}
timeout-minutes: 60
strategy:
fail-fast: false
@@ -280,8 +312,8 @@ jobs:
done
publish-versions:
- needs: [prepare, backfill]
- runs-on: ubuntu-latest
+ needs: [prepare, runner-info, backfill]
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'2+4', '8+16') }}
name: "Publish versions.json"
# Shares registry-build.yml's group, so it never runs alongside a registry
publish to the same
# bucket. Only this job: the backfill matrix writes per-version paths of
its own, and putting
diff --git a/.github/workflows/registry-build.yml
b/.github/workflows/registry-build.yml
index 7afc284682f..f9a9f5a4275 100644
--- a/.github/workflows/registry-build.yml
+++ b/.github/workflows/registry-build.yml
@@ -90,7 +90,40 @@ permissions:
packages: read
jobs:
+ # Picks the runner the way CI does: RunsOn in apache/airflow, public runners
when the repository
+ # variable USE_PUBLIC_RUNNERS is "true". Runs the workflow's own breeze, as
the registry job checks
+ # out a ref whose breeze may predate RunsOn.
+ runner-info:
+ timeout-minutes: 10
+ name: "Runner info"
+ runs-on: ["ubuntu-slim"]
+ permissions:
+ contents: read
+ outputs:
+ runner-type: ${{ steps.selective-checks.outputs.runner-type }}
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ free-space: "false"
+ - name: "Selective checks"
+ id: selective-checks
+ env:
+ COMMIT_REF: "${{ github.sha }}"
+ VERBOSE: "false"
+ # Only what the runner choice reads: the full context would carry
the dispatch inputs,
+ # and this job runs before the committer allowlist of the registry
job.
+ GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{
github.ref_name }}"}'
+ PLATFORM: "linux/amd64"
+ USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
+ run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
build-and-publish-registry:
+ needs: [runner-info]
timeout-minutes: 45
name: "Build & Publish Registry"
# An incremental build downloads the bucket's providers.json and
modules.json, merges its own
@@ -122,7 +155,7 @@ jobs:
"utkarsharma2",
"vincbeck"
]'), github.event.sender.login)
- runs-on: ubuntu-latest
+ runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0],
'8+16', '32+64') }}
env:
SCARF_ANALYTICS: "false"
DO_NOT_TRACK: "1"
@@ -162,10 +195,12 @@ jobs:
persist-credentials: false
path: workflow-version
+ # From the workflow's version, not the checked-out ref: an older
make_mnt_writeable.sh empties
+ # /mnt, which on RunsOn holds the runner's own disks.
- name: "Make /mnt writeable and cleanup"
- run: ./scripts/ci/make_mnt_writeable.sh
+ run: ./workflow-version/scripts/ci/make_mnt_writeable.sh
- name: "Free up disk space"
- run: ./scripts/tools/free_up_disk_space.sh
+ run: ./workflow-version/scripts/tools/free_up_disk_space.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
