This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch publish-docs-self-hosted-runners in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1c54f523b260a4201e89d94206920e4393b84f23 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Oct 31 20:10:31 2026 +0100 Run docs publishing and provider registry workflows on self-hosted runners Publishing docs and the provider registry builds a full CI image on public runners, which are slower and queue behind the rest of the project's jobs. These workflows now pick their runners the same way CI does, so they use RunsOn in apache/airflow and keep USE_PUBLIC_RUNNERS as the switch back. The registry build checks out the ref being published, and older refs ship a make_mnt_writeable.sh that empties /mnt - on RunsOn that is the runner's own disks - so the runner-preparation scripts now come from the workflow's version. Generated-by: Claude Opus 5 --- .github/workflows/publish-docs-to-s3.yml | 52 ++++++++++++++++++++++++++------ .github/workflows/registry-backfill.yml | 42 +++++++++++++++++++++++--- .github/workflows/registry-build.yml | 41 +++++++++++++++++++++++-- 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index d69319ad315..833aef3810d 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -72,6 +72,38 @@ on: # yamllint disable-line rule:truthy permissions: contents: read jobs: + # Picks the runner the way CI does: RunsOn in apache/airflow, public runners when the repository + # variable USE_PUBLIC_RUNNERS is "true". A job of its own because build-info checks out the ref + # being published, whose breeze may predate RunsOn; this one runs the workflow's own breeze. + runner-info: + timeout-minutes: 10 + name: "Runner info" + runs-on: ["ubuntu-slim"] + permissions: + contents: read + outputs: + runner-type: ${{ steps.selective-checks.outputs.runner-type }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + free-space: "false" + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + # Only what the runner choice reads: the full context would carry the dispatch inputs, + # and this job runs before build-info's committer allowlist. + GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{ github.ref_name }}"}' + PLATFORM: "linux/amd64" + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} build-info: timeout-minutes: 10 name: "Build Info" @@ -282,10 +314,10 @@ jobs: echo "publish-dag-schema=${PUBLISH_DAG}" >> ${GITHUB_OUTPUT} build-docs: - needs: [build-info] + needs: [build-info, runner-info] timeout-minutes: 150 name: "Build documentation" - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '8+16', '32+64') }} permissions: contents: read packages: read @@ -473,10 +505,10 @@ jobs: overwrite: 'true' build-java-sdk-docs: - needs: [build-info] + needs: [build-info, runner-info] timeout-minutes: 30 name: "Build Java SDK Javadoc" - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '4+8', '16+32') }} permissions: contents: read packages: read @@ -539,10 +571,10 @@ jobs: overwrite: 'true' build-ts-sdk-docs: - needs: [build-info] + needs: [build-info, runner-info] timeout-minutes: 30 name: "Build TypeScript SDK API reference" - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '4+8', '16+32') }} permissions: contents: read packages: read @@ -604,7 +636,7 @@ jobs: overwrite: 'true' publish-docs-to-s3: - needs: [build-docs, build-java-sdk-docs, build-ts-sdk-docs, build-info, update-registry] + needs: [build-docs, build-java-sdk-docs, build-ts-sdk-docs, build-info, runner-info, update-registry] # `update-registry` is skipped for non-provider distributions, and a job whose dependency was # skipped is skipped in turn, so its result is checked rather than implied - which means the # two that are not conditional have to be asserted here as well. @@ -626,7 +658,7 @@ jobs: id-token: write contents: read packages: write - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '4+8', '16+32') }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -775,7 +807,7 @@ jobs: --destination-location "${SCHEMAS_DESTINATION}" "${args[@]}" update-packages-metadata: - needs: [build-info, publish-docs-to-s3] + needs: [build-info, runner-info, publish-docs-to-s3] # Keyed on the sync step rather than the job, so a schema publish failing after a complete sync # still gets its docs into the metadata, while a partial sync is not advertised. if: ${{ !cancelled() && needs.publish-docs-to-s3.outputs.docs-synced == 'true' }} @@ -789,7 +821,7 @@ jobs: cancel-in-progress: false permissions: contents: read - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '2+4', '8+16') }} steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/registry-backfill.yml b/.github/workflows/registry-backfill.yml index 6dd2967594a..58085f9ae61 100644 --- a/.github/workflows/registry-backfill.yml +++ b/.github/workflows/registry-backfill.yml @@ -42,7 +42,39 @@ permissions: packages: read jobs: + # Picks the runner the way CI does: RunsOn in apache/airflow, public runners when the repository + # variable USE_PUBLIC_RUNNERS is "true". + runner-info: + timeout-minutes: 10 + name: "Runner info" + runs-on: ["ubuntu-slim"] + permissions: + contents: read + outputs: + runner-type: ${{ steps.selective-checks.outputs.runner-type }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + free-space: "false" + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + # Only what the runner choice reads: the full context would carry the dispatch inputs, + # and this job runs before the committer allowlist of the image build. + GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{ github.ref_name }}"}' + PLATFORM: "linux/amd64" + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} build-ci-image: + needs: [runner-info] name: "Build CI image" uses: ./.github/workflows/ci-image-build.yml permissions: @@ -65,7 +97,7 @@ jobs: "vincbeck" ]'), github.event.sender.login) with: - runners: '["ubuntu-22.04"]' + runners: ${{ needs.runner-info.outputs.runner-type }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" @@ -115,8 +147,8 @@ jobs: >> "${GITHUB_OUTPUT}" backfill: - needs: [prepare, build-ci-image] - runs-on: ubuntu-latest + needs: [prepare, runner-info, build-ci-image] + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 60 strategy: fail-fast: false @@ -280,8 +312,8 @@ jobs: done publish-versions: - needs: [prepare, backfill] - runs-on: ubuntu-latest + needs: [prepare, runner-info, backfill] + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '2+4', '8+16') }} name: "Publish versions.json" # Shares registry-build.yml's group, so it never runs alongside a registry publish to the same # bucket. Only this job: the backfill matrix writes per-version paths of its own, and putting diff --git a/.github/workflows/registry-build.yml b/.github/workflows/registry-build.yml index 7afc284682f..f9a9f5a4275 100644 --- a/.github/workflows/registry-build.yml +++ b/.github/workflows/registry-build.yml @@ -90,7 +90,40 @@ permissions: packages: read jobs: + # Picks the runner the way CI does: RunsOn in apache/airflow, public runners when the repository + # variable USE_PUBLIC_RUNNERS is "true". Runs the workflow's own breeze, as the registry job checks + # out a ref whose breeze may predate RunsOn. + runner-info: + timeout-minutes: 10 + name: "Runner info" + runs-on: ["ubuntu-slim"] + permissions: + contents: read + outputs: + runner-type: ${{ steps.selective-checks.outputs.runner-type }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + free-space: "false" + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + # Only what the runner choice reads: the full context would carry the dispatch inputs, + # and this job runs before the committer allowlist of the registry job. + GITHUB_CONTEXT: '{"run_id": "${{ github.run_id }}", "ref_name": "${{ github.ref_name }}"}' + PLATFORM: "linux/amd64" + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} build-and-publish-registry: + needs: [runner-info] timeout-minutes: 45 name: "Build & Publish Registry" # An incremental build downloads the bucket's providers.json and modules.json, merges its own @@ -122,7 +155,7 @@ jobs: "utkarsharma2", "vincbeck" ]'), github.event.sender.login) - runs-on: ubuntu-latest + runs-on: ${{ format(fromJSON(needs.runner-info.outputs.runner-type)[0], '8+16', '32+64') }} env: SCARF_ANALYTICS: "false" DO_NOT_TRACK: "1" @@ -162,10 +195,12 @@ jobs: persist-credentials: false path: workflow-version + # From the workflow's version, not the checked-out ref: an older make_mnt_writeable.sh empties + # /mnt, which on RunsOn holds the runner's own disks. - name: "Make /mnt writeable and cleanup" - run: ./scripts/ci/make_mnt_writeable.sh + run: ./workflow-version/scripts/ci/make_mnt_writeable.sh - name: "Free up disk space" - run: ./scripts/tools/free_up_disk_space.sh + run: ./workflow-version/scripts/tools/free_up_disk_space.sh - name: "Install Breeze" uses: ./.github/actions/breeze with:
