This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c6449eb429f Build the docs CI image once and let the registry share it
(#70618)
c6449eb429f is described below
commit c6449eb429fd045037644dbc42986875fccec1f6
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Jul 28 21:19:05 2026 +0200
Build the docs CI image once and let the registry share it (#70618)
* Build the docs CI image once and let the registry share it
A docs publish paid for two CI image builds of the same sources, one for
the docs and one for the provider registry, and only started the second
once the docs were already live. Building it once up front and handing the
same image to both lets them run side by side, which is most of the wait a
release manager sits through after the docs have landed.
Keeping the mount cache means consecutive publishes stop re-downloading the
whole dependency set, and pinning both to one Python removes the second
image that only differed by interpreter.
* Retry the docs image build without a cache when the cached one fails
The build the docs job used to run inline fell back to a plain buildx build
whenever an old ref's pyproject.toml had drifted far enough from main that
main's registry cache could not satisfy it. Moving the build into its own
job dropped that safety net, which would turn a slow-but-correct build of
an old ref into a hard failure.
* Retry inside the image build instead of chaining a second build job
Expressing the no-cache retry as its own job put a permanently-skipped job
in every docs publish and forced both consumers to reason about which of
two builds produced the image. The retry belongs where the build is, and
callers that do not want it are unaffected because it is off by default.
---
.github/workflows/ci-image-build.yml | 19 +++++++-
.github/workflows/publish-docs-to-s3.yml | 77 ++++++++++++++++++++++----------
.github/workflows/registry-build.yml | 54 +++++++++++++++++++---
3 files changed, 118 insertions(+), 32 deletions(-)
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index e7c644568a6..f9aaa3aa473 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -94,6 +94,11 @@ on: # yamllint disable-line rule:truthy
description: "Docker cache specification to build the image (registry,
local, disabled)."
required: true
type: string
+ retry-without-cache:
+ description: "Retry the build once with the cache disabled if the
cached build fails (true/false)"
+ required: false
+ default: "false"
+ type: string
disable-airflow-repo-cache:
description: "Disable airflow repo cache read from main."
required: true
@@ -169,9 +174,19 @@ jobs:
- name: >
Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} image
- run: >
- breeze ci-image build --platform "${PLATFORM}"
+ run: |
+ if breeze ci-image build --platform "${PLATFORM}"; then
+ exit 0
+ fi
+ if [[ "${RETRY_WITHOUT_CACHE}" != "true" ]]; then
+ exit 1
+ fi
+ # Reached when the ref's pyproject.toml has drifted far enough from
the branch the
+ # cache was built for that the cached layers cannot satisfy it.
Slow, but correct.
+ echo "Build with ${DOCKER_CACHE} cache failed - retrying with the
cache disabled."
+ DOCKER_CACHE="disabled" breeze ci-image build --platform
"${PLATFORM}"
env:
+ RETRY_WITHOUT_CACHE: ${{ inputs.retry-without-cache }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index eeb58b1759d..da8c17f1085 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -98,7 +98,9 @@ jobs:
publish-supervisor-schema: ${{
steps.parameters.outputs.publish-supervisor-schema }}
# yamllint disable rule:line-length
skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder &&
'--skip-write-to-stable-folder' || '' }}
- default-python-version: "3.10"
+ # The docs build and the registry extraction share one CI image, so they
share its
+ # Python too. 3.12 is what the registry has always extracted on.
+ default-python-version: "3.12"
registry-providers: ${{
steps.derive_registry_inputs.outputs.registry-providers }}
registry-full-build: ${{
steps.derive_registry_inputs.outputs.registry-full-build }}
if: contains(fromJSON('[
@@ -226,8 +228,39 @@ jobs:
echo "publish-execution-api-schema=${PUBLISH_EXEC}" >>
${GITHUB_OUTPUT}
echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT}
- build-docs:
+ build-ci-image:
+ name: "Build CI image"
needs: [build-info]
+ uses: ./.github/workflows/ci-image-build.yml
+ permissions:
+ contents: read
+ packages: write
+ with:
+ runners: '["ubuntu-22.04"]'
+ platform: "linux/amd64"
+ # Built from the docs ref so the image matches the sources being
documented, and
+ # cached against main's registry cache, which the regular Test workflow
keeps warm.
+ checkout-ref: ${{ inputs.ref }}
+ push-image: "false"
+ upload-image-artifact: "true"
+ # Leaves the BuildKit mount cache behind for the next docs/registry run
on this branch;
+ # ci-image-build.yml always restores it, so consecutive publishes stop
re-downloading
+ # the whole dependency set.
+ upload-mount-cache-artifact: "true"
+ python-versions: ${{ format('["{0}"]',
needs.build-info.outputs.default-python-version) }}
+ branch: "main"
+ constraints-branch: "constraints-main"
+ use-uv: "true"
+ upgrade-to-newer-dependencies: "false"
+ docker-cache: "registry"
+ disable-airflow-repo-cache: "false"
+ # Building against main's cache can fail outright when the ref's
pyproject.toml has
+ # diverged from main — the case the docs job's inline `breeze ci-image
build || docker
+ # buildx build` fallback used to cover.
+ retry-without-cache: "true"
+
+ build-docs:
+ needs: [build-info, build-ci-image]
timeout-minutes: 150
name: "Build documentation"
runs-on: ubuntu-latest
@@ -293,29 +326,20 @@ jobs:
uses: ./.github/actions/breeze
with:
python-version: "${{ needs.build-info.outputs.default-python-version
}}"
- - name: "Login to ghcr.io"
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- ACTOR: ${{ github.actor }}
- run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}"
--password-stdin
- - name: "Building image from the ${{ inputs.ref }} reference"
+ # The image comes from the `build-ci-image` job, which builds it from
this same ref and
+ # stashes it. The registry build restores the very same stash, so a run
builds it once.
+ - name: "Restore CI docker image linux/amd64:${{
needs.build-info.outputs.default-python-version }}"
+ uses:
apache/infrastructure-actions/stash/restore@49df447b39b18354895520e0a63731b7cad7cbec
+ with:
+ key: ci-image-save-v3-linux/amd64-${{
needs.build-info.outputs.default-python-version }}
+ path: "/mnt/"
+ only-current-branch: 'true'
+ fail-on-download: 'true'
+ - name: "Load CI image linux/amd64:${{
needs.build-info.outputs.default-python-version }}"
env:
- INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }}
- INCLUDE_COMMITS: ${{ startsWith(inputs.ref, 'providers') && 'true'
|| 'false' }}
- # The regular `breeze ci-image build` path uses the registry cache
pushed by the last
- # successful "main" Test workflow (DOCKER_CACHE=registry,
--cache-from=<main>:cache-<platform>),
- # which is the fast path for builds from main or main-like refs. When
building from an old
- # ref whose pyproject.toml has diverged from main, reusing the main
cache can fail the build,
- # so we fall back to a plain buildx build with no cache-from.
+ PYTHON: ${{ needs.build-info.outputs.default-python-version }}
run: >
- breeze ci-image build ||
- docker buildx build --load --builder default --progress=auto --pull
- --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg
AIRFLOW_PRE_CACHED_PIP_PACKAGES=false
- --build-arg AIRFLOW_USE_UV=true
- --build-arg BUILD_PROGRESS=auto --build-arg
INSTALL_MYSQL_CLIENT_TYPE=mariadb
- --build-arg VERSION_SUFFIX_FOR_PYPI=dev0
- -t
"ghcr.io/apache/airflow/main/ci/python${PYTHON_MAJOR_MINOR_VERSION}:latest"
--target main .
- -f Dockerfile.ci --platform linux/amd64
+ breeze ci-image load --platform "linux/amd64" --python "${PYTHON}"
--image-file-dir "/mnt"
- name: "Restore docs inventory cache"
uses:
apache/infrastructure-actions/stash/restore@49df447b39b18354895520e0a63731b7cad7cbec
with:
@@ -606,7 +630,9 @@ jobs:
--destination-location "${SCHEMAS_DESTINATION}" "${args[@]}"
update-registry:
- needs: [publish-docs-to-s3, build-info]
+ # Runs alongside the docs build rather than after it: the registry reads
nothing the docs
+ # publish produces, so waiting only added its whole duration to the
release manager's wait.
+ needs: [build-info, build-ci-image]
if: needs.build-info.outputs.registry-providers != '' ||
needs.build-info.outputs.registry-full-build == 'true'
name: "Update Provider Registry"
permissions:
@@ -616,6 +642,9 @@ jobs:
with:
destination: ${{ needs.build-info.outputs.destination }}
provider: ${{ needs.build-info.outputs.registry-providers }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ # `build-ci-image` already built and stashed the image this run.
+ ci-image-already-built: true
secrets:
DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/registry-build.yml
b/.github/workflows/registry-build.yml
index 472e0c185c1..e1932365173 100644
--- a/.github/workflows/registry-build.yml
+++ b/.github/workflows/registry-build.yml
@@ -33,6 +33,11 @@ on: # yamllint disable-line rule:truthy
required: false
type: string
default: ""
+ python-version:
+ description: "Python version of the CI image the extraction runs in"
+ required: false
+ type: string
+ default: "3.12"
workflow_call:
inputs:
destination:
@@ -45,6 +50,16 @@ on: # yamllint disable-line rule:truthy
required: false
type: string
default: ""
+ python-version:
+ description: "Python version of the CI image the extraction runs in"
+ required: false
+ type: string
+ default: "3.12"
+ ci-image-already-built:
+ description: "Caller has already built and stashed the CI image for
this run"
+ required: false
+ type: boolean
+ default: false
secrets:
DOCS_AWS_ACCESS_KEY_ID:
required: true
@@ -62,7 +77,10 @@ jobs:
permissions:
contents: read
packages: write
+ # Skipped when the caller stashed an image for this run; a standalone
dispatch has no
+ # such image and still builds its own.
if: >
+ (inputs.ci-image-already-built != true) && (
github.event_name == 'workflow_call' ||
contains(fromJSON('[
"ashb",
@@ -78,14 +96,15 @@ jobs:
"uranusjr",
"utkarsharma2",
"vincbeck"
- ]'), github.event.sender.login)
+ ]'), github.event.sender.login))
with:
runners: '["ubuntu-22.04"]'
platform: "linux/amd64"
push-image: "false"
upload-image-artifact: "true"
- upload-mount-cache-artifact: "false"
- python-versions: '["3.12"]'
+ # Kept so a standalone registry dispatch reuses the mount cache its last
run left behind.
+ upload-mount-cache-artifact: "true"
+ python-versions: ${{ format('["{0}"]', inputs.python-version) }}
branch: "main"
constraints-branch: "constraints-main"
use-uv: "true"
@@ -97,6 +116,27 @@ jobs:
timeout-minutes: 45
name: "Build & Publish Registry"
needs: [build-ci-image]
+ # `build-ci-image` is skipped when the caller stashed the image, so this
cannot simply
+ # inherit its result — but it must keep enforcing the same committer
allowlist that
+ # skipping that job used to enforce for us on a dispatch.
+ if: >
+ !cancelled() && needs.build-ci-image.result != 'failure' && (
+ github.event_name == 'workflow_call' ||
+ contains(fromJSON('[
+ "ashb",
+ "bugraoz93",
+ "eladkal",
+ "ephraimbuddy",
+ "jedcunningham",
+ "jscheffl",
+ "kaxil",
+ "pierrejeambrun",
+ "shahar1",
+ "potiuk",
+ "uranusjr",
+ "utkarsharma2",
+ "vincbeck"
+ ]'), github.event.sender.login))
runs-on: ubuntu-latest
env:
SCARF_ANALYTICS: "false"
@@ -125,7 +165,7 @@ jobs:
- name: "Prepare breeze & CI image"
uses: ./.github/actions/prepare_breeze_and_image
with:
- python: "3.12"
+ python: "${{ inputs.python-version }}"
platform: "linux/amd64"
use-uv: "true"
make-mnt-writeable-and-cleanup: "true"
@@ -183,6 +223,7 @@ jobs:
env:
PROVIDER: ${{ inputs.provider }}
DESTINATION: ${{ inputs.destination }}
+ PYTHON_VERSION: ${{ inputs.python-version }}
run: |
# Staging dispatches preview unreleased providers (maintainers want
to
# verify newly-bumped versions look right before tagging). Live
builds
@@ -193,9 +234,10 @@ jobs:
ALLOW_UNRELEASED="--allow-unreleased"
fi
if [[ -n "${PROVIDER}" ]]; then
- breeze registry extract-data --python 3.12 --provider
"${PROVIDER}" ${ALLOW_UNRELEASED}
+ breeze registry extract-data --python "${PYTHON_VERSION}" \
+ --provider "${PROVIDER}" ${ALLOW_UNRELEASED}
else
- breeze registry extract-data --python 3.12 ${ALLOW_UNRELEASED}
+ breeze registry extract-data --python "${PYTHON_VERSION}"
${ALLOW_UNRELEASED}
fi
# --- Incremental: merge new data with existing ---
