This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch prod-packages-early in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 806f156e606c19a4a89569e34df36f7afe337c63 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 2 00:09:37 2026 +0100 Build the PROD packages beside the CI image and constraints The PROD image build first built the Airflow and provider wheels, but only after the CI image and generate-constraints had finished, although the wheels need neither. In a RunsOn pull request run that put them on the chain that ends the run: CI image to minute 9.1, constraints to 15.2, wheels 15.5 to 17.9, PROD image to 29.6, Kubernetes tests to 41.3. The CI workflows now build the wheels in their own job right after build-info and tell the PROD image build they are already uploaded; the other callers of prod-image-build.yml still build them as before. Generated-by: Claude Opus 5 --- .github/workflows/ci-amd.yml | 19 +++- .github/workflows/ci-arm.yml | 19 +++- .github/workflows/prod-image-build.yml | 96 ++++------------ .github/workflows/prod-packages-build.yml | 129 ++++++++++++++++++++++ scripts/tests/ci/test_prek_cache_optimizations.py | 2 +- 5 files changed, 187 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 71619921033..1feaf21abe4 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -1130,9 +1130,23 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} + build-prod-packages: + name: Build PROD packages + needs: [build-info] + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + upload-package-artifact: "true" + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + build-prod-images: name: Build PROD images - needs: [build-info, build-ci-images, generate-constraints] + needs: [build-info, build-ci-images, build-prod-packages, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -1155,6 +1169,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + packages-prebuilt: "true" additional-prod-image-tests: name: "Additional PROD image tests" @@ -1601,6 +1616,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - ci-image-checks @@ -1668,6 +1684,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - check-timed-out-jobs diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index e6072ea6fcf..5386e40bb09 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -1006,9 +1006,23 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} + build-prod-packages: + name: Build PROD packages + needs: [build-info] + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + upload-package-artifact: "true" + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + build-prod-images: name: Build PROD images - needs: [build-info, build-ci-images, generate-constraints] + needs: [build-info, build-ci-images, build-prod-packages, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -1031,6 +1045,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + packages-prebuilt: "true" additional-prod-image-tests: name: "Additional PROD image tests" @@ -1477,6 +1492,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - ci-image-checks @@ -1543,6 +1559,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - check-timed-out-jobs diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 5cac2e2817d..1df788fac5e 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -113,86 +113,28 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a prod-image build (true/false)" required: true type: string + packages-prebuilt: + description: "Whether the caller already uploaded the prod-packages artifact (true/false)" + required: false + default: "false" + type: string permissions: contents: read jobs: build-prod-packages: name: "Build Airflow and provider distributions" - timeout-minutes: 10 - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} - if: inputs.prod-image-build == 'true' - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - if: inputs.upload-package-artifact == 'true' - - name: "Checkout target branch" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: "Make /mnt writeable" - run: ./scripts/ci/make_mnt_writeable.sh - if: inputs.upload-package-artifact == 'true' - - name: "Move docker to /mnt" - run: ./scripts/ci/move_docker_to_mnt.sh - if: inputs.upload-package-artifact == 'true' - - name: "Cleanup dist and context file" - shell: bash - run: rm -fv ./dist/* ./docker-context-files/* - if: inputs.upload-package-artifact == 'true' - - name: "Install Breeze" - uses: ./.github/actions/breeze - if: inputs.upload-package-artifact == 'true' - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - platform: ${{ inputs.platform }} - save-cache: false - # The hatch_build.py files of the fab, edge3 and common.ai providers run these hooks. - hooks: compile-fab-assets compile-edge-assets compile-common-ai-provider-assets - if: inputs.upload-package-artifact == 'true' - - name: "Prepare providers packages - all providers built from sources" - shell: bash - run: > - breeze release-management prepare-provider-distributions - --distributions-list-file ./prod_image_installed_providers.txt - --distribution-format wheel --include-not-ready-providers --skip-tag-check - if: > - inputs.upload-package-artifact == 'true' && inputs.branch == 'main' - - name: "Prepare providers packages with only new versions of providers" - shell: bash - run: > - breeze release-management prepare-provider-distributions - --distributions-list-file ./prod_image_installed_providers.txt - --distribution-format wheel --include-not-ready-providers --github-repository apache/airflow - if: > - inputs.upload-package-artifact == 'true' && inputs.branch != 'main' - - name: "Prepare airflow package" - shell: bash - run: > - breeze release-management prepare-airflow-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Prepare task-sdk package" - shell: bash - run: > - breeze release-management prepare-task-sdk-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Prepare airflow-ctl package" - shell: bash - run: > - breeze release-management prepare-airflow-ctl-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Upload prepared packages as artifacts" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: prod-packages - path: ./dist - retention-days: 7 - if-no-files-found: error - if: inputs.upload-package-artifact == 'true' + # The CI workflows build the packages before calling this workflow; see prod-packages-build.yml. + if: inputs.packages-prebuilt != 'true' + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} + default-python-version: ${{ inputs.default-python-version }} + branch: ${{ inputs.branch }} + upload-package-artifact: ${{ inputs.upload-package-artifact }} + prod-image-build: ${{ inputs.prod-image-build }} build-prod-images: strategy: @@ -204,6 +146,10 @@ jobs: runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} needs: - build-prod-packages + # A skipped build-prod-packages means the caller built the packages already. + if: >- + !cancelled() && inputs.prod-image-build == 'true' && + (needs.build-prod-packages.result == 'success' || inputs.packages-prebuilt == 'true') env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" diff --git a/.github/workflows/prod-packages-build.yml b/.github/workflows/prod-packages-build.yml new file mode 100644 index 00000000000..c80bfd816db --- /dev/null +++ b/.github/workflows/prod-packages-build.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# Builds the Airflow, task-sdk, airflow-ctl and provider wheels the PROD image installs. It reads no +# constraints and no CI image, so the CI workflows start it right after build-info, beside the CI +# image build and generate-constraints, instead of as the first step of the PROD image build. +name: Build PROD packages +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string + default-python-version: + description: "Python version used to build the packages" + required: true + type: string + branch: + description: "Branch used to run the CI jobs in (main/v*_*_test)." + required: true + type: string + upload-package-artifact: + description: "Whether to build and upload the packages as the prod-packages artifact (true/false)" + required: true + type: string + prod-image-build: + description: "Whether this is a prod-image build (true/false)" + required: true + type: string +permissions: + contents: read +jobs: + build-prod-packages: + name: "Build Airflow and provider distributions" + timeout-minutes: 10 + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + if: inputs.prod-image-build == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + if: inputs.upload-package-artifact == 'true' + - name: "Checkout target branch" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: "Make /mnt writeable" + run: ./scripts/ci/make_mnt_writeable.sh + if: inputs.upload-package-artifact == 'true' + - name: "Move docker to /mnt" + run: ./scripts/ci/move_docker_to_mnt.sh + if: inputs.upload-package-artifact == 'true' + - name: "Cleanup dist and context file" + shell: bash + run: rm -fv ./dist/* ./docker-context-files/* + if: inputs.upload-package-artifact == 'true' + - name: "Install Breeze" + uses: ./.github/actions/breeze + if: inputs.upload-package-artifact == 'true' + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + platform: ${{ inputs.platform }} + save-cache: false + # The hatch_build.py files of the fab, edge3 and common.ai providers run these hooks. + hooks: compile-fab-assets compile-edge-assets compile-common-ai-provider-assets + if: inputs.upload-package-artifact == 'true' + - name: "Prepare providers packages - all providers built from sources" + shell: bash + run: > + breeze release-management prepare-provider-distributions + --distributions-list-file ./prod_image_installed_providers.txt + --distribution-format wheel --include-not-ready-providers --skip-tag-check + if: > + inputs.upload-package-artifact == 'true' && inputs.branch == 'main' + - name: "Prepare providers packages with only new versions of providers" + shell: bash + run: > + breeze release-management prepare-provider-distributions + --distributions-list-file ./prod_image_installed_providers.txt + --distribution-format wheel --include-not-ready-providers --github-repository apache/airflow + if: > + inputs.upload-package-artifact == 'true' && inputs.branch != 'main' + - name: "Prepare airflow package" + shell: bash + run: > + breeze release-management prepare-airflow-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Prepare task-sdk package" + shell: bash + run: > + breeze release-management prepare-task-sdk-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Prepare airflow-ctl package" + shell: bash + run: > + breeze release-management prepare-airflow-ctl-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Upload prepared packages as artifacts" + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: prod-packages + path: ./dist + retention-days: 7 + if-no-files-found: error + if: inputs.upload-package-artifact == 'true' diff --git a/scripts/tests/ci/test_prek_cache_optimizations.py b/scripts/tests/ci/test_prek_cache_optimizations.py index 8d8449153ce..6310637a751 100644 --- a/scripts/tests/ci/test_prek_cache_optimizations.py +++ b/scripts/tests/ci/test_prek_cache_optimizations.py @@ -270,7 +270,7 @@ def test_selected_hooks_match_the_hooks_each_job_runs(): "ci-arm.yml:migration-round-trip", "ci-arm.yml:mypy-providers", "generate-constraints.yml:generate-constraints-matrix", - "prod-image-build.yml:build-prod-packages", + "prod-packages-build.yml:build-prod-packages", "test-providers.yml:prepare-install-verify-provider-distributions", "test-providers.yml:providers-compatibility-tests-matrix", ]
