This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch constraints-generated-per-release in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 44a3056475d1c9be1747c08b94458cb3718c4803 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Aug 4 02:02:58 2026 +0200 Resolve the constraints a release ships instead of tagging whatever exists The constraints published with a version were whatever the constraints-X-Y branch happened to hold when the release ran - a resolution made by the last CI build, for the sources at that moment, rather than for the version being released. The candidate tagged that tip and the final release retagged the candidate, so no release ever resolved constraints of its own. A candidate now resolves them allowing pre-releases: the providers of the wave being voted on exist on PyPI only as rc versions, so constraints that refuse pre-releases cannot describe what a tester is asked to install. They land on a branch of the candidate's own, leaving the branch every other build reads where it was. The final release cannot promote those by retagging - a released version must never pin an rc - so it resolves the same set again, without pre-releases, and commits onto constraints-X-Y, which is what makes the released constraints the baseline everything downstream reads. Which of the two happens is derived from the version, so the stage cannot be set inconsistently with it. The work runs on CI runners rather than on the release manager's machine, which would otherwise need a CI image for every supported Python before it could cut a release, and is reachable on its own through `breeze workflow-run release-constraints` for redoing a candidate's constraints or producing them for a release cut before this existed. --- .github/workflows/generate-constraints.yml | 14 +- .github/workflows/release-constraints.yml | 246 +++++++++++++++++++++ dev/README_RELEASE_AIRFLOW.md | 29 ++- ...put_release-management_generate-constraints.svg | 24 +- ...put_release-management_generate-constraints.txt | 2 +- dev/breeze/doc/images/output_workflow-run.svg | 14 +- dev/breeze/doc/images/output_workflow-run.txt | 2 +- .../output_workflow-run_release-constraints.svg | 138 ++++++++++++ .../output_workflow-run_release-constraints.txt | 1 + .../commands/release_candidate_command.py | 31 +-- .../src/airflow_breeze/commands/release_command.py | 48 +--- .../commands/release_management_commands.py | 4 + .../airflow_breeze/commands/workflow_commands.py | 44 ++++ .../commands/workflow_commands_config.py | 12 +- .../airflow_breeze/utils/release_constraints.py | 61 +++++ dev/breeze/tests/test_release_command.py | 57 ++--- dev/breeze/tests/test_release_constraints.py | 91 ++++++++ scripts/in_container/run_generate_constraints.py | 20 ++ 18 files changed, 705 insertions(+), 133 deletions(-) diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 28c078a3563..2e205a89ffa 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -44,6 +44,13 @@ on: # yamllint disable-line rule:truthy description: "Whether to generate PyPI constraints (true/false)" required: true type: string + allow-pre-releases: + description: > + Whether the PyPI constraints may pin pre-releases (true/false). Set when constraints are + cut for a release candidate, whose providers are on PyPI only as rc versions. + required: false + default: "false" + type: string debug-resources: description: "Whether to run in debug mode (true/false)" required: true @@ -137,9 +144,14 @@ jobs: - name: "PyPI constraints" shell: bash timeout-minutes: 25 + env: + # Deliberately not named ALLOW_PRE_RELEASES: that is the option's own envvar, and having + # both paths set it would make it unclear which one is in force. Empty on a normal run, + # so the unquoted expansion below contributes no argument at all. + PRE_RELEASE_FLAG: ${{ inputs.allow-pre-releases == 'true' && '--allow-pre-releases' || '' }} run: | breeze release-management generate-constraints --airflow-constraints-mode constraints \ - --answer yes --python "${PYTHON_VERSION}" + --answer yes --python "${PYTHON_VERSION}" ${PRE_RELEASE_FLAG} if: inputs.generate-pypi-constraints == 'true' - name: "Check for provider downgrade alert" id: downgrade-alert diff --git a/.github/workflows/release-constraints.yml b/.github/workflows/release-constraints.yml new file mode 100644 index 00000000000..0b6f3198344 --- /dev/null +++ b/.github/workflows/release-constraints.yml @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Resolves the constraints that a release ships, rather than tagging whatever the +# `constraints-X-Y` branch happened to hold when the release ran. +# +# The stage is derived from the version, so the two cannot be mismatched by hand: +# +# * a candidate (`3.1.3rc1`) resolves with pre-releases allowed - the providers of the wave +# being voted on exist on PyPI only as rc versions - and lands on a branch of its own, so a +# candidate never moves the branch every other build reads; +# * a final (`3.1.3`) resolves without them, against the providers now published as finals, and +# commits onto `constraints-X-Y` itself, which is what makes the released constraints the +# baseline everything downstream reads. +# +# A final therefore cannot be produced by retagging a candidate: a released version must never +# pin an rc. +--- +name: Release constraints +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + version: + description: "Version the constraints belong to, e.g. 3.1.3rc1 or 3.1.3" + required: true + type: string + ref: + description: "Ref the constraints are resolved from, e.g. v3-1-stable or the release tag" + required: true + type: string +permissions: + contents: read +concurrency: + group: release-constraints-${{ inputs.version }} + cancel-in-progress: false +jobs: + build-info: + timeout-minutes: 10 + name: "Build info" + runs-on: ["ubuntu-22.04"] + if: contains(fromJSON('[ + "ashb", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2", + "vincbeck", + ]'), github.event.sender.login) + outputs: + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + constraints-branch: ${{ steps.stage.outputs.constraints-branch }} + target-branch: ${{ steps.stage.outputs.target-branch }} + allow-pre-releases: ${{ steps.stage.outputs.allow-pre-releases }} + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ inputs.ref }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + id: breeze + - name: "Save github context to file" + # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the + # github context to a file, and the single-quoted heredoc makes the zizmor + # template-injection finding a false positive (no bash expansion happens inside it). + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "[]" + COMMIT_REF: "${{ inputs.ref }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: "Derive the release stage from the version" + id: stage + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$ ]]; then + echo "'${VERSION}' is not a release version - expected X.Y.Z or X.Y.ZrcN." >&2 + exit 1 + fi + constraints_branch="constraints-$(echo "${VERSION}" | cut -d. -f1)-$(echo "${VERSION}" | cut -d. -f2)" + echo "constraints-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}" + if [[ "${VERSION}" == *rc* ]]; then + echo "allow-pre-releases=true" >> "${GITHUB_OUTPUT}" + echo "target-branch=constraints-${VERSION}" >> "${GITHUB_OUTPUT}" + else + echo "allow-pre-releases=false" >> "${GITHUB_OUTPUT}" + echo "target-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}" + fi + - name: "Parameters summary" + shell: bash + env: + VERSION: ${{ inputs.version }} + REF: ${{ inputs.ref }} + CONSTRAINTS_BRANCH: ${{ steps.stage.outputs.constraints-branch }} + TARGET_BRANCH: ${{ steps.stage.outputs.target-branch }} + ALLOW_PRE_RELEASES: ${{ steps.stage.outputs.allow-pre-releases }} + run: | + { + echo "## Release constraints" + echo "" + echo "| Parameter | Value |" + echo "|---|---|" + echo "| Version | \`${VERSION}\` |" + echo "| Resolved from ref | \`${REF}\` |" + echo "| Branched off | \`${CONSTRAINTS_BRANCH}\` |" + echo "| Committed to | \`${TARGET_BRANCH}\` |" + echo "| Pre-releases allowed | \`${ALLOW_PRE_RELEASES}\` |" + echo "| Tagged | \`constraints-${VERSION}\` |" + } | tee -a "${GITHUB_STEP_SUMMARY}" + + build-ci-images: + name: "Build CI images" + needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + packages: write + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: "false" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + checkout-ref: ${{ inputs.ref }} + use-uv: "true" + upgrade-to-newer-dependencies: "false" + docker-cache: "registry" + disable-airflow-repo-cache: "false" + + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + generate-pypi-constraints: "true" + # Only the PyPI constraints are what a release ships; the other modes serve CI, and + # regenerating them here would move them for reasons unrelated to the release. + generate-no-providers-constraints: "false" + allow-pre-releases: ${{ needs.build-info.outputs.allow-pre-releases }} + debug-resources: "false" + checkout-ref: ${{ inputs.ref }} + use-uv: "true" + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + publish-constraints: + runs-on: ["ubuntu-22.04"] + timeout-minutes: 20 + name: "Publish and tag constraints" + needs: [build-info, generate-constraints] + permissions: + contents: write + packages: read + env: + VERSION: ${{ inputs.version }} + CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.constraints-branch }} + TARGET_BRANCH: ${{ needs.build-info.outputs.target-branch }} + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ inputs.ref }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + # Branched off the shared constraints branch in both cases; what differs is where the commit + # ends up, which the checkout below decides. + - name: "Checkout ${{ needs.build-info.outputs.constraints-branch }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: "constraints" + ref: ${{ needs.build-info.outputs.constraints-branch }} + persist-credentials: true + fetch-depth: 0 + - name: "Download constraints from the generate-constraints job" + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: constraints-* + path: ./files + - name: "Switch to ${{ needs.build-info.outputs.target-branch }}" + working-directory: "constraints" + shell: bash + run: git switch -c "${TARGET_BRANCH}" 2>/dev/null || git switch "${TARGET_BRANCH}" + - name: "Diff in constraints for Python: ${{ needs.build-info.outputs.python-versions-list-as-string }}" + run: ./scripts/ci/constraints/ci_diff_constraints.sh + - name: "Commit changed constraint files" + run: ./scripts/ci/constraints/ci_commit_constraints.sh + - name: "Push ${{ needs.build-info.outputs.target-branch }}" + working-directory: "constraints" + shell: bash + run: git push origin "${TARGET_BRANCH}" + - name: "Tag constraints-${{ inputs.version }}" + working-directory: "constraints" + shell: bash + run: | + git tag -a "constraints-${VERSION}" -m "Constraints for Apache Airflow ${VERSION}" + git push origin "constraints-${VERSION}" + - name: "Summary" + shell: bash + run: | + { + echo "Constraints for \`${VERSION}\` are on \`${TARGET_BRANCH}\`," + echo "tagged \`constraints-${VERSION}\`." + } | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 33881515df9..0cb5621ee85 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -1372,15 +1372,28 @@ breeze release-management start-release \ Note: The `--task-sdk-version` parameter is optional. If you are releasing Airflow without a corresponding Task SDK release, you can omit this parameter. -Note: When it reaches the constraints step, `start-release` asks whether to base the final -`constraints-${VERSION}` tag on the latest `constraints-X-Y` branch tip instead of the RC -constraints tag. The RC constraints are frozen when the RC is cut, so if any providers were -released (or constraints were otherwise refreshed - see +Note: When it reaches the constraints step, `start-release` resolves the constraints again rather +than promoting the ones the RC was cut with. The RC constraints deliberately pin pre-releases - +that wave's providers exist on PyPI only as `rcN` versions at candidate time - so they can never +become the released constraints by retagging. The regeneration runs without pre-releases against +the same providers now published as finals, commits the result onto the `constraints-X-Y` branch, +pushes it, and tags it `constraints-${VERSION}`. That commit is what makes the released +constraints the new baseline, so refreshing the branch beforehand (see [MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md](MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md)) -after the last RC and you want the released constraints to reflect that, answer **yes** to tag the -`constraints-X-Y` branch tip. Otherwise (the default) the final tag matches the RC exactly. If you -do refresh, run the `Update constraints` workflow from `main` with `ref` set to the ref you are -releasing (typically `v3-*-stable`) **before** running `start-release`. +is no longer necessary. + +The resolution runs on CI runners through the `Release constraints` workflow, so the release +manager's machine does not need a CI image for every supported Python. `start-release` triggers it +and waits. You can also run it on its own - to redo a candidate's constraints, or to produce them +for a release cut before this existed: + +```shell script +breeze workflow-run release-constraints --version ${VERSION} --ref v3-1-stable +``` + +The workflow derives the stage from `--version` alone: `3.1.3rc1` resolves with pre-releases and +lands on `constraints-3.1.3rc1`, `3.1.3` resolves without them and commits onto `constraints-3-1`. +There is no separate switch that could disagree with the version. 4. Make sure to update Airflow version in ``v3-*-test`` branch after cherry-picking to X.Y.1 in diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.svg b/dev/breeze/doc/images/output_release-management_generate-constraints.svg index 9a2a1328ea1..30fd74577a7 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.svg +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 757.5999999999999" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 806.4" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-generate-constraints-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="706.5999999999999" /> + <rect x="0" y="0" width="1463.0" height="755.4" /> </clipPath> <clipPath id="breeze-release-management-generate-constraints-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -129,9 +129,15 @@ <clipPath id="breeze-release-management-generate-constraints-line-27"> <rect x="0" y="660.3" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-28"> + <rect x="0" y="684.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-29"> + <rect x="0" y="709.1" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="755.6" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="804.4" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -165,11 +171,13 @@ </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="532.4" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-21)">--skip-cleanup   </text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="532.4" textLeng [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-22)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)"> </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-23)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="581.2" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="581.2" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-24)">--verbose</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="605.6" textLength="24.4" clip-path="ur [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="630" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-25)">--dry-run</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="630" textLength="24.4" clip-path="url(#bre [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-26)">--answer </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="654.4" textLength="24.4" clip-pat [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="678.8" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-27)">--help   </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="678.8" textLength="24.4 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-28)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="244" clip-path="url(#breeze-release-management-generate-constraints-line-24)">--allow-pre-releases</text><text class="breeze-release-management-generate-constraints-r1" x="341.6" y="605.6" textLength="1098" clip [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r1" x="341.6" y="630" textLength="427" clip-path="url(#breeze-release-management-generate-constraints-line-25)">true automatically for pre-release </text><text class="breeze-release-management-generate-constraints-r4" x="768.6" [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="244" clip-path="url(#breeze-release-management-generate-constraints-line-26)">--verbose           </text><text class="breeze-release-management-generate-con [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="678.8" textLength="244" clip-path="url(#breeze-release-management-generate-constraints-line-27)">--dry-run           </text><text class="breeze-release-management-generate-con [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="244" clip-path="url(#breeze-release-management-generate-constraints-line-28)">--answer            </text><text class="breeze-release-management-generat [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="244" clip-path="url(#breeze-release-management-generate-constraints-line-29)">--help              </text><text class="breeze-release-manageme [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.txt b/dev/breeze/doc/images/output_release-management_generate-constraints.txt index 6b4ef06da4a..f65abff1032 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.txt +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.txt @@ -1 +1 @@ -8f5434ddc6564b1a2b026a63424eecea +102e1e067b55e8748782d783dc134948 diff --git a/dev/breeze/doc/images/output_workflow-run.svg b/dev/breeze/doc/images/output_workflow-run.svg index af211f19a8e..f8f7cb779f0 100644 --- a/dev/breeze/doc/images/output_workflow-run.svg +++ b/dev/breeze/doc/images/output_workflow-run.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 318.4" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 342.79999999999995" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -42,7 +42,7 @@ <defs> <clipPath id="breeze-workflow-run-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="267.4" /> + <rect x="0" y="0" width="1463.0" height="291.79999999999995" /> </clipPath> <clipPath id="breeze-workflow-run-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -74,9 +74,12 @@ <clipPath id="breeze-workflow-run-line-9"> <rect x="0" y="221.1" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-workflow-run-line-10"> + <rect x="0" y="245.5" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="316.4" rx="8"/><text class="breeze-workflow-run-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: workflow-run</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="340.8" rx="8"/><text class="breeze-workflow-run-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: workflow-run</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -95,8 +98,9 @@ </text><text class="breeze-workflow-run-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-workflow-run-line-6)">│</text><text class="breeze-workflow-run-r4" x="24.4" y="166.4" textLength="73.2" clip-path="url(#breeze-workflow-run-line-6)">--help</text><text class="breeze-workflow-run-r6" x="122" y="166.4" textLength="24.4" clip-path="url(#breeze-workflow-run-line-6)">-h</text><text class="breeze-workflow-run-r1" x="170.8" y="166.4" textLength="329.4" clip-path="url(#breeze-wor [...] </text><text class="breeze-workflow-run-r5" x="0" y="190.8" textLength="1464" clip-path="url(#breeze-workflow-run-line-7)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-r1" x="1464" y="190.8" textLength="12.2" clip-path="url(#breeze-workflow-run-line-7)"> </text><text class="breeze-workflow-run-r5" x="0" y="215.2" textLength="24.4" clip-path="url(#breeze-workflow-run-line-8)">╭─</text><text class="breeze-workflow-run-r5" x="24.4" y="215.2" textLength="512.4" clip-path="url(#breeze-workflow-run-line-8)"> Airflow github actions workflow commands </text><text class="breeze-workflow-run-r5" x="536.8" y="215.2" textLength="902.8" clip-path="url(#breeze-workflow-run-line-8)">──────────────────────────────────────── [...] -</text><text class="breeze-workflow-run-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-line-9)">│</text><text class="breeze-workflow-run-r4" x="24.4" y="239.6" textLength="366" clip-path="url(#breeze-workflow-run-line-9)">publish-docs                  </text><text class="breeze-workflow-run-r1" x="414.8" y="239.6" textLength="1024.8" clip-path="url(#breeze-workflow-run-lin [...] -</text><text class="breeze-workflow-run-r5" x="0" y="264" textLength="1464" clip-path="url(#breeze-workflow-run-line-10)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-r1" x="1464" y="264" textLength="12.2" clip-path="url(#breeze-workflow-run-line-10)"> +</text><text class="breeze-workflow-run-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-line-9)">│</text><text class="breeze-workflow-run-r4" x="24.4" y="239.6" textLength="256.2" clip-path="url(#breeze-workflow-run-line-9)">publish-docs         </text><text class="breeze-workflow-run-r1" x="305" y="239.6" textLength="1134.6" clip-path="url(#breeze-workflow-run-line-9)">Trigger publish docs to S3&# [...] +</text><text class="breeze-workflow-run-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-workflow-run-line-10)">│</text><text class="breeze-workflow-run-r4" x="24.4" y="264" textLength="256.2" clip-path="url(#breeze-workflow-run-line-10)">release-constraints  </text><text class="breeze-workflow-run-r1" x="305" y="264" textLength="1134.6" clip-path="url(#breeze-workflow-run-line-10)">Trigger the workflow that resolves, publishes and t [...] +</text><text class="breeze-workflow-run-r5" x="0" y="288.4" textLength="1464" clip-path="url(#breeze-workflow-run-line-11)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-r1" x="1464" y="288.4" textLength="12.2" clip-path="url(#breeze-workflow-run-line-11)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_workflow-run.txt b/dev/breeze/doc/images/output_workflow-run.txt index 096f1b4e226..606ed71654d 100644 --- a/dev/breeze/doc/images/output_workflow-run.txt +++ b/dev/breeze/doc/images/output_workflow-run.txt @@ -1 +1 @@ -348e7af5d380b4c7dd9452e48dfc4abf +cfbf0a5234b3a9689a01d164e3e91abf diff --git a/dev/breeze/doc/images/output_workflow-run_release-constraints.svg b/dev/breeze/doc/images/output_workflow-run_release-constraints.svg new file mode 100644 index 00000000000..e20e2d2696b --- /dev/null +++ b/dev/breeze/doc/images/output_workflow-run_release-constraints.svg @@ -0,0 +1,138 @@ +<svg class="rich-terminal" viewBox="0 0 1482 513.5999999999999" xmlns="http://www.w3.org/2000/svg";> + <!-- Generated with Rich https://www.textualize.io --> + <style> + + @font-face { + font-family: "Fira Code"; + src: local("FiraCode-Regular"), + url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2";) format("woff2"), + url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff";) format("woff"); + font-style: normal; + font-weight: 400; + } + @font-face { + font-family: "Fira Code"; + src: local("FiraCode-Bold"), + url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2";) format("woff2"), + url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff";) format("woff"); + font-style: bold; + font-weight: 700; + } + + .breeze-workflow-run-release-constraints-matrix { + font-family: Fira Code, monospace; + font-size: 20px; + line-height: 24.4px; + font-variant-east-asian: full-width; + } + + .breeze-workflow-run-release-constraints-title { + font-size: 18px; + font-weight: bold; + font-family: arial; + } + + .breeze-workflow-run-release-constraints-r1 { fill: #c5c8c6 } +.breeze-workflow-run-release-constraints-r2 { fill: #d0b344 } +.breeze-workflow-run-release-constraints-r3 { fill: #c5c8c6;font-weight: bold } +.breeze-workflow-run-release-constraints-r4 { fill: #68a0b3;font-weight: bold } +.breeze-workflow-run-release-constraints-r5 { fill: #868887 } +.breeze-workflow-run-release-constraints-r6 { fill: #cc555a } +.breeze-workflow-run-release-constraints-r7 { fill: #8a4346 } +.breeze-workflow-run-release-constraints-r8 { fill: #8d7b39 } +.breeze-workflow-run-release-constraints-r9 { fill: #98a84b;font-weight: bold } + </style> + + <defs> + <clipPath id="breeze-workflow-run-release-constraints-clip-terminal"> + <rect x="0" y="0" width="1463.0" height="462.59999999999997" /> + </clipPath> + <clipPath id="breeze-workflow-run-release-constraints-line-0"> + <rect x="0" y="1.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-1"> + <rect x="0" y="25.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-2"> + <rect x="0" y="50.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-3"> + <rect x="0" y="74.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-4"> + <rect x="0" y="99.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-5"> + <rect x="0" y="123.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-6"> + <rect x="0" y="147.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-7"> + <rect x="0" y="172.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-8"> + <rect x="0" y="196.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-9"> + <rect x="0" y="221.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-10"> + <rect x="0" y="245.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-11"> + <rect x="0" y="269.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-12"> + <rect x="0" y="294.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-13"> + <rect x="0" y="318.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-14"> + <rect x="0" y="343.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-15"> + <rect x="0" y="367.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-16"> + <rect x="0" y="391.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-release-constraints-line-17"> + <rect x="0" y="416.3" width="1464" height="24.65"/> + </clipPath> + </defs> + + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="511.6" rx="8"/><text class="breeze-workflow-run-release-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: workflow-run release-constraints</text> + <g transform="translate(26,22)"> + <circle cx="0" cy="0" r="7" fill="#ff5f57"/> + <circle cx="22" cy="0" r="7" fill="#febc2e"/> + <circle cx="44" cy="0" r="7" fill="#28c840"/> + </g> + + <g transform="translate(9, 41)" clip-path="url(#breeze-workflow-run-release-constraints-clip-terminal)"> + + <g class="breeze-workflow-run-release-constraints-matrix"> + <text class="breeze-workflow-run-release-constraints-r1" x="1464" y="20" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-0)"> +</text><text class="breeze-workflow-run-release-constraints-r2" x="12.2" y="44.4" textLength="73.2" clip-path="url(#breeze-workflow-run-release-constraints-line-1)">Usage:</text><text class="breeze-workflow-run-release-constraints-r3" x="97.6" y="44.4" textLength="475.8" clip-path="url(#breeze-workflow-run-release-constraints-line-1)">breeze workflow-run release-constraints</text><text class="breeze-workflow-run-release-constraints-r1" x="585.6" y="44.4" textLength="12.2" clip- [...] +</text><text class="breeze-workflow-run-release-constraints-r1" x="1464" y="68.8" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-2)"> +</text><text class="breeze-workflow-run-release-constraints-r1" x="12.2" y="93.2" textLength="1037" clip-path="url(#breeze-workflow-run-release-constraints-line-3)">Trigger the workflow that resolves, publishes and tags the constraints for a release.</text><text class="breeze-workflow-run-release-constraints-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-3)"> +</text><text class="breeze-workflow-run-release-constraints-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-4)"> +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-workflow-run-release-constraints-line-5)">╭─</text><text class="breeze-workflow-run-release-constraints-r5" x="24.4" y="142" textLength="292.8" clip-path="url(#breeze-workflow-run-release-constraints-line-5)"> Constraints to resolve </text><text class="breeze-workflow-run-release-constraints-r5" x="317.2" y="142" textLength="1122.4" clip-path="url(#br [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-6)">│</text><text class="breeze-workflow-run-release-constraints-r6" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-6)">*</text><text class="breeze-workflow-run-release-constraints-r4" x="61" y="166.4" textLength="207.4" clip-path="url(#breeze-workflow-run-release-constraints-line- [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-7)">│</text><text class="breeze-workflow-run-release-constraints-r1" x="292.8" y="190.8" textLength="1146.8" clip-path="url(#breeze-workflow-run-release-constraints-line-7)">and lands on a branch of its own; a final (3.1.3) resolves without them and commits& [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-8)">│</text><text class="breeze-workflow-run-release-constraints-r1" x="292.8" y="215.2" textLength="1146.8" clip-path="url(#breeze-workflow-run-release-constraints-line-8)">constraints-X-Y. The stage is derived from this, so it cannot be set inconsistently.   [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-9)">│</text><text class="breeze-workflow-run-release-constraints-r7" x="292.8" y="239.6" textLength="122" clip-path="url(#breeze-workflow-run-release-constraints-line-9)">[required]</text><text class="breeze-workflow-run-release-constraints-r8" x="427" y="239.6" textLength="73.2" clip-path="url(#breeze-workflow-run-release-constrai [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-10)">│</text><text class="breeze-workflow-run-release-constraints-r6" x="24.4" y="264" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-10)">*</text><text class="breeze-workflow-run-release-constraints-r4" x="61" y="264" textLength="207.4" clip-path="url(#breeze-workflow-run-release-constraints-line-10)" [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-11)">│</text><text class="breeze-workflow-run-release-constraints-r8" x="292.8" y="288.4" textLength="73.2" clip-path="url(#breeze-workflow-run-release-constraints-line-11)">(TEXT)</text><text class="breeze-workflow-run-release-constraints-r5" x="1451.8" y="288.4" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constr [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-12)">│</text><text class="breeze-workflow-run-release-constraints-r4" x="61" y="312.8" textLength="207.4" clip-path="url(#breeze-workflow-run-release-constraints-line-12)">--workflow-branch</text><text class="breeze-workflow-run-release-constraints-r1" x="292.8" y="312.8" textLength="1146.8" clip-path="url(#breeze-workflow-run-rele [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-13)">│</text><text class="breeze-workflow-run-release-constraints-r1" x="292.8" y="337.2" textLength="1146.8" clip-path="url(#breeze-workflow-run-release-constraints-line-13)">want: unlike the docs build, the constraints do not have to be produced by the wor [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-14)">│</text><text class="breeze-workflow-run-release-constraints-r1" x="292.8" y="361.6" textLength="402.6" clip-path="url(#breeze-workflow-run-release-constraints-line-14)">stood at the ref being released. </text><text class="breeze-workflow-run-release-constraints-r5" x="695.4" y="361.6" textLength= [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-workflow-run-release-constraints-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-release-constraints-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-15)"> +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="410.4" textLength="24.4" clip-path="url(#breeze-workflow-run-release-constraints-line-16)">╭─</text><text class="breeze-workflow-run-release-constraints-r5" x="24.4" y="410.4" textLength="195.2" clip-path="url(#breeze-workflow-run-release-constraints-line-16)"> Common options </text><text class="breeze-workflow-run-release-constraints-r5" x="219.6" y="410.4" textLength="1220" clip-path="url(#breeze-wo [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-17)">│</text><text class="breeze-workflow-run-release-constraints-r4" x="24.4" y="434.8" textLength="73.2" clip-path="url(#breeze-workflow-run-release-constraints-line-17)">--help</text><text class="breeze-workflow-run-release-constraints-r9" x="122" y="434.8" textLength="24.4" clip-path="url(#breeze-workflow-run-release-constraint [...] +</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="459.2" textLength="1464" clip-path="url(#breeze-workflow-run-release-constraints-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-release-constraints-r1" x="1464" y="459.2" textLength="12.2" clip-path="url(#breeze-workflow-run-release-constraints-line-18)"> +</text> + </g> + </g> +</svg> diff --git a/dev/breeze/doc/images/output_workflow-run_release-constraints.txt b/dev/breeze/doc/images/output_workflow-run_release-constraints.txt new file mode 100644 index 00000000000..871ae3667b4 --- /dev/null +++ b/dev/breeze/doc/images/output_workflow-run_release-constraints.txt @@ -0,0 +1 @@ +14273c8c9c667e0004c0a1379d53da58 diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index 5175e6e40ce..f6ac278bb35 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -49,6 +49,7 @@ from airflow_breeze.utils.path_utils import ( AIRFLOW_ROOT_PATH, OUT_PATH, ) +from airflow_breeze.utils.release_constraints import publish_constraints from airflow_breeze.utils.reproducible import get_source_date_epoch, repack_deterministically from airflow_breeze.utils.run_utils import run_command from airflow_breeze.utils.shared_options import get_dry_run @@ -442,28 +443,10 @@ def sign_the_release(repo_root): console_print("[success]Release signed") -def tag_and_push_constraints(version, version_branch, remote_name): - if confirm_action("Do you want to tag and push constraints?"): - run_command( - ["git", "checkout", f"{remote_name}/constraints-{version_branch}"], - check=True, - ) - run_command( - [ - "git", - "tag", - "-s", - f"constraints-{version}", - "-m", - f"Constraints for Apache Airflow {version}", - ], - check=True, - ) - run_command( - ["git", "push", remote_name, "tag", f"constraints-{version}"], - check=True, - ) - console_print("[success]Constraints tagged and pushed") +def generate_and_push_constraints(version, version_branch): + # Resolved from the stable branch the candidate was cut from, so the constraints describe the + # sources being voted on. The workflow reads "rcN" and allows pre-releases accordingly. + publish_constraints(version=version, ref=f"v{version_branch}-stable") def clone_asf_repo(version, repo_root): @@ -845,8 +828,8 @@ def publish_release_candidate( test_airflow() # Sign the release sign_the_release(airflow_repo_root) - # Tag and push constraints - tag_and_push_constraints(version, version_branch, remote_name) + # Generate, publish and tag the constraints for this candidate + generate_and_push_constraints(version, version_branch) # Clone the asf repo clone_asf_repo(version, airflow_repo_root) # Move artifacts to SVN diff --git a/dev/breeze/src/airflow_breeze/commands/release_command.py b/dev/breeze/src/airflow_breeze/commands/release_command.py index 1b93f9c28ac..ffa97a01f63 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_command.py @@ -29,6 +29,7 @@ from airflow_breeze.utils.confirm import confirm_action from airflow_breeze.utils.console import console_print from airflow_breeze.utils.environment_check import is_ci_environment from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH +from airflow_breeze.utils.release_constraints import publish_constraints from airflow_breeze.utils.run_utils import run_command # Pattern to match Airflow release versions (e.g., "3.0.5") @@ -343,45 +344,12 @@ def upload_to_pypi(version, task_sdk_version=None): ) -def get_constraints_branch_for_version(version: str) -> str: +def regenerate_constraints(version): + # The candidate's constraints pinned pre-releases, so they cannot be promoted by retagging - a + # released version must never point at rc pins. The workflow reads a version without "rcN" and + # resolves again without them, against the providers now published as finals. major, minor = version.split(".")[:2] - return f"constraints-{major}-{minor}" - - -def retag_constraints(release_candidate, version): - # By default the final ``constraints-<version>`` tag is created from the RC constraints tag. - # If the constraints were refreshed after the last RC (e.g. to pick up newly released - # providers - see dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md) the tip of the - # ``constraints-X-Y`` branch is newer than the RC tag and should be tagged instead. - constraints_branch = get_constraints_branch_for_version(version) - source_ref = f"constraints-{release_candidate}" - if confirm_action( - f"Base the final constraints on the latest '{constraints_branch}' branch tip instead of the " - f"'{source_ref}' tag? Choose yes if you refreshed constraints after {release_candidate}." - ): - run_command(["git", "fetch", "origin", constraints_branch], check=True) - source_ref = f"origin/{constraints_branch}" - if confirm_action(f"Retag constraints from {source_ref} as {version}?"): - run_command( - ["git", "checkout", source_ref], - check=True, - ) - run_command( - [ - "git", - "tag", - "-s", - f"constraints-{version}", - "-m", - f"Constraints for Apache Airflow {version}", - ], - check=True, - ) - if confirm_action(f"Push constraints-{version} tag to GitHub?"): - run_command( - ["git", "push", "origin", "tag", f"constraints-{version}"], - check=True, - ) + publish_constraints(version=version, ref=f"v{major}-{minor}-stable") def tag_and_push_latest_constraint(version): @@ -567,8 +535,8 @@ def airflow_release(version, task_sdk_version): # Change Directory to airflow os.chdir(airflow_repo_root) - # Retag and push the constraint file - retag_constraints(release_candidate, version) + # Regenerate, publish and tag the constraints for the final version + regenerate_constraints(version) tag_and_push_latest_constraint(version) # Push tag for final version diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index c7efa5a2fd3..52ed5a6001a 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -1548,6 +1548,7 @@ def tag_providers( @option_debug_resources @option_python_versions @option_airflow_constraints_mode_ci +@option_allow_pre_releases @option_github_repository @option_use_uv @option_verbose @@ -1555,6 +1556,7 @@ def tag_providers( @option_answer def generate_constraints( airflow_constraints_mode: str, + allow_pre_releases: bool, debug_resources: bool, github_repository: str, parallelism: int, @@ -1599,6 +1601,7 @@ def generate_constraints( shell_params_list = [ ShellParams( airflow_constraints_mode=airflow_constraints_mode, + allow_pre_releases=allow_pre_releases, github_repository=github_repository, python=python, use_uv=use_uv, @@ -1617,6 +1620,7 @@ def generate_constraints( else: shell_params = ShellParams( airflow_constraints_mode=airflow_constraints_mode, + allow_pre_releases=allow_pre_releases, github_repository=github_repository, python=python, use_uv=use_uv, diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py index 2330edfcba6..d6c98827b42 100644 --- a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py @@ -34,8 +34,13 @@ WORKFLOW_NAME_MAPS = { "publish-docs": "publish-docs-to-s3.yml", "airflow-refresh-site": "build.yml", "sync-s3-to-github": "s3-to-github.yml", + "release-constraints": "release-constraints.yml", } +# X.Y.Z or X.Y.ZrcN - the workflow derives the release stage from which of the two it is given, +# so there is no separate switch that could disagree with the version. +RELEASE_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+(rc\d+)?$") + APACHE_AIRFLOW_REPO = "apache/airflow" APACHE_AIRFLOW_SITE_REPO = "apache/airflow-site" APACHE_AIRFLOW_SITE_ARCHIVE_REPO = "apache/airflow-site-archive" @@ -272,3 +277,42 @@ def workflow_run_publish( **workflow_fields, monitor=False, ) + + +@workflow_run_group.command( + name="release-constraints", + help="Trigger the workflow that resolves, publishes and tags the constraints for a release.", +) [email protected]( + "--version", + help="Version the constraints belong to. A candidate (3.1.3rc1) resolves with pre-releases " + "allowed and lands on a branch of its own; a final (3.1.3) resolves without them and commits " + "onto constraints-X-Y. The stage is derived from this, so it cannot be set inconsistently.", + required=True, +) [email protected]( + "--ref", + help="Git ref the constraints are resolved from, e.g. 'v3-1-stable' or the release tag.", + required=True, +) [email protected]( + "--workflow-branch", + help="Git ref the workflow DEFINITION runs from. Defaults to 'main', which is normally what " + "you want: unlike the docs build, the constraints do not have to be produced by the workflow " + "as it stood at the ref being released.", + default="main", + show_default=True, +) +def workflow_run_release_constraints(version: str, ref: str, workflow_branch: str): + if not RELEASE_VERSION_PATTERN.match(version): + console_print(f"[red]Error: '{version}' is not a release version - expected X.Y.Z or X.Y.ZrcN.[/red]") + sys.exit(1) + stage = "candidate" if "rc" in version else "final" + console_print(f"[blue]Triggering constraints generation for the {stage} {version} from {ref}[/blue]") + trigger_workflow_and_monitor( + workflow_name=WORKFLOW_NAME_MAPS["release-constraints"], + repo=APACHE_AIRFLOW_REPO, + branch=workflow_branch, + version=version, + ref=ref, + ) diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py index 239a5e46a6a..9c6d0c99277 100644 --- a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py @@ -18,7 +18,7 @@ from __future__ import annotations WORKFLOW_RUN_COMMANDS: dict[str, str | list[str]] = { "name": "Airflow github actions workflow commands", - "commands": ["publish-docs"], + "commands": ["publish-docs", "release-constraints"], } WORKFLOW_RUN_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { @@ -55,3 +55,13 @@ WORKFLOW_RUN_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { }, ], } +WORKFLOW_RUN_PARAMETERS["breeze workflow-run release-constraints"] = [ + { + "name": "Constraints to resolve", + "options": [ + "--version", + "--ref", + "--workflow-branch", + ], + }, +] diff --git a/dev/breeze/src/airflow_breeze/utils/release_constraints.py b/dev/breeze/src/airflow_breeze/utils/release_constraints.py new file mode 100644 index 00000000000..466ab689120 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/release_constraints.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Handing the constraints of a release over to the workflow that resolves them. + +A release used to tag whatever the ``constraints-X-Y`` branch happened to hold, so the constraints +shipped with a version were a resolution made by the last CI build rather than one made for that +version. They are resolved per release now, by the ``release-constraints`` workflow - on runners +rather than on the release manager's machine, which would otherwise need a CI image for every +supported Python before it could cut a release. + +The workflow derives the stage from the version it is given, so nothing here has to pass a +separate switch that could disagree with it. +""" + +from __future__ import annotations + +from airflow_breeze.utils.confirm import confirm_action +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.gh_workflow_utils import trigger_workflow_and_monitor + +RELEASE_CONSTRAINTS_WORKFLOW = "release-constraints.yml" +APACHE_AIRFLOW_REPO = "apache/airflow" + + +def publish_constraints(*, version: str, ref: str, workflow_branch: str = "main") -> None: + """Resolve, publish and tag the constraints belonging to ``version``. + + ``version`` alone decides what happens: a candidate (``3.1.3rc1``) resolves with pre-releases + allowed - the providers of the wave being voted on are on PyPI only as rc versions - and lands + on a branch of its own, leaving the branch every other build reads where it was. A final + (``3.1.3``) resolves without them and commits onto ``constraints-X-Y``, which is what makes the + released constraints the baseline everything downstream reads. + """ + stage = "candidate" if "rc" in version else "final" + if not confirm_action( + f"Trigger the release-constraints workflow for the {stage} {version} (resolved from {ref})?" + ): + return + console_print(f"[info]Resolving constraints for {version} from {ref} - this runs on CI runners.") + trigger_workflow_and_monitor( + workflow_name=RELEASE_CONSTRAINTS_WORKFLOW, + repo=APACHE_AIRFLOW_REPO, + branch=workflow_branch, + version=version, + ref=ref, + ) + console_print(f"[success]Constraints for {version} published and tagged 'constraints-{version}'") diff --git a/dev/breeze/tests/test_release_command.py b/dev/breeze/tests/test_release_command.py index 1e023e6bf5b..c1d89193e4f 100644 --- a/dev/breeze/tests/test_release_command.py +++ b/dev/breeze/tests/test_release_command.py @@ -540,53 +540,22 @@ def test_remove_old_release_no_task_sdk_version(monkeypatch, release_cmd): assert chdir_calls == [svn_release_repo, "/original/dir"] [email protected]( - ("version", "expected_branch"), - [ - ("3.3.0", "constraints-3-3"), - ("3.0.5", "constraints-3-0"), - ("3.12.10", "constraints-3-12"), - ], -) -def test_get_constraints_branch_for_version(release_cmd, version, expected_branch): - assert release_cmd.get_constraints_branch_for_version(version) == expected_branch - - -def test_retag_constraints_from_rc_tag_by_default(monkeypatch, release_cmd): - run_command_calls: list[list[str]] = [] - - def fake_confirm_action(prompt: str, **_kwargs) -> bool: - # Decline basing on the branch tip; accept the retag and the push. - return not prompt.startswith("Base the final constraints on the latest") - - monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action) - monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd)) - - release_cmd.retag_constraints("3.3.0rc2", "3.3.0") +def test_regenerate_constraints_resolves_from_the_stable_branch(monkeypatch, release_cmd): + """The final release resolves from the branch it is cut from, not from main.""" + calls: dict[str, object] = {} + monkeypatch.setattr(release_cmd, "publish_constraints", lambda **kw: calls.update(kw)) - assert ["git", "checkout", "constraints-3.3.0rc2"] in run_command_calls - assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in ( - run_command_calls - ) - assert ["git", "push", "origin", "tag", "constraints-3.3.0"] in run_command_calls - # The branch tip is not fetched when basing on the RC tag. - assert not any(cmd[:2] == ["git", "fetch"] for cmd in run_command_calls) + release_cmd.regenerate_constraints("3.3.0") + assert calls == {"version": "3.3.0", "ref": "v3-3-stable"} -def test_retag_constraints_from_branch_tip_when_confirmed(monkeypatch, release_cmd): - run_command_calls: list[list[str]] = [] - def fake_confirm_action(prompt: str, **_kwargs) -> bool: - return True +def test_regenerate_constraints_passes_a_final_version(monkeypatch, release_cmd): + """No rc suffix reaches the workflow, which is what makes it refuse pre-releases.""" + calls: dict[str, object] = {} + monkeypatch.setattr(release_cmd, "publish_constraints", lambda **kw: calls.update(kw)) - monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action) - monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd)) - - release_cmd.retag_constraints("3.3.0rc2", "3.3.0") + release_cmd.regenerate_constraints("3.0.5") - assert ["git", "fetch", "origin", "constraints-3-3"] in run_command_calls - assert ["git", "checkout", "origin/constraints-3-3"] in run_command_calls - assert ["git", "checkout", "constraints-3.3.0rc2"] not in run_command_calls - assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in ( - run_command_calls - ) + assert "rc" not in str(calls["version"]) + assert calls["ref"] == "v3-0-stable" diff --git a/dev/breeze/tests/test_release_constraints.py b/dev/breeze/tests/test_release_constraints.py new file mode 100644 index 00000000000..db914d97722 --- /dev/null +++ b/dev/breeze/tests/test_release_constraints.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow_breeze.utils import release_constraints + + [email protected] +def triggered(monkeypatch): + """Capture what would have been dispatched, with the prompt answered yes.""" + calls: list[dict] = [] + monkeypatch.setattr(release_constraints, "confirm_action", lambda *a, **kw: True) + monkeypatch.setattr( + release_constraints, "trigger_workflow_and_monitor", lambda **kwargs: calls.append(kwargs) + ) + return calls + + [email protected]( + ("version", "ref"), + [ + pytest.param("3.1.3rc1", "v3-1-stable", id="candidate"), + pytest.param("3.1.3", "v3-1-stable", id="final"), + ], +) +def test_version_and_ref_are_passed_through(triggered, version, ref): + release_constraints.publish_constraints(version=version, ref=ref) + + assert len(triggered) == 1 + assert triggered[0]["version"] == version + assert triggered[0]["ref"] == ref + assert triggered[0]["workflow_name"] == "release-constraints.yml" + assert triggered[0]["repo"] == "apache/airflow" + + +def test_workflow_definition_defaults_to_main(triggered): + # Unlike the docs build, constraints need not be produced by the workflow as it stood at the + # ref being released - so this defaults to main rather than to the ref. + release_constraints.publish_constraints(version="3.1.3", ref="v3-1-stable") + + assert triggered[0]["branch"] == "main" + + +def test_workflow_definition_can_be_overridden(triggered): + release_constraints.publish_constraints( + version="3.1.3", ref="v3-1-stable", workflow_branch="my-fix-branch" + ) + + assert triggered[0]["branch"] == "my-fix-branch" + + +def test_nothing_is_dispatched_when_declined(monkeypatch, triggered): + monkeypatch.setattr(release_constraints, "confirm_action", lambda *a, **kw: False) + + release_constraints.publish_constraints(version="3.1.3", ref="v3-1-stable") + + assert triggered == [] + + [email protected]( + ("version", "expected_stage"), + [ + pytest.param("3.1.3rc2", "candidate", id="rc-is-a-candidate"), + pytest.param("3.1.3", "final", id="plain-version-is-final"), + ], +) +def test_the_prompt_names_the_stage_the_version_implies(monkeypatch, version, expected_stage): + """The stage is never passed separately, so the operator sees what the version implies.""" + prompts: list[str] = [] + monkeypatch.setattr(release_constraints, "confirm_action", lambda prompt, **kw: prompts.append(prompt)) + monkeypatch.setattr(release_constraints, "trigger_workflow_and_monitor", lambda **kw: None) + + release_constraints.publish_constraints(version=version, ref="v3-1-stable") + + assert expected_stage in prompts[0] diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index 6310ec1a052..cc20f08d271 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -130,6 +130,7 @@ PYPI_PROVIDERS_CONSTRAINTS_PREFIX = f""" @dataclass class ConfigParams: airflow_constraints_mode: str + allow_pre_releases: bool constraints_github_repository: str default_constraints_branch: str github_actions: bool @@ -521,6 +522,14 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None: "gremlinpython>=3.8.0", ] + # Constraints cut for a release candidate have to pin the candidates themselves - the providers + # for that wave exist on PyPI only as rcN versions, and uv will not resolve to a pre-release + # unless asked. The final release regenerates these without the flag, so a released constraints + # file can never carry an rc pin. + pre_release_flags = ["--pre"] if config_params.allow_pre_releases else [] + if config_params.allow_pre_releases: + console.print("[bright_blue]Allowing pre-release versions of airflow and providers") + result = run_command( cmd=[ "uv", @@ -534,6 +543,7 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None: f"apache-airflow-task-sdk=={AIRFLOW_TASK_SDK_VERSION}", "./airflow-ctl", *additional_constraints_for_highest_resolution, + *pre_release_flags, "--reinstall", # We need to pull the provider distributions from PyPI or dist, not the local ones "--resolution", "highest", @@ -632,8 +642,17 @@ ALLOWED_CONSTRAINTS_MODES = ["constraints", "constraints-source-providers", "con help="Use uv instead of pip as packaging tool.", envvar="USE_UV", ) [email protected]( + "--allow-pre-releases", + is_flag=True, + default=False, + help="Allow pre-release versions of Airflow and providers to be pinned. Used when constraints " + "are generated for a release candidate, whose providers are only on PyPI as rc versions.", + envvar="ALLOW_PRE_RELEASES", +) def generate_constraints( airflow_constraints_mode: str, + allow_pre_releases: bool, constraints_github_repository: str, default_constraints_branch: str, github_actions: bool, @@ -642,6 +661,7 @@ def generate_constraints( ) -> None: config_params = ConfigParams( airflow_constraints_mode=airflow_constraints_mode, + allow_pre_releases=allow_pre_releases, constraints_github_repository=constraints_github_repository, default_constraints_branch=default_constraints_branch, github_actions=github_actions,
