This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fix-constraints-rc-pins in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 52edefc4d4aaa22ec0281125ea0bfb04e24ce9e0 Author: Jarek Potiuk <[email protected]> AuthorDate: Thu Oct 29 00:50:13 2026 +0100 Keep release-candidate provider pins until each is released or yanked Constraints for an Airflow release candidate pin the provider wave being voted on at its rc versions, and commit them onto the shared constraints-X-Y branch. The next regular generation for the vX-Y-test branch resolves without pre-releases, so it saw the older finals as a provider downgrade and failed - the v3-4-test canary has been red since 3.4.0rc1 for exactly that reason, although nothing was yanked. The final release has the opposite gap: it ignored the candidates, so a provider rc that was neither released nor yanked was silently passed over and the released constraints never recorded that decision. It now has to be made, and the run says which providers it is still waiting on. The alert that should have reported the canary failure never reached Slack because the action was not told which API method to call, and the twice-daily CI notification announced "All passing" because it read a run that was still in progress, whose missing conclusion looked like no failures. Generated-by: Claude Opus 5 --- .github/workflows/generate-constraints.yml | 14 +- .github/workflows/release-constraints.yml | 3 + dev/README_RELEASE_AIRFLOW.md | 37 ++++- dev/breeze/doc/09_release_management_tasks.rst | 15 +- ...put_release-management_generate-constraints.svg | 78 ++++++---- ...put_release-management_generate-constraints.txt | 2 +- .../src/airflow_breeze/commands/common_options.py | 7 + .../commands/release_management_commands.py | 5 + .../commands/release_management_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 2 + .../src/airflow_breeze/utils/workflow_status.py | 4 + dev/breeze/tests/test_shell_params.py | 6 + dev/breeze/tests/test_workflow_status.py | 35 +++++ scripts/in_container/run_generate_constraints.py | 129 ++++++++++++++-- .../in_container/test_run_generate_constraints.py | 167 ++++++++++++++++++++- 15 files changed, 458 insertions(+), 47 deletions(-) diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 17a460aaa68..fc801924ccd 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -53,6 +53,14 @@ on: # yamllint disable-line rule:truthy required: false default: "false" type: string + fail-on-pending-provider-candidates: + description: > + Fail the PyPI constraints when a provider's newest version is a release candidate that was + neither released nor yanked (true/false). Set when constraints are cut for a final release, + which must not pin a candidate nor silently pass over one. + required: false + default: "false" + type: string pypi-providers-only: description: > Resolve against providers exactly as published on PyPI (true/false). Skips building the @@ -168,9 +176,12 @@ jobs: # both paths set it would make it unclear which one is in force. Empty on a normal run, # so the unquoted expansion below contributes no argument at all. PRE_RELEASE_FLAG: ${{ inputs.allow-pre-releases == 'true' && '--allow-pre-releases' || '' }} + PENDING_CANDIDATES_FLAG: >- + ${{ inputs.fail-on-pending-provider-candidates == 'true' + && '--fail-on-pending-provider-candidates' || '' }} run: | breeze release-management generate-constraints --airflow-constraints-mode constraints \ - --answer yes --python "${PYTHON_VERSION}" ${PRE_RELEASE_FLAG} + --answer yes --python "${PYTHON_VERSION}" ${PRE_RELEASE_FLAG} ${PENDING_CANDIDATES_FLAG} if: inputs.generate-pypi-constraints == 'true' - name: "Check for provider downgrade alert" id: downgrade-alert @@ -191,6 +202,7 @@ jobs: if: always() && steps.downgrade-alert.outputs.notify == 'true' uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: + method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} payload-file-path: ${{ steps.downgrade-alert.outputs.message-file }} - name: "Upload constraint artifacts" diff --git a/.github/workflows/release-constraints.yml b/.github/workflows/release-constraints.yml index fe74f5527f7..faa75605bd5 100644 --- a/.github/workflows/release-constraints.yml +++ b/.github/workflows/release-constraints.yml @@ -188,6 +188,9 @@ jobs: # the constraints, which is exactly the version the release needed pinned. pypi-providers-only: "true" allow-pre-releases: ${{ needs.build-info.outputs.allow-pre-releases }} + # A final must not be cut while a provider candidate is still neither released nor yanked. + fail-on-pending-provider-candidates: >- + ${{ needs.build-info.outputs.allow-pre-releases == 'true' && 'false' || 'true' }} debug-resources: "false" checkout-ref: ${{ inputs.ref }} use-uv: "true" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index be218b3f6eb..8b89af8f6eb 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -46,6 +46,7 @@ - [Verify the release candidate by Contributors](#verify-the-release-candidate-by-contributors) - [Installing release candidate in your local virtual environment](#installing-release-candidate-in-your-local-virtual-environment) - [Publish the final Apache Airflow release](#publish-the-final-apache-airflow-release) + - [Release or yank the outstanding provider candidates](#release-or-yank-the-outstanding-provider-candidates) - [Summarize the voting for the Apache Airflow release](#summarize-the-voting-for-the-apache-airflow-release) - [Publish release to SVN](#publish-release-to-svn) - [Manually prepare production Docker Image](#manually-prepare-production-docker-image) @@ -726,6 +727,11 @@ still works but is no longer recommended. `constraints-${VERSION_RC}`. There is no `constraints-${VERSION_RC}` branch — a branch with the same name as the tag would make the name ambiguous, so the workflow fails if it finds one. + The regular constraints generation for the `vX-Y-test` branch, which runs without + pre-releases, keeps the `rcN` provider pins it finds on `constraints-X-Y` for as long as PyPI + serves them, so it does not undo the candidate's constraints while the vote is on. A pinned + candidate gives way once its final is published, and is dropped if it is yanked. + **Testing the start-rc-process command:** Before running the actual release command, you can safely test it using: @@ -1448,6 +1454,28 @@ simple instructions described in # Publish the final Apache Airflow release +## Release or yank the outstanding provider candidates + +The constraints of the final release cannot pin a provider release candidate, and they will not +silently pass over one either. If the candidate's constraints pinned providers at `rcN` versions +(see the `start-rc-process` note above), every one of them must be cleared before `start-release` +reaches its constraints step: + +* **release it** - the provider vote passed and its final version is published on PyPI, or +* **yank it** on PyPI - it is not going to be released; yanked versions are skipped. + +Until then, the `Release constraints` workflow for the final version fails, listing every +provider whose newest version on PyPI is a candidate that is neither released nor yanked. To see +which providers the candidate pinned: + +```shell script +curl -s "https://raw.githubusercontent.com/apache/airflow/constraints-${VERSION_RC}/constraints-3.13.txt"; \ + | grep -E '^apache-airflow-providers-.*(a|b|rc)[0-9]+$' +``` + +Agree with the providers' release manager on the decision early in the vote, so the final +release does not wait on it. + ## Summarize the voting for the Apache Airflow release Once the vote has been passed, you will need to send a result vote to [email protected]: @@ -1564,9 +1592,12 @@ The providers are pinned rather than resolved: every `apache-airflow-providers-* `==` at the newest version PyPI can install, so the constraints record what is actually published instead of whatever the resolver settles on. A candidate takes the wave's `rcN` versions along with it - a pre-release only wins by sorting above every final release, so a provider without a -candidate in the wave keeps its release. A final ignores candidates entirely, which is what makes -it impossible for a released constraints file to carry an `rc` pin. Nothing else in the dependency -graph can resolve to a pre-release, since no other requirement mentions one. +candidate in the wave keeps its release. A final never pins a candidate, which is what makes it +impossible for a released constraints file to carry an `rc` pin - and it fails instead of passing +one over: a provider whose newest version on PyPI is a candidate must be released or yanked first +(see [Release or yank the outstanding provider candidates](#release-or-yank-the-outstanding-provider-candidates)). +Yanked versions are skipped. Nothing else in the dependency graph can resolve to a pre-release, +since no other requirement mentions one. Re-running the workflow for the same candidate adds another commit onto `constraints-X-Y` and re-creates the candidate's tag on it, so redoing a candidate's constraints just works. If a diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index b24c38e9ebe..431784aeb16 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -693,6 +693,13 @@ Constraints are generated separately for each python version and there are separ providers. If you want to manage Airflow separately and then add providers individually, you can use those. +In the ``constraints`` mode every provider is pinned at the newest version PyPI holds for it. Pre-releases +are only picked up with ``--allow-pre-releases`` (the constraints of a release candidate), except for the +candidates the current constraints branch already pins: those are kept while PyPI still serves them, so a +regular run does not undo a release candidate's constraints. ``--fail-on-pending-provider-candidates`` +(the constraints of a final release) fails instead when a provider's newest version is a candidate that +was neither released nor yanked. + These are all available flags of ``generate-constraints`` command: .. image:: ./images/output_release-management_generate-constraints.svg @@ -1128,9 +1135,11 @@ the workflow existed. The stage is derived from ``--version`` alone, so it cannot be set inconsistently with it. A candidate (``3.1.3rc1``) resolves allowing pre-releases for ``apache-airflow-providers-*`` — the providers of the wave -being voted on are on PyPI only as ``rcN`` versions — and lands on a branch of its own, leaving the shared -``constraints-X-Y`` branch where it was. A final (``3.1.3``) resolves without them and commits onto -``constraints-X-Y``, which is what makes the released constraints the baseline everything downstream reads. +being voted on are on PyPI only as ``rcN`` versions. A final (``3.1.3``) resolves without them, and fails +while any provider's newest version on PyPI is still a candidate: each one has to be released or yanked +first (yanked versions are skipped), so the released constraints neither pin a candidate nor silently pass +over one. Both commit onto ``constraints-X-Y`` and tag the result ``constraints-<VERSION>``; the final's +commit is what makes the released constraints the baseline everything downstream reads. These are all available flags of ``workflow-run release-constraints`` command: diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.svg b/dev/breeze/doc/images/output_release-management_generate-constraints.svg index 43fcc697029..4bdb49fab4c 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.svg +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 806.4" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 952.8" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-generate-constraints-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="755.4" /> + <rect x="0" y="0" width="1463.0" height="901.8" /> </clipPath> <clipPath id="breeze-release-management-generate-constraints-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -135,9 +135,27 @@ <clipPath id="breeze-release-management-generate-constraints-line-29"> <rect x="0" y="709.1" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-30"> + <rect x="0" y="733.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-31"> + <rect x="0" y="757.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-32"> + <rect x="0" y="782.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-33"> + <rect x="0" y="806.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-34"> + <rect x="0" y="831.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-35"> + <rect x="0" y="855.5" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="804.4" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="950.8" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -153,31 +171,37 @@ </text><text class="breeze-release-management-generate-constraints-r1" x="12.2" y="93.2" textLength="1000.4" clip-path="url(#breeze-release-management-generate-constraints-line-3)">Generates pinned constraint files with all extras from pyproject.toml in parallel.</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-3)"> </text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-4)"> </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-5)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="142" textLength="341.6" clip-path="url(#breeze-release-management-generate-constraints-line-5)"> Generate constraints flags </text><text class="breeze-release-management-generate-constraints-r5" x="366" y="142" t [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-6)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="166.4" textLength="317.2" clip-path="url(#breeze-release-management-generate-constraints-line-6)">--airflow-constraints-mode</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="166.4" textLength="671" [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-7)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="190.8" textLength="353.8" clip-path="url(#breeze-release-management-generate-constraints-line-7)">constraints-source-providers]</text><text class="breeze-release-management-generate-constraints-r6" x="780.8" y="190.8" textLength=" [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-8)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="215.2" textLength="305" clip-path="url(#breeze-release-management-generate-constraints-line-8)">constraints-no-providers)</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y="215.2" textLength="12.2" [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-9)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="239.6" textLength="317.2" clip-path="url(#breeze-release-management-generate-constraints-line-9)">--allow-pre-releases      </text><text class="breeze-release-management-generate-constraints-r1" x="414 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-10)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="264" textLength="512.4" clip-path="url(#breeze-release-management-generate-constraints-line-10)">Set to true automatically for pre-release </text><text class="breeze-release-management-generate-constra [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-11)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="288.4" textLength="317.2" clip-path="url(#breeze-release-management-generate-constraints-line-11)">--github-repository       </text><text class="breeze-release-management-generate-constraints-r7" [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="317.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">--python                  </text><text cl [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="256.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">| 3.12 | 3.13 | 3.14)</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y= [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="361.6" textLength="97.6" clip-path="url(#breeze-release-management-generate-constraints-line-14)">--use-uv</text><text class="breeze-release-management-generate-constraints-r1" x="122" y="361.6" textLength="12.2" clip-path="url(#b [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-16)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="410.4" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-16)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="410.4" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="434.8" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-17)">--debug-resources</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="434.8" textLength="768.6" clip [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="459.2" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-18)">--parallelism    </text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="459.2" tex [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r6" x="256.2" y="483.6" textLength="170.8" clip-path="url(#breeze-release-management-generate-constraints-line-19)">RANGE 1<=x<=8)</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y="483.6" textLength="1 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-20)">--python-versions</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="508" textLength="963.8" clip-path= [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">│</text><text class="breeze-release-management-generate-constraints-r5" x="256.2" y="532.4" textLength="183" clip-path="url(#breeze-release-management-generate-constraints-line-21)">3.12 3.13 3.14]</text><text class="breeze-release-management-generate-constraints-r6" x="451.4" y="532.4" textLength="73.2 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="556.8" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-22)">--run-in-parallel</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="556.8" textLength="854" clip-p [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="581.2" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-23)">--skip-cleanup   </text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="581.2" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-24)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-25)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="630" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="630" textLength="12 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-26)">--verbose</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="654.4" textLength="24.4" clip-path="ur [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="678.8" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-27)">--dry-run</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="678.8" textLength="24.4" clip-path="ur [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-28)">--answer </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="703.2" textLength="24.4" clip-pat [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-29)">--help   </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="727.6" textLength="24.4 [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-6)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="166.4" textLength="451.4" clip-path="url(#breeze-release-management-generate-constraints-line-6)">--airflow-constraints-mode           </text><text class="breeze-release-manage [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-7)">│</text><text class="breeze-release-management-generate-constraints-r5" x="549" y="190.8" textLength="353.8" clip-path="url(#breeze-release-management-generate-constraints-line-7)">constraints-source-providers]</text><text class="breeze-release-management-generate-constraints-r6" x="915" y="190.8" textLength="524. [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-8)">│</text><text class="breeze-release-management-generate-constraints-r6" x="549" y="215.2" textLength="329.4" clip-path="url(#breeze-release-management-generate-constraints-line-8)">| constraints-no-providers)</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y="215.2" textLength [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-9)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="239.6" textLength="451.4" clip-path="url(#breeze-release-management-generate-constraints-line-9)">--allow-pre-releases                 </text><tex [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-10)">│</text><text class="breeze-release-management-generate-constraints-r1" x="549" y="264" textLength="890.6" clip-path="url(#breeze-release-management-generate-constraints-line-10)">installed. Set to true automatically for pre-release          [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-11)">│</text><text class="breeze-release-management-generate-constraints-r4" x="549" y="288.4" textLength="256.2" clip-path="url(#breeze-release-management-generate-constraints-line-11)">--use-airflow-version</text><text class="breeze-release-management-generate-constraints-r1" x="805.2" y="288.4" textLength="634.4" c [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="451.4" clip-path="url(#breeze-release-management-generate-constraints-line-12)">--fail-on-pending-provider-candidates</text><text class="breeze-release-management-generate-constraints-r1" x="549" y="312.8" textL [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r1" x="549" y="337.2" textLength="890.6" clip-path="url(#breeze-release-management-generate-constraints-line-13)">was neither released nor yanked. Set when generating the constraints of a</te [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r1" x="549" y="361.6" textLength="890.6" clip-path="url(#breeze-release-management-generate-constraints-line-14)">final Airflow release.                 [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="451.4" clip-path="url(#breeze-release-management-generate-constraints-line-15)">--github-repository                  </text>< [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r5" x="549" y="410.4" textLength="183" clip-path="url(#breeze-release-management-generate-constraints-line-16)">apache/airflow]</text><text class="breeze-release-management-generate-constraints-r6" x="744.2" y="410.4" textLength="73.2" clip-path= [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="434.8" textLength="451.4" clip-path="url(#breeze-release-management-generate-constraints-line-17)">--python                     [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r5" x="549" y="459.2" textLength="61" clip-path="url(#breeze-release-management-generate-constraints-line-18)">3.11]</text><text class="breeze-release-management-generate-constraints-r6" x="622.2" y="459.2" textLength="353.8" clip-path="url(#bree [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="97.6" clip-path="url(#breeze-release-management-generate-constraints-line-19)">--use-uv</text><text class="breeze-release-management-generate-constraints-r1" x="122" y="483.6" textLength="12.2" clip-path="url(#b [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r5" x="549" y="508" textLength="85.4" clip-path="url(#breeze-release-management-generate-constraints-line-20)">use-uv]</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y="508" textLength="12.2" clip-path="url(#breeze [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-22)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="556.8" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-22)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="556.8" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="581.2" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-23)">--debug-resources</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="581.2" textLength="768.6" clip [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-24)">--parallelism    </text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="605.6" tex [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r6" x="256.2" y="630" textLength="170.8" clip-path="url(#breeze-release-management-generate-constraints-line-25)">RANGE 1<=x<=8)</text><text class="breeze-release-management-generate-constraints-r5" x="1451.8" y="630" textLength="12.2" c [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-26)">--python-versions</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="654.4" textLength="963.8" clip [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r5" x="256.2" y="678.8" textLength="183" clip-path="url(#breeze-release-management-generate-constraints-line-27)">3.12 3.13 3.14]</text><text class="breeze-release-management-generate-constraints-r6" x="451.4" y="678.8" textLength="73.2 [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-28)">--run-in-parallel</text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="703.2" textLength="854" clip-p [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="207.4" clip-path="url(#breeze-release-management-generate-constraints-line-29)">--skip-cleanup   </text><text class="breeze-release-management-generate-constraints-r1" x="256.2" y="727.6" textLeng [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-31)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="776.4" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="776.4" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="800.8" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-32)">--verbose</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="800.8" textLength="24.4" clip-path="ur [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="825.2" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-33)">--dry-run</text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="825.2" textLength="24.4" clip-path="ur [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="849.6" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-34)">--answer </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="849.6" textLength="24.4" clip-pat [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="874" textLength="109.8" clip-path="url(#breeze-release-management-generate-constraints-line-35)">--help   </text><text class="breeze-release-management-generate-constraints-r7" x="158.6" y="874" textLength="24.4" clip [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="898.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-36)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.txt b/dev/breeze/doc/images/output_release-management_generate-constraints.txt index 0e5763e946b..57f2cb408cf 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.txt +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.txt @@ -1 +1 @@ -f80db442347ca0dd7d7ca368405ffeee +42ead896784a045bf1fb7a94c8681b6e diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py index cc3aabf5408..4d5d614f37f 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -484,6 +484,13 @@ option_allow_pre_releases = click.option( is_flag=True, envvar="ALLOW_PRE_RELEASES", ) +option_fail_on_pending_provider_candidates = click.option( + "--fail-on-pending-provider-candidates", + help="Fail when a provider's newest version on PyPI is a release candidate that was neither " + "released nor yanked. Set when generating the constraints of a final Airflow release.", + is_flag=True, + envvar="FAIL_ON_PENDING_PROVIDER_CANDIDATES", +) option_airflow_version = click.option( "-A", "--airflow-version", diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 2fb8a69a419..eea8d9ce160 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -56,6 +56,7 @@ from airflow_breeze.commands.common_options import ( option_commit_sha, option_debug_resources, option_dry_run, + option_fail_on_pending_provider_candidates, option_github_repository, option_github_token, option_include_not_ready_providers, @@ -1590,6 +1591,7 @@ def tag_providers( @option_python_versions @option_airflow_constraints_mode_ci @option_allow_pre_releases +@option_fail_on_pending_provider_candidates @option_github_repository @option_use_uv @option_verbose @@ -1599,6 +1601,7 @@ def generate_constraints( airflow_constraints_mode: str, allow_pre_releases: bool, debug_resources: bool, + fail_on_pending_provider_candidates: bool, github_repository: str, parallelism: int, python: str, @@ -1643,6 +1646,7 @@ def generate_constraints( ShellParams( airflow_constraints_mode=airflow_constraints_mode, allow_pre_releases=allow_pre_releases, + fail_on_pending_provider_candidates=fail_on_pending_provider_candidates, github_repository=github_repository, python=python, use_uv=use_uv, @@ -1662,6 +1666,7 @@ def generate_constraints( shell_params = ShellParams( airflow_constraints_mode=airflow_constraints_mode, allow_pre_releases=allow_pre_releases, + fail_on_pending_provider_candidates=fail_on_pending_provider_candidates, github_repository=github_repository, python=python, use_uv=use_uv, diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 8c2a3850935..305e1f01d93 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -352,6 +352,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "options": [ "--airflow-constraints-mode", "--allow-pre-releases", + "--fail-on-pending-provider-candidates", "--github-repository", "--python", "--use-uv", diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6d117084825..8c779099f7d 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -195,6 +195,7 @@ class ShellParams: excluded_providers: str = "" executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR extra_args: tuple = () + fail_on_pending_provider_candidates: bool = False force_build: bool = False force_sa_warnings: bool = True force_lowest_dependencies: bool = False @@ -708,6 +709,7 @@ services: _set_var(_env, "FLOWER_HOST_PORT", None, FLOWER_HOST_PORT) _set_var(_env, "GREMLIN_HOST_PORT", None, GREMLIN_HOST_PORT) _set_var(_env, "EXCLUDED_PROVIDERS", self.excluded_providers) + _set_var(_env, "FAIL_ON_PENDING_PROVIDER_CANDIDATES", self.fail_on_pending_provider_candidates) _set_var(_env, "FORCE_LOWEST_DEPENDENCIES", self.force_lowest_dependencies) _set_var(_env, "SQLALCHEMY_WARN_20", self.force_sa_warnings) _set_var(_env, "GITHUB_ACTIONS", self.github_actions) diff --git a/dev/breeze/src/airflow_breeze/utils/workflow_status.py b/dev/breeze/src/airflow_breeze/utils/workflow_status.py index 2770ff22e4a..ba525ad22c8 100644 --- a/dev/breeze/src/airflow_breeze/utils/workflow_status.py +++ b/dev/breeze/src/airflow_breeze/utils/workflow_status.py @@ -53,6 +53,10 @@ def workflow_status( workflow_id, "--branch", branch, + # A run still in progress has no conclusion yet, which would read as "no failures" and + # announce a recovery that has not happened. + "--status", + "completed", "--limit", "1", # Limit to the most recent run "--repo", diff --git a/dev/breeze/tests/test_shell_params.py b/dev/breeze/tests/test_shell_params.py index 78e3d56d691..47655d7942a 100644 --- a/dev/breeze/tests/test_shell_params.py +++ b/dev/breeze/tests/test_shell_params.py @@ -304,6 +304,12 @@ def test_worktree_labels_are_derived_from_checkout(tmp_path, monkeypatch, linked {"POSTGRES_DATA_VOLUME_PATH": "/var/lib/postgresql/data"}, id="POSTGRES_DATA_VOLUME_PATH ignores the empty version of non-postgres backends", ), + pytest.param( + {}, + {"fail_on_pending_provider_candidates": True}, + {"FAIL_ON_PENDING_PROVIDER_CANDIDATES": "true"}, + id="FAIL_ON_PENDING_PROVIDER_CANDIDATES reaches the constraints generation", + ), ], ) def test_shell_params_to_env_var_conversion( diff --git a/dev/breeze/tests/test_workflow_status.py b/dev/breeze/tests/test_workflow_status.py new file mode 100644 index 00000000000..c28fededffc --- /dev/null +++ b/dev/breeze/tests/test_workflow_status.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import subprocess +from unittest import mock + +from airflow_breeze.utils import workflow_status + + [email protected](workflow_status.subprocess, "run", autospec=True) +def test_reads_only_completed_runs(mock_run): + """A run still in progress has no conclusion yet and must not be reported as all passing.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout=b'[{"conclusion": "failure", "url": "u", "databaseId": 1}]' + ) + + workflow_status.workflow_status("v3-4-test", "ci-amd.yml") + + cmd = mock_run.call_args.args[0] + assert cmd[cmd.index("--status") + 1] == "completed" diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index 5ca30fc59f1..717fb8c7923 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -136,6 +136,7 @@ class ConfigParams: allow_pre_releases: bool constraints_github_repository: str default_constraints_branch: str + fail_on_pending_provider_candidates: bool github_actions: bool python: str @@ -344,6 +345,24 @@ def _read_provider_versions_from_constraints(constraints_file: Path) -> dict[str return provider_versions +def _is_pre_release(version: str) -> bool: + try: + return Version(version).is_prerelease + except InvalidVersion: + return False + + +def find_pre_release_provider_pins(constraints_file: Path) -> dict[str, str]: + """Return the providers ``constraints_file`` pins at a pre-release, e.g. by a release candidate.""" + if not constraints_file.exists(): + return {} + return { + provider: version + for provider, version in _read_provider_versions_from_constraints(constraints_file).items() + if _is_pre_release(version) + } + + def write_provider_downgrade_slack_message( config_params: ConfigParams, downgraded: list[tuple[str, str, str]] ) -> None: @@ -406,6 +425,32 @@ def write_provider_downgrade_slack_message( console.print(f"[yellow]Wrote provider downgrade Slack payload to {slack_message_file}") +def check_baseline_pre_releases_unchanged( + config_params: ConfigParams, kept_pre_releases: dict[str, str] +) -> None: + """ + Fail if the candidates pinned by the baseline changed while the constraints were being resolved. + + The pins are chosen from the baseline read before the resolution, which takes minutes. A release + candidate's constraints published in the meantime would otherwise be overwritten by this run's + older pins - and, the baseline now pinning candidates, without the downgrade check noticing. + """ + current_pre_releases = find_pre_release_provider_pins(config_params.latest_constraints_file) + if current_pre_releases == kept_pre_releases: + return + console.print( + "[red]The release candidates pinned by the constraints branch changed while the constraints " + "were being generated:[/]" + ) + for provider in sorted(kept_pre_releases.keys() | current_pre_releases.keys()): + console.print( + f"[red] * {provider}: {kept_pre_releases.get(provider, '-')} -> " + f"{current_pre_releases.get(provider, '-')}" + ) + console.print("[yellow]Run the generation again so it starts from the updated constraints.") + sys.exit(1) + + def check_providers_not_downgraded(config_params: ConfigParams) -> None: """ Fail generation if any released provider is downgraded compared to the latest constraints. @@ -416,7 +461,9 @@ def check_providers_not_downgraded(config_params: ConfigParams) -> None: being published. A release candidate is exempt: it pins the wave being voted on, which sorts below the released - versions a constraints branch may already carry, so the comparison says nothing there. + versions a constraints branch may already carry, so the comparison says nothing there. So is a + provider the baseline pins at a pre-release: the generation keeps that pin while PyPI still + serves it, so a lower version means it was yanked - the outcome of a failed vote, not a break. """ if config_params.allow_pre_releases: console.print( @@ -432,7 +479,7 @@ def check_providers_not_downgraded(config_params: ConfigParams) -> None: downgraded: list[tuple[str, str, str]] = [] for provider, latest_version in latest_versions.items(): current_version = current_versions.get(provider) - if current_version is None: + if current_version is None or _is_pre_release(latest_version): continue try: if Version(current_version) < Version(latest_version): @@ -512,13 +559,18 @@ def is_file_installable(pypi_file: dict, target_python: Version) -> bool: def find_newest_version_in_pypi( - distribution: str, python_version: str, allow_pre_releases: bool + distribution: str, + python_version: str, + allow_pre_releases: bool, + kept_pre_release: str | None = None, ) -> str | None: """Return the newest version of ``distribution`` PyPI can install for ``python_version``. A pre-release is only ever considered when ``allow_pre_releases`` is set, and even then it wins only by sorting above every final release - so a provider with a candidate in the wave being - voted on resolves to that candidate while one without keeps its last release. Versions PyPI can + voted on resolves to that candidate while one without keeps its last release. The one exception + is ``kept_pre_release``, the candidate the baseline constraints already pin: it stays eligible so + that a run without pre-releases does not move the provider back below it. Versions PyPI can no longer serve for the Python being resolved - fully yanked, or excluded by ``requires_python`` - are passed over, because pinning one leaves the resolution nothing to install. ``None`` (nothing installable at all, e.g. a provider whose first release is still in this wave) leaves @@ -530,13 +582,14 @@ def find_newest_version_in_pypi( return None response.raise_for_status() target_python = Version(python_version) + kept_version = Version(kept_pre_release) if kept_pre_release else None newest_version: Version | None = None for version, files in response.json().get("releases", {}).items(): try: parsed_version = Version(version) except InvalidVersion: continue - if parsed_version.is_prerelease and not allow_pre_releases: + if parsed_version.is_prerelease and not allow_pre_releases and parsed_version != kept_version: continue if not any(is_file_installable(pypi_file, target_python) for pypi_file in files): continue @@ -548,7 +601,30 @@ def find_newest_version_in_pypi( return str(newest_version) -def build_pinned_provider_requirements(python_version: str, allow_pre_releases: bool) -> list[str]: +def exit_on_pending_provider_candidates(pending: list[tuple[str, str]]) -> None: + """Stop a final release's constraints while a provider candidate is neither released nor yanked.""" + console.print( + "[red]These providers have a release candidate on PyPI that is newer than their latest " + "final release:[/]" + ) + for distribution, version in pending: + console.print(f"[red] * {distribution}=={version}") + console.print( + "[yellow]Constraints for a final Airflow release cannot pin a release candidate, and passing " + "over it would publish constraints that silently ignore it. Clear each candidate first, " + "then run the release constraints again:[/]\n" + "[yellow] * release it - publish the final version once its vote has passed, or[/]\n" + "[yellow] * yank it on PyPI if it is not going to be released - yanked versions are skipped.[/]" + ) + sys.exit(1) + + +def build_pinned_provider_requirements( + python_version: str, + allow_pre_releases: bool, + kept_pre_releases: dict[str, str] | None = None, + fail_on_pending_candidates: bool = False, +) -> list[str]: """Exact pins for every active provider, at the newest version PyPI holds for it. Naming the versions retrieved from PyPI leaves the providers nothing to resolve: the constraints @@ -558,8 +634,14 @@ def build_pinned_provider_requirements(python_version: str, allow_pre_releases: satisfying the lower bounds. It also keeps pre-releases confined to the providers: no third-party dependency can answer with one, because no requirement here mentions a pre-release of anything else. + + ``kept_pre_releases`` maps providers to the candidates the baseline constraints pin; those are + kept while PyPI serves them. With ``fail_on_pending_candidates`` - the constraints of a final + release - the lookup sees candidates too, and any provider whose newest version is one fails the + run instead: it must be released or yanked first. """ distributions = get_all_active_provider_distributions(python_version) + kept_pre_releases = kept_pre_releases or {} console.print(f"[bright_blue]Retrieving the newest PyPI version of {len(distributions)} providers.") with ThreadPoolExecutor(max_workers=PYPI_LOOKUP_PARALLELISM) as executor: newest_versions = list( @@ -567,9 +649,18 @@ def build_pinned_provider_requirements(python_version: str, allow_pre_releases: find_newest_version_in_pypi, distributions, repeat(python_version), - repeat(allow_pre_releases), + repeat(allow_pre_releases or fail_on_pending_candidates), + [kept_pre_releases.get(distribution) for distribution in distributions], ) ) + if fail_on_pending_candidates: + pending = [ + (distribution, version) + for distribution, version in zip(distributions, newest_versions) + if version is not None and _is_pre_release(version) + ] + if pending: + exit_on_pending_provider_candidates(pending) return [ f"{distribution}=={version}" for distribution, version in zip(distributions, newest_versions) @@ -656,10 +747,17 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None: ] # Every run pins the providers at the versions PyPI holds when it is generated. Only a run for - # a release candidate considers the rc versions of the wave being voted on; a final regenerates - # these against the published releases, so a released constraints file can never carry an rc pin. + # a release candidate picks up the rc versions of the wave being voted on. A regular run keeps + # the candidates the baseline already pins, so it does not undo a candidate's constraints while + # the vote is on. A final fails while any provider still has a candidate that was neither + # released nor yanked, so a released constraints file can never carry an rc pin. + download_latest_constraint_file(config_params) + kept_pre_releases = find_pre_release_provider_pins(config_params.latest_constraints_file) pinned_provider_requirements = build_pinned_provider_requirements( - config_params.python, config_params.allow_pre_releases + config_params.python, + config_params.allow_pre_releases, + kept_pre_releases=kept_pre_releases, + fail_on_pending_candidates=config_params.fail_on_pending_provider_candidates, ) console.print( f"[bright_blue]Pinning {len(pinned_provider_requirements)} provider distributions to the " @@ -713,6 +811,7 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None: config_params, constraints_file, distributions_to_exclude_from_constraints ) download_latest_constraint_file(config_params) + check_baseline_pre_releases_unchanged(config_params, kept_pre_releases) diff_constraints(config_params) check_providers_not_downgraded(config_params) @@ -789,11 +888,20 @@ ALLOWED_CONSTRAINTS_MODES = ["constraints", "constraints-source-providers", "con "only on PyPI as rc versions.", envvar="ALLOW_PRE_RELEASES", ) [email protected]( + "--fail-on-pending-provider-candidates", + is_flag=True, + default=False, + help="Fail when a provider's newest version on PyPI is a release candidate that was neither " + "released nor yanked. Used when constraints are generated for a final Airflow release.", + envvar="FAIL_ON_PENDING_PROVIDER_CANDIDATES", +) def generate_constraints( airflow_constraints_mode: str, allow_pre_releases: bool, constraints_github_repository: str, default_constraints_branch: str, + fail_on_pending_provider_candidates: bool, github_actions: bool, python: str, use_uv: bool, @@ -803,6 +911,7 @@ def generate_constraints( allow_pre_releases=allow_pre_releases, constraints_github_repository=constraints_github_repository, default_constraints_branch=default_constraints_branch, + fail_on_pending_provider_candidates=fail_on_pending_provider_candidates, github_actions=github_actions, python=python, ) diff --git a/scripts/tests/in_container/test_run_generate_constraints.py b/scripts/tests/in_container/test_run_generate_constraints.py index f01653a422d..e01f62df34c 100644 --- a/scripts/tests/in_container/test_run_generate_constraints.py +++ b/scripts/tests/in_container/test_run_generate_constraints.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import re from pathlib import Path from types import SimpleNamespace @@ -73,6 +74,47 @@ class TestReadProviderVersionsFromConstraints: } +class TestFindPreReleaseProviderPins: + def test_returns_only_the_providers_pinned_at_a_pre_release(self, tmp_path): + constraints = tmp_path / "constraints.txt" + constraints.write_text( + "opentelemetry-exporter-prometheus==0.65b0\n" + "apache-airflow-providers-amazon==9.39.0\n" + "apache-airflow-providers-common-ai==1.1.0rc1\n" + "apache-airflow-providers-openlineage==2.21.0b1\n" + "apache-airflow-providers-broken==not-a-version\n" + ) + + assert m.find_pre_release_provider_pins(constraints) == { + "apache-airflow-providers-common-ai": "1.1.0rc1", + "apache-airflow-providers-openlineage": "2.21.0b1", + } + + def test_missing_baseline_has_no_pins(self, tmp_path): + assert m.find_pre_release_provider_pins(tmp_path / "missing.txt") == {} + + +class TestCheckBaselinePreReleasesUnchanged: + def test_passes_when_the_baseline_candidates_did_not_change(self, tmp_path): + latest = tmp_path / "latest.txt" + latest.write_text( + "apache-airflow-providers-common-ai==1.1.0rc1\napache-airflow-providers-amazon==9.0.1\n" + ) + + m.check_baseline_pre_releases_unchanged( + _config(latest, tmp_path / "current.txt"), {"apache-airflow-providers-common-ai": "1.1.0rc1"} + ) + + def test_fails_when_a_candidate_was_published_during_the_generation(self, tmp_path): + latest = tmp_path / "latest.txt" + latest.write_text("apache-airflow-providers-common-ai==1.1.0rc1\n") + + with pytest.raises(SystemExit) as exc_info: + m.check_baseline_pre_releases_unchanged(_config(latest, tmp_path / "current.txt"), {}) + + assert exc_info.value.code == 1 + + class TestCheckProvidersNotDowngraded: def test_exits_and_writes_slack_payload_when_a_provider_is_downgraded(self, tmp_path, monkeypatch): monkeypatch.delenv("SLACK_CHANNEL", raising=False) @@ -115,6 +157,16 @@ class TestCheckProvidersNotDowngraded: assert not (tmp_path / "provider-downgrade-slack-message.json").exists() + def test_a_candidate_in_the_baseline_is_not_a_downgrade(self, tmp_path): + latest = tmp_path / "latest.txt" + current = tmp_path / "current.txt" + latest.write_text("apache-airflow-providers-common-ai==1.1.0rc1\n") + current.write_text("apache-airflow-providers-common-ai==1.0.0\n") + + m.check_providers_not_downgraded(_config(latest, current)) + + assert not (tmp_path / "provider-downgrade-slack-message.json").exists() + def test_passes_when_no_provider_is_downgraded(self, tmp_path): latest = tmp_path / "latest.txt" current = tmp_path / "current.txt" @@ -219,6 +271,40 @@ class TestFindNewestVersionInPypi: assert newest == expected + @pytest.mark.parametrize( + "releases, expected", + [ + pytest.param( + {"1.0.0": [_file()], "1.1.0rc1": [_file()]}, + "1.1.0rc1", + id="candidate-the-baseline-pins-is-kept", + ), + pytest.param( + {"1.0.0": [_file()], "1.1.0rc1": [_file()], "1.1.0rc2": [_file()]}, + "1.1.0rc1", + id="other-candidates-stay-ignored", + ), + pytest.param( + {"1.0.0": [_file()], "1.1.0rc1": [_file()], "1.1.0": [_file()]}, + "1.1.0", + id="released-candidate-gives-way-to-its-release", + ), + pytest.param( + {"1.0.0": [_file()], "1.1.0rc1": [_file(yanked=True)]}, + "1.0.0", + id="yanked-candidate-is-dropped", + ), + ], + ) + def test_keeps_the_candidate_the_baseline_pins(self, monkeypatch, releases, expected): + monkeypatch.setattr(m.requests, "get", lambda url, timeout: _FakeResponse({"releases": releases})) + + newest = m.find_newest_version_in_pypi( + "apache-airflow-providers-common-ai", "3.11", False, kept_pre_release="1.1.0rc1" + ) + + assert newest == expected + def test_unpublished_distribution_is_unpinned(self, monkeypatch): monkeypatch.setattr(m.requests, "get", lambda url, timeout: _FakeResponse({}, status_code=404)) @@ -244,7 +330,9 @@ class TestBuildPinnedProviderRequirements: "apache-airflow-providers-brand-new": None, } monkeypatch.setattr( - m, "find_newest_version_in_pypi", lambda dist, python_version, allow_pre_releases: newest[dist] + m, + "find_newest_version_in_pypi", + lambda dist, python_version, allow_pre_releases, kept_pre_release: newest[dist], ) assert m.build_pinned_provider_requirements("3.11", True) == [ @@ -261,7 +349,7 @@ class TestBuildPinnedProviderRequirements: seen_for_distributions.append(python_version) return ["apache-airflow-providers-amazon"] - def fake_lookup(distribution, python_version, allow_pre_releases): + def fake_lookup(distribution, python_version, allow_pre_releases, kept_pre_release): seen_for_lookup.append((python_version, allow_pre_releases)) return "9.0.0" @@ -273,6 +361,81 @@ class TestBuildPinnedProviderRequirements: assert seen_for_distributions == ["3.14"] assert seen_for_lookup == [("3.14", False)] + def test_baseline_candidates_reach_the_lookup_of_their_provider(self, monkeypatch): + seen: dict[str, str | None] = {} + + def fake_lookup(distribution, python_version, allow_pre_releases, kept_pre_release): + seen[distribution] = kept_pre_release + return kept_pre_release or "9.0.0" + + monkeypatch.setattr( + m, + "get_all_active_provider_distributions", + lambda python_version=None: [ + "apache-airflow-providers-amazon", + "apache-airflow-providers-common-ai", + ], + ) + monkeypatch.setattr(m, "find_newest_version_in_pypi", fake_lookup) + + pins = m.build_pinned_provider_requirements( + "3.11", False, kept_pre_releases={"apache-airflow-providers-common-ai": "1.1.0rc1"} + ) + + assert seen == { + "apache-airflow-providers-amazon": None, + "apache-airflow-providers-common-ai": "1.1.0rc1", + } + assert pins == [ + "apache-airflow-providers-amazon==9.0.0", + "apache-airflow-providers-common-ai==1.1.0rc1", + ] + + def test_final_release_fails_on_a_candidate_neither_released_nor_yanked(self, monkeypatch, capsys): + seen_allow_pre_releases: list[bool] = [] + newest = { + "apache-airflow-providers-amazon": "9.39.0", + "apache-airflow-providers-common-ai": "1.1.0rc1", + "apache-airflow-providers-microsoft-azure": "15.3.0rc1", + } + + def fake_lookup(distribution, python_version, allow_pre_releases, kept_pre_release): + seen_allow_pre_releases.append(allow_pre_releases) + return newest[distribution] + + monkeypatch.setattr( + m, "get_all_active_provider_distributions", lambda python_version=None: list(newest) + ) + monkeypatch.setattr(m, "find_newest_version_in_pypi", fake_lookup) + + with pytest.raises(SystemExit) as exc_info: + m.build_pinned_provider_requirements("3.11", False, fail_on_pending_candidates=True) + + assert exc_info.value.code == 1 + assert set(seen_allow_pre_releases) == {True} + output = re.sub(r"\x1b\[[0-9;]*m", "", capsys.readouterr().out) + assert "apache-airflow-providers-common-ai==1.1.0rc1" in output + assert "apache-airflow-providers-microsoft-azure==15.3.0rc1" in output + assert "apache-airflow-providers-amazon" not in output + assert "release it" in output + assert "yank it" in output + + def test_final_release_passes_once_every_candidate_is_released_or_yanked(self, monkeypatch): + monkeypatch.setattr( + m, + "get_all_active_provider_distributions", + lambda python_version=None: ["apache-airflow-providers-common-ai"], + ) + monkeypatch.setattr( + m, + "find_newest_version_in_pypi", + lambda dist, python_version, allow_pre_releases, kept_pre_release: "1.1.0", + ) + + assert m.build_pinned_provider_requirements("3.11", False, fail_on_pending_candidates=True) == [ + "apache-airflow-providers-common-ai==1.1.0" + ] + class TestDownloadLatestConstraintFile: @staticmethod
