This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 923c798d1bf [v3-4-test] Let a final release override the
pending-provider-candidate guard (#76592) (#76597)
923c798d1bf is described below
commit 923c798d1bfee0d7521f3aeecb68a4a244bb7a05
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Nov 6 15:50:58 2026 +0000
[v3-4-test] Let a final release override the pending-provider-candidate
guard (#76592) (#76597)
* Let a final release override the pending-provider-candidate guard
A final release-constraints run fails when a provider's newest PyPI version
is a
pending rc, so a release never ships constraints that silently drop its own
provider wave (#75919). But an independent provider wave can overlap a core
release - e.g. an accelerated release landing during a wave of breaking
provider
majors it was never tested against - and there the release should pin the
currently-released providers and ship.
Expose fail-on-pending-provider-candidates as an optional override on the
release-constraints workflow. start-release asks before resolving, and the
standalone 'breeze workflow-run release-constraints' takes the flag. Default
behaviour is unchanged: a final still fails on a pending candidate unless
the RM
opts out, and the constraints never pin a release candidate either way.
* Simplify the override prompt and fix its tests
The default (no) keeps the guard; yes pins the currently-released
providers. Mock
the prompt in the regenerate_constraints tests and cover both answers.
* Only prompt for the override when a provider candidate is actually pending
Check PyPI for providers whose newest version is a pending rc, and offer the
override only when the final constraints run would actually fail - no
prompt on a
normal release. Addresses review feedback. Detection reused from #76593.
* handling review comments
---------
(cherry picked from commit e0b7126d57325d35ec8acaacb1a5b22b1ac0b9c4)
Co-authored-by: Rahul Vats <[email protected]>
Co-authored-by: Claude <[email protected]>
Co-authored-by: Amogh Desai <[email protected]>
Co-authored-by: Amogh Desai <[email protected]>
---
.github/workflows/release-constraints.yml | 16 ++++-
dev/README_RELEASE_AIRFLOW.md | 16 +++++
.../output_workflow-run_release-constraints.svg | 52 ++++++++++-----
.../output_workflow-run_release-constraints.txt | 2 +-
.../src/airflow_breeze/commands/release_command.py | 37 +++++++++--
.../airflow_breeze/commands/workflow_commands.py | 14 +++-
.../commands/workflow_commands_config.py | 1 +
.../airflow_breeze/utils/release_constraints.py | 67 ++++++++++++++++++-
dev/breeze/tests/test_release_command.py | 77 +++++++++++++++++++++-
dev/breeze/tests/test_release_constraints.py | 65 ++++++++++++++++++
dev/breeze/tests/test_workflow_commands.py | 21 ++++++
11 files changed, 342 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/release-constraints.yml
b/.github/workflows/release-constraints.yml
index cceb80cd5ee..78e71dc6dbf 100644
--- a/.github/workflows/release-constraints.yml
+++ b/.github/workflows/release-constraints.yml
@@ -43,6 +43,14 @@ on: # yamllint disable-line rule:truthy
description: "Ref the constraints are resolved from, e.g. v3-3-stable
or the release tag"
required: true
type: string
+ fail-on-pending-provider-candidates:
+ description: >-
+ Override the final-release guard. Empty derives it from the version
(a final fails when a
+ provider's newest PyPI version is a pending rc; a pre-release does
not). Set to 'false' to
+ pin the currently-released providers and proceed through an
overlapping provider wave.
+ required: false
+ default: ""
+ type: string
permissions:
contents: read
concurrency:
@@ -187,9 +195,13 @@ jobs:
# the constraints, which is exactly the version the release needed
pinned.
pypi-providers-only: "true"
allow-pre-releases: ${{ needs.build-info.outputs.allow-pre-releases }}
- # A final must not be cut while a provider candidate is still neither
released nor yanked.
+ # A final must not be cut while a provider candidate is still neither
released nor yanked -
+ # unless the release manager overrides it to pin the currently-released
providers instead (for
+ # a final that deliberately ships past an overlapping, independent
provider wave).
fail-on-pending-provider-candidates: >-
- ${{ needs.build-info.outputs.allow-pre-releases == 'true' && 'false'
|| 'true' }}
+ ${{ inputs.fail-on-pending-provider-candidates != ''
+ && inputs.fail-on-pending-provider-candidates
+ || (needs.build-info.outputs.allow-pre-releases == 'true' && 'false'
|| 'true') }}
debug-resources: "false"
checkout-ref: ${{ inputs.ref }}
use-uv: "true"
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 506dd7ee8d8..b8e8bc9b371 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -1523,6 +1523,22 @@ curl -s
"https://raw.githubusercontent.com/apache/airflow/constraints-${VERSION_
Agree with the providers' release manager on the decision early in the vote,
so the final
release does not wait on it.
+If the provider wave is independent of this release and you deliberately want
to ship the final
+against the currently-released providers - for example an accelerated core
release that overlaps a
+wave carrying breaking provider majors the release was not tested against -
you can override the
+guard. ``start-release`` checks for pending provider candidates before
anything is uploaded and
+asks whether to proceed; answer ``y`` to pin the currently-released providers,
or ``n`` to stop the
+release until the candidates are released or yanked. The standalone workflow
takes the same override:
+
+```shell script
+breeze workflow-run release-constraints --version ${VERSION} --ref v3-1-stable
\
+ --fail-on-pending-provider-candidates false
+```
+
+The constraints still never pin a release candidate; the override only lets
the run pin the newest
+*final* of each provider instead of stopping. The pending candidates are
simply left out, exactly
+as they were before they were cut.
+
## Summarize the voting for the Apache Airflow release
Once the vote has been passed, you will need to send a result vote to
[email protected]:
diff --git a/dev/breeze/doc/images/output_workflow-run_release-constraints.svg
b/dev/breeze/doc/images/output_workflow-run_release-constraints.svg
index e20e2d2696b..fbc946f168f 100644
--- a/dev/breeze/doc/images/output_workflow-run_release-constraints.svg
+++ b/dev/breeze/doc/images/output_workflow-run_release-constraints.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 513.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 635.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -45,7 +45,7 @@
<defs>
<clipPath id="breeze-workflow-run-release-constraints-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="462.59999999999997" />
+ <rect x="0" y="0" width="1463.0" height="584.5999999999999" />
</clipPath>
<clipPath id="breeze-workflow-run-release-constraints-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -101,9 +101,24 @@
<clipPath id="breeze-workflow-run-release-constraints-line-17">
<rect x="0" y="416.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-workflow-run-release-constraints-line-18">
+ <rect x="0" y="440.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-release-constraints-line-19">
+ <rect x="0" y="465.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-release-constraints-line-20">
+ <rect x="0" y="489.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-release-constraints-line-21">
+ <rect x="0" y="513.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-release-constraints-line-22">
+ <rect x="0" y="538.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="511.6" rx="8"/><text
class="breeze-workflow-run-release-constraints-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: workflow-run release-constraints</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="633.6" rx="8"/><text
class="breeze-workflow-run-release-constraints-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: workflow-run release-constraints</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -119,19 +134,24 @@
</text><text class="breeze-workflow-run-release-constraints-r1" x="12.2"
y="93.2" textLength="1037"
clip-path="url(#breeze-workflow-run-release-constraints-line-3)">Trigger the workflow that resolves, publishes and tags the constraints for a release.</text><text
class="breeze-workflow-run-release-constraints-r1" x="1464" y="93.2"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-3)">
</text><text class="breeze-workflow-run-release-constraints-r1" x="1464"
y="117.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-4)">
</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="142"
textLength="24.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-5)">╭─</text><text
class="breeze-workflow-run-release-constraints-r5" x="24.4" y="142"
textLength="292.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-5)"> Constraints to resolve </text><text
class="breeze-workflow-run-release-constraints-r5" x="317.2" y="142"
textLength="1122.4" clip-path="url(#br [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="166.4" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-6)">│</text><text
class="breeze-workflow-run-release-constraints-r6" x="24.4" y="166.4"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-6)">*</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="166.4"
textLength="207.4"
clip-path="url(#breeze-workflow-run-release-constraints-line- [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="190.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-7)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="292.8" y="190.8"
textLength="1146.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-7)">and lands on a branch of its own; a final (3.1.3) resolves without them and commits&
[...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-8)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="292.8" y="215.2"
textLength="1146.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-8)">constraints-X-Y. The stage is derived from this, so it cannot be set inconsistently.  
[...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-9)">│</text><text
class="breeze-workflow-run-release-constraints-r7" x="292.8" y="239.6"
textLength="122"
clip-path="url(#breeze-workflow-run-release-constraints-line-9)">[required]</text><text
class="breeze-workflow-run-release-constraints-r8" x="427" y="239.6"
textLength="73.2" clip-path="url(#breeze-workflow-run-release-constrai [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="264"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)">│</text><text
class="breeze-workflow-run-release-constraints-r6" x="24.4" y="264"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)">*</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="264"
textLength="207.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)" [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-11)">│</text><text
class="breeze-workflow-run-release-constraints-r8" x="292.8" y="288.4"
textLength="73.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-11)">(TEXT)</text><text
class="breeze-workflow-run-release-constraints-r5" x="1451.8" y="288.4"
textLength="12.2" clip-path="url(#breeze-workflow-run-release-constr [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-12)">│</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="312.8"
textLength="207.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-12)">--workflow-branch</text><text
class="breeze-workflow-run-release-constraints-r1" x="292.8" y="312.8"
textLength="1146.8" clip-path="url(#breeze-workflow-run-rele [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-13)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="292.8" y="337.2"
textLength="1146.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-13)">want: unlike the docs build, the constraints do not have to be produced by the wor
[...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-14)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="292.8" y="361.6"
textLength="402.6"
clip-path="url(#breeze-workflow-run-release-constraints-line-14)">stood at the ref being released. </text><text
class="breeze-workflow-run-release-constraints-r5" x="695.4" y="361.6"
textLength= [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="386"
textLength="1464"
clip-path="url(#breeze-workflow-run-release-constraints-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-release-constraints-r1" x="1464" y="386"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-15)">
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="410.4" textLength="24.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-16)">╭─</text><text
class="breeze-workflow-run-release-constraints-r5" x="24.4" y="410.4"
textLength="195.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-16)"> Common options </text><text
class="breeze-workflow-run-release-constraints-r5" x="219.6" y="410.4"
textLength="1220" clip-path="url(#breeze-wo [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-17)">│</text><text
class="breeze-workflow-run-release-constraints-r4" x="24.4" y="434.8"
textLength="73.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-17)">--help</text><text
class="breeze-workflow-run-release-constraints-r9" x="122" y="434.8"
textLength="24.4" clip-path="url(#breeze-workflow-run-release-constraint [...]
-</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="459.2" textLength="1464"
clip-path="url(#breeze-workflow-run-release-constraints-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-release-constraints-r1" x="1464" y="459.2"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-18)">
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="166.4" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-6)">│</text><text
class="breeze-workflow-run-release-constraints-r6" x="24.4" y="166.4"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-6)">*</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="166.4"
textLength="451.4"
clip-path="url(#breeze-workflow-run-release-constraints-line- [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="190.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-7)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="190.8"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-7)">pre-releases allowed and lands on a branch of its own; a final (3.1.3)    </text
[...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-8)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="215.2"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-8)">resolves without them and commits onto constraints-X-Y. The stage is      </text><tex
[...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-9)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="239.6"
textLength="671"
clip-path="url(#breeze-workflow-run-release-constraints-line-9)">derived from this, so it cannot be set inconsistently. </text><text
class="breeze-workflow-run-release-constraints-r7 [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="264"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)">│</text><text
class="breeze-workflow-run-release-constraints-r6" x="24.4" y="264"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)">*</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="264"
textLength="451.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-10)" [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-11)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="288.4"
textLength="158.6"
clip-path="url(#breeze-workflow-run-release-constraints-line-11)">release tag. </text><text
class="breeze-workflow-run-release-constraints-r7" x="695.4" y="288.4"
textLength="122" clip-path="url(#breeze-workflow-ru [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-12)">│</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="312.8"
textLength="451.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-12)">--workflow-branch                    </text><text
class="bree [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-13)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="337.2"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-13)">normally what you want: unlike the docs build, the constraints do not have</text><text
class="bree [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-14)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="361.6"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-14)">to be produced by the workflow as it stood at the ref being released.    
[...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="386"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-15)">│</text><text
class="breeze-workflow-run-release-constraints-r5" x="536.8" y="386"
textLength="183"
clip-path="url(#breeze-workflow-run-release-constraints-line-15)">[default: main]</text><text
class="breeze-workflow-run-release-constraints-r8" x="732" y="386"
textLength="73.2" clip-path="url(#breeze-workflow-run-release-co [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-16)">│</text><text
class="breeze-workflow-run-release-constraints-r4" x="61" y="410.4"
textLength="451.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-16)">--fail-on-pending-provider-candidates</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="410.4"
textLength="902.8" clip-path="url(#breez [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-17)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="434.8"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-17)">final fails when a provider's newest PyPI version is a pending rc). Set   </te
[...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="459.2" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-18)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="459.2"
textLength="902.8"
clip-path="url(#breeze-workflow-run-release-constraints-line-18)">'false' to pin the currently-released providers and proceed through an    </text><t
[...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="483.6" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-19)">│</text><text
class="breeze-workflow-run-release-constraints-r1" x="536.8" y="483.6"
textLength="329.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-19)">overlapping provider wave. </text><text
class="breeze-workflow-run-release-constraints-r8" x="866.2" y="483.6"
textLength="146.4" clip-path="ur [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0" y="508"
textLength="1464"
clip-path="url(#breeze-workflow-run-release-constraints-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-release-constraints-r1" x="1464" y="508"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-20)">
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="532.4" textLength="24.4"
clip-path="url(#breeze-workflow-run-release-constraints-line-21)">╭─</text><text
class="breeze-workflow-run-release-constraints-r5" x="24.4" y="532.4"
textLength="195.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-21)"> Common options </text><text
class="breeze-workflow-run-release-constraints-r5" x="219.6" y="532.4"
textLength="1220" clip-path="url(#breeze-wo [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="556.8" textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-22)">│</text><text
class="breeze-workflow-run-release-constraints-r4" x="24.4" y="556.8"
textLength="73.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-22)">--help</text><text
class="breeze-workflow-run-release-constraints-r9" x="122" y="556.8"
textLength="24.4" clip-path="url(#breeze-workflow-run-release-constraint [...]
+</text><text class="breeze-workflow-run-release-constraints-r5" x="0"
y="581.2" textLength="1464"
clip-path="url(#breeze-workflow-run-release-constraints-line-23)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-release-constraints-r1" x="1464" y="581.2"
textLength="12.2"
clip-path="url(#breeze-workflow-run-release-constraints-line-23)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_workflow-run_release-constraints.txt
b/dev/breeze/doc/images/output_workflow-run_release-constraints.txt
index 871ae3667b4..a8be1202c38 100644
--- a/dev/breeze/doc/images/output_workflow-run_release-constraints.txt
+++ b/dev/breeze/doc/images/output_workflow-run_release-constraints.txt
@@ -1 +1 @@
-14273c8c9c667e0004c0a1379d53da58
+a692fbae91ea1080aa71e1e6936a6d94
diff --git a/dev/breeze/src/airflow_breeze/commands/release_command.py
b/dev/breeze/src/airflow_breeze/commands/release_command.py
index ffa97a01f63..c6a9701ec08 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_command.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_command.py
@@ -29,7 +29,10 @@ from airflow_breeze.utils.confirm import confirm_action
from airflow_breeze.utils.console import console_print
from airflow_breeze.utils.environment_check import is_ci_environment
from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH
-from airflow_breeze.utils.release_constraints import publish_constraints
+from airflow_breeze.utils.release_constraints import (
+ find_pending_provider_candidates,
+ publish_constraints,
+)
from airflow_breeze.utils.run_utils import run_command
# Pattern to match Airflow release versions (e.g., "3.0.5")
@@ -344,12 +347,36 @@ def upload_to_pypi(version, task_sdk_version=None):
)
-def regenerate_constraints(version):
+def decide_pending_candidates_override() -> bool | None:
+ """Settle the pending-provider-candidate guard before anything
irreversible happens.
+
+ The constraints are resolved after the PyPI upload, so a guard that trips
there leaves the
+ release published without constraints. Returns ``False`` to pin the
currently-released providers,
+ or ``None`` to let the workflow derive the guard when nothing is pending.
+ """
+ pending = find_pending_provider_candidates()
+ if not pending:
+ return None
+ console_print("[warning]These providers have a release candidate on PyPI
newer than their latest final:")
+ for distribution, candidate in pending:
+ console_print(f"[warning] * {distribution}=={candidate}")
+ if confirm_action(
+ "Pin the currently-released providers and proceed even though a
provider wave is still in rc?"
+ ):
+ return False
+ exit("Release or yank the pending provider candidates, then start the
release again.")
+
+
+def regenerate_constraints(version, fail_on_pending_provider_candidates: bool
| None = None):
# The candidate's constraints pinned pre-releases, so they cannot be
promoted by retagging - a
# released version must never point at rc pins. The workflow reads a
version without "rcN" and
# resolves again without them, against the providers now published as
finals.
major, minor = version.split(".")[:2]
- publish_constraints(version=version, ref=f"v{major}-{minor}-stable")
+ publish_constraints(
+ version=version,
+ ref=f"v{major}-{minor}-stable",
+
fail_on_pending_provider_candidates=fail_on_pending_provider_candidates,
+ )
def tag_and_push_latest_constraint(version):
@@ -446,6 +473,8 @@ def airflow_release(version, task_sdk_version):
# Final confirmation
confirm_action("Pushes will be made to origin. Do you want to continue?",
abort=True)
+ fail_on_pending_provider_candidates = decide_pending_candidates_override()
+
# Clone the asf repo
os.chdir("..")
working_dir = os.getcwd()
@@ -536,7 +565,7 @@ def airflow_release(version, task_sdk_version):
os.chdir(airflow_repo_root)
# Regenerate, publish and tag the constraints for the final version
- regenerate_constraints(version)
+ regenerate_constraints(version, fail_on_pending_provider_candidates)
tag_and_push_latest_constraint(version)
# Push tag for final version
diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
index a1769f5a6cc..dcbf5d3f640 100644
--- a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
@@ -481,18 +481,30 @@ def check_providers_are_in_site_index(doc_packages:
tuple[str, ...], exclude_doc
default="main",
show_default=True,
)
-def workflow_run_release_constraints(version: str, ref: str, workflow_branch:
str):
[email protected](
+ "--fail-on-pending-provider-candidates",
+ type=click.Choice(["true", "false"]),
+ default=None,
+ help="Override the final-release guard. Unset derives it from the version
(a final fails when a "
+ "provider's newest PyPI version is a pending rc). Set 'false' to pin the
currently-released "
+ "providers and proceed through an overlapping provider wave.",
+)
+def workflow_run_release_constraints(
+ version: str, ref: str, workflow_branch: str,
fail_on_pending_provider_candidates: str | None
+):
if not RELEASE_VERSION_PATTERN.match(version):
console_print(f"[red]Error: '{version}' is not a release version -
expected X.Y.Z or X.Y.ZrcN.[/red]")
sys.exit(1)
stage = "candidate" if "rc" in version else "final"
console_print(f"[blue]Triggering constraints generation for the {stage}
{version} from {ref}[/blue]")
+ overrides = {"fail-on-pending-provider-candidates":
fail_on_pending_provider_candidates or ""}
trigger_workflow_and_monitor(
workflow_name=WORKFLOW_NAME_MAPS["release-constraints"],
repo=APACHE_AIRFLOW_REPO,
branch=workflow_branch,
version=version,
ref=ref,
+ **overrides,
)
diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
index 5b4eb705780..3b8e9047489 100644
--- a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
@@ -69,6 +69,7 @@ WORKFLOW_RUN_PARAMETERS["breeze workflow-run
release-constraints"] = [
"--version",
"--ref",
"--workflow-branch",
+ "--fail-on-pending-provider-candidates",
],
},
]
diff --git a/dev/breeze/src/airflow_breeze/utils/release_constraints.py
b/dev/breeze/src/airflow_breeze/utils/release_constraints.py
index 2565e430608..e4544ad8202 100644
--- a/dev/breeze/src/airflow_breeze/utils/release_constraints.py
+++ b/dev/breeze/src/airflow_breeze/utils/release_constraints.py
@@ -28,15 +28,67 @@ separate switch that could disagree with it.
from __future__ import annotations
+from concurrent.futures import ThreadPoolExecutor
+
from airflow_breeze.utils.confirm import confirm_action
from airflow_breeze.utils.console import console_print
from airflow_breeze.utils.gh_workflow_utils import trigger_workflow_and_monitor
RELEASE_CONSTRAINTS_WORKFLOW = "release-constraints.yml"
APACHE_AIRFLOW_REPO = "apache/airflow"
+PYPI_JSON_API_URL = "https://pypi.org/pypi/{distribution}/json";
+PYPI_LOOKUP_PARALLELISM = 16
+
+
+def _find_newest_published_version(distribution: str) -> str | None:
+ """Newest non-yanked version of ``distribution`` on PyPI, candidates
included."""
+ import requests
+ from packaging.version import InvalidVersion, Version
+
+ response =
requests.get(PYPI_JSON_API_URL.format(distribution=distribution), timeout=60)
+ if response.status_code == 404:
+ return None
+ response.raise_for_status()
+ newest: Version | None = None
+ for version, files in response.json().get("releases", {}).items():
+ try:
+ parsed = Version(version)
+ except InvalidVersion:
+ continue
+ if not any(not pypi_file.get("yanked") for pypi_file in files):
+ continue
+ if newest is None or parsed > newest:
+ newest = parsed
+ return str(newest) if newest else None
+
+def find_pending_provider_candidates() -> list[tuple[str, str]]:
+ """Providers whose newest PyPI version is a candidate - what the final
constraints run refuses."""
+ from packaging.version import Version
+
+ from airflow_breeze.utils.provider_dependencies import
get_provider_dependencies
+
+ distributions = [
+ f"apache-airflow-providers-{provider_id.replace('.', '-')}"
+ for provider_id, metadata in
sorted(get_provider_dependencies().items())
+ if metadata["state"] == "ready"
+ ]
+ with ThreadPoolExecutor(max_workers=PYPI_LOOKUP_PARALLELISM) as executor:
+ newest_versions = list(executor.map(_find_newest_published_version,
distributions))
+ return [
+ (distribution, version)
+ for distribution, version in zip(distributions, newest_versions)
+ if version is not None and Version(version).is_prerelease
+ ]
-def publish_constraints(*, version: str, ref: str, workflow_branch: str =
"main") -> None:
+
+def publish_constraints(
+ *,
+ version: str,
+ ref: str,
+ workflow_branch: str = "main",
+ fail_on_pending_provider_candidates: bool | None = None,
+) -> None:
"""Resolve, publish and tag the constraints belonging to ``version``.
``version`` alone decides what happens: a candidate (``3.3.1rc1``) pins
the providers at the
@@ -45,6 +97,11 @@ def publish_constraints(*, version: str, ref: str,
workflow_branch: str = "main"
the same candidate replaces that branch and its tag, so a wave can be
re-cut. A final
(``3.3.1``) resolves against the published releases and commits onto
``constraints-X-Y``, which
is what makes the released constraints the baseline everything downstream
reads.
+
+ ``fail_on_pending_provider_candidates`` overrides the workflow's own
derivation. Left ``None``
+ the workflow derives it from the version (a final fails when a provider's
newest PyPI version is
+ a pending rc); ``False`` pins the currently-released providers and
proceeds, for a final that
+ deliberately ships past an overlapping, independent provider wave.
"""
from packaging.version import Version
@@ -60,11 +117,19 @@ def publish_constraints(*, version: str, ref: str,
workflow_branch: str = "main"
):
return
console_print(f"[info]Resolving constraints for {version} from {ref} -
this runs on CI runners.")
+ # The workflow input name is hyphenated, so it cannot be a keyword
argument; "" lets the
+ # workflow derive the guard from the version, which is the default when no
override is given.
+ overrides = {
+ "fail-on-pending-provider-candidates": ""
+ if fail_on_pending_provider_candidates is None
+ else str(fail_on_pending_provider_candidates).lower()
+ }
trigger_workflow_and_monitor(
workflow_name=RELEASE_CONSTRAINTS_WORKFLOW,
repo=APACHE_AIRFLOW_REPO,
branch=workflow_branch,
version=version,
ref=ref,
+ **overrides,
)
console_print(f"[success]Constraints for {version} published and tagged
'constraints-{version}'")
diff --git a/dev/breeze/tests/test_release_command.py
b/dev/breeze/tests/test_release_command.py
index c1d89193e4f..5becca58768 100644
--- a/dev/breeze/tests/test_release_command.py
+++ b/dev/breeze/tests/test_release_command.py
@@ -547,7 +547,11 @@ def
test_regenerate_constraints_resolves_from_the_stable_branch(monkeypatch, rel
release_cmd.regenerate_constraints("3.3.0")
- assert calls == {"version": "3.3.0", "ref": "v3-3-stable"}
+ assert calls == {
+ "version": "3.3.0",
+ "ref": "v3-3-stable",
+ "fail_on_pending_provider_candidates": None,
+ }
def test_regenerate_constraints_passes_a_final_version(monkeypatch,
release_cmd):
@@ -559,3 +563,74 @@ def
test_regenerate_constraints_passes_a_final_version(monkeypatch, release_cmd)
assert "rc" not in str(calls["version"])
assert calls["ref"] == "v3-0-stable"
+
+
[email protected]("override", [None, False])
+def test_regenerate_constraints_passes_the_override_through(monkeypatch,
release_cmd, override):
+ calls: dict[str, object] = {}
+ monkeypatch.setattr(release_cmd, "publish_constraints", lambda **kw:
calls.update(kw))
+
+ release_cmd.regenerate_constraints("3.3.0", override)
+
+ assert calls["fail_on_pending_provider_candidates"] is override
+
+
+def
test_decide_override_does_not_prompt_without_pending_candidates(monkeypatch,
release_cmd):
+ monkeypatch.setattr(release_cmd, "find_pending_provider_candidates",
lambda: [])
+
+ def fail_if_called(*a, **k):
+ raise AssertionError("confirm_action must not be called when nothing
is pending")
+
+ monkeypatch.setattr(release_cmd, "confirm_action", fail_if_called)
+
+ assert release_cmd.decide_pending_candidates_override() is None
+
+
+def test_decide_override_pins_released_providers_when_accepted(monkeypatch,
release_cmd):
+ monkeypatch.setattr(
+ release_cmd,
+ "find_pending_provider_candidates",
+ lambda: [("apache-airflow-providers-standard", "2.0.0rc1")],
+ )
+ monkeypatch.setattr(release_cmd, "confirm_action", lambda *a, **k: True)
+
+ assert release_cmd.decide_pending_candidates_override() is False
+
+
+def test_decide_override_stops_the_release_when_declined(monkeypatch,
release_cmd):
+ """Keeping the guard on would only fail after the PyPI upload, so
declining stops here."""
+ monkeypatch.setattr(
+ release_cmd,
+ "find_pending_provider_candidates",
+ lambda: [("apache-airflow-providers-standard", "2.0.0rc1")],
+ )
+ monkeypatch.setattr(release_cmd, "confirm_action", lambda *a, **k: False)
+
+ with pytest.raises(SystemExit):
+ release_cmd.decide_pending_candidates_override()
+
+
+class _StopRelease(Exception):
+ pass
+
+
+def test_airflow_release_decides_override_before_touching_svn(monkeypatch,
release_cmd):
+ """The override is settled before the first irreversible step, not after
the PyPI upload."""
+ calls: list[str] = []
+ monkeypatch.setattr(release_cmd, "confirm_action", lambda *a, **kw: True)
+ monkeypatch.setattr(release_cmd, "run_command", lambda *a, **kw: None)
+ monkeypatch.setattr(release_cmd.os, "chdir", lambda *a: None)
+ monkeypatch.setattr(
+ release_cmd, "decide_pending_candidates_override", lambda:
calls.append("override") or None
+ )
+
+ def stop_at_clone(*a, **kw):
+ calls.append("clone")
+ raise _StopRelease
+
+ monkeypatch.setattr(release_cmd, "clone_asf_repo", stop_at_clone)
+
+ with pytest.raises(_StopRelease):
+ release_cmd.airflow_release.callback("3.4.0", None)
+
+ assert calls == ["override", "clone"]
diff --git a/dev/breeze/tests/test_release_constraints.py
b/dev/breeze/tests/test_release_constraints.py
index 84e6b2144a9..13e61664940 100644
--- a/dev/breeze/tests/test_release_constraints.py
+++ b/dev/breeze/tests/test_release_constraints.py
@@ -16,7 +16,10 @@
# under the License.
from __future__ import annotations
+from unittest import mock
+
import pytest
+import requests
from airflow_breeze.utils import release_constraints
@@ -90,3 +93,65 @@ def
test_the_prompt_names_the_stage_the_version_implies(monkeypatch, version, ex
release_constraints.publish_constraints(version=version, ref="v3-1-stable")
assert expected_stage in prompts[0]
+
+
[email protected](
+ ("override", "expected"),
+ [
+ pytest.param(None, "", id="derive-from-version"),
+ pytest.param(False, "false", id="pin-released-providers"),
+ pytest.param(True, "true", id="force-guard"),
+ ],
+)
+def test_override_is_passed_to_the_workflow_input(triggered, override,
expected):
+ release_constraints.publish_constraints(
+ version="3.4.0", ref="v3-4-stable",
fail_on_pending_provider_candidates=override
+ )
+
+ assert triggered[0]["fail-on-pending-provider-candidates"] == expected
+
+
[email protected](
+ ("releases", "expected"),
+ [
+ pytest.param({"1.0.0": [{}], "2.0.0rc1": [{}]}, "2.0.0rc1",
id="candidate-newest"),
+ pytest.param({"1.0.0rc1": [{}], "1.0.0": [{}]}, "1.0.0",
id="candidate-released"),
+ pytest.param({"1.0.0": [{}], "2.0.0rc1": [{"yanked": True}]}, "1.0.0",
id="candidate-yanked"),
+ pytest.param({"1.0.0": [{}], "2.0.0": []}, "1.0.0", id="no-files"),
+ pytest.param({"not-a-version": [{}]}, None, id="nothing-usable"),
+ ],
+)
[email protected](requests, "get", autospec=True)
+def test_find_newest_published_version(mock_get, releases, expected):
+ mock_get.return_value.status_code = 200
+ mock_get.return_value.json.return_value = {"releases": releases}
+
+ assert
release_constraints._find_newest_published_version("apache-airflow-providers-x")
== expected
+
+
[email protected](requests, "get", autospec=True)
+def test_find_newest_published_version_unpublished(mock_get):
+ mock_get.return_value.status_code = 404
+
+ assert
release_constraints._find_newest_published_version("apache-airflow-providers-x")
is None
+
+
[email protected]("airflow_breeze.utils.provider_dependencies.get_provider_dependencies",
autospec=True)
+def
test_find_pending_provider_candidates_checks_only_ready_providers(mock_deps,
monkeypatch):
+ mock_deps.return_value = {
+ "amazon": {"state": "ready"},
+ "apache.hive": {"state": "ready"},
+ "google": {"state": "ready"},
+ "old": {"state": "suspended"},
+ "new": {"state": "not-ready"},
+ }
+ newest = {
+ "apache-airflow-providers-amazon": "10.0.0rc1",
+ "apache-airflow-providers-apache-hive": "9.0.0",
+ "apache-airflow-providers-google": None,
+ }
+ monkeypatch.setattr(release_constraints, "_find_newest_published_version",
newest.__getitem__)
+
+ assert release_constraints.find_pending_provider_candidates() == [
+ ("apache-airflow-providers-amazon", "10.0.0rc1")
+ ]
diff --git a/dev/breeze/tests/test_workflow_commands.py
b/dev/breeze/tests/test_workflow_commands.py
index 36c81673eb8..2170356ceac 100644
--- a/dev/breeze/tests/test_workflow_commands.py
+++ b/dev/breeze/tests/test_workflow_commands.py
@@ -30,6 +30,7 @@ from airflow_breeze.commands.workflow_commands import (
get_lines_added_to_site_index,
workflow_run_mirror_base_images,
workflow_run_publish,
+ workflow_run_release_constraints,
workflow_run_sync_staging_to_main,
workflow_run_upgrade_check,
)
@@ -446,3 +447,23 @@ def
test_upgrade_check_offers_only_branches_with_an_upgrade_workflow(mock_trigge
assert result.exit_code == 2
mock_trigger.assert_not_called()
+
+
[email protected](
+ ("args", "expected"),
+ [
+ pytest.param([], "", id="unset"),
+ pytest.param(["--fail-on-pending-provider-candidates", "false"],
"false", id="false"),
+ pytest.param(["--fail-on-pending-provider-candidates", "true"],
"true", id="true"),
+ ],
+)
+@patch("airflow_breeze.commands.workflow_commands.trigger_workflow_and_monitor",
autospec=True)
+def test_release_constraints_passes_the_override_flag(mock_trigger, args,
expected):
+ result = CliRunner().invoke(
+ workflow_run_release_constraints,
+ ["--version", "3.4.0", "--ref", "v3-4-stable", *args],
+ catch_exceptions=False,
+ )
+
+ assert result.exit_code == 0
+ assert
mock_trigger.call_args.kwargs["fail-on-pending-provider-candidates"] == expected
