This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 681651d7e7c Add option --version-suffix-for-local (#43769)
681651d7e7c is described below
commit 681651d7e7c2a4521fda45736d80470069136974
Author: perry2of5 <[email protected]>
AuthorDate: Sun Nov 17 18:45:45 2024 -0800
Add option --version-suffix-for-local (#43769)
---
contributing-docs/11_provider_packages.rst | 53 ++++++++++++++++-----
...elease-management_prepare-provider-packages.svg | 52 +++++++++++++++------
...elease-management_prepare-provider-packages.txt | 2 +-
.../commands/release_management_commands.py | 21 +++++++--
.../commands/release_management_commands_config.py | 1 +
.../prepare_providers/provider_packages.py | 6 ++-
dev/breeze/src/airflow_breeze/utils/packages.py | 34 ++++++++++++--
.../src/airflow_breeze/utils/version_utils.py | 54 ++++++++++++++++++++++
8 files changed, 189 insertions(+), 34 deletions(-)
diff --git a/contributing-docs/11_provider_packages.rst
b/contributing-docs/11_provider_packages.rst
index f59d5a49c7d..1b70d91205d 100644
--- a/contributing-docs/11_provider_packages.rst
+++ b/contributing-docs/11_provider_packages.rst
@@ -118,29 +118,60 @@ in development mode - then capabilities of your provider
will be discovered by a
the provider among other providers in ``airflow providers`` command output.
-Local Development Release of a Specific Provider
+Local Release of a Specific Provider
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
When you develop a provider, you can release it locally and test it in your
Airflow environment. This should
-be accomplished using breeze. Choose a suffix for the release such as "dev1"
and run the breeze build for
+be accomplished using breeze. Choose a suffix for the release such as
"patch.asb.1" and run the breeze build for
that provider. Remember Provider IDs use a dot ('.') for directory separators
so the Provider ID for the
-Microsoft Azure provider is 'microsoft.azure'. This can be provided in the
PACKAGE_LIST environment variable
-or passed on the command line.
+Microsoft Azure provider is 'microsoft.azure'. The provider IDs to build can
be provided in the PACKAGE_LIST
+environment variable or passed on the command line.
-``export PACKAGE_LIST=microsoft.azure``
+.. code-block:: bash
+
+ export PACKAGE_LIST=microsoft.azure
Then build the provider (you don't need to pass the package ID if you set the
environment variable above):
-```bash
-breeze release-management prepare-provider-packages \
- --package-format both --version-suffix-for-pypi=dev1 \
- --skip-tag-check microsoft.azure
-```
+.. code-block:: bash
+
+ breeze release-management prepare-provider-packages \
+ --package-format both \
+ --version-suffix-for-local=patch.asb.1 \
+ microsoft.azure
+
Finally, copy the wheel file from the dist directory to the a directory your
airflow deployment can use.
If this is ~/airflow/test-airflow/local_providers, you can use the following
command:
-``cp
dist/apache_airflow_providers_microsoft_azure-10.5.2+dev1-py3-none-any.whl
~/airflow/test-airflow/local_providers/``
+``cp
dist/apache_airflow_providers_microsoft_azure-10.5.2+patch.asb.1-none-any.whl
~/airflow/test-airflow/local_providers/``
+
+If you want to build a local version of a version already released to PyPI,
such as rc1, then you can combine
+the PyPI suffix flag --version-suffix-for-pypi with the local suffix flag
--version-suffix-for-local. For example:
+
+.. code-block:: bash
+
+ breeze release-management prepare-provider-packages \
+ --package-format both \
+ --version-suffix-for-pypi rc1 \
+ --version-suffix-for-local=patch.asb.1 \
+ microsoft.azure
+
+
+The above would result in a wheel file
+
+
apache_airflow_providers_microsoft_azure-10.5.2rc1+patch.asb.1-py3-none-any.whl
+
+Builds using a local suffix will not check to see if a release has already
been made. This is useful for testing.
+
+Local versions can also be built using the version-suffix-for-pypi flag
although using the version-suffix-for-local
+flag is preferred. To build with the version-suffix-for-pypi flag, use the
following command:
+
+.. code-block:: bash
+
+ breeze release-management prepare-provider-packages \
+ --package-format both --version-suffix-for-pypi=dev1 \
+ --skip-tag-check microsoft.azure
Naming Conventions for provider packages
diff --git
a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg
b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg
index 3cd8a8a819c..3acf07c540c 100644
---
a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg
+++
b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1001.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 1148.0"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath
id="breeze-release-management-prepare-provider-packages-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="950.5999999999999" />
+ <rect x="0" y="0" width="1463.0" height="1097.0" />
</clipPath>
<clipPath id="breeze-release-management-prepare-provider-packages-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -159,9 +159,27 @@
<clipPath id="breeze-release-management-prepare-provider-packages-line-37">
<rect x="0" y="904.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-38">
+ <rect x="0" y="928.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-39">
+ <rect x="0" y="953.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-40">
+ <rect x="0" y="977.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-41">
+ <rect x="0" y="1001.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-42">
+ <rect x="0" y="1026.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-43">
+ <rect x="0" y="1050.7" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="999.6" rx="8"/><text
class="breeze-release-management-prepare-provider-packages-title"
fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: release-management prepare-provider-packages</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1146" rx="8"/><text
class="breeze-release-management-prepare-provider-packages-title"
fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: release-management prepare-provider-packages</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -199,17 +217,23 @@
</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="630" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="630" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">debugging and developing changes to the build process.    &
[...]
</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-26)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="654.4" textLength="195.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-26)">--skip-tag-check</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="654.4" [...]
</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-27)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="678.8" textLength="305"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-27)">--version-suffix-for-pypi</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8" y=
[...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="703.2" textLength="170.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">--package-list</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="703.2" t [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="727.6" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="727.6" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">documentation building, and document publishing. It is an easier 
[...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="752" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="752" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">adding individual packages as arguments to every command. This overri
[...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="776.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="776.4" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">packages passed as arguments.           
[...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="800.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-32)">│</text><text
class="breeze-release-management-prepare-provider-packages-r7" x="475.8"
y="800.8" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-32)">(TEXT)                 &#
[...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="825.2" textLength="1464"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-33)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="1464"
y="825.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-33)">
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="849.6" textLength="24.4"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-34)">╭─</text><text
class="breeze-release-management-prepare-provider-packages-r5" x="24.4"
y="849.6" textLength="195.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-34)"> Common options </text><text
class="breeze-release-management-prepare-provider-packages-r5" x=" [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="874" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-35)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="874" textLength="109.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-35)">--dry-run</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="874" textLength=" [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="898.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-36)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="898.4" textLength="109.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-36)">--verbose</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="898.4" textLe [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="922.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-37)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="922.8" textLength="73.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-37)">--help</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="922.8" textLength [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="947.2" textLength="1464"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-38)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="1464"
y="947.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-38)">
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="703.2" textLength="317.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">--version-suffix-for-local</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8" [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="727.6" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="727.6" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">The suffix must contain only ascii letters, numbers, and periods.
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="752" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="752" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">character must be an ascii letter or number and the last ch
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="776.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="776.4" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">ascii letter or number. Note: the local suffix will be appen
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="800.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-32)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="800.8" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-32)">suffix if both are provided.          &#
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="825.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-33)">│</text><text
class="breeze-release-management-prepare-provider-packages-r7" x="475.8"
y="825.2" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-33)">(TEXT)                 &#
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="849.6" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-34)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="849.6" textLength="170.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-34)">--package-list</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="849.6" t [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="874" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-35)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="874" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-35)">documentation building, and document publishing. It is an easier alte
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="898.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-36)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="898.4" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-36)">adding individual packages as arguments to every command. This ov
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="922.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-37)">│</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="475.8"
y="922.8" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-37)">packages passed as arguments.           
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="947.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-38)">│</text><text
class="breeze-release-management-prepare-provider-packages-r7" x="475.8"
y="947.2" textLength="963.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-38)">(TEXT)                 &#
[...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="971.6" textLength="1464"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-39)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="1464"
y="971.6" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-39)">
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="996" textLength="24.4"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-40)">╭─</text><text
class="breeze-release-management-prepare-provider-packages-r5" x="24.4"
y="996" textLength="195.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-40)"> Common options </text><text
class="breeze-release-management-prepare-provider-packages-r5" x="219. [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="1020.4" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-41)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="1020.4" textLength="109.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-41)">--dry-run</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="1020.4" tex [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="1044.8" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-42)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="1044.8" textLength="109.8"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-42)">--verbose</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="1044.8" tex [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="1069.2" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-43)">│</text><text
class="breeze-release-management-prepare-provider-packages-r4" x="24.4"
y="1069.2" textLength="73.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-43)">--help</text><text
class="breeze-release-management-prepare-provider-packages-r6" x="158.6"
y="1069.2" textLen [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5"
x="0" y="1093.6" textLength="1464"
clip-path="url(#breeze-release-management-prepare-provider-packages-line-44)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-prepare-provider-packages-r1" x="1464"
y="1093.6" textLength="12.2"
clip-path="url(#breeze-release-management-prepare-provider-packages-l [...]
</text>
</g>
</g>
diff --git
a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt
b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt
index 5caa5aa1be3..93b4263cd1a 100644
---
a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt
+++
b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt
@@ -1 +1 @@
-a781b53f55fe962ebab27068bcd96e44
+85b9590e16c1986679675b5e605844da
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 7f9228ff45a..394cf12014c 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -157,7 +157,12 @@ from airflow_breeze.utils.run_utils import (
run_command,
)
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
-from airflow_breeze.utils.version_utils import get_latest_airflow_version,
get_latest_helm_chart_version
+from airflow_breeze.utils.version_utils import (
+ create_package_version,
+ get_latest_airflow_version,
+ get_latest_helm_chart_version,
+ is_local_package_version,
+)
from airflow_breeze.utils.versions import is_pre_release
from airflow_breeze.utils.virtualenv_utils import create_pip_command,
create_venv
@@ -864,6 +869,15 @@ def basic_provider_checks(provider_package_id: str) ->
dict[str, Any]:
is_flag=True,
help="Skip checking if the tag already exists in the remote repository",
)
[email protected](
+ "--version-suffix-for-local",
+ default=None,
+ show_default=False,
+ help="Version suffix for local builds. Do not provide the leading plus
sign ('+'). The suffix must "
+ "contain only ascii letters, numbers, and periods. The first character
must be an ascii letter or number "
+ "and the last character must be an ascii letter or number. Note: the local
suffix will be appended after "
+ "the PyPi suffix if both are provided.",
+)
@click.option(
"--skip-deleting-generated-files",
default=False,
@@ -904,6 +918,7 @@ def prepare_provider_packages(
skip_deleting_generated_files: bool,
skip_tag_check: bool,
version_suffix_for_pypi: str,
+ version_suffix_for_local: str,
):
perform_environment_checks()
fix_ownership_using_docker()
@@ -926,7 +941,8 @@ def prepare_provider_packages(
include_removed=include_removed_providers,
include_not_ready=include_not_ready_providers,
)
- if not skip_tag_check:
+ package_version = create_package_version(version_suffix_for_pypi,
version_suffix_for_local)
+ if not skip_tag_check and not is_local_package_version(package_version):
run_command(["git", "remote", "rm", "apache-https-for-providers"],
check=False, stderr=DEVNULL)
make_sure_remote_apache_exists_and_fetch(github_repository=github_repository)
success_packages = []
@@ -939,7 +955,6 @@ def prepare_provider_packages(
shutil.rmtree(DIST_DIR, ignore_errors=True)
DIST_DIR.mkdir(parents=True, exist_ok=True)
for provider_id in packages_list:
- package_version = version_suffix_for_pypi
try:
basic_provider_checks(provider_id)
if not skip_tag_check:
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index b232666aba7..9cd77b823eb 100644
---
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -214,6 +214,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str
| list[str]]]] = {
"--skip-deleting-generated-files",
"--skip-tag-check",
"--version-suffix-for-pypi",
+ "--version-suffix-for-local",
"--package-list",
],
}
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
index ebe89402949..d007e914df2 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
@@ -39,6 +39,7 @@ from airflow_breeze.utils.packages import (
)
from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_SRC,
AIRFLOW_SOURCES_ROOT
from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.version_utils import is_local_package_version
LICENCE_RST = """
.. Licensed to the Apache Software Foundation (ASF) under one
@@ -161,8 +162,11 @@ def should_skip_the_package(provider_id: str,
version_suffix: str) -> tuple[bool
For RC and official releases we check if the "officially released" version
exists
and skip the released if it was. This allows to skip packages that have
not been
marked for release in this wave. For "dev" suffixes, we always build all
packages.
+ A local version of an RC release will always be built.
"""
- if version_suffix != "" and not version_suffix.startswith("rc"):
+ if version_suffix != "" and (
+ not version_suffix.startswith("rc") or
is_local_package_version(version_suffix)
+ ):
return False, version_suffix
if version_suffix == "":
current_tag = get_latest_provider_tag(provider_id, "")
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 9f1eae36938..427d4b5976a 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -48,6 +48,7 @@ from airflow_breeze.utils.publish_docs_helpers import (
get_provider_yaml_paths,
)
from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.version_utils import remove_local_version_suffix
from airflow_breeze.utils.versions import get_version_tag,
strip_leading_zeros_from_version
MIN_AIRFLOW_VERSION = "2.8.0"
@@ -420,7 +421,9 @@ def get_dist_package_name_prefix(provider_id: str) -> str:
def apply_version_suffix(install_clause: str, version_suffix: str) -> str:
- if install_clause.startswith("apache-airflow") and ">=" in install_clause
and version_suffix:
+ # Need to resolve a version suffix based on PyPi versions, but can ignore
local version suffix.
+ pypi_version_suffix = remove_local_version_suffix(version_suffix)
+ if pypi_version_suffix and install_clause.startswith("apache-airflow") and
">=" in install_clause:
# Applies version suffix to the apache-airflow and provider package
dependencies to make
# sure that pre-release versions have correct limits - this address
the issue with how
# pip handles pre-release versions when packages are pre-release and
refer to each other - we
@@ -429,6 +432,8 @@ def apply_version_suffix(install_clause: str,
version_suffix: str) -> str:
# For example `apache-airflow-providers-fab==2.0.0.dev0` should refer
to
# `apache-airflow>=2.9.0.dev0` and not `apache-airflow>=2.9.0` because
both packages are
# released together and >= 2.9.0 is not correct reference for
2.9.0.dev0 version of Airflow.
+ # This assumes a local release, one where the suffix starts with a
plus sign, uses the last
+ # version of the dependency, so it is not necessary to add the suffix
to the dependency.
prefix, version = install_clause.split(">=")
# If version has a upper limit (e.g. ">=2.10.0,<3.0"), we need to cut
this off not to fail
if "," in version:
@@ -437,9 +442,9 @@ def apply_version_suffix(install_clause: str,
version_suffix: str) -> str:
base_version = Version(version).base_version
# always use `pre-release`+ `0` as the version suffix
- version_suffix = version_suffix.rstrip("0123456789") + "0"
+ pypi_version_suffix = pypi_version_suffix.rstrip("0123456789") + "0"
- target_version = Version(str(base_version) + "." + version_suffix)
+ target_version = Version(str(base_version) + "." + pypi_version_suffix)
return prefix + ">=" + str(target_version)
return install_clause
@@ -590,6 +595,27 @@ def
get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]
return PROVIDER_DEPENDENCIES[provider_package_id]["cross-providers-deps"]
+def format_version_suffix(version_suffix: str) -> str:
+ """
+ Formats the version suffix by adding a dot prefix unless it is a local
prefix. If no version suffix is
+ passed in, an empty string is returned.
+
+ Args:
+ version_suffix (str): The version suffix to be formatted.
+
+ Returns:
+ str: The formatted version suffix.
+
+ """
+ if version_suffix:
+ if "." == version_suffix[0] or "+" == version_suffix[0]:
+ return version_suffix
+ else:
+ return f".{version_suffix}"
+ else:
+ return ""
+
+
def get_provider_jinja_context(
provider_id: str,
current_release_version: str,
@@ -609,7 +635,7 @@ def get_provider_jinja_context(
"FULL_PACKAGE_NAME": provider_details.full_package_name,
"RELEASE": current_release_version,
"RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros,
- "VERSION_SUFFIX": f".{version_suffix}" if version_suffix else "",
+ "VERSION_SUFFIX": format_version_suffix(version_suffix),
"PIP_REQUIREMENTS":
get_provider_requirements(provider_details.provider_id),
"PROVIDER_DESCRIPTION": provider_details.provider_description,
"INSTALL_REQUIREMENTS": get_install_requirements(
diff --git a/dev/breeze/src/airflow_breeze/utils/version_utils.py
b/dev/breeze/src/airflow_breeze/utils/version_utils.py
index 7b41fa46bdc..4ed887dc1ea 100644
--- a/dev/breeze/src/airflow_breeze/utils/version_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/version_utils.py
@@ -35,3 +35,57 @@ def get_latest_airflow_version():
response.raise_for_status()
latest_released_version = response.json()["info"]["version"]
return latest_released_version
+
+
+def create_package_version(version_suffix_for_pypi: str,
version_suffix_for_local: str) -> str:
+ """
+ Creates a package version by combining the version suffix for PyPI and the
version suffix for local. If
+ either one is an empty string, it is ignored. If the local suffix does not
have a leading plus sign,
+ the leading plus sign will be added.
+
+ Args:
+ version_suffix_for_pypi (str): The version suffix for PyPI.
+ version_suffix_for_local (str): The version suffix for local.
+
+ Returns:
+ str: The combined package version.
+
+ """
+ # if there is no local version suffix, return the PyPi version suffix
+ if not version_suffix_for_local:
+ return version_suffix_for_pypi
+
+ # ensure the local version suffix starts with a plus sign
+ if version_suffix_for_local[0] != "+":
+ version_suffix_for_local = "+" + version_suffix_for_local
+
+ # if there is a PyPi version suffix, return the combined version.
Otherwise just return the local version.
+ if version_suffix_for_pypi:
+ return version_suffix_for_pypi + version_suffix_for_local
+ else:
+ return version_suffix_for_local
+
+
+def remove_local_version_suffix(version_suffix: str) -> str:
+ if "+" in version_suffix:
+ return version_suffix.split("+")[0]
+ else:
+ return version_suffix
+
+
+def is_local_package_version(version_suffix: str) -> bool:
+ """
+ Check if the given version suffix is a local version suffix. A local
version suffix will contain a
+ plus sign ('+'). This function does not guarantee that the version suffix
is a valid local version suffix.
+
+ Args:
+ version_suffix (str): The version suffix to check.
+
+ Returns:
+ bool: True if the version suffix contains a '+', False otherwise.
Please note this does not
+ guarantee that the version suffix is a valid local version suffix.
+ """
+ if version_suffix and ("+" in version_suffix):
+ return True
+ else:
+ return False
