This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d8537857036a36dcfda171395092f932770d6dda Author: Jarek Potiuk <[email protected]> AuthorDate: Wed Dec 27 09:05:54 2023 +0100 Automate rcN calculation when releasing provider packages (#36441) This change will automatically generate the **right** rcN package when prepareing the packages for PyPI. This allows to have pretty much continuous release process for voting over the provider packages. Simply when an rcN candidate is not released, it will be automatically included in the next wave of packages with rcN+1 version - unless during provider package generation the version will be bumped to MAJOR or MINOR due to new changes. This allows for the workflow where in every new wave we always generate all provider packages ready for release. (cherry picked from commit 6f5a50ea10842c2bb4b6bdc1e28dfaa680536d5a) --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 17 ++++++--------- .../commands/release_management_commands.py | 9 +++++--- .../prepare_providers/provider_packages.py | 25 ++++++++++++++++------ 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index e53ef22d1a..554197b40f 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -206,7 +206,7 @@ are not generated. Release notes are only generated, when the latest version of yet have a corresponding TAG. The tags for providers is of the form ``providers-<PROVIDER_ID>/<VERSION>`` for example -``providers-amazon/1.0.0``. During releasing, the RC1/RC2 tags are created (for example +``providers-amazon/1.0.0``. During releasing, the `rc*` tags are created (for example ``providers-amazon/1.0.0rc1``). Details about maintaining the SEMVER version are going to be discussed and implemented in @@ -372,8 +372,8 @@ so you need to use `--version-suffix-for-pypi` switch to prepare those packages. Note that these are different packages than the ones used for SVN upload though they should be generated from the same sources. -* Generate the packages with the right RC version (specify the version suffix with PyPI switch). Note that -this will clean up dist folder before generating the packages, so you will only have the right packages there. +* Generate the packages with the rc1 version (specify the version suffix with PyPI switch). Note that +you should clean up dist folder before generating the packages, so you will only have the right packages there. ```shell script rm -rf ${AIRFLOW_REPO_ROOT}/dist/* @@ -389,13 +389,10 @@ breeze release-management prepare-provider-packages \ --version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE .... ``` -In case you are ALSO releasing RC2, RC3, etc. for selected packages, they will be skipped automatically because -the `rc1` tag will be created for them already. In this case you should specify the ``rc*`` that you want to -build and specify the package id's you want to build. - -```shell script -breeze release-management prepare-provider-packages --version-suffix-for-pypi rc2 --package-format both PACKAGE -``` +In case some packages already had rc1 suffix prepared and released, and they still need to be released, they +will have automatically appropriate rcN suffix added to them. The suffix will be increased for each release +candidate and checked if tag has been already created for that release candidate. If yes, the suffix will be +increased until the tag is not found. * Verify the artifacts that would be uploaded: diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 21a2e17987..8d1ee6eaa6 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -563,17 +563,20 @@ def prepare_provider_packages( shutil.rmtree(DIST_DIR, ignore_errors=True) DIST_DIR.mkdir(parents=True, exist_ok=True) for provider_id in packages_list: + package_version = version_suffix_for_pypi try: basic_provider_checks(provider_id) - if not skip_tag_check and should_skip_the_package(provider_id, version_suffix_for_pypi): - continue + if not skip_tag_check: + should_skip, package_version = should_skip_the_package(provider_id, package_version) + if should_skip: + continue get_console().print() with ci_group(f"Preparing provider package [special]{provider_id}"): get_console().print() target_provider_root_sources_path = copy_provider_sources_to_target(provider_id) generate_build_files( provider_id=provider_id, - version_suffix=version_suffix_for_pypi, + version_suffix=package_version, target_provider_root_sources_path=target_provider_root_sources_path, ) cleanup_build_remnants(target_provider_root_sources_path) diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py index 231bb7ff87..e8d99e6ed6 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py @@ -155,19 +155,30 @@ def generate_build_files(provider_id: str, version_suffix: str, target_provider_ get_console().print(f"\n[info]Generated package build files for {provider_id}[/]\n") -def should_skip_the_package(provider_id: str, version_suffix: str) -> bool: - """Return True if the package should be skipped. +def should_skip_the_package(provider_id: str, version_suffix: str) -> tuple[bool, str]: + """Return True, version if the package should be skipped and False, good version suffix if not. For RC and official releases we check if the "officially released" version exists and skip the released if it was. This allows to skip packages that have not been marked for release in this wave. For "dev" suffixes, we always build all packages. """ - if version_suffix.startswith("rc") or version_suffix == "": - current_tag = get_latest_provider_tag(provider_id, version_suffix) + if version_suffix != "" and not version_suffix.startswith("rc"): + return False, version_suffix + if version_suffix == "": + current_tag = get_latest_provider_tag(provider_id, "") if tag_exists_for_provider(provider_id, current_tag): - get_console().print(f"[warning]The tag {current_tag} exists. Skipping the package.[/]") - return True - return False + get_console().print(f"[warning]The 'final' tag {current_tag} exists. Skipping the package.[/]") + return True, version_suffix + return False, version_suffix + # version_suffix starts with "rc" + current_version = int(version_suffix[2:]) + while True: + current_tag = get_latest_provider_tag(provider_id, f"rc{current_version}") + if tag_exists_for_provider(provider_id, current_tag): + current_version += 1 + get_console().print(f"[warning]The tag {current_tag} exists. Checking rc{current_version}.[/]") + else: + return False, f"rc{current_version}" def cleanup_build_remnants(target_provider_root_sources_path: Path):
