This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 61dc4cacacf1478076ab0a2fb775df684591c1b0 Author: Jarek Potiuk <[email protected]> AuthorDate: Thu Jul 15 16:17:18 2021 +0200 Fixes "development" and "rc" cross dependencies between providers (#17023) In case we have additional dependencies between providers released at the same time (for example we need to release sftp and ssh packages now where sftp package depends on release of ssh at the same time) we have to add suffix to the version of the additional_dependency. PIP does not take into account unfortunately that development dependencies should likely be considered as fulfilling the requirement of >=. For example if you have: sftp depends on ssh>=2.1.0 and you release ssh 2.1.0.dev0 at the same time the ssh>=2.1.0 condition is not fulfilled. Same case will be with rc1. Therefore we need to add the suffix in such cross-provider dependencies to be able to install them in CI and in rc candidates. In the future we might ask PIP to change behaviour in such case. (cherry picked from commit 07e0a67bf641d25b0237b457f2585788fa4c8ada) --- dev/provider_packages/prepare_provider_packages.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 16bd046..e36fcf8 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -357,11 +357,12 @@ def get_long_description(provider_package_id: str) -> str: return long_description -def get_install_requirements(provider_package_id: str) -> List[str]: +def get_install_requirements(provider_package_id: str, version_suffix: str) -> List[str]: """ Returns install requirements for the package. :param provider_package_id: id of the provider package + :param version_suffix: optional version suffix for packages :return: install requirements of the package """ @@ -369,7 +370,21 @@ def get_install_requirements(provider_package_id: str) -> List[str]: provider_yaml = get_provider_yaml(provider_package_id) install_requires = [] if "additional-dependencies" in provider_yaml: - install_requires = provider_yaml['additional-dependencies'] + additional_dependencies = provider_yaml['additional-dependencies'] + if version_suffix: + # In case we are preparing "rc" or dev0 packages, we should also + # make sure that cross-dependency with Airflow or Airflow Providers will + # contain the version suffix, otherwise we will have conflicting dependencies. + # For example if (in sftp) we have ssh>=2.0.1 and release ssh==2.0.1 + # we want to turn this into ssh>=2.0.1.dev0 if we build dev0 version of the packages + # or >=2.0.1rc1 if we build rc1 version of the packages. + for dependency in additional_dependencies: + if dependency.startswith("apache-airflow") and ">=" in dependency: + dependency = dependency + version_suffix + install_requires.append(dependency) + else: + install_requires.extend(additional_dependencies) + install_requires.extend(dependencies) return install_requires @@ -1513,7 +1528,7 @@ def get_provider_jinja_context( "PROVIDERS_FOLDER": "providers", "PROVIDER_DESCRIPTION": provider_details.provider_description, "INSTALL_REQUIREMENTS": get_install_requirements( - provider_package_id=provider_details.provider_package_id + provider_package_id=provider_details.provider_package_id, version_suffix=version_suffix ), "SETUP_REQUIREMENTS": get_setup_requirements(), "EXTRAS_REQUIREMENTS": get_package_extras(provider_package_id=provider_details.provider_package_id),
