This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b20d7cfcc5 Use `packaging` for parse requirements in documentation
generation (#35718)
b20d7cfcc5 is described below
commit b20d7cfcc5607171c2dc8923b01f9f36d825cf21
Author: Andrey Anshin <[email protected]>
AuthorDate: Sat Nov 18 16:56:21 2023 +0400
Use `packaging` for parse requirements in documentation generation (#35718)
---
.../prepare_providers/provider_documentation.py | 44 ++++++++++++---
dev/breeze/tests/test_provider_documentation.py | 64 ++++++++++++++++++----
2 files changed, 88 insertions(+), 20 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
index 6842d1c4fd..5cddc80d92 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
@@ -32,6 +32,7 @@ from typing import Any, Iterable, NamedTuple
import jinja2
import semver
+from packaging.requirements import Requirement
from rich.syntax import Syntax
from airflow_breeze.global_constants import PROVIDER_DEPENDENCIES
@@ -115,6 +116,34 @@ class Change(NamedTuple):
pr: str | None
+class PipRequirements(NamedTuple):
+ """Store details about python packages"""
+
+ package: str
+ version_required: str
+
+ @classmethod
+ def from_requirement(cls, requirement_string: str) -> PipRequirements:
+ req = Requirement(requirement_string)
+
+ package = req.name
+ if req.extras:
+ # Sort extras by name
+ package += f"[{','.join(sorted(req.extras))}]"
+
+ version_required = ""
+ if req.specifier:
+ # String representation of `packaging.specifiers.SpecifierSet`
sorted by the operator
+ # which might not looking good, e.g. '>=5.3.0,<6,!=5.3.3,!=5.3.2'
transform into the
+ # '!=5.3.3,!=5.3.2,<6,>=5.3.0'. Instead of that we sort by version
and resulting string would be
+ # '>=5.3.0,!=5.3.2,!=5.3.3,<6'
+ version_required = ",".join(map(str, sorted(req.specifier,
key=lambda spec: spec.version)))
+ if req.marker:
+ version_required += f"; {req.marker}"
+
+ return cls(package=package, version_required=version_required.strip())
+
+
class TypeOfChange(Enum):
DOCUMENTATION = "d"
BUGFIX = "b"
@@ -566,15 +595,12 @@ def _convert_pip_requirements_to_table(requirements:
Iterable[str], markdown: bo
headers = ["PIP package", "Version required"]
table_data = []
for dependency in requirements:
- found = re.match(r"(^[^<=>~!]*)([^<=>~!]?.*)$", dependency)
- if found:
- package = found.group(1)
- version_required = found.group(2)
- if version_required != "":
- version_required = f"`{version_required}`" if markdown else
f"``{version_required}``"
- table_data.append((f"`{package}`" if markdown else
f"``{package}``", version_required))
- else:
- table_data.append((dependency, ""))
+ req = PipRequirements.from_requirement(dependency)
+ formatted_package = f"`{req.package}`" if markdown else
f"``{req.package}``"
+ formatted_version = ""
+ if req.version_required:
+ formatted_version = f"`{req.version_required}`" if markdown else
f"``{req.version_required}``"
+ table_data.append((formatted_package, formatted_version))
return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown
else "rst")
diff --git a/dev/breeze/tests/test_provider_documentation.py
b/dev/breeze/tests/test_provider_documentation.py
index cf1a1e3024..764038769d 100644
--- a/dev/breeze/tests/test_provider_documentation.py
+++ b/dev/breeze/tests/test_provider_documentation.py
@@ -22,6 +22,7 @@ import pytest
from airflow_breeze.prepare_providers.provider_documentation import (
Change,
+ PipRequirements,
_convert_git_changes_to_table,
_convert_pip_requirements_to_table,
_find_insertion_index_for_version,
@@ -236,33 +237,74 @@ def test_convert_git_changes_to_table(input: str, output:
str, markdown: bool, c
assert list_of_changes[2].pr == "12346"
[email protected](
+ "requirement_string, expected",
+ [
+ pytest.param("apache-airflow", ("apache-airflow", ""),
id="no-version-specifier"),
+ pytest.param(
+ "apache-airflow <2.7,>=2.5", ("apache-airflow", ">=2.5,<2.7"),
id="range-version-specifier"
+ ),
+ pytest.param("watchtower~=3.0.1", ("watchtower", "~=3.0.1"),
id="compat-version-specifier"),
+ pytest.param("PyGithub!=1.58", ("PyGithub", "!=1.58"),
id="not-equal-version-specifier"),
+ pytest.param(
+ "apache-airflow[amazon,google,microsoft.azure,docker]>2.7.0",
+ ("apache-airflow[amazon,docker,google,microsoft.azure]", ">2.7.0"),
+ id="package-with-extra",
+ ),
+ pytest.param(
+ 'mysql-connector-python>=8.0.11; platform_machine != "aarch64"',
+ ("mysql-connector-python", '>=8.0.11; platform_machine !=
"aarch64"'),
+ id="version-with-platform-marker",
+ ),
+ pytest.param(
+ "backports.zoneinfo>=0.2.1;python_version<'3.9'",
+ ("backports.zoneinfo", '>=0.2.1; python_version < "3.9"'),
+ id="version-with-python-marker",
+ ),
+ pytest.param(
+ "celery>=5.3.0,<6,!=5.3.3,!=5.3.2",
+ ("celery", ">=5.3.0,!=5.3.2,!=5.3.3,<6"),
+ id="complex-version-specifier",
+ ),
+ pytest.param(
+ "apache-airflow; python_version<'3.12' or platform_machine !=
'i386'",
+ ("apache-airflow", '; python_version < "3.12" or platform_machine
!= "i386"'),
+ id="no-version-specifier-with-complex-marker",
+ ),
+ ],
+)
+def test_parse_pip_requirements_parse(requirement_string, expected):
+ assert PipRequirements.from_requirement(requirement_string) == expected
+
+
@pytest.mark.parametrize(
"requirements, markdown, table",
[
(
- ["apache-airflow>2.5.0"],
+ ["apache-airflow>2.5.0", "apache-airflow-providers-http"],
False,
"""
-================== ==================
-PIP package Version required
-================== ==================
-``apache-airflow`` ``>2.5.0``
-================== ==================
+================================= ==================
+PIP package Version required
+================================= ==================
+``apache-airflow`` ``>2.5.0``
+``apache-airflow-providers-http``
+================================= ==================
""",
),
(
- ["apache-airflow>2.5.0"],
+ ["apache-airflow>2.5.0", "apache-airflow-providers-http"],
True,
"""
-| PIP package | Version required |
-|:-----------------|:-------------------|
-| `apache-airflow` | `>2.5.0` |
+| PIP package | Version required |
+|:--------------------------------|:-------------------|
+| `apache-airflow` | `>2.5.0` |
+| `apache-airflow-providers-http` | |
""",
),
],
)
def test_convert_pip_requirements_to_table(requirements: Iterable[str],
markdown: bool, table: str):
- print(_convert_pip_requirements_to_table(requirements, markdown))
assert _convert_pip_requirements_to_table(requirements, markdown).strip()
== table.strip()
