Dev-iL commented on code in PR #74556: URL: https://github.com/apache/airflow/pull/74556#discussion_r4308425815
########## scripts/docker/filter_ci_install_set.py: ########## @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Compute the CI image install set for one Python minor without excluded workspace members. + +``uv sync --all-packages`` installs every workspace member, including the dev groups of providers +excluded for a Python minor (``Requires-Python`` carries ``!=<minor>``). Their third-party +dependencies would still be installed, and some do not build for the new interpreter. + +The helper resolves the requirements of the members that are not excluded, not the lock's export +graph. Each kept member becomes a local ``file://`` requirement, and the requirements of each kept +member's ``dev`` group (the root's ``ci-image`` group for the root) are added verbatim, with +excluded members removed. ``uv pip compile`` resolves that input for the target interpreter and +platform, so markers apply per edge. The lock's pins are passed as constraints, so every version +matches the lock. Output is the third-party pins and the editable member directories. With +``--hashes``, each pin carries the lock export's ``--hash`` options so the install can be run +under ``--require-hashes``, which is the check ``uv sync --frozen`` would have done. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tomllib +from glob import glob +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse + +MEMBER_DEFAULT_GROUP = "dev" +ROOT_CI_GROUP = "ci-image" +MINOR_VERSION = re.compile(r"\d+\.\d+") +REQUIREMENT_NAME = re.compile(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)") +RESOLVED_MEMBER = re.compile(r"^([^\s@]+)\s*@\s*(file://\S+)$") +PINNED_NAME = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*(?:==|@)") +PINNED_VERSION = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)==([^\s;]+)") +HASH_OPTION = re.compile(r"--hash=[A-Za-z0-9_]+:[A-Fa-f0-9]+") +GLOB_CHARACTERS = frozenset("*?[") +PYTHON_PLATFORMS = {"x86_64": "x86_64-manylinux_2_28", "aarch64": "aarch64-manylinux_2_28"} + + +def normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def requirement_name(entry: str) -> str: + match = REQUIREMENT_NAME.match(entry) + if not match: + raise ValueError(f"Cannot read a requirement name from {entry!r}") + return normalize_name(match.group(1)) + + +def is_python_minor_excluded(requires_python: str, minor: str) -> bool: + # The lookahead keeps longer versions such as !=3.150 or !=3.15.1 from matching a whole-minor exclusion. + return re.search(rf"!=\s*{re.escape(minor)}(\.\*)?(?![\d.])", requires_python) is not None + + +def load_manifest(directory: Path) -> dict[str, Any]: + with (directory / "pyproject.toml").open("rb") as manifest: + return tomllib.load(manifest) + + +def has_glob_characters(pattern: str) -> bool: + return any(character in GLOB_CHARACTERS for character in pattern) + + +def find_workspace_members(root: Path, patterns: list[str]) -> list[Path]: + members: list[Path] = [] + for pattern in patterns: + matches = sorted(glob(str(root / pattern))) + if not matches and not has_glob_characters(pattern): + raise ValueError(f"Workspace member {pattern!r} does not exist under {root}") + for match in matches: + candidate = Path(match) + if (candidate / "pyproject.toml").is_file(): + members.append(candidate) + elif not has_glob_characters(pattern): + raise ValueError(f"Workspace member {pattern!r} has no pyproject.toml") + return members + + +def expand_group(groups: dict[str, list[Any]], group: str, seen: set[str] | None = None) -> list[str]: + """Return the raw requirement strings of a dependency group, following ``include-group`` entries.""" + if seen is None: + seen = set() + if group in seen: + return [] + seen.add(group) + entries: list[str] = [] + for entry in groups.get(group, []): + if isinstance(entry, dict): + entries.extend(expand_group(groups, entry["include-group"], seen)) + else: + entries.append(entry) + return entries + + +def split_members(root: Path, minor: str) -> tuple[dict[str, Path], set[str]]: + """Return the members kept for installation (name -> directory) and the excluded member names.""" + patterns = load_manifest(root)["tool"]["uv"]["workspace"]["members"] + kept: dict[str, Path] = {} + excluded: set[str] = set() + for member in find_workspace_members(root, patterns): + project = load_manifest(member).get("project", {}) + name = normalize_name(project.get("name", member.name)) + if is_python_minor_excluded(project.get("requires-python", ""), minor): + excluded.add(name) + else: + kept[name] = member + return kept, excluded Review Comment: Removed; only providers appear in that metadata, so the root, `airflow-core` and `task-sdk` can no longer drop out whatever their `requires-python` says. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
