Hello,
On 19/10/2023 09:36:53+0200, Julien Stephan wrote:
> add support for PEP517 [1]
>
> if a pyproject.toml file is found, use it to create the recipe,
> otherwise fallback to the old setup.py method.
>
> [YOCTO #14737]
>
> [1]: https://peps.python.org/pep-0517/
>
> Signed-off-by: Julien Stephan <[email protected]>
> ---
> .../lib/recipetool/create_buildsys_python.py | 234 +++++++++++++++++-
> 1 file changed, 233 insertions(+), 1 deletion(-)
>
> diff --git a/scripts/lib/recipetool/create_buildsys_python.py
> b/scripts/lib/recipetool/create_buildsys_python.py
> index 69f6f5ca511..0b601d50a4b 100644
> --- a/scripts/lib/recipetool/create_buildsys_python.py
> +++ b/scripts/lib/recipetool/create_buildsys_python.py
> @@ -18,6 +18,7 @@ import os
> import re
> import sys
> import subprocess
> +import toml
This fails on the autobuilders because we don't have the toml module installed
so I guess you need to add a dependency.
> from recipetool.create import RecipeHandler
>
> logger = logging.getLogger('recipetool')
> @@ -656,6 +657,235 @@ class PythonSetupPyRecipeHandler(PythonRecipeHandler):
>
> handled.append('buildsystem')
>
> +class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
> + """Base class to support PEP517 and PEP518
> +
> + PEP517 https://peps.python.org/pep-0517/#source-trees
> + PEP518 https://peps.python.org/pep-0518/#build-system-table
> + """
> +
> + # PEP621:
> https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
> + # add only the ones that map to a BB var
> + # potentially missing: optional-dependencies
> + bbvar_map = {
> + "name": "PN",
> + "version": "PV",
> + "Homepage": "HOMEPAGE",
> + "description": "SUMMARY",
> + "license": "LICENSE",
> + "dependencies": "RDEPENDS:${PN}",
> + "requires": "DEPENDS",
> + }
> +
> + replacements = [
> + ("license", r" +$", ""),
> + ("license", r"^ +", ""),
> + ("license", r" ", "-"),
> + ("license", r"^GNU-", ""),
> + ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
> + ("license", r"^UNKNOWN$", ""),
> + # Remove currently unhandled version numbers from these variables
> + ("requires", r"\[[^\]]+\]$", ""),
> + ("requires", r"^([^><= ]+).*", r"\1"),
> + ("dependencies", r"\[[^\]]+\]$", ""),
> + ("dependencies", r"^([^><= ]+).*", r"\1"),
> + ]
> +
> + build_backend_map = {
> + "setuptools.build_meta": "python_setuptools_build_meta",
> + "poetry.core.masonry.api": "python_poetry_core",
> + "flit_core.buildapi": "python_flit_core",
> + }
> +
> + excluded_native_pkgdeps = [
> + # already provided by python_setuptools_build_meta.bbclass
> + "python3-setuptools-native",
> + "python3-wheel-native",
> + # already provided by python_poetry_core.bbclass
> + "python3-poetry-core-native",
> + # already provided by python_flit_core.bbclass
> + "python3-flit-core-native",
> + ]
> +
> + # add here a list of known and often used packages and the corresponding
> bitbake package
> + known_deps_map = {
> + "setuptools": "python3-setuptools",
> + "wheel": "python3-wheel",
> + "poetry-core": "python3-poetry-core",
> + "flit_core": "python3-flit-core",
> + "setuptools-scm": "python3-setuptools-scm",
> + }
> +
> + def __init__(self):
> + pass
> +
> + def process(self, srctree, classes, lines_before, lines_after, handled,
> extravalues):
> + info = {}
> +
> + if 'buildsystem' in handled:
> + return False
> +
> + # Check for non-zero size setup.py files
> + setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
> + for fn in setupfiles:
> + if os.path.getsize(fn):
> + break
> + else:
> + return False
> +
> + setupscript = os.path.join(srctree, "pyproject.toml")
> +
> + try:
> + config = self.parse_pyproject_toml(setupscript)
> + build_backend = config["build-system"]["build-backend"]
> + if build_backend in self.build_backend_map:
> + classes.append(self.build_backend_map[build_backend])
> + else:
> + logger.error(
> + "Unsupported build-backend: %s, cannot use
> pyproject.toml. Will try to use legacy setup.py"
> + % build_backend
> + )
> + return False
> +
> + licfile = ""
> + if "project" in config:
> + for field, values in config["project"].items():
> + if field == "license":
> + value = values.get("text", "")
> + if not value:
> + licfile = values.get("file", "")
> + elif isinstance(values, dict):
> + for k, v in values.items():
> + info[k] = v
> + continue
> + else:
> + value = values
> +
> + info[field] = value
> +
> + # Grab the license value before applying replacements
> + license_str = info.get("license", "").strip()
> +
> + if license_str:
> + for i, line in enumerate(lines_before):
> + if line.startswith("##LICENSE_PLACEHOLDER##"):
> + lines_before.insert(
> + i, "# NOTE: License in pyproject.toml is: %s" %
> license_str
> + )
> + break
> +
> + info["requires"] = config["build-system"]["requires"]
> +
> + self.apply_info_replacements(info)
> +
> + if "classifiers" in info:
> + license = self.handle_classifier_license(
> + info["classifiers"], info.get("license", "")
> + )
> + if license:
> + if licfile:
> + lines = []
> + md5value = bb.utils.md5_file(os.path.join(srctree,
> licfile))
> + lines.append('LICENSE = "%s"' % license)
> + lines.append(
> + 'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
> + % (licfile, md5value)
> + )
> + lines.append("")
> +
> + # Replace the placeholder so we get the values in
> the right place in the recipe file
> + try:
> + pos =
> lines_before.index("##LICENSE_PLACEHOLDER##")
> + except ValueError:
> + pos = -1
> + if pos == -1:
> + lines_before.extend(lines)
> + else:
> + lines_before[pos : pos + 1] = lines
> +
> + handled.append(("license", [license, licfile,
> md5value]))
> + else:
> + info["license"] = license
> +
> + provided_packages = self.parse_pkgdata_for_python_packages()
> + provided_packages.update(self.known_deps_map)
> + native_mapped_deps, native_unmapped_deps = set(), set()
> + mapped_deps, unmapped_deps = set(), set()
> +
> + if "requires" in info:
> + for require in info["requires"]:
> + mapped = provided_packages.get(require)
> +
> + if mapped:
> + logger.error("Mapped %s to %s" % (require, mapped))
> + native_mapped_deps.add(mapped)
> + else:
> + logger.error("Could not map %s" % require)
> + native_unmapped_deps.add(require)
> +
> + info.pop("requires")
> +
> + if native_mapped_deps != set():
> + native_mapped_deps = {
> + item + "-native" for item in native_mapped_deps
> + }
> + native_mapped_deps -= set(self.excluded_native_pkgdeps)
> + if native_mapped_deps != set():
> + info["requires"] = "
> ".join(sorted(native_mapped_deps))
> +
> + if native_unmapped_deps:
> + lines_after.append("")
> + lines_after.append(
> + "# WARNING: We were unable to map the following
> python package/module"
> + )
> + lines_after.append(
> + "# dependencies to the bitbake packages which
> include them:"
> + )
> + lines_after.extend(
> + "# {}".format(d) for d in
> sorted(native_unmapped_deps)
> + )
> +
> + if "dependencies" in info:
> + for dependency in info["dependencies"]:
> + mapped = provided_packages.get(dependency)
> + if mapped:
> + logger.error("Mapped %s to %s" % (dependency,
> mapped))
> + mapped_deps.add(mapped)
> + else:
> + logger.error("Could not map %s" % dependency)
> + unmapped_deps.add(dependency)
> +
> + info.pop("dependencies")
> +
> + if mapped_deps != set():
> + if mapped_deps != set():
> + info["dependencies"] = " ".join(sorted(mapped_deps))
> +
> + if unmapped_deps:
> + lines_after.append("")
> + lines_after.append(
> + "# WARNING: We were unable to map the following
> python package/module"
> + )
> + lines_after.append(
> + "# runtime dependencies to the bitbake packages
> which include them:"
> + )
> + lines_after.extend(
> + "# {}".format(d) for d in sorted(unmapped_deps)
> + )
> +
> + self.map_info_to_bbvar(info, extravalues)
> +
> + handled.append("buildsystem")
> + except Exception:
> + logger.exception("Failed to parse pyproject.toml")
> + return False
> +
> + def parse_pyproject_toml(self, setupscript):
> + with open(setupscript, "r") as f:
> + config = toml.load(f)
> + return config
> +
> +
> def gather_setup_info(fileobj):
> parsed = ast.parse(fileobj.read(), fileobj.name)
> visitor = SetupScriptVisitor()
> @@ -769,5 +999,7 @@ def has_non_literals(value):
>
>
> def register_recipe_handlers(handlers):
> - # We need to make sure this is ahead of the makefile fallback handler
> + # We need to make sure these are ahead of the makefile fallback handler
> + # and the pyproject.toml handler ahead of the setup.py handler
> + handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
> handlers.append((PythonSetupPyRecipeHandler(), 70))
> --
> 2.42.0
>
>
>
>
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189462):
https://lists.openembedded.org/g/openembedded-core/message/189462
Mute This Topic: https://lists.openembedded.org/mt/102055999/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
