On Mon, Dec 4, 2023 at 7:59 AM Julien Stephan <[email protected]> wrote:
> Today, we can use devtool/recipetool to create recipes for python projects > using the github url or the direct release tarball of the project, but the > create_buildsys_python plugin doesn't support the pypi class, since we > cannot > know from the extracted source if the package is available on pypi or not. > > By implementing the new optional process_url callback, we can detect > that the url is a pypi one (i.e 'https://pypi.org/project/<package>') > and retrieve the release tarball location. > Also detect if the url points to a release tarball hosted on > "files.pythonhosted.iorg" (i.e https://files.pythonhosted.org/packages/.. > .) > > In both cases, adds the pypi class, remove 'S' and 'SRC_URIxxx' > variables from the created recipe as they will be handled by the pypi class > and add the PYPI_PACKAGE variable > > This helps to produce cleaner recipes when package is hosted on pypi. > > If the url points to a github url or a release tarball not coming from > "files.pythonhosted.org", the created recipe is the same as before. > One can also use the newly added "--no-pypi" switch to NOT inherit > from pypi class on matching url, to keep legacy behaviour. > > To create a recipe for a pypi package, one can now use one of the > new following syntax (using recipetool create / devtool add): > > * recipetool create https://pypi.org/project/<package> > * recipetool create https://pypi.org/project/<package>/<version> > * recipetool create https://pypi.org/project/<package> --version <version> > > or the old syntax: > * recipetool create https://files.pythonhosted.org/packages/<...> > > Signed-off-by: Julien Stephan <[email protected]> > --- > scripts/lib/devtool/standard.py | 3 + > scripts/lib/recipetool/create.py | 1 + > .../lib/recipetool/create_buildsys_python.py | 72 +++++++++++++++++++ > 3 files changed, 76 insertions(+) > > diff --git a/scripts/lib/devtool/standard.py > b/scripts/lib/devtool/standard.py > index d53fb810071..f18ebaa70d3 100644 > --- a/scripts/lib/devtool/standard.py > +++ b/scripts/lib/devtool/standard.py > @@ -147,6 +147,8 @@ def add(args, config, basepath, workspace): > extracmdopts += ' -a' > if args.npm_dev: > extracmdopts += ' --npm-dev' > + if args.no_pypi: > + extracmdopts += ' --no-pypi' > if args.mirrors: > extracmdopts += ' --mirrors' > if args.srcrev: > @@ -2260,6 +2262,7 @@ def register_commands(subparsers, context): > group.add_argument('--no-same-dir', help='Force build in a separate > build directory', action="store_true") > parser_add.add_argument('--fetch', '-f', help='Fetch the specified > URI and extract it to create the source tree (deprecated - pass as > positional argument instead)', metavar='URI') > parser_add.add_argument('--npm-dev', help='For npm, also fetch > devDependencies', action="store_true") > + parser_add.add_argument('--no-pypi', help='Do not inherit pypi > class', action="store_true") > parser_add.add_argument('--version', '-V', help='Version to use > within recipe (PV)') > parser_add.add_argument('--no-git', '-g', help='If fetching source, > do not set up source tree as a git repository', action="store_true") > group = parser_add.add_mutually_exclusive_group() > diff --git a/scripts/lib/recipetool/create.py > b/scripts/lib/recipetool/create.py > index 5c5ac7ae403..963aa91421e 100644 > --- a/scripts/lib/recipetool/create.py > +++ b/scripts/lib/recipetool/create.py > @@ -1413,6 +1413,7 @@ def register_commands(subparsers): > parser_create.add_argument('-B', '--srcbranch', help='Branch in > source repository if fetching from an SCM such as git (default master)') > parser_create.add_argument('--keep-temp', action="store_true", > help='Keep temporary directory (for debugging)') > parser_create.add_argument('--npm-dev', action="store_true", > help='For npm, also fetch devDependencies') > + parser_create.add_argument('--no-pypi', action="store_true", help='Do > not inherit pypi class') > parser_create.add_argument('--devtool', action="store_true", > help=argparse.SUPPRESS) > parser_create.add_argument('--mirrors', action="store_true", > help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by > default).') > parser_create.set_defaults(func=create_recipe) > diff --git a/scripts/lib/recipetool/create_buildsys_python.py > b/scripts/lib/recipetool/create_buildsys_python.py > index b620e3271b1..5e07222ece1 100644 > --- a/scripts/lib/recipetool/create_buildsys_python.py > +++ b/scripts/lib/recipetool/create_buildsys_python.py > @@ -18,7 +18,11 @@ import os > import re > import sys > import subprocess > +import json > +import urllib.request > from recipetool.create import RecipeHandler > +from urllib.parse import urldefrag > +from recipetool.create import determine_from_url > > logger = logging.getLogger('recipetool') > > @@ -111,6 +115,74 @@ class PythonRecipeHandler(RecipeHandler): > def __init__(self): > pass > > + def process_url(self, args, classes, handled, extravalues): > + """ > + Convert any pypi url https://pypi.org/project/<package>/<version> > into https://files.pythonhosted.org/packages/source/... > + which corresponds to the archive location, and add pypi class > + """ > + > + if 'url' in handled: > + return None > + > + fetch_uri = None > + source = args.source > + required_version = args.version if args.version else None > + match = re.match(r'https?://pypi.org/project/([ > <http://pypi.org/project/(%5B>^/]+)(?:/([^/]+))?/?$', > urldefrag(source)[0]) > + if match: > + package = match.group(1) > + version = match.group(2) if match.group(2) else > required_version > + > + json_url = f"https://pypi.org/pypi/%s/json"; % package > + response = urllib.request.urlopen(json_url) > + if response.status == 200: > + data = json.loads(response.read()) > + if not version: > + # grab latest version > + version = data["info"]["version"] > + pypi_package = data["info"]["name"] > + for release in reversed(data["releases"][version]): > + if release["packagetype"] == "sdist": > + fetch_uri = release["url"] > + break > + else: > + logger.warning("Cannot handle pypi url %s: cannot fetch > package information using %s", source, json_url) > + return None > + else: > + match = re.match(r'^https?:// > files.pythonhosted.org/packages.*/(.*)-.*$', source) > + if match: > + fetch_uri = source > + pypi_package = match.group(1) > + _, version = determine_from_url(fetch_uri) > + > + if match and not args.no_pypi: > + if required_version and version != required_version: > + raise Exception("Version specified using --version/-V > (%s) and version specified in the url (%s) do not match" % > (required_version, version)) > + # This is optionnal if BPN looks like "python-<pypi_package>" > or "python3-<pypi_package>" (see pypi.bbclass) > + # but at this point we cannot know because because user can > specify the output name of the recipe on the command line > + extravalues["PYPI_PACKAGE"] = pypi_package > + # If the tarball extension is not 'tar.gz' (default value in > pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe > + pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % > (pypi_package, version), fetch_uri) > + if pypi_package_ext: > + pypi_package_ext = pypi_package_ext.group(1) > + if pypi_package_ext != "tar.gz": > + extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext > + > + # Pypi class will handle S and SRC_URIxxx variables, so > remove them > + # TODO: allow oe.recipeutils.patch_recipe_lines() to accept > regexp so we can simplify the following to: > + # extravalues['SRC_URI(?:\[.*?\])?'] = None > + extravalues['S'] = None > + extravalues['SRC_URI'] = None > + extravalues['SRC_URI[md5sum]'] = None > + extravalues['SRC_URI[sha1sum]'] = None > + extravalues['SRC_URI[sha256sum]'] = None > + extravalues['SRC_URI[sha384sum]'] = None > + extravalues['SRC_URI[sha512sum]'] = None > + this was not correct. The pypi.bbclass does not create the SRC_URI[sha256sum]. It does create the SRC_URI. So new recipes will not have any check on the validity of the downloaded tarball. The only test case which checked for this was with —nopypi https://git.yoctoproject.org/poky-contrib/tree/meta/lib/oeqa/selftest/cases/recipetool.py#n752 > + classes.append('pypi') > + > + handled.append('url') > + return fetch_uri > + > def handle_classifier_license(self, classifiers, > existing_licenses=""): > > licenses = [] > -- > 2.42.0 > > > > >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#192092): https://lists.openembedded.org/g/openembedded-core/message/192092 Mute This Topic: https://lists.openembedded.org/mt/102972952/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
