Re: [OE-core] [RFC PATCH 12/15] recipetool: npm: Add dependencies to SRC_URI and auto select classes
A description of the changes and how the new code works is missing. How is
the SRC_URI formed? How is the appropriate class selected?
Alex
On Wed, 24 Nov 2021 at 15:48, Stefan Herbrechtsmeier <
stefan.herbrechtsmeier-...@weidmueller.com> wrote:
> From: Stefan Herbrechtsmeier
>
> Signed-off-by: Stefan Herbrechtsmeier <
> stefan.herbrechtsme...@weidmueller.com>
>
> ---
>
> scripts/lib/recipetool/create_npm.py | 243 ---
> 1 file changed, 222 insertions(+), 21 deletions(-)
>
> diff --git a/scripts/lib/recipetool/create_npm.py
> b/scripts/lib/recipetool/create_npm.py
> index 3394a89970..296b84340e 100644
> --- a/scripts/lib/recipetool/create_npm.py
> +++ b/scripts/lib/recipetool/create_npm.py
> @@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler):
> name = name.strip("-")
> return name
>
> +@staticmethod
> +def _node_recipe_name(name):
> +"""Generate a OE friendly Node.js recipe name"""
> +name = NpmRecipeHandler._npm_name(name)
> +if not name.startswith("node-"):
> +name = "node-" + name
> +return name
> +
> @staticmethod
> def _get_registry(lines):
> """Get the registry value from the 'npm://registry' url"""
> @@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler):
>
> return registry
>
> +@staticmethod
> +def _get_srcdir(lines):
> +"""Get the source directory value from the url"""
> +srcdir = ""
> +
> +def _handle_srcdir(varname, origvalue, op, newlines):
> +nonlocal srcdir
> +if origvalue.startswith("${WORKDIR}"):
> +srcdir = origvalue[11:]
> +else:
> +srcdir = "${BP}"
> +
> +return origvalue, None, 0, True
> +
> +bb.utils.edit_metadata(lines, ["S"], _handle_srcdir)
> +
> +return srcdir
> +
> @staticmethod
> def _ensure_npm():
> """Check if the 'npm' command is available in the recipes"""
> @@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler):
>
> return os.path.join(srctree, "npm-shrinkwrap.json")
>
> +def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir):
> +"""
> +Extract package urls from shrinkwrap dependencies
> +"""
> +
> +urls = []
> +
> +def _populate_modules(name, params, deptree):
> +from bb.fetch2 import URI
> +from bb.fetch2.npm import npm_integrity
> +from bb.fetch2.npm import npm_localfile
> +
> +destsubdirs = [os.path.join("node_modules", dep) for dep in
> deptree]
> +destsuffix = os.path.join(srcdir, *destsubdirs)
> +
> +dev = params.get("dev", False)
> +integrity = params.get("integrity")
> +resolved = params.get("resolved")
> +version = params.get("version")
> +requires = params.get("requires", {})
> +
> +# Handle registry sources
> +if bb.utils.is_semver(version) and integrity:
> +# Skip dependencies without url
> +if not resolved:
> +return
> +
> +pkgv = version
> +
> +uri = URI(resolved)
> +uri.params["downloadfilename"] = npm_localfile(name,
> version)
> +
> +checksum_name, checksum_expected =
> npm_integrity(integrity)
> +uri.params[checksum_name] = checksum_expected
> +
> +uri.params["subdir"] = destsuffix
> +uri.params["striplevel"] = "1"
> +
> +url = str(uri)
> +
> +# Handle http tarball sources
> +elif version.startswith("http") and integrity:
> +checksum_name, checksum_expected =
> npm_integrity(integrity)
> +
> +pkgv = checksum_expected[:13]
> +
> +uri = URI(version)
> +uri.params["downloadfilename"] = npm_localfile(name, pkgv)
> +
> +uri.params[checksum_name] = checksum_expected
> +
> +uri.params["destsuffix"] = destsuffix
> +uri.params["striplevel"] = "1"
> +
> +url = str(uri)
> +
> +# Handle git sources
> +elif version.startswith("git"):
> +if version.startswith("github:"):
> +version = "git+https://github.com/; +
> version[len("github:"):]
> +regex = re.compile(r"""
> +^
> +git\+
> +(?P[a-z]+)
> +://
> +(?P[^#]+)
> +\#
> +(?P[0-9a-f]+)
> +$
> +""", re.VERBOSE)
> +
> +match = regex.match(version)
> +
> +if not match:
> +raise Exception("Invalid git url: %s - %s" %
> (version, url))
> +
> +groups = match.groupdict()
> +
> +pkgv =
[OE-core] [RFC PATCH 12/15] recipetool: npm: Add dependencies to SRC_URI and auto select classes
From: Stefan Herbrechtsmeier
Signed-off-by: Stefan Herbrechtsmeier
---
scripts/lib/recipetool/create_npm.py | 243 ---
1 file changed, 222 insertions(+), 21 deletions(-)
diff --git a/scripts/lib/recipetool/create_npm.py
b/scripts/lib/recipetool/create_npm.py
index 3394a89970..296b84340e 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler):
name = name.strip("-")
return name
+@staticmethod
+def _node_recipe_name(name):
+"""Generate a OE friendly Node.js recipe name"""
+name = NpmRecipeHandler._npm_name(name)
+if not name.startswith("node-"):
+name = "node-" + name
+return name
+
@staticmethod
def _get_registry(lines):
"""Get the registry value from the 'npm://registry' url"""
@@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler):
return registry
+@staticmethod
+def _get_srcdir(lines):
+"""Get the source directory value from the url"""
+srcdir = ""
+
+def _handle_srcdir(varname, origvalue, op, newlines):
+nonlocal srcdir
+if origvalue.startswith("${WORKDIR}"):
+srcdir = origvalue[11:]
+else:
+srcdir = "${BP}"
+
+return origvalue, None, 0, True
+
+bb.utils.edit_metadata(lines, ["S"], _handle_srcdir)
+
+return srcdir
+
@staticmethod
def _ensure_npm():
"""Check if the 'npm' command is available in the recipes"""
@@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler):
return os.path.join(srctree, "npm-shrinkwrap.json")
+def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir):
+"""
+Extract package urls from shrinkwrap dependencies
+"""
+
+urls = []
+
+def _populate_modules(name, params, deptree):
+from bb.fetch2 import URI
+from bb.fetch2.npm import npm_integrity
+from bb.fetch2.npm import npm_localfile
+
+destsubdirs = [os.path.join("node_modules", dep) for dep in
deptree]
+destsuffix = os.path.join(srcdir, *destsubdirs)
+
+dev = params.get("dev", False)
+integrity = params.get("integrity")
+resolved = params.get("resolved")
+version = params.get("version")
+requires = params.get("requires", {})
+
+# Handle registry sources
+if bb.utils.is_semver(version) and integrity:
+# Skip dependencies without url
+if not resolved:
+return
+
+pkgv = version
+
+uri = URI(resolved)
+uri.params["downloadfilename"] = npm_localfile(name, version)
+
+checksum_name, checksum_expected = npm_integrity(integrity)
+uri.params[checksum_name] = checksum_expected
+
+uri.params["subdir"] = destsuffix
+uri.params["striplevel"] = "1"
+
+url = str(uri)
+
+# Handle http tarball sources
+elif version.startswith("http") and integrity:
+checksum_name, checksum_expected = npm_integrity(integrity)
+
+pkgv = checksum_expected[:13]
+
+uri = URI(version)
+uri.params["downloadfilename"] = npm_localfile(name, pkgv)
+
+uri.params[checksum_name] = checksum_expected
+
+uri.params["destsuffix"] = destsuffix
+uri.params["striplevel"] = "1"
+
+url = str(uri)
+
+# Handle git sources
+elif version.startswith("git"):
+if version.startswith("github:"):
+version = "git+https://github.com/; +
version[len("github:"):]
+regex = re.compile(r"""
+^
+git\+
+(?P[a-z]+)
+://
+(?P[^#]+)
+\#
+(?P[0-9a-f]+)
+$
+""", re.VERBOSE)
+
+match = regex.match(version)
+
+if not match:
+raise Exception("Invalid git url: %s - %s" % (version,
url))
+
+groups = match.groupdict()
+
+pkgv = str(groups["rev"])[:10]
+
+uri = URI("git://" + str(groups["url"]))
+uri.params["destsuffix"] = destsuffix
+uri.params["nobranch"] = "1"
+uri.params["protocol"] = str(groups["protocol"])
+uri.params["rev"] = str(groups["rev"])
+
+url = str(uri)
+
+else:
+raise Exception("Unsupported dependency: %s - %s" % (name,
version))
+
+# Set package version in shrinkwrap for dependency resolution
+params["pkgv"] = pkgv
