On 2020-12-02 11:25 a.m., [email protected] wrote:
Task do_configure delivered by npm.bbclass takes a lot of time, the
longer shrinkwrap dependency list the longer time needed for
do_configure to execute.
Following improvement allows to execute do_configure on any available
CPI threads.
Thanks for the patch Michal.
Can you resend to:
[email protected]
using git send-email as per:
https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
Thanks,
../Randy
diff --git a/bitbake/lib/bb/fetch2/npmsw.py
b/bitbake/lib/bb/fetch2/npmsw.py
index 0c3511d8ab..0e6075859c 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
from bb.fetch2.npm import npm_localfile
from bb.fetch2.npm import npm_unpack
from bb.utils import is_semver
+from concurrent.futures import ThreadPoolExecutor
-def foreach_dependencies(shrinkwrap, callback=None, dev=False):
+def foreach_dependencies(shrinkwrap, callback=None, dev=False,
executor=None):
"""
Run a callback for each dependencies of a shrinkwrap file.
The callback is using the format:
@@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap,
callback=None, dev=False):
continue
elif deps[name].get("bundled", False):
continue
- callback(name, deps[name], subtree)
+ if executor:
+ executor.submit(callback, name=name,
params=deps[name], deptree=subtree)
+ else:
+ callback(name, deps[name], subtree)
_walk_deps(shrinkwrap.get("dependencies", {}), [])
diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
index 068032a1e5..1869b63cdb 100644
--- a/meta/classes/npm.bbclass
+++ b/meta/classes/npm.bbclass
@@ -80,6 +80,7 @@ python npm_do_configure() {
from bb.fetch2.npm import npm_unpack
from bb.fetch2.npmsw import foreach_dependencies
from bb.progress import OutOfProgressHandler
+ from concurrent.futures import ThreadPoolExecutor
bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
@@ -164,9 +165,11 @@ python npm_do_configure() {
progress_done += 1
progress.write("%d/%d" % (progress_done, progress_total))
+ executor = ThreadPoolExecutor(max_workers = os.cpu_count())
dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
- foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
+ foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
executor)
+ executor.shutdown(True)
# Configure the main package
with tempfile.TemporaryDirectory() as tmpdir:
--
# Randy MacLeod
# Wind River Linux
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#145177):
https://lists.openembedded.org/g/openembedded-core/message/145177
Mute This Topic: https://lists.openembedded.org/mt/78663507/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
