Hi Michal, Thanks, this is definitely helpful.
Have you checked if your improvement is tested. I think it should be tested by the test_devtool_add_npm under meta/lib/oeqa/selftest/cases/devtool.py See: - https://wiki.yoctoproject.org/wiki/Oe-selftest - https://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=22dd46cc34629d0750177fddff2e1c178c854340 Best Regards, Jean-Marie On Wed, Dec 2, 2020 at 5:26 PM <[email protected]> wrote: > Task do_configure delivered by npm.bbclass takes a lot of time, the longer > shrinkwrap dependency list the longer time needed for do_configure to > execute. > Following improvement allows to execute do_configure on any available CPI > threads. > > diff --git a/bitbake/lib/bb/fetch2/npmsw.py > b/bitbake/lib/bb/fetch2/npmsw.py > index 0c3511d8ab..0e6075859c 100644 > --- a/bitbake/lib/bb/fetch2/npmsw.py > +++ b/bitbake/lib/bb/fetch2/npmsw.py > @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity > from bb.fetch2.npm import npm_localfile > from bb.fetch2.npm import npm_unpack > from bb.utils import is_semver > +from concurrent.futures import ThreadPoolExecutor > > -def foreach_dependencies(shrinkwrap, callback=None, dev=False): > +def foreach_dependencies(shrinkwrap, callback=None, dev=False, > executor=None): > """ > Run a callback for each dependencies of a shrinkwrap file. > The callback is using the format: > @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None, > dev=False): > continue > elif deps[name].get("bundled", False): > continue > - callback(name, deps[name], subtree) > + if executor: > + executor.submit(callback, name=name, > params=deps[name], deptree=subtree) > + else: > + callback(name, deps[name], subtree) > > _walk_deps(shrinkwrap.get("dependencies", {}), []) > > diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass > index 068032a1e5..1869b63cdb 100644 > --- a/meta/classes/npm.bbclass > +++ b/meta/classes/npm.bbclass > @@ -80,6 +80,7 @@ python npm_do_configure() { > from bb.fetch2.npm import npm_unpack > from bb.fetch2.npmsw import foreach_dependencies > from bb.progress import OutOfProgressHandler > + from concurrent.futures import ThreadPoolExecutor > > bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True) > bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True) > @@ -164,9 +165,11 @@ python npm_do_configure() { > progress_done += 1 > progress.write("%d/%d" % (progress_done, progress_total)) > > + executor = ThreadPoolExecutor(max_workers = os.cpu_count()) > dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) > foreach_dependencies(orig_shrinkwrap, _count_dependency, dev) > - foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev) > + foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev, > executor) > + executor.shutdown(True) > > # Configure the main package > with tempfile.TemporaryDirectory() as tmpdir: > > > > > > >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#145180): https://lists.openembedded.org/g/openembedded-core/message/145180 Mute This Topic: https://lists.openembedded.org/mt/78663507/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
