On Feb 17, 2018 18:25, "Khem Raj" <raj.k...@gmail.com> wrote:
On 2/16/18 12:14 PM, Martin Jansa wrote: > + v = max(limit, v) > > this seems to be doing the opposite of what the limit was used for before: > > - v = min(64, int(v)) > > moreover, I think the limit should be based on ncpu() instead of hard coding it. I don't know about that. Doesn't PARALLEL_MAKE already default to something based on ncpu()? My understanding of the limit was that some recipes (boost? waf?) have a limit on the amount of parallism they can reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j 1000". As long as that is true I think this is correct. Whether those recipes still need to limit their parallism is another discussion. On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhac...@gmail.com <mailto: > jpewhac...@gmail.com>> wrote: > > The code to extract the integer number of parallel build threads and > construct a new argument from them has started to be copied in multiple > locations, so create two new helper utilities to aid recipes. > > The first helper (parallel_make()) extracts the integer number of > parallel build threads from PARALLEL_MAKE. The second > (parallel_make_argument()) does the same and then puts the result back > into a format string, optionally clamping it to some maximum value. > > Additionally, rework the oe-core recipes that were manually doing this > to use the new helper utilities. > > Signed-off-by: Joshua Watt <jpewhac...@gmail.com > <mailto:jpewhac...@gmail.com>> > > --- > meta/classes/go.bbclass | 19 +--------------- > meta/classes/waf.bbclass | 24 +------------------- > meta/lib/oe/utils.py | 43 > ++++++++++++++++++++++++++++++++++++ > meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +- > > meta/recipes-support/boost/boost.inc | 24 +------------------- > 5 files changed, 47 insertions(+), 65 deletions(-) > > diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass > index 09b01a84c37..7ecd8c92541 100644 > --- a/meta/classes/go.bbclass > +++ b/meta/classes/go.bbclass > @@ -1,23 +1,6 @@ > inherit goarch ptest > > -def get_go_parallel_make(d): > - pm = (d.getVar('PARALLEL_MAKE') or '').split() > - # look for '-j' and throw other options (e.g. '-l') away > - # because they might have a different meaning in golang > - while pm: > - opt = pm.pop(0) > - if opt == '-j': > - v = pm.pop(0) > - elif opt.startswith('-j'): > - v = opt[2:].strip() > - else: > - continue > - > - return '-p %d' % int(v) > - > - return "" > - > -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" > +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p > %d')}" > > GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" > GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go" > diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass > index bdbdc56767c..f9a851d0759 100644 > --- a/meta/classes/waf.bbclass > +++ b/meta/classes/waf.bbclass > @@ -3,28 +3,6 @@ DISABLE_STATIC = "" > > EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" > > -def get_waf_parallel_make(d): > - pm = d.getVar('PARALLEL_MAKE') > - if pm: > - # look for '-j' and throw other options (e.g. '-l') away > - # because they might have different meaning in bjam > - pm = pm.split() > - while pm: > - v = None > - opt = pm.pop(0) > - if opt == '-j': > - v = pm.pop(0) > - elif opt.startswith('-j'): > - v = opt[2:].strip() > - else: > - v = None > - > - if v: > - v = min(64, int(v)) > - return '-j' + str(v) > - > - return "" > - > python waf_preconfigure() { > import subprocess > from distutils.version import StrictVersion > @@ -47,7 +25,7 @@ waf_do_configure() { > > do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" > waf_do_compile() { > - ${S}/waf build ${@get_waf_parallel_make(d)} > + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', > limit=64)} > } > > waf_do_install() { > diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py > index 7a79d752b69..ec91927233a 100644 > --- a/meta/lib/oe/utils.py > +++ b/meta/lib/oe/utils.py > @@ -156,6 +156,49 @@ def any_distro_features(d, features, > truevalue="1", falsevalue=""): > """ > return bb.utils.contains_any("DISTRO_FEATURES", features, > truevalue, falsevalue, d) > > +def parallel_make(d): > + """ > + Return the integer value for the number of parallel threads to > use when > + building, scraped out of PARALLEL_MAKE. If no parallelization > option is > + found, returns None > + > + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an > integer. > + """ > + pm = (d.getVar('PARALLEL_MAKE') or '').split() > + # look for '-j' and throw other options (e.g. '-l') away > + while pm: > + opt = pm.pop(0) > + if opt == '-j': > + v = pm.pop(0) > + elif opt.startswith('-j'): > + v = opt[2:].strip() > + else: > + continue > + > + return int(v) > + > + return None > + > +def parallel_make_argument(d, fmt, limit=None): > + """ > + Helper utility to construct a parallel make argument from the > number of > + parallel threads specified in PARALLEL_MAKE. > + > + Returns the input format string `fmt` where a single '%d' will > be expanded > + with the number of parallel threads to use. If `limit` is > specified, the > + number of parallel threads will be no larger than it. If no > parallelization > + option is found in PARALLEL_MAKE, returns an empty string > + > + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n > %d") will return > + "-n 10" > + """ > + v = parallel_make(d) > + if v: > + if limit: > + v = max(limit, v) > + return fmt % v > + return '' > + > def packages_filter_out_system(d): > """ > Return a list of packages from PACKAGES with the "system" > packages such as > diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> > b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> > index fa0d66291d1..8750b3c528d 100644 > --- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> > +++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> > > @@ -151,7 +151,7 @@ do_compile_class-native() { > > do_compile_class-target() { > export LFLAGS="${LDFLAGS}" > - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" > + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}" > OVMF_ARCH="X64" > if [ "${TARGET_ARCH}" != "x86_64" ] ; then > OVMF_ARCH="IA32" > diff --git a/meta/recipes-support/boost/boost.inc > b/meta/recipes-support/boost/boost.inc > index 41fc90fb211..0461ec6fcf1 100644 > --- a/meta/recipes-support/boost/boost.inc > +++ b/meta/recipes-support/boost/boost.inc > @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \ > > # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, > greater parallelism causes bjam to segfault or to ignore -j > # https://svn.boost.org/trac/boost/ticket/7634 > <https://svn.boost.org/trac/boost/ticket/7634> > -def get_boost_parallel_make(d): > - pm = d.getVar('PARALLEL_MAKE') > - if pm: > - # look for '-j' and throw other options (e.g. '-l') away > - # because they might have different meaning in bjam > - pm = pm.split() > - while pm: > - v = None > - opt = pm.pop(0) > - if opt == '-j': > - v = pm.pop(0) > - elif opt.startswith('-j'): > - v = opt[2:].strip() > - else: > - v = None > - > - if v: > - v = min(64, int(v)) > - return '-j' + str(v) > - > - return "" > - > -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" > +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, > '-j%d', limit=64)}" > BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \ > ${BJAM_TOOLS} \ > -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam > \ > -- > 2.14.3 > > -- > _______________________________________________ > Openembedded-core mailing list > Openembedded-core@lists.openembedded.org > <mailto:Openembedded-core@lists.openembedded.org> > http://lists.openembedded.org/mailman/listinfo/openembedded-core > <http://lists.openembedded.org/mailman/listinfo/openembedded-core> > > > > -- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
-- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core