On Sat, Feb 17, 2018 at 4:48 PM Joshua Watt <jpewhac...@gmail.com> wrote:
> > > On Feb 17, 2018 18:25, "Khem Raj" <raj.k...@gmail.com> wrote: > > > > On 2/16/18 12:14 PM, Martin Jansa wrote: > >> + v = max(limit, v) >> >> this seems to be doing the opposite of what the limit was used for before: >> >> - v = min(64, int(v)) >> >> > moreover, I think the limit should be based on ncpu() instead of hard > coding it. > > > I don't know about that. Doesn't PARALLEL_MAKE already default to > something based on ncpu()? My understanding of the limit was that some > recipes (boost? waf?) have a limit on the amount of parallism they can > reasonably stand, and you don't want those recipes to break if someone sets > PARALLEL_MAKE="-j > 1000". > Ah yes that’s a fair point > > As long as that is true I think this is correct. Whether those recipes > still need to limit their parallism is another discussion. > > > On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhac...@gmail.com >> <mailto:jpewhac...@gmail.com>> wrote: >> >> The code to extract the integer number of parallel build threads and >> construct a new argument from them has started to be copied in >> multiple >> locations, so create two new helper utilities to aid recipes. >> >> The first helper (parallel_make()) extracts the integer number of >> parallel build threads from PARALLEL_MAKE. The second >> (parallel_make_argument()) does the same and then puts the result back >> into a format string, optionally clamping it to some maximum value. >> >> Additionally, rework the oe-core recipes that were manually doing this >> to use the new helper utilities. >> >> Signed-off-by: Joshua Watt <jpewhac...@gmail.com >> <mailto:jpewhac...@gmail.com>> >> >> --- >> meta/classes/go.bbclass | 19 +--------------- >> meta/classes/waf.bbclass | 24 +------------------- >> meta/lib/oe/utils.py | 43 >> ++++++++++++++++++++++++++++++++++++ >> meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +- >> >> meta/recipes-support/boost/boost.inc | 24 +------------------- >> 5 files changed, 47 insertions(+), 65 deletions(-) >> >> diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass >> index 09b01a84c37..7ecd8c92541 100644 >> --- a/meta/classes/go.bbclass >> +++ b/meta/classes/go.bbclass >> @@ -1,23 +1,6 @@ >> inherit goarch ptest >> >> -def get_go_parallel_make(d): >> - pm = (d.getVar('PARALLEL_MAKE') or '').split() >> - # look for '-j' and throw other options (e.g. '-l') away >> - # because they might have a different meaning in golang >> - while pm: >> - opt = pm.pop(0) >> - if opt == '-j': >> - v = pm.pop(0) >> - elif opt.startswith('-j'): >> - v = opt[2:].strip() >> - else: >> - continue >> - >> - return '-p %d' % int(v) >> - >> - return "" >> - >> -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" >> +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p >> %d')}" >> >> GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" >> GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go" >> diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass >> index bdbdc56767c..f9a851d0759 100644 >> --- a/meta/classes/waf.bbclass >> +++ b/meta/classes/waf.bbclass >> @@ -3,28 +3,6 @@ DISABLE_STATIC = "" >> >> EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" >> >> -def get_waf_parallel_make(d): >> - pm = d.getVar('PARALLEL_MAKE') >> - if pm: >> - # look for '-j' and throw other options (e.g. '-l') away >> - # because they might have different meaning in bjam >> - pm = pm.split() >> - while pm: >> - v = None >> - opt = pm.pop(0) >> - if opt == '-j': >> - v = pm.pop(0) >> - elif opt.startswith('-j'): >> - v = opt[2:].strip() >> - else: >> - v = None >> - >> - if v: >> - v = min(64, int(v)) >> - return '-j' + str(v) >> - >> - return "" >> - >> python waf_preconfigure() { >> import subprocess >> from distutils.version import StrictVersion >> @@ -47,7 +25,7 @@ waf_do_configure() { >> >> do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" >> waf_do_compile() { >> - ${S}/waf build ${@get_waf_parallel_make(d)} >> + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', >> limit=64)} >> } >> >> waf_do_install() { >> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py >> index 7a79d752b69..ec91927233a 100644 >> --- a/meta/lib/oe/utils.py >> +++ b/meta/lib/oe/utils.py >> @@ -156,6 +156,49 @@ def any_distro_features(d, features, >> truevalue="1", falsevalue=""): >> """ >> return bb.utils.contains_any("DISTRO_FEATURES", features, >> truevalue, falsevalue, d) >> >> +def parallel_make(d): >> + """ >> + Return the integer value for the number of parallel threads to >> use when >> + building, scraped out of PARALLEL_MAKE. If no parallelization >> option is >> + found, returns None >> + >> + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an >> integer. >> + """ >> + pm = (d.getVar('PARALLEL_MAKE') or '').split() >> + # look for '-j' and throw other options (e.g. '-l') away >> + while pm: >> + opt = pm.pop(0) >> + if opt == '-j': >> + v = pm.pop(0) >> + elif opt.startswith('-j'): >> + v = opt[2:].strip() >> + else: >> + continue >> + >> + return int(v) >> + >> + return None >> + >> +def parallel_make_argument(d, fmt, limit=None): >> + """ >> + Helper utility to construct a parallel make argument from the >> number of >> + parallel threads specified in PARALLEL_MAKE. >> + >> + Returns the input format string `fmt` where a single '%d' will >> be expanded >> + with the number of parallel threads to use. If `limit` is >> specified, the >> + number of parallel threads will be no larger than it. If no >> parallelization >> + option is found in PARALLEL_MAKE, returns an empty string >> + >> + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n >> %d") will return >> + "-n 10" >> + """ >> + v = parallel_make(d) >> + if v: >> + if limit: >> + v = max(limit, v) >> + return fmt % v >> + return '' >> + >> def packages_filter_out_system(d): >> """ >> Return a list of packages from PACKAGES with the "system" >> packages such as >> diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> >> b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> >> index fa0d66291d1..8750b3c528d 100644 >> --- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> >> +++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> >> >> @@ -151,7 +151,7 @@ do_compile_class-native() { >> >> do_compile_class-target() { >> export LFLAGS="${LDFLAGS}" >> - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" >> + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}" >> OVMF_ARCH="X64" >> if [ "${TARGET_ARCH}" != "x86_64" ] ; then >> OVMF_ARCH="IA32" >> diff --git a/meta/recipes-support/boost/boost.inc >> b/meta/recipes-support/boost/boost.inc >> index 41fc90fb211..0461ec6fcf1 100644 >> --- a/meta/recipes-support/boost/boost.inc >> +++ b/meta/recipes-support/boost/boost.inc >> @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \ >> >> # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, >> greater parallelism causes bjam to segfault or to ignore -j >> # https://svn.boost.org/trac/boost/ticket/7634 >> <https://svn.boost.org/trac/boost/ticket/7634> >> -def get_boost_parallel_make(d): >> - pm = d.getVar('PARALLEL_MAKE') >> - if pm: >> - # look for '-j' and throw other options (e.g. '-l') away >> - # because they might have different meaning in bjam >> - pm = pm.split() >> - while pm: >> - v = None >> - opt = pm.pop(0) >> - if opt == '-j': >> - v = pm.pop(0) >> - elif opt.startswith('-j'): >> - v = opt[2:].strip() >> - else: >> - v = None >> - >> - if v: >> - v = min(64, int(v)) >> - return '-j' + str(v) >> - >> - return "" >> - >> -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" >> +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, >> '-j%d', limit=64)}" >> BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \ >> ${BJAM_TOOLS} \ >> -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam >> \ >> -- >> 2.14.3 >> >> -- >> _______________________________________________ >> Openembedded-core mailing list >> Openembedded-core@lists.openembedded.org >> <mailto:Openembedded-core@lists.openembedded.org> >> http://lists.openembedded.org/mailman/listinfo/openembedded-core >> <http://lists.openembedded.org/mailman/listinfo/openembedded-core> >> >> >> >> -- > _______________________________________________ > Openembedded-core mailing list > Openembedded-core@lists.openembedded.org > > http://lists.openembedded.org/mailman/listinfo/openembedded-core > > >
-- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core