On Fri, Sep 25, 2015 at 11:34 AM, Khem Raj <[email protected]> wrote: > >> On Sep 25, 2015, at 11:24 AM, Andre McCurdy <[email protected]> wrote: >> >> On Thu, Sep 24, 2015 at 9:59 PM, Khem Raj <[email protected]> wrote: >>> pxz results in significant time saving when generating xz filetypes for >>> images due to parallelization support >> >> It looks like the version of xz utils already packaged in oe-core >> supports multiple threads via the -T option. See XZ_THREADS in >> image_types.bbclass. > > the option -T is there for long time but does it work ?
I don't know. The -T option isn't supported by the version of xz packaged in Ubuntu 14.04, so maybe it has issues (oe-core commit cfd201ec suggests that xz -T 0 can use a lot of memory). pxz just looks like a simple wrapper on top of liblzma though, so I'd be surprised if it worked fundamentally better than xz. How much memory did your pxz test use? From the pxz source it looks like the -M option to limit memory usage is accepted but ignored... >>> a simple test on ubuntu build host with 16 CPUs >>> >>> time xz -M 50% -f -k -c -e -9 --check=crc32 >>> CX041AEI_PROD_default_20150610000105sdy-dbg.rootfs.cpio >>> >CX041AEI_PROD_default_20150610000105sdy-dbg.rootfs.cpio.xz >>> >>> real >>> 23m42.299s >>> user 23m36.947s >>> sys 0m5.101s >>> >>> time pxz -M 50% -f -k -c -e -9 --check=crc32 >>> CX041AEI_PROD_default_20150610000105sdy-dbg.rootfs.cpio >>> >CX041AEI_PROD_default_20150610000105sdy-dbg.rootfs.cpio.xz >>> >>> real2m59.666s >>> user 24m38.529s >>> sys 0m10.056s >>> >>> Signed-off-by: Khem Raj <[email protected] >>> --- >>> meta/classes/image_types.bbclass | 4 ++-- >>> meta/recipes-extended/pxz/pxz_git.bb | 31 +++++++++++++++++++++++++++++++ >>> 2 files changed, 33 insertions(+), 2 deletions(-) >>> create mode 100644 meta/recipes-extended/pxz/pxz_git.bb >>> >>> diff --git a/meta/classes/image_types.bbclass >>> b/meta/classes/image_types.bbclass >>> index 306403e..d766cd2 100644 >>> --- a/meta/classes/image_types.bbclass >>> +++ b/meta/classes/image_types.bbclass >>> @@ -233,13 +233,13 @@ COMPRESSIONTYPES = "gz bz2 lzma xz lz4 sum" >>> COMPRESS_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}.rootfs.${type}" >>> COMPRESS_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}.rootfs.${type} > >>> ${IMAGE_NAME}.rootfs.${type}.gz" >>> COMPRESS_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}.rootfs.${type}" >>> -COMPRESS_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} >>> --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}.rootfs.${type} > >>> ${IMAGE_NAME}.rootfs.${type}.xz" >>> +COMPRESS_CMD_xz = "pxz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} >>> --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}.rootfs.${type} > >>> ${IMAGE_NAME}.rootfs.${type}.xz" >>> COMPRESS_CMD_lz4 = "lz4c -9 -c ${IMAGE_NAME}.rootfs.${type} > >>> ${IMAGE_NAME}.rootfs.${type}.lz4" >>> COMPRESS_CMD_sum = "sumtool -i ${IMAGE_NAME}.rootfs.${type} -o >>> ${IMAGE_NAME}.rootfs.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" >>> COMPRESS_DEPENDS_lzma = "xz-native" >>> COMPRESS_DEPENDS_gz = "" >>> COMPRESS_DEPENDS_bz2 = "pbzip2-native" >>> -COMPRESS_DEPENDS_xz = "xz-native" >>> +COMPRESS_DEPENDS_xz = "pxz-native" >>> COMPRESS_DEPENDS_lz4 = "lz4-native" >>> COMPRESS_DEPENDS_sum = "mtd-utils-native" >>> >>> diff --git a/meta/recipes-extended/pxz/pxz_git.bb >>> b/meta/recipes-extended/pxz/pxz_git.bb >>> new file mode 100644 >>> index 0000000..fe21be6 >>> --- /dev/null >>> +++ b/meta/recipes-extended/pxz/pxz_git.bb >>> @@ -0,0 +1,31 @@ >>> +# Copyright (C) 2015 Khem Raj <[email protected]> >>> +# Released under the MIT license (see COPYING.MIT for the terms) >>> + >>> +SUMMARY = "Parallel LZMA compressor compatible with XZ" >>> +DESCRIPTION = "Parallel XZ is a compression utility that takes advantage >>> of running LZMA compression of different parts of an input file on multiple >>> cores and processors simultaneously. Its primary goal is to utilize all >>> resources to speed up compression time with minimal possible influence on >>> compression ratio" >>> +HOMEPAGE = "https://jnovy.fedorapeople.org/pxz/"; >>> +LICENSE = "GPL-2.0+" >>> +LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263" >>> +SECTION = "console/utils" >>> +DEPENDS = "xz" >>> + >>> +SRCREV = "ae808463c2950edfdedb8fb49f95006db0a18667" >>> +PV = "4.999.9beta+git${SRCPV}" >>> +SRC_URI = "git://github.com/jnovy/pxz.git" >>> + >>> +S = "${WORKDIR}/git" >>> + >>> +CFLAGS_append = " -fopenmp -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE" >>> +LDFLAGS_append = " -llzma" >>> + >>> +do_compile () { >>> + oe_runmake >>> +} >>> + >>> +do_install () { >>> + oe_runmake DESTDIR=${D} INSTALL="install -p" >>> +} >>> + >>> +deltask do_configure >>> + >>> +BBCLASSEXTEND = "native" >>> -- >>> 2.5.3 >>> >>> -- >>> _______________________________________________ >>> Openembedded-core mailing list >>> [email protected] >>> http://lists.openembedded.org/mailman/listinfo/openembedded-core > -- _______________________________________________ Openembedded-core mailing list [email protected] http://lists.openembedded.org/mailman/listinfo/openembedded-core
