Converts IPK package generation to use zstd instead of xz. zstd has a much larger compression/speed tradeoff range allowing users to choose what suits them best. In addition, it support parallel compression for additional compression speed up.
A new variable called ZSTD_DEFAULTS is provided to set the defaults for places that want to use zstd for compression; the zst image conversion command is also modified to use this. Finally, in order for this to function properly, opkg must include zstd support, so it is enabled all the time with no PACKAGECONFIG to turn it off. Signed-off-by: Joshua Watt <[email protected]> --- meta/classes-global/package_ipk.bbclass | 2 +- meta/classes-recipe/image_types.bbclass | 4 +--- meta/conf/bitbake.conf | 4 ++++ meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass index 71ffdd522ac..025dba49313 100644 --- a/meta/classes-global/package_ipk.bbclass +++ b/meta/classes-global/package_ipk.bbclass @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" diff --git a/meta/classes-recipe/image_types.bbclass b/meta/classes-recipe/image_types.bbclass index d615b41ed1e..3733bdfc20c 100644 --- a/meta/classes-recipe/image_types.bbclass +++ b/meta/classes-recipe/image_types.bbclass @@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" 7ZIP_COMPRESSION_METHOD ?= "BZip2" 7ZIP_EXTENSION ?= "7z" -ZSTD_COMPRESSION_LEVEL ?= "-3" - JFFS2_SUM_EXTRA_ARGS ?= "" IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}" @@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} ${IMAGE_NAME}.${type}.l CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip ${IMAGE_NAME}.${type}" CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} ${IMAGE_NAME}.${type}" -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.md5sum" CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.sha1sum" diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index 55166d01784..4b1fe28c85a 100644 --- a/meta/conf/bitbake.conf +++ b/meta/conf/bitbake.conf @@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" ZSTD_THREADS[vardepvalue] = "1" +ZSTD_COMPRESSION_LEVEL ?= "-3" +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_COMPRESSION_LEVEL ZSTD_THREADS" + # Limit the number of threads that OpenMP libraries will use. Otherwise they # may fallback to using all CPUs export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}" diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py index e6f9c08e2b6..8fcbad56aab 100644 --- a/meta/lib/oe/package_manager/ipk/__init__.py +++ b/meta/lib/oe/package_manager/ipk/__init__.py @@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - data_tar = 'data.tar.xz' + data_tar = 'data.tar.zst' try: cmd = [ar_cmd, 'x', pkg_path] @@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) return tmp_dir diff --git a/meta/recipes-devtools/opkg/opkg_0.6.2.bb b/meta/recipes-devtools/opkg/opkg_0.6.2.bb index b2e675ae48b..71c0b44aed0 100644 --- a/meta/recipes-devtools/opkg/opkg_0.6.2.bb +++ b/meta/recipes-devtools/opkg/opkg_0.6.2.bb @@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" -DEPENDS = "libarchive" +DEPENDS = "libarchive zstd" PE = "1" @@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" -EXTRA_OECONF:class-native = "--localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" +EXTRA_OECONF = "--enable-zstd" +EXTRA_OECONF:append:class-native = " --localstatedir=/${@os.path.relpath('${localstatedir}', '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', '${STAGING_DIR_NATIVE}')}" do_install:append () { install -d ${D}${sysconfdir}/opkg -- 2.34.1
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#192546): https://lists.openembedded.org/g/openembedded-core/message/192546 Mute This Topic: https://lists.openembedded.org/mt/103198693/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
