Hi Josh, This still causes reproducibility issues:
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4113/steps/13/logs/stdio https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20231219-mrw5vs1g/packages/diff-html/ On 18/12/2023 08:37:00-0700, Joshua Watt wrote: > Converts IPK package generation to use zstd instead of xz. zstd has a > much larger compression/speed tradeoff range allowing users to choose > what suits them best, and fast decompression speeds. It also continues > to support parallel compression as xz did. > > A new variable called ZSTD_DEFAULTS is provided to set the defaults for > places that want to use zstd for compression; the zst image conversion > command is also modified to use this. > > Finally, in order for this to function properly, opkg must include zstd > support, so it is enabled all the time with no PACKAGECONFIG to turn it > off. > > Signed-off-by: Joshua Watt <[email protected]> > --- > meta/classes-global/package_ipk.bbclass | 2 +- > meta/classes-recipe/image_types.bbclass | 4 +--- > meta/conf/bitbake.conf | 4 ++++ > meta/lib/oe/package_manager/ipk/__init__.py | 4 ++-- > meta/recipes-devtools/opkg/opkg_0.6.2.bb | 6 +++--- > 5 files changed, 11 insertions(+), 9 deletions(-) > > diff --git a/meta/classes-global/package_ipk.bbclass > b/meta/classes-global/package_ipk.bbclass > index 71ffdd522ac..025dba49313 100644 > --- a/meta/classes-global/package_ipk.bbclass > +++ b/meta/classes-global/package_ipk.bbclass > @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf" > PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" > > # Program to be used to build opkg packages > -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' > +OPKGBUILDCMD ??= 'opkg-build -Z zstd -a "${ZSTD_DEFAULTS}"' > > OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" > OPKG_ARGS += "${@['', > '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" > diff --git a/meta/classes-recipe/image_types.bbclass > b/meta/classes-recipe/image_types.bbclass > index d615b41ed1e..3733bdfc20c 100644 > --- a/meta/classes-recipe/image_types.bbclass > +++ b/meta/classes-recipe/image_types.bbclass > @@ -63,8 +63,6 @@ ZIP_COMPRESSION_LEVEL ?= "-9" > 7ZIP_COMPRESSION_METHOD ?= "BZip2" > 7ZIP_EXTENSION ?= "7z" > > -ZSTD_COMPRESSION_LEVEL ?= "-3" > - > JFFS2_SUM_EXTRA_ARGS ?= "" > IMAGE_CMD:jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime > --output=${IMGDEPLOYDIR}/${IMAGE_NAME}.jffs2 ${EXTRA_IMAGECMD}" > > @@ -337,7 +335,7 @@ CONVERSION_CMD:lz4 = "lz4 -9 -z -l ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.l > CONVERSION_CMD:lzo = "lzop -9 ${IMAGE_NAME}.${type}" > CONVERSION_CMD:zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}.${type}.zip > ${IMAGE_NAME}.${type}" > CONVERSION_CMD:7zip = "7za a -mx=${7ZIP_COMPRESSION_LEVEL} > -mm=${7ZIP_COMPRESSION_METHOD} ${IMAGE_NAME}.${type}.${7ZIP_EXTENSION} > ${IMAGE_NAME}.${type}" > -CONVERSION_CMD:zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} > ${IMAGE_NAME}.${type} > ${IMAGE_NAME}.${type}.zst" > +CONVERSION_CMD:zst = "zstd -f -k -c ${ZSTD_DEFAULTS} ${IMAGE_NAME}.${type} > > ${IMAGE_NAME}.${type}.zst" > CONVERSION_CMD:sum = "sumtool -i ${IMAGE_NAME}.${type} -o > ${IMAGE_NAME}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" > CONVERSION_CMD:md5sum = "md5sum ${IMAGE_NAME}.${type} > > ${IMAGE_NAME}.${type}.md5sum" > CONVERSION_CMD:sha1sum = "sha1sum ${IMAGE_NAME}.${type} > > ${IMAGE_NAME}.${type}.sha1sum" > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index 83b12cbc158..b3f1f18a0d7 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -878,6 +878,10 @@ XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS" > ZSTD_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}" > ZSTD_THREADS[vardepvalue] = "1" > > +ZSTD_COMPRESSION_LEVEL ?= "-3" > +ZSTD_DEFAULTS ?= "--threads=${ZSTD_THREADS} ${ZSTD_COMPRESSION_LEVEL}" > +ZSTD_DEFAULTS[vardepsexclude] = "ZSTD_THREADS" > + > # Limit the number of threads that OpenMP libraries will use. Otherwise they > # may fallback to using all CPUs > export OMP_NUM_THREADS = "${BB_NUMBER_THREADS}" > diff --git a/meta/lib/oe/package_manager/ipk/__init__.py > b/meta/lib/oe/package_manager/ipk/__init__.py > index e6f9c08e2b6..8fcbad56aab 100644 > --- a/meta/lib/oe/package_manager/ipk/__init__.py > +++ b/meta/lib/oe/package_manager/ipk/__init__.py > @@ -133,7 +133,7 @@ class OpkgDpkgPM(PackageManager): > tmp_dir = tempfile.mkdtemp() > current_dir = os.getcwd() > os.chdir(tmp_dir) > - data_tar = 'data.tar.xz' > + data_tar = 'data.tar.zst' > > try: > cmd = [ar_cmd, 'x', pkg_path] > @@ -505,6 +505,6 @@ class OpkgPM(OpkgDpkgPM): > "trying to extract the package." % pkg) > > tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) > - bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) > + bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) > > return tmp_dir > diff --git a/meta/recipes-devtools/opkg/opkg_0.6.2.bb > b/meta/recipes-devtools/opkg/opkg_0.6.2.bb > index b2e675ae48b..71c0b44aed0 100644 > --- a/meta/recipes-devtools/opkg/opkg_0.6.2.bb > +++ b/meta/recipes-devtools/opkg/opkg_0.6.2.bb > @@ -8,7 +8,7 @@ LICENSE = "GPL-2.0-or-later" > LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f \ > > file://src/opkg.c;beginline=4;endline=18;md5=d6200b0f2b41dee278aa5fad333eecae" > > -DEPENDS = "libarchive" > +DEPENDS = "libarchive zstd" > > PE = "1" > > @@ -40,10 +40,10 @@ PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\ > PACKAGECONFIG[curl] = "--enable-curl,--disable-curl,curl" > PACKAGECONFIG[ssl-curl] = "--enable-ssl-curl,--disable-ssl-curl,curl openssl" > PACKAGECONFIG[sha256] = "--enable-sha256,--disable-sha256" > -PACKAGECONFIG[zstd] = "--enable-zstd,--disable-zstd,zstd" > PACKAGECONFIG[libsolv] = "--with-libsolv,--without-libsolv,libsolv" > > -EXTRA_OECONF:class-native = > "--localstatedir=/${@os.path.relpath('${localstatedir}', > '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', > '${STAGING_DIR_NATIVE}')}" > +EXTRA_OECONF = "--enable-zstd" > +EXTRA_OECONF:append:class-native = " > --localstatedir=/${@os.path.relpath('${localstatedir}', > '${STAGING_DIR_NATIVE}')} --sysconfdir=/${@os.path.relpath('${sysconfdir}', > '${STAGING_DIR_NATIVE}')}" > > do_install:append () { > install -d ${D}${sysconfdir}/opkg > -- > 2.34.1 > > > > -- Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#192778): https://lists.openembedded.org/g/openembedded-core/message/192778 Mute This Topic: https://lists.openembedded.org/mt/103244564/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
