I noticed that xz and zstd are in HOSTTOOLS but we don't have them in ASSUME_PROVIDED. I ran some tests and now understand what the pros/cons of the situation are and I thought it worth documenting. At the end of the email, I've included a patch which people can experiment with if they're interested.
There are three things these tools are used for, compression, decompression and libraries. For decompression, we can use them from HOSTTOOLS. The lack of the entry in ASSUME_PROVIDED means we currently have to build the tool before that decompression can happen, which does slow builds a bit as more and more recipe source uses those formats. All versions of the tool should be able to decompress any versions output. For compression, the output can vary by version. This means for reproducibility, we need to use a specific version for compression else our builds don't reproduce. For libraries, we need the headers and need to use our built version. The patch below handles libraries but not compression and therefore fails reproducibility tests. There are also transient dependencies for recipes like sysstat which compresses manpages and searches for xz. There is no xz-native dependency so it's output is not reproducible and just happens to work through indirect dependencies in master today. One other consideration is that as the files are installed into the sysroot, the libraries need to go first, then the binaries that link against them. The opposite is true for removing them and this piece is probably harder. The issue is the binary can be called even when the dependency isn't active due to the HOSTTOOLS entries. This does mean there are potential race issues. The big win for builds would be being able to decompress without using a built recipe. I'm therefore wondering about reworking the patch below to use XXX-decompress instead of XXX-libs. I'll continue to ponder but wanted to document the thoughts/issues. Cheers, Richard [PATCH] bitbake.conf: Add xz-native and zstd-native to ASSUME_PROVIDED These have been in HOSTTOOLS for a while, they can be added to ASSUME_PROVIDED to avoid the recipe builds and streamline builds a bit. Signed-off-by: Richard Purdie <[email protected]> --- meta/conf/bitbake.conf | 2 ++ meta/recipes-core/zlib/zlib_1.3.2.bb | 1 + meta/recipes-devtools/cmake/cmake-native_4.3.2.bb | 2 +- meta/recipes-devtools/cmake/cmake_4.3.2.bb | 2 +- meta/recipes-devtools/dpkg/dpkg.inc | 2 +- meta/recipes-devtools/file/file_5.47.bb | 10 +++++----- .../squashfs-tools/squashfs-tools_4.7.5.bb | 8 ++++---- meta/recipes-extended/bzip2/bzip2_1.0.8.bb | 1 + meta/recipes-extended/xz/xz_5.8.3.bb | 2 ++ meta/recipes-extended/zstd/zstd_1.5.7.bb | 2 ++ meta/recipes-kernel/kmod/kmod_34.2.bb | 6 +++--- 11 files changed, 23 insertions(+), 15 deletions(-) diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index 4eb235da60a..96197edeb41 100644 --- a/meta/conf/bitbake.conf +++ b/meta/conf/bitbake.conf @@ -234,6 +234,8 @@ ASSUME_PROVIDED = "\ virtual/libiconv-native \ virtual/libintl-native \ wget-native \ + xz-native \ + zstd-native \ " # gzip-native should be listed above? diff --git a/meta/recipes-core/zlib/zlib_1.3.2.bb b/meta/recipes-core/zlib/zlib_1.3.2.bb index 8a23d98cd3e..aa9ed1b6157 100644 --- a/meta/recipes-core/zlib/zlib_1.3.2.bb +++ b/meta/recipes-core/zlib/zlib_1.3.2.bb @@ -3,6 +3,7 @@ DESCRIPTION = "Zlib is a general-purpose, patent-free, lossless data compression library which is used by many different programs." HOMEPAGE = "http://zlib.net/"; SECTION = "libs" +PROVIDES += "zlib-libs" LICENSE = "Zlib" LIC_FILES_CHKSUM = "file://zlib.h;beginline=6;endline=23;md5=5377232268e952e9ef63bc555f7aa6c0" diff --git a/meta/recipes-devtools/cmake/cmake-native_4.3.2.bb b/meta/recipes-devtools/cmake/cmake-native_4.3.2.bb index 9ab1f0ce738..3a3c62578c0 100644 --- a/meta/recipes-devtools/cmake/cmake-native_4.3.2.bb +++ b/meta/recipes-devtools/cmake/cmake-native_4.3.2.bb @@ -1,7 +1,7 @@ require cmake.inc inherit native -DEPENDS += "bzip2-replacement-native xz-native zlib-native ncurses-native zstd-native openssl-native" +DEPENDS += "bzip2-libs-native xz-libs-native zlib-libs-native ncurses-native zstd-libs-native openssl-native" SRC_URI += "file://OEToolchainConfig.cmake \ file://environment.d-cmake.sh \ diff --git a/meta/recipes-devtools/cmake/cmake_4.3.2.bb b/meta/recipes-devtools/cmake/cmake_4.3.2.bb index e6abc06c85c..0734ff9c2a8 100644 --- a/meta/recipes-devtools/cmake/cmake_4.3.2.bb +++ b/meta/recipes-devtools/cmake/cmake_4.3.2.bb @@ -2,7 +2,7 @@ require cmake.inc inherit cmake bash-completion -DEPENDS += "curl expat zlib libarchive xz ncurses bzip2 libuv" +DEPENDS += "curl expat zlib libarchive xz-libs ncurses bzip2 libuv" SRC_URI:append:class-nativesdk = " \ file://OEToolchainConfig.cmake \ diff --git a/meta/recipes-devtools/dpkg/dpkg.inc b/meta/recipes-devtools/dpkg/dpkg.inc index b4e9667107e..93b442f3930 100644 --- a/meta/recipes-devtools/dpkg/dpkg.inc +++ b/meta/recipes-devtools/dpkg/dpkg.inc @@ -31,7 +31,7 @@ EXTRA_OECONF:append:class-target = " --disable-update-alternatives DEB_HOST_ARCH EXTRA_OECONF:append:class-nativesdk = " --disable-update-alternatives DEB_HOST_ARCH=${DPKG_ARCH}" PACKAGECONFIG = "liblzma" -PACKAGECONFIG[liblzma] = "--with-liblzma,--without-liblzma, xz" +PACKAGECONFIG[liblzma] = "--with-liblzma,--without-liblzma, xz-libs" #autotools.bbclass default AUTOTOOLS_AUXDIR is ${S}, we need to under ${S}/build-aux AUTOTOOLS_AUXDIR = "${S}/build-aux" diff --git a/meta/recipes-devtools/file/file_5.47.bb b/meta/recipes-devtools/file/file_5.47.bb index 528ed49a120..0db886cb914 100644 --- a/meta/recipes-devtools/file/file_5.47.bb +++ b/meta/recipes-devtools/file/file_5.47.bb @@ -8,8 +8,8 @@ SECTION = "console/utils" LICENSE = "BSD-2-Clause" LIC_FILES_CHKSUM = "file://COPYING;beginline=2;md5=0251eaec1188b20d9a72c502ecfdda1b" -DEPENDS = "file-replacement-native" -DEPENDS:class-native = "bzip2-replacement-native" +DEPENDS:class-target = "file-replacement-native" +DEPENDS:class-nativesdk = "file-replacement-native" SRC_URI = "git://github.com/file/file.git;branch=master;protocol=https \ " @@ -19,10 +19,10 @@ SRCREV = "7d20612996567ecedec7f5c58f7bf15c2cf42c19" inherit autotools update-alternatives PACKAGECONFIG ??= "bz2 lzma zlib zstdlib lzlib" -PACKAGECONFIG[bz2] = "--enable-bzlib, --disable-bzlib, bzip2" -PACKAGECONFIG[lzma] = "--enable-xzlib, --disable-xzlib, xz" +PACKAGECONFIG[bz2] = "--enable-bzlib, --disable-bzlib, bzip2-libs" +PACKAGECONFIG[lzma] = "--enable-xzlib, --disable-xzlib, xz-libs" PACKAGECONFIG[zlib] = "--enable-zlib, --disable-zlib, zlib" -PACKAGECONFIG[zstdlib] = "--enable-zstdlib, --disable-zstdlib, zstd" +PACKAGECONFIG[zstdlib] = "--enable-zstdlib, --disable-zstdlib, zstd-libs" PACKAGECONFIG[lzlib] = "--enable-lzlib, --disable-lzlib, lzlib" PACKAGECONFIG[seccomp] = "--enable-libseccomp, --disable-libseccomp, libseccomp" diff --git a/meta/recipes-devtools/squashfs-tools/squashfs-tools_4.7.5.bb b/meta/recipes-devtools/squashfs-tools/squashfs-tools_4.7.5.bb index 9a1ebd575c1..f965998774b 100644 --- a/meta/recipes-devtools/squashfs-tools/squashfs-tools_4.7.5.bb +++ b/meta/recipes-devtools/squashfs-tools/squashfs-tools_4.7.5.bb @@ -13,12 +13,12 @@ UPSTREAM_CHECK_GITTAGREGEX = "(?P<pver>(\d+(\.\d+)+))" EXTRA_OEMAKE = "${PACKAGECONFIG_CONFARGS}" PACKAGECONFIG ??= "gzip lz4 xz zstd xattr" -PACKAGECONFIG[gzip] = "GZIP_SUPPORT=1,GZIP_SUPPORT=0,zlib" -PACKAGECONFIG[xz] = "XZ_SUPPORT=1,XZ_SUPPORT=0,xz" +PACKAGECONFIG[gzip] = "GZIP_SUPPORT=1,GZIP_SUPPORT=0,zlib-libs" +PACKAGECONFIG[xz] = "XZ_SUPPORT=1,XZ_SUPPORT=0,xz-libs" PACKAGECONFIG[lzo] = "LZO_SUPPORT=1,LZO_SUPPORT=0,lzo" PACKAGECONFIG[lz4] = "LZ4_SUPPORT=1,LZ4_SUPPORT=0,lz4" -PACKAGECONFIG[lzma] = "LZMA_XZ_SUPPORT=1,LZMA_XZ_SUPPORT=0,xz" -PACKAGECONFIG[zstd] = "ZSTD_SUPPORT=1,ZSTD_SUPPORT=0,zstd" +PACKAGECONFIG[lzma] = "LZMA_XZ_SUPPORT=1,LZMA_XZ_SUPPORT=0,xz-libs" +PACKAGECONFIG[zstd] = "ZSTD_SUPPORT=1,ZSTD_SUPPORT=0,zstd-libs" PACKAGECONFIG[xattr] = "XATTR_SUPPORT=1,XATTR_SUPPORT=0,attr" do_compile() { diff --git a/meta/recipes-extended/bzip2/bzip2_1.0.8.bb b/meta/recipes-extended/bzip2/bzip2_1.0.8.bb index 9cecf6a331b..47dfe8160f2 100644 --- a/meta/recipes-extended/bzip2/bzip2_1.0.8.bb +++ b/meta/recipes-extended/bzip2/bzip2_1.0.8.bb @@ -68,5 +68,6 @@ RDEPENDS:${PN}-ptest += "make bash" CVE_PRODUCT = "bzip:bzip2" +PROVIDES += "bzip2-libs" PROVIDES:append:class-native = " bzip2-replacement-native" BBCLASSEXTEND = "native nativesdk" diff --git a/meta/recipes-extended/xz/xz_5.8.3.bb b/meta/recipes-extended/xz/xz_5.8.3.bb index 74efe561c6b..f253c764006 100644 --- a/meta/recipes-extended/xz/xz_5.8.3.bb +++ b/meta/recipes-extended/xz/xz_5.8.3.bb @@ -3,6 +3,8 @@ HOMEPAGE = "https://tukaani.org/xz/"; DESCRIPTION = "XZ Utils is free general-purpose data compression software with a high compression ratio. XZ Utils were written for POSIX-like systems, but also work on some not-so-POSIX systems. XZ Utils are the successor to LZMA Utils." SECTION = "base" +PROVIDES += "xz-libs" + # The source includes bits of 0BSD, GPL-2.0, GPL-3.0, LGPL-2.1-or-later, but the # only file which is GPL-3.0 is an m4 macro which isn't shipped in any of our # packages, and the LGPL bits are under lib/, which appears to be used for diff --git a/meta/recipes-extended/zstd/zstd_1.5.7.bb b/meta/recipes-extended/zstd/zstd_1.5.7.bb index 66e326f386e..2c9e500d651 100644 --- a/meta/recipes-extended/zstd/zstd_1.5.7.bb +++ b/meta/recipes-extended/zstd/zstd_1.5.7.bb @@ -5,6 +5,8 @@ It's backed by a very fast entropy stage, provided by Huff0 and FSE library." HOMEPAGE = "http://www.zstd.net/"; SECTION = "console/utils" +PROVIDES += "zstd-libs" + LICENSE = "BSD-3-Clause | GPL-2.0-only" LIC_FILES_CHKSUM = "file://LICENSE;md5=0822a32f7acdbe013606746641746ee8 \ file://COPYING;md5=39bba7d2cf0ba1036f2a6e2be52fe3f0 \ diff --git a/meta/recipes-kernel/kmod/kmod_34.2.bb b/meta/recipes-kernel/kmod/kmod_34.2.bb index a61466cefbf..c36ee3d4b60 100644 --- a/meta/recipes-kernel/kmod/kmod_34.2.bb +++ b/meta/recipes-kernel/kmod/kmod_34.2.bb @@ -34,9 +34,9 @@ PACKAGECONFIG[debug] = "-Ddebug-messages=true,-Ddebug-messages=false" PACKAGECONFIG[logging] = " -Dlogging=true,-Dlogging=false" PACKAGECONFIG[manpages] = "-Dmanpages=true,-Dmanpages=false,scdoc-native" PACKAGECONFIG[openssl] = "-Dopenssl=enabled,-Dopenssl=disabled,openssl" -PACKAGECONFIG[xz] = "-Dxz=enabled,-Dxz=disabled,xz" -PACKAGECONFIG[zlib] = "-Dzlib=enabled,-Dzlib=disabled,zlib" -PACKAGECONFIG[zstd] = "-Dzstd=enabled,-Dzstd=disabled,zstd" +PACKAGECONFIG[xz] = "-Dxz=enabled,-Dxz=disabled,xz-libs" +PACKAGECONFIG[zlib] = "-Dzlib=enabled,-Dzlib=disabled,zlib-libs" +PACKAGECONFIG[zstd] = "-Dzstd=enabled,-Dzstd=disabled,zstd-libs" PROVIDES += "module-init-tools-insmod-static module-init-tools-depmod module-init-tools" RPROVIDES:${PN} += "module-init-tools-insmod-static module-init-tools-depmod module-init-tools"
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#2392): https://lists.openembedded.org/g/openembedded-architecture/message/2392 Mute This Topic: https://lists.openembedded.org/mt/119659553/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-architecture/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
