subject:"\[OE\-core\] \[PATCH\] sstate\: Switch to ZStandard compressor support"

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

2021-10-12 Thread Henry Kleynhans

From: Henry Kleynhans 

This patch switches the compressor from Gzip to ZStandard for ssate cache
files.

Zstandard compression provides a significant improvement in
decompression speed as well as improvement in compression speed and disk
usage over the 'tgz' format in use.  Furthermore, its configurable
compression level offers a trade-off between time spent compressing
sstate cache files and disk space used by those files.  The reduced disk
usage also contributes to saving network traffic for those sharing their
sstate cache with others.

Zstandard should therefore be a good choice when:
* disk space is at a premium
* network speed / resources are limited
* the CI server can sstate packages can be created at high compression
* less CPU on the build server should be used for sstate decompression

Signed-off-by: Henry Kleynhans 
---
 meta/classes/sstate.bbclass | 29 +--
 meta/lib/oeqa/selftest/cases/signing.py |  8 ++---
 meta/lib/oeqa/selftest/cases/sstatetests.py | 24 ++---
 scripts/sstate-cache-management.sh  | 40 ++---
 4 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 7f4b1f6804..fe61323010 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1,17 +1,19 @@
 SSTATE_VERSION = "5"
 
+SSTATE_ZSTD_CLEVEL ??= "8"
+
 SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
 SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
 
 def generate_sstatefn(spec, hash, taskname, siginfo, d):
 if taskname is None:
return ""
-extension = ".tgz"
+extension = ".tar.zst"
 # 8 chars reserved for siginfo
 limit = 254 - 8
 if siginfo:
 limit = 254
-extension = ".tgz.siginfo"
+extension = ".tar.zst.siginfo"
 if not hash:
 hash = "INVALID"
 fn = spec + hash + "_" + taskname + extension
@@ -37,7 +39,7 @@ SSTATE_PKGNAME= 
"${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
 SSTATE_PKG= "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
+SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
 
 # explicitly make PV to depend on evaluated value of PV variable
 PV[vardepvalue] = "${PV}"
@@ -832,23 +834,24 @@ sstate_create_package () {
mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
TFILE=`mktemp ${SSTATE_PKG}.`
 
-   # Use pigz if available
-   OPT="-czS"
-   if [ -x "$(command -v pigz)" ]; then
-   OPT="-I pigz -cS"
+   OPT="-cS"
+   ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
fi
 
# Need to handle empty directories
if [ "$(ls -A)" ]; then
set +e
-   tar $OPT -f $TFILE *
+   tar -I "$ZSTD" $OPT -f $TFILE *
ret=$?
if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
exit 1
fi
set -e
else
-   tar $OPT --file=$TFILE --files-from=/dev/null
+   tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
fi
chmod 0664 $TFILE
# Skip if it was already created by some other process
@@ -887,7 +890,13 @@ python sstate_report_unihash() {
 # Will be run from within SSTATE_INSTDIR.
 #
 sstate_unpack_package () {
-   tar -xvzf ${SSTATE_PKG}
+   ZSTD="zstd -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -p ${ZSTD_THREADS}"
+   fi
+
+   tar -I "$ZSTD" -xvf ${SSTATE_PKG}
# update .siginfo atime on local/NFS mirror
[ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h 
${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
# Use "! -w ||" to return true for read only files
diff --git a/meta/lib/oeqa/selftest/cases/signing.py 
b/meta/lib/oeqa/selftest/cases/signing.py
index af7a0b8b45..6f3d4aeae9 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -159,13 +159,13 @@ class Signing(OESelftestTestCase):
 bitbake('-c clean %s' % test_recipe)
 bitbake('-c populate_lic %s' % test_recipe)
 
-recipe_sig = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tgz.sig')
-recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz')
+recipe_sig = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tar.zst.sig')
+recipe_archive = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tar.zst')

Re: [OE-core] [PATCH] sstate: Switch to ZStandard compressor support

2021-10-11 Thread Jose Quaresma

Hi Henry,

Henry Kleynhans  escreveu no dia segunda,
11/10/2021 à(s) 18:12:

> From: Henry Kleynhans 
>
> This patch switches the compressor from Gzip to ZStandard for ssate cache
> files.
>
> Zstandard compression provides a significant improvement in
> decompression speed as well as improvement in compression speed and disk
> usage over the 'tgz' format in use.  Furthermore, its configurable
> compression level offers a trade-off between time spent compressing
> sstate cache files and disk space used by those files.  The reduced disk
> usage also contributes to saving network traffic for those sharing their
> sstate cache with others.
>
> Zstandard should therefore be a good choice when:
> * disk space is at a premium
> * network speed / resources are limited
> * the CI server can sstate packages can be created at high compression
> * less CPU on the build server should be used for sstate decompression
>
> Signed-off-by: Henry Kleynhans 
> ---
>  meta/classes/sstate.bbclass | 29 +--
>  meta/lib/oeqa/selftest/cases/signing.py |  8 ++---
>  meta/lib/oeqa/selftest/cases/sstatetests.py | 24 ++---
>  scripts/sstate-cache-management.sh  | 40 ++---
>  4 files changed, 55 insertions(+), 46 deletions(-)
>
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 7f4b1f6804..3a5195267a 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -1,17 +1,19 @@
>  SSTATE_VERSION = "5"
>
> +SSTATE_ZSTD_CLEVEL = "8"
>

Giving the user the ability to adjust the compression level can be useful.

Using a weak default value (?? = "8") gives the possibility to use a
default value (?=) in local.conf
that can be overwritten at recipe level.

Sets only a default value on the class can be more appropriate and fits
most use cases.

SSTATE_ZSTD_CLEVEL ?= "8"


> +
>  SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
>  SSTATE_MANFILEPREFIX =
> "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
>
>  def generate_sstatefn(spec, hash, taskname, siginfo, d):
>  if taskname is None:
> return ""
> -extension = ".tgz"
> +extension = ".tar.zst"
>  # 8 chars reserved for siginfo
>  limit = 254 - 8
>  if siginfo:
>  limit = 254
> -extension = ".tgz.siginfo"
> +extension = ".tar.zst.siginfo"
>  if not hash:
>  hash = "INVALID"
>  fn = spec + hash + "_" + taskname + extension
> @@ -37,7 +39,7 @@ SSTATE_PKGNAME=
> "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
>  SSTATE_PKG= "${SSTATE_DIR}/${SSTATE_PKGNAME}"
>  SSTATE_EXTRAPATH   = ""
>  SSTATE_EXTRAPATHWILDCARD = ""
> -SSTATE_PATHSPEC   =
> "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
> +SSTATE_PATHSPEC   =
> "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
>
>  # explicitly make PV to depend on evaluated value of PV variable
>  PV[vardepvalue] = "${PV}"
> @@ -832,23 +834,24 @@ sstate_create_package () {
> mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
> TFILE=`mktemp ${SSTATE_PKG}.`
>
> -   # Use pigz if available
> -   OPT="-czS"
> -   if [ -x "$(command -v pigz)" ]; then
> -   OPT="-I pigz -cS"
> +   OPT="-cS"
> +   ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
> +   # Use pzstd if available
> +   if [ -x "$(command -v pzstd)" ]; then
> +   ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
> fi
>
> # Need to handle empty directories
> if [ "$(ls -A)" ]; then
> set +e
> -   tar $OPT -f $TFILE *
> +   tar -I "$ZSTD" $OPT -f $TFILE *
> ret=$?
> if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
> exit 1
> fi
> set -e
> else
> -   tar $OPT --file=$TFILE --files-from=/dev/null
> +   tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
> fi
> chmod 0664 $TFILE
> # Skip if it was already created by some other process
> @@ -887,7 +890,13 @@ python sstate_report_unihash() {
>  # Will be run from within SSTATE_INSTDIR.
>  #
>  sstate_unpack_package () {
> -   tar -xvzf ${SSTATE_PKG}
> +   ZSTD="zstd -T${ZSTD_THREADS}"
> +   # Use pzstd if available
> +   if [ -x "$(command -v pzstd)" ]; then
> +   ZSTD="pzstd -p ${ZSTD_THREADS}"
> +   fi
> +
> +   tar -I "$ZSTD" -xvf ${SSTATE_PKG}
> # update .siginfo atime on local/NFS mirror
> [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [
> -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
> # Use "! -w ||" to return true for read only files
> diff --git a/meta/lib/oeqa/selftest/cases/signing.py
> b/meta/lib/oeqa/selftest/cases/signing.py
> index af7a0b8b45..6f3d4aeae9 100644
> ---

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

2021-10-11 Thread Henry Kleynhans

From: Henry Kleynhans 

This patch switches the compressor from Gzip to ZStandard for ssate cache
files.

Zstandard compression provides a significant improvement in
decompression speed as well as improvement in compression speed and disk
usage over the 'tgz' format in use.  Furthermore, its configurable
compression level offers a trade-off between time spent compressing
sstate cache files and disk space used by those files.  The reduced disk
usage also contributes to saving network traffic for those sharing their
sstate cache with others.

Zstandard should therefore be a good choice when:
* disk space is at a premium
* network speed / resources are limited
* the CI server can sstate packages can be created at high compression
* less CPU on the build server should be used for sstate decompression

Signed-off-by: Henry Kleynhans 
---
 meta/classes/sstate.bbclass | 29 +--
 meta/lib/oeqa/selftest/cases/signing.py |  8 ++---
 meta/lib/oeqa/selftest/cases/sstatetests.py | 24 ++---
 scripts/sstate-cache-management.sh  | 40 ++---
 4 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 7f4b1f6804..3a5195267a 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1,17 +1,19 @@
 SSTATE_VERSION = "5"
 
+SSTATE_ZSTD_CLEVEL = "8"
+
 SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
 SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
 
 def generate_sstatefn(spec, hash, taskname, siginfo, d):
 if taskname is None:
return ""
-extension = ".tgz"
+extension = ".tar.zst"
 # 8 chars reserved for siginfo
 limit = 254 - 8
 if siginfo:
 limit = 254
-extension = ".tgz.siginfo"
+extension = ".tar.zst.siginfo"
 if not hash:
 hash = "INVALID"
 fn = spec + hash + "_" + taskname + extension
@@ -37,7 +39,7 @@ SSTATE_PKGNAME= 
"${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
 SSTATE_PKG= "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
+SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
 
 # explicitly make PV to depend on evaluated value of PV variable
 PV[vardepvalue] = "${PV}"
@@ -832,23 +834,24 @@ sstate_create_package () {
mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
TFILE=`mktemp ${SSTATE_PKG}.`
 
-   # Use pigz if available
-   OPT="-czS"
-   if [ -x "$(command -v pigz)" ]; then
-   OPT="-I pigz -cS"
+   OPT="-cS"
+   ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
fi
 
# Need to handle empty directories
if [ "$(ls -A)" ]; then
set +e
-   tar $OPT -f $TFILE *
+   tar -I "$ZSTD" $OPT -f $TFILE *
ret=$?
if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
exit 1
fi
set -e
else
-   tar $OPT --file=$TFILE --files-from=/dev/null
+   tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
fi
chmod 0664 $TFILE
# Skip if it was already created by some other process
@@ -887,7 +890,13 @@ python sstate_report_unihash() {
 # Will be run from within SSTATE_INSTDIR.
 #
 sstate_unpack_package () {
-   tar -xvzf ${SSTATE_PKG}
+   ZSTD="zstd -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -p ${ZSTD_THREADS}"
+   fi
+
+   tar -I "$ZSTD" -xvf ${SSTATE_PKG}
# update .siginfo atime on local/NFS mirror
[ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h 
${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
# Use "! -w ||" to return true for read only files
diff --git a/meta/lib/oeqa/selftest/cases/signing.py 
b/meta/lib/oeqa/selftest/cases/signing.py
index af7a0b8b45..6f3d4aeae9 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -159,13 +159,13 @@ class Signing(OESelftestTestCase):
 bitbake('-c clean %s' % test_recipe)
 bitbake('-c populate_lic %s' % test_recipe)
 
-recipe_sig = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tgz.sig')
-recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz')
+recipe_sig = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tar.zst.sig')
+recipe_archive = glob.glob(sstatedir + 
'/*/*/*:ed:*_populate_lic.tar.zst')

Re: [OE-core] [PATCH] sstate: Switch to ZStandard compressor support

2021-10-06 Thread Alexandre Belloni

Hello Henry,

On 04/10/2021 14:38:58+0100, Henry Kleynhans wrote:
> From: Henry Kleynhans 
> 
> This patch switches the compressor from Gzip to ZStandard for ssate cache
> files.
> 
> Zstandard compression provides a significant improvement in
> decompression speed as well as improvement in compression speed and disk
> usage over the 'tgz' format in use.  Furthermore, its configurable
> compression level offers a trade-off between time spent compressing
> sstate cache files and disk space used by those files.  The reduced disk
> usage also contributes to saving network traffic for those sharing their
> sstate cache with others.
> 
> Zstandard should therefore be a good choice when:
> * disk space is at a premium
> * network speed / resources are limited
> * the CI server can sstate packages can be created at high compression
> * less CPU on the build server should be used for sstate decompression
> 
> Signed-off-by: Henry Kleynhans 
> ---
>  meta/classes/sstate.bbclass| 29 ++
>  scripts/sstate-cache-management.sh | 40 +++---
>  2 files changed, 39 insertions(+), 30 deletions(-)
> 
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 92a73114bb..3a67aaba19 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -1,17 +1,19 @@
>  SSTATE_VERSION = "3"
>  
> +SSTATE_ZSTD_CLEVEL = "8"
> +
>  SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
>  SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
>  
>  def generate_sstatefn(spec, hash, taskname, siginfo, d):
>  if taskname is None:
> return ""
> -extension = ".tgz"
> +extension = ".tar.zst"
>  # 8 chars reserved for siginfo
>  limit = 254 - 8
>  if siginfo:
>  limit = 254
> -extension = ".tgz.siginfo"
> +extension = ".tar.zst.siginfo"
>  if not hash:
>  hash = "INVALID"
>  fn = spec + hash + "_" + taskname + extension
> @@ -37,7 +39,7 @@ SSTATE_PKGNAME= 
> "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
>  SSTATE_PKG= "${SSTATE_DIR}/${SSTATE_PKGNAME}"
>  SSTATE_EXTRAPATH   = ""
>  SSTATE_EXTRAPATHWILDCARD = ""
> -SSTATE_PATHSPEC   = 
> "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
> +SSTATE_PATHSPEC   = 
> "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
>  

I believe this is the cause of those failures:

https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/2671/steps/15/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/2640/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/2662/steps/15/logs/stdio

2021-10-06 12:38:04,114 - oe-selftest - INFO - 
testtools.testresult.real._StringException: Traceback (most recent call last):
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/selftest/cases/sstatetests.py",
 line 117, in test_cleansstate_task_distro_nonspecific
self.run_test_cleansstate_task(['linux-libc-headers'], 
distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/selftest/cases/sstatetests.py",
 line 102, in run_test_cleansstate_task
self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s 
(%s)" % (', '.join(map(str, targets)), str(tgz_created)))
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/buildtools/sysroots/x86_64-pokysdk-linux/usr/lib/python3.9/unittest/case.py",
 line 682, in assertTrue
raise self.failureException(msg)
AssertionError: [] is not true : Could not find sstate .tgz files for: 
linux-libc-headers ([])

2021-10-06 12:40:57,420 - oe-selftest - INFO - 
testtools.testresult.real._StringException: Traceback (most recent call last):
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/selftest/cases/sstatetests.py",
 line 158, in test_rebuild_distro_specific_sstate_cross_native_targets
self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + 
self.tune_arch, 'binutils-native'], temp_sstate_location=True)
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/meta/lib/oeqa/selftest/cases/sstatetests.py",
 line 140, in run_test_rebuild_distro_specific_sstate
self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate 
files were created for: %s" % ', '.join(map(str, targets)))
  File 
"/home/pokybuild/yocto-worker/oe-selftest-centos/build/buildtools/sysroots/x86_64-pokysdk-linux/usr/lib/python3.9/unittest/case.py",
 line 682, in assertTrue
raise self.failureException(msg)
AssertionError: False is not true : Not all sstate files were created for: 
binutils-cross-x86_64, binutils-native


>  # explicitly make PV to depend on evaluated value of PV variable
>  PV[vardepvalue] = "${PV}"
> @@ -825,23 +827,24 @@

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

2021-10-04 Thread Henry Kleynhans

From: Henry Kleynhans 

This patch switches the compressor from Gzip to ZStandard for ssate cache
files.

Zstandard compression provides a significant improvement in
decompression speed as well as improvement in compression speed and disk
usage over the 'tgz' format in use.  Furthermore, its configurable
compression level offers a trade-off between time spent compressing
sstate cache files and disk space used by those files.  The reduced disk
usage also contributes to saving network traffic for those sharing their
sstate cache with others.

Zstandard should therefore be a good choice when:
* disk space is at a premium
* network speed / resources are limited
* the CI server can sstate packages can be created at high compression
* less CPU on the build server should be used for sstate decompression

Signed-off-by: Henry Kleynhans 
---
 meta/classes/sstate.bbclass| 29 ++
 scripts/sstate-cache-management.sh | 40 +++---
 2 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 92a73114bb..3a67aaba19 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1,17 +1,19 @@
 SSTATE_VERSION = "3"
 
+SSTATE_ZSTD_CLEVEL = "8"
+
 SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
 SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
 
 def generate_sstatefn(spec, hash, taskname, siginfo, d):
 if taskname is None:
return ""
-extension = ".tgz"
+extension = ".tar.zst"
 # 8 chars reserved for siginfo
 limit = 254 - 8
 if siginfo:
 limit = 254
-extension = ".tgz.siginfo"
+extension = ".tar.zst.siginfo"
 if not hash:
 hash = "INVALID"
 fn = spec + hash + "_" + taskname + extension
@@ -37,7 +39,7 @@ SSTATE_PKGNAME= 
"${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
 SSTATE_PKG= "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
+SSTATE_PATHSPEC   = 
"${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
 
 # explicitly make PV to depend on evaluated value of PV variable
 PV[vardepvalue] = "${PV}"
@@ -825,23 +827,24 @@ sstate_create_package () {
mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
TFILE=`mktemp ${SSTATE_PKG}.`
 
-   # Use pigz if available
-   OPT="-czS"
-   if [ -x "$(command -v pigz)" ]; then
-   OPT="-I pigz -cS"
+   OPT="-cS"
+   ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
fi
 
# Need to handle empty directories
if [ "$(ls -A)" ]; then
set +e
-   tar $OPT -f $TFILE *
+   tar -I "$ZSTD" $OPT -f $TFILE *
ret=$?
if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
exit 1
fi
set -e
else
-   tar $OPT --file=$TFILE --files-from=/dev/null
+   tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
fi
chmod 0664 $TFILE
# Skip if it was already created by some other process
@@ -880,7 +883,13 @@ python sstate_report_unihash() {
 # Will be run from within SSTATE_INSTDIR.
 #
 sstate_unpack_package () {
-   tar -xvzf ${SSTATE_PKG}
+   ZSTD="zstd -T${ZSTD_THREADS}"
+   # Use pzstd if available
+   if [ -x "$(command -v pzstd)" ]; then
+   ZSTD="pzstd -p ${ZSTD_THREADS}"
+   fi
+
+   tar -I "$ZSTD" -xvf ${SSTATE_PKG}
# update .siginfo atime on local/NFS mirror
[ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h 
${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
# Use "! -w ||" to return true for read only files
diff --git a/scripts/sstate-cache-management.sh 
b/scripts/sstate-cache-management.sh
index f1706a2229..d39671f7c6 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -114,7 +114,7 @@ echo_error () {
 # * Add .done/.siginfo to the remove list
 # * Add destination of symlink to the remove list
 #
-# $1: output file, others: sstate cache file (.tgz)
+# $1: output file, others: sstate cache file (.tar.zst)
 gen_rmlist (){
   local rmlist_file="$1"
   shift
@@ -131,13 +131,13 @@ gen_rmlist (){
   dest="`readlink -e $i`"
   if [ -n "$dest" ]; then
   echo $dest >> $rmlist_file
-  # Remove the .siginfo when .tgz is removed
+  # Remove the .siginfo when .tar.zst is removed
   if [ -f "$dest.siginfo" ]; then
   echo $dest.siginfo >>

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

Re: [OE-core] [PATCH] sstate: Switch to ZStandard compressor support

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

Re: [OE-core] [PATCH] sstate: Switch to ZStandard compressor support

[OE-core] [PATCH] sstate: Switch to ZStandard compressor support

5 matches

Site Navigation

Mail list logo

Footer information