Newer kernels might be compressed using ZSTD add support to libdwfl open
so we can can automatically read ZSTD compressed files and kernel images.
The support is very similar to the bzip2 and lzma support, but slightly
different. With a bit more macros it could maybe have used the gzip.c
USE_INFLATE code path. But I felt that the many macros didn't really help
understand the code. So the unzip routine has a slightly different code
path for ZSTD.
Signed-off-by: Mark Wielaard <m...@klomp.org>
---
ChangeLog | 4 ++
config/ChangeLog | 5 ++
config/elfutils.spec.in | 2 +
config/libdw.pc.in | 4 +-
configure.ac | 13 ++++-
libdwfl/ChangeLog | 11 ++++
libdwfl/Makefile.am | 3 ++
libdwfl/gzip.c | 79 +++++++++++++++++++++++++++-
libdwfl/libdwflP.h | 5 ++
libdwfl/open.c | 6 +++
libdwfl/zstd.c | 4 ++
tests/ChangeLog | 6 +++
tests/Makefile.am | 5 ++
tests/run-readelf-compressed-zstd.sh | 39 ++++++++++++++
14 files changed, 180 insertions(+), 6 deletions(-)
create mode 100644 libdwfl/zstd.c
create mode 100755 tests/run-readelf-compressed-zstd.sh
diff --git a/ChangeLog b/ChangeLog
index 094a798a..021b06f3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2020-09-18 Mark Wielaard <m...@klomp.org>
+
+ * configure.ac: Check availability of libzstd and zstd.
+
2020-09-08 Mark Wielaard <m...@klomp.org>
* configure.ac: Set version to 0.181.
diff --git a/config/ChangeLog b/config/ChangeLog
index 1cb3d204..c8e4fcd4 100644
--- a/config/ChangeLog
+++ b/config/ChangeLog
@@ -1,3 +1,8 @@
+2020-09-18 Mark Wielaard <m...@klomp.org>
+
+ * elfutils.spec.in: Add BuildRequires for libzstd-devel and zstd.
+ * libdw.pc.in: Requires.private libzstd.
+
2020-09-08 Mark Wielaard <m...@klomp.org>
* elfutils.spec.in: Update for 0.181.
diff --git a/config/elfutils.spec.in b/config/elfutils.spec.in
index 95f63f5a..37af1b07 100644
--- a/config/elfutils.spec.in
+++ b/config/elfutils.spec.in
@@ -24,6 +24,7 @@ BuildRequires: flex
BuildRequires: zlib-devel
BuildRequires: bzip2-devel
BuildRequires: xz-devel
+BuildRequires: libzstd-devel
# For debuginfod
BuildRequires: pkgconfig(libmicrohttpd) >= 0.9.33
@@ -33,6 +34,7 @@ BuildRequires: pkgconfig(libarchive) >= 3.1.2
# For tests need to bunzip2 test files.
BuildRequires: bzip2
+BuildRequires: zstd
# For the run-debuginfod-find.sh test case in %check for /usr/sbin/ss
BuildRequires: iproute
BuildRequires: bsdtar
diff --git a/config/libdw.pc.in b/config/libdw.pc.in
index 3fc283db..2e83a432 100644
--- a/config/libdw.pc.in
+++ b/config/libdw.pc.in
@@ -17,6 +17,6 @@ Requires: libelf = @VERSION@
# We support various compressed ELF images, but don't export any of the
# data structures or functions. zlib (gz) is always required, bzip2 (bz2)
-# and lzma (xz) are optional. But bzip2 doesn't have a pkg-config file.
-Requires.private: zlib @LIBLZMA@
+# lzma (xz) and zstd () are optional. But bzip2 doesn't have a pkg-config file.
+Requires.private: zlib @LIBLZMA@ @LIBZSTD@
Libs.private: @BZ2_LIB@
diff --git a/configure.ac b/configure.ac
index bf833872..1b794df3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -397,8 +397,8 @@ eu_ZIPLIB(zlib,ZLIB,z,gzdirect,gzip)
AS_IF([test "x$with_zlib" = xno], [AC_MSG_ERROR([zlib not found but is
required])])
LIBS="$save_LIBS"
-dnl Test for bzlib and xz/lzma, gives BZLIB/LZMALIB .am
-dnl conditional and config.h USE_BZLIB/USE_LZMALIB #define.
+dnl Test for bzlib and xz/lzma/zstd, gives BZLIB/LZMALIB/ZSTD .am
+dnl conditional and config.h USE_BZLIB/USE_LZMALIB/USE_ZSTD #define.
save_LIBS="$LIBS"
LIBS=
eu_ZIPLIB(bzlib,BZLIB,bz2,BZ2_bzdopen,bzip2)
@@ -408,6 +408,9 @@ AC_SUBST([BZ2_LIB])
eu_ZIPLIB(lzma,LZMA,lzma,lzma_auto_decoder,[LZMA (xz)])
AS_IF([test "x$with_lzma" = xyes], [LIBLZMA="liblzma"], [LIBLZMA=""])
AC_SUBST([LIBLZMA])
+eu_ZIPLIB(zstd,ZSTD,zstd,ZSTD_decompress,[ZSTD (zst)])
+AS_IF([test "x$with_zstd" = xyes], [LIBZSTD="libzstd"], [LIBLZSTD=""])
+AC_SUBST([LIBZSTD])
zip_LIBS="$LIBS"
LIBS="$save_LIBS"
AC_SUBST([zip_LIBS])
@@ -677,6 +680,10 @@ if test "$HAVE_BUNZIP2" = "no"; then
AC_MSG_WARN([No bunzip2, needed to run make check])
fi
+# For tests that need to use zstd compression
+AC_CHECK_PROG(HAVE_ZSTD, zstd, yes, no)
+AM_CONDITIONAL([HAVE_ZSTD],[test "x$HAVE_ZSTD" = "xyes"])
+
# Look for libcurl for libdebuginfod minimum version as per rhel7.
AC_ARG_ENABLE([libdebuginfod],AC_HELP_STRING([--enable-libdebuginfod], [Build
debuginfod client library (can be =dummy)]))
AS_IF([test "x$enable_libdebuginfod" != "xno"], [
@@ -742,6 +749,7 @@ AC_MSG_NOTICE([
gzip support : ${with_zlib}
bzip2 support : ${with_bzlib}
lzma/xz support : ${with_lzma}
+ zstd support : ${with_zstd}
libstdc++ demangle support : ${enable_demangler}
File textrel check : ${enable_textrelcheck}
Symbol versioning : ${enable_symbol_versioning}
@@ -759,6 +767,7 @@ AC_MSG_NOTICE([
EXTRA TEST FEATURES (used with make check)
have bunzip2 installed (required) : ${HAVE_BUNZIP2}
+ have zstd installed : ${HAVE_ZSTD}
debug branch prediction : ${use_debugpred}
gprof support : ${use_gprof}
gcov support : ${use_gcov}
diff --git a/libdwfl/ChangeLog b/libdwfl/ChangeLog
index ca10ce88..253ae069 100644
--- a/libdwfl/ChangeLog
+++ b/libdwfl/ChangeLog
@@ -1,3 +1,14 @@
+2020-09-18 Mark Wielaard <m...@klomp.org>
+
+ * zstd.c: New file.
+ * libdwflP.h: Add DWFL_E_ZSTD and __libdw_unzstd.
+ * Makefile.am (libdwfl_a_SOURCES): add zstd.c if ZSTD.
+ * gzip.c: Add defines and includes for ZSTD.
+ (zlib_fail): Don't define for ZSTD.
+ (unzip): Change pread_retry failure from zlib_fail to fail.
+ Add ZSTD support.
+ * open.c (decompress): Also try __libdw_unzstd.
+
2020-08-20 Dmitry V. Levin <l...@altlinux.org>
* Makefile.am (libdwfl_a_SOURCES): Conditionalize
diff --git a/libdwfl/Makefile.am b/libdwfl/Makefile.am
index 1de05492..a0013e41 100644
--- a/libdwfl/Makefile.am
+++ b/libdwfl/Makefile.am
@@ -78,6 +78,9 @@ endif
if LZMA
libdwfl_a_SOURCES += lzma.c
endif
+if ZSTD
+libdwfl_a_SOURCES += zstd.c
+endif
if LIBDEBUGINFOD
libdwfl_a_SOURCES += debuginfod-client.c
endif
diff --git a/libdwfl/gzip.c b/libdwfl/gzip.c
index e9988cc2..ba8ecfba 100644
--- a/libdwfl/gzip.c
+++ b/libdwfl/gzip.c
@@ -48,6 +48,12 @@
# define inflateInit(z) lzma_auto_decoder (z, 1 << 30, 0)
# define do_inflate(z) lzma_code (z, LZMA_RUN)
# define inflateEnd(z) lzma_end (z)
+#elif defined ZSTD
+# define USE_INFLATE 1
+# include <zstd.h>
+# define unzip __libdw_unzstd
+# define DWFL_E_ZLIB DWFL_E_ZSTD
+# define MAGIC "\x28\xb5\x2f\xfd"
#elif defined BZLIB
# define USE_INFLATE 1
# include <bzlib.h>
@@ -119,6 +125,7 @@ fail (struct unzip_state *state, Dwfl_Error failure)
return failure;
}
+#ifndef ZSTD
static inline Dwfl_Error
zlib_fail (struct unzip_state *state, int result)
{
@@ -132,6 +139,7 @@ zlib_fail (struct unzip_state *state, int result)
return fail (state, DWFL_E_ZLIB);
}
}
+#endif
#if !USE_INFLATE
static Dwfl_Error
@@ -197,7 +205,7 @@ unzip (int fd, off_t start_offset,
ssize_t n = pread_retry (fd, state.input_buffer, READ_SIZE,
start_offset);
if (unlikely (n < 0))
- return zlib_fail (&state, Z (ERRNO));
+ return fail (&state, DWFL_E_ERRNO);
state.input_pos = n;
mapped = state.input_buffer;
@@ -223,7 +231,74 @@ unzip (int fd, off_t start_offset,
/* Not a compressed file. */
return DWFL_E_BADELF;
-#if USE_INFLATE
+#ifdef ZSTD
+ /* special case for libzstd since it is slightly different from the
+ API provided by bzlib and liblzma. */
+
+ void *next_in = mapped;
+ size_t avail_in = state.mapped_size;
+ void *next_out = NULL;
+ size_t avail_out = 0;
+ size_t total_out = 0;
+
+ size_t result;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+ if (dctx == NULL)
+ return fail (&state, DWFL_E_NOMEM);
+
+ do
+ {
+ if (avail_in == 0 && state.input_buffer != NULL)
+ {
+ ssize_t n = pread_retry (fd, state.input_buffer, READ_SIZE,
+ start_offset + state.input_pos);
+ if (unlikely (n < 0))
+ {
+ ZSTD_freeDCtx (dctx);
+ return fail (&state, DWFL_E_ERRNO);
+ }
+ next_in = state.input_buffer;
+ avail_in = n;
+ state.input_pos += n;
+ }
+ if (avail_out == 0)
+ {
+ ptrdiff_t pos = (void *) next_out - state.buffer;
+ if (!bigger_buffer (&state, avail_in))
+ {
+ ZSTD_freeDCtx (dctx);
+ return fail (&state, DWFL_E_NOMEM);
+ }
+ next_out = state.buffer + pos;
+ avail_out = state.size - pos;
+ }
+
+ ZSTD_inBuffer input = { next_in, avail_in, 0 };
+ ZSTD_outBuffer output = { next_out, avail_out, 0 };
+ result = ZSTD_decompressStream (dctx, &output, &input);
+
+ if (! ZSTD_isError (result))
+ {
+ total_out += output.pos;
+ next_out += output.pos;
+ avail_out -= output.pos;
+ next_in += input.pos;
+ avail_in -= input.pos;
+ }
+
+ if (result == 0)
+ break;
+ }
+ while (avail_in > 0 && ! ZSTD_isError (result));
+
+ ZSTD_freeDCtx (dctx);
+
+ if (ZSTD_isError (result))
+ return fail (&state, DWFL_E_ZSTD);
+
+ smaller_buffer (&state, total_out);
+
+#elif USE_INFLATE
/* This style actually only works with bzlib and liblzma.
The stupid zlib interface has nothing to grok the
diff --git a/libdwfl/libdwflP.h b/libdwfl/libdwflP.h
index ad6779ad..4c6fcb28 100644
--- a/libdwfl/libdwflP.h
+++ b/libdwfl/libdwflP.h
@@ -61,6 +61,7 @@ typedef struct Dwfl_Process Dwfl_Process;
DWFL_ERROR (ZLIB, N_("gzip decompression failed")) \
DWFL_ERROR (BZLIB, N_("bzip2 decompression failed")) \
DWFL_ERROR (LZMA, N_("LZMA decompression failed")) \
+ DWFL_ERROR (ZSTD, N_("zstd decompression failed")) \
DWFL_ERROR (UNKNOWN_MACHINE, N_("no support library found for machine")) \
DWFL_ERROR (NOREL, N_("Callbacks missing for ET_REL file")) \
DWFL_ERROR (BADRELTYPE, N_("Unsupported relocation type")) \
@@ -612,6 +613,10 @@ extern Dwfl_Error __libdw_unlzma (int fd, off_t
start_offset,
void *mapped, size_t mapped_size,
void **whole, size_t *whole_size)
internal_function;
+extern Dwfl_Error __libdw_unzstd (int fd, off_t start_offset,
+ void *mapped, size_t mapped_size,
+ void **whole, size_t *whole_size)
+ internal_function;
/* Skip the image header before a file image: updates *START_OFFSET. */
extern Dwfl_Error __libdw_image_header (int fd, off_t *start_offset,
diff --git a/libdwfl/open.c b/libdwfl/open.c
index 35fc5283..77bd2bd9 100644
--- a/libdwfl/open.c
+++ b/libdwfl/open.c
@@ -44,6 +44,10 @@
# define __libdw_unlzma(...) DWFL_E_BADELF
#endif
+#if !USE_ZSTD
+# define __libdw_unzstd(...) DWFL_E_BADELF
+#endif
+
/* Consumes and replaces *ELF only on success. */
static Dwfl_Error
decompress (int fd __attribute__ ((unused)), Elf **elf)
@@ -64,6 +68,8 @@ decompress (int fd __attribute__ ((unused)), Elf **elf)
error = __libdw_bunzip2 (fd, offset, mapped, mapped_size, &buffer, &size);
if (error == DWFL_E_BADELF)
error = __libdw_unlzma (fd, offset, mapped, mapped_size, &buffer, &size);
+ if (error == DWFL_E_BADELF)
+ error = __libdw_unzstd (fd, offset, mapped, mapped_size, &buffer, &size);
if (error == DWFL_E_NOERROR)
{
diff --git a/libdwfl/zstd.c b/libdwfl/zstd.c
new file mode 100644
index 00000000..dc4d5238
--- /dev/null
+++ b/libdwfl/zstd.c
@@ -0,0 +1,4 @@
+/* libzstd is pretty close to zlib and bzlib. */
+
+#define ZSTD
+#include "gzip.c"
diff --git a/tests/ChangeLog b/tests/ChangeLog
index 5f2b1449..5a8b5899 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,9 @@
+2020-09-18 Mark Wielaard <m...@klomp.org>
+
+ * run-readelf-compressed-zstd.sh: New test.
+ * Makefile.am (EXTRA_DISTS): Add run-readelf-compressed-zstd.sh.
+ (TESTS): Add run-readelf-compressed-zstd.sh if HAVE_ZSTD.
+
2020-09-03 Mark Wielaard <m...@klomp.org>
* run-readelf-frames.sh: New test.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4629ce64..9d0707da 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -204,6 +204,10 @@ if LZMA
TESTS += run-readelf-s.sh run-dwflsyms.sh
endif
+if HAVE_ZSTD
+TESTS += run-readelf-compressed-zstd.sh
+endif
+
if HAVE_LIBASM
check_PROGRAMS += $(asm_TESTS)
TESTS += $(asm_TESTS) run-disasm-bpf.sh
@@ -256,6 +260,7 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
run-nm-syms.sh testfilesyms32.bz2 testfilesyms64.bz2 \
run-nm-self.sh run-readelf-self.sh run-readelf-info-plus.sh \
run-readelf-compressed.sh \
+ run-readelf-compressed-zstd.sh \
run-readelf-const-values.sh testfile-const-values.debug.bz2 \
run-addrcfi.sh run-dwarfcfi.sh \
testfile11-debugframe.bz2 testfile12-debugframe.bz2 \
diff --git a/tests/run-readelf-compressed-zstd.sh
b/tests/run-readelf-compressed-zstd.sh
new file mode 100755
index 00000000..96208092
--- /dev/null
+++ b/tests/run-readelf-compressed-zstd.sh
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Copyright (C) 2018 Red Hat, Inc.
+# This file is part of elfutils.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# elfutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+. $srcdir/test-subr.sh
+
+if ! grep -q -F '#define USE_ZSTD' ${abs_top_builddir}/config.h; then
+ echo "elfutils built without zstd support"
+ exit 77
+fi
+
+# See run-strip-reloc.sh
+testfiles hello_i386.ko
+
+tempfiles hello_i386.ko.zst readelf.out.1 readelf.out.2
+
+testrun ${abs_top_builddir}/src/readelf -a hello_i386.ko > readelf.out.1
+zstd hello_i386.ko
+testrun ${abs_top_builddir}/src/readelf -a hello_i386.ko.zst > readelf.out.2
+
+diff -u readelf.out.1 readelf.out.2
+if [ $? != 0 ]; then
+ exit 1;
+fi
+
+exit 0
--
2.18.4
