[systemd-devel] [PATCH 1/2] journal: add LZ4 as optional compressor
Add liblz4 as an optional dependency when requested with --enable-lz4, and
use it in preference to liblzma for journal blob and coredump compression. To
retain backwards compatibility, XZ is used to decompress old blobs.
Things will function correctly only with lz4-119.
Based on the benchmarks found on the web, lz4 seems to be the best choice for
quick compressors atm. If we decide to go this way, I'll work with upstream
to improve their packaging practices, and at least use pkg-config.
---
Makefile.am | 11 +-
configure.ac | 15 +-
src/journal/compress.c | 382 +++
src/journal/compress.h | 65 +++-
src/journal/coredump.c | 10 +-
src/journal/coredumpctl.c| 2 +-
src/journal/journal-def.h| 28 +++-
src/journal/journal-file.c | 99 ++-
src/journal/journal-file.h | 10 +-
src/journal/journal-verify.c | 51 --
src/journal/sd-journal.c | 36 ++--
src/journal/test-compress.c | 158 +-
12 files changed, 688 insertions(+), 179 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 55a7546eef..c85d66ef32 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3526,14 +3526,12 @@ test_catalog_CPPFLAGS = \
test_catalog_LDADD = \
libsystemd-journal-core.la
-if HAVE_XZ
test_compress_SOURCES = \
src/journal/test-compress.c
test_compress_LDADD = \
libsystemd-journal-internal.la \
libsystemd-shared.la
-endif
libsystemd_journal_core_la_SOURCES = \
src/journal/journald-kmsg.c \
@@ -3617,9 +3615,7 @@ tests += \
test-mmap-cache \
test-catalog
-if HAVE_XZ
tests += test-compress
-endif
pkginclude_HEADERS += \
src/systemd/sd-journal.h \
@@ -3652,10 +3648,10 @@ libsystemd_journal_internal_la_CFLAGS = \
libsystemd_journal_internal_la_LIBADD =
-if HAVE_XZ
libsystemd_journal_internal_la_SOURCES += \
src/journal/compress.c
+if HAVE_XZ
libsystemd_journal_internal_la_CFLAGS += \
$(XZ_CFLAGS)
@@ -3663,6 +3659,11 @@ libsystemd_journal_internal_la_LIBADD += \
$(XZ_LIBS)
endif
+if HAVE_LZ4
+libsystemd_journal_internal_la_LIBADD += \
+ -llz4
+endif
+
if HAVE_GCRYPT
libsystemd_journal_internal_la_SOURCES += \
src/journal/journal-authenticate.c \
diff --git a/configure.ac b/configure.ac
index 93aba06739..7a04ca4fa0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -503,14 +503,24 @@ have_xz=no
AC_ARG_ENABLE(xz, AS_HELP_STRING([--disable-xz], [Disable optional XZ
support]))
if test x$enable_xz != xno; then
PKG_CHECK_MODULES(XZ, [ liblzma ],
-[AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available])
have_xz=yes], have_xz=no)
+[AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available])
have_xz=yes])
if test x$have_xz = xno -a x$enable_xz = xyes; then
-AC_MSG_ERROR([*** Xz support requested but libraries not
found])
+AC_MSG_ERROR([*** XZ support requested but libraries not
found])
fi
fi
AM_CONDITIONAL(HAVE_XZ, [test $have_xz = yes])
#
--
+have_lz4=no
+AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4
support]))
+AS_IF([test x$enable_lz4 == xyes], [
+AC_CHECK_HEADERS(lz4.h,
+ [AC_DEFINE(HAVE_LZ4, 1, [Define in LZ4 is available])
have_lz4=yes],
+ [AC_MSG_ERROR([*** LZ4 support requested but headers not
found])])
+])
+AM_CONDITIONAL(HAVE_LZ4, [test $have_lz4 = yes])
+
+#
--
AC_ARG_ENABLE([pam],
AS_HELP_STRING([--disable-pam],[Disable optional PAM support]),
[case ${enableval} in
@@ -1266,6 +1276,7 @@ AC_MSG_RESULT([
SECCOMP: ${have_seccomp}
SMACK: ${have_smack}
XZ: ${have_xz}
+LZ4: ${have_lz4}
ACL: ${have_acl}
GCRYPT: ${have_gcrypt}
QRENCODE:${have_qrencode}
diff --git a/src/journal/compress.c b/src/journal/compress.c
index 37c55a8728..f7fc665e9c 100644
--- a/src/journal/compress.c
+++ b/src/journal/compress.c
@@ -23,13 +23,30 @@
#include stdlib.h
#include string.h
#include unistd.h
-#include lzma.h
+
+#ifdef HAVE_XZ
+# include lzma.h
+#endif
+
+#ifdef HAVE_LZ4
+# include lz4.h
+#endif
#include compress.h
#include macro.h
#include util.h
+#include sparse-endian.h
+#include journal-def.h
+
+static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
+[OBJECT_COMPRESSED_XZ] = XZ,
+[OBJECT_COMPRESSED_LZ4] = LZ4,
+};
+
+DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
-bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t
*dst_size) {
+int compress_blob_xz(const void *src, uint64_t src_size,
[systemd-devel] [PATCH 2/2] compress: add benchmark-style test
This is useful to test the behaviour of the compressor for various buffer
sizes.
Time is limited to a minute per compression, since otherwise, when LZ4
takes more than a second which is necessary to reduce the noise, XZ
takes more than 10 minutes.
% build/test-compress-benchmark (without time limit)
XZ: compressed decompressed 2535300963 bytes in 794.57s (3.04MiB/s), mean
compresion 99.95%, skipped 3570 bytes
LZ4: compressed decompressed 2535303543 bytes in 1.56s (1550.07MiB/s), mean
compresion 99.60%, skipped 990 bytes
% build/test-compress-benchmark (with time limit)
XZ: compressed decompressed 174321481 bytes in 60.02s (2.77MiB/s), mean
compresion 99.76%, skipped 3570 bytes
LZ4: compressed decompressed 2535303543 bytes in 1.63s (1480.83MiB/s), mean
compresion 99.60%, skipped 990 bytes
It appears that there's a bug in lzma_end where it leaks 32 bytes.
---
.gitignore| 1 +
Makefile.am | 11 +++-
src/journal/test-compress-benchmark.c | 113 ++
3 files changed, 124 insertions(+), 1 deletion(-)
create mode 100644 src/journal/test-compress-benchmark.c
diff --git a/.gitignore b/.gitignore
index 7c19bcbea3..c225638e03 100644
--- a/.gitignore
+++ b/.gitignore
@@ -142,6 +142,7 @@
/test-cgroup-mask
/test-cgroup-util
/test-compress
+/test-compress-benchmark
/test-conf-files
/test-coredump-vacuum
/test-daemon
diff --git a/Makefile.am b/Makefile.am
index c85d66ef32..9562444c69 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3533,6 +3533,13 @@ test_compress_LDADD = \
libsystemd-journal-internal.la \
libsystemd-shared.la
+test_compress_benchmark_SOURCES = \
+ src/journal/test-compress-benchmark.c
+
+test_compress_benchmark_LDADD = \
+ libsystemd-journal-internal.la \
+ libsystemd-shared.la
+
libsystemd_journal_core_la_SOURCES = \
src/journal/journald-kmsg.c \
src/journal/journald-kmsg.h \
@@ -3615,7 +3622,9 @@ tests += \
test-mmap-cache \
test-catalog
-tests += test-compress
+tests += \
+ test-compress \
+ test-compress-benchmark
pkginclude_HEADERS += \
src/systemd/sd-journal.h \
diff --git a/src/journal/test-compress-benchmark.c
b/src/journal/test-compress-benchmark.c
new file mode 100644
index 00..96f8bb5abc
--- /dev/null
+++ b/src/journal/test-compress-benchmark.c
@@ -0,0 +1,113 @@
+/***
+ This file is part of systemd
+
+ Copyright 2014 Zbigniew Jędrzejewski-Szmek
+
+ systemd is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1 of the License, or
+ (at your option) any later version.
+
+ systemd is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with systemd; If not, see http://www.gnu.org/licenses/.
+***/
+
+#include compress.h
+#include util.h
+#include macro.h
+
+typedef int (compress_t)(const void *src, uint64_t src_size, void *dst,
uint64_t *dst_size);
+typedef int (decompress_t)(const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t*
dst_size, uint64_t dst_max);
+
+#define MAX_SIZE (1024*1024LU)
+
+static char* make_buf(size_t count) {
+char *buf;
+size_t i;
+
+buf = malloc(count);
+assert(buf);
+
+for (i = 0; i count; i++)
+buf[i] = 'a' + i % ('z' - 'a' + 1);
+
+return buf;
+}
+
+static void test_compress_decompress(const char* label,
+ compress_t compress, decompress_t
decompress) {
+usec_t n, n2;
+float dt;
+
+_cleanup_free_ char *text, *buf;
+_cleanup_free_ void *buf2 = NULL;
+size_t buf2_allocated = 0;
+size_t skipped = 0, compressed = 0, total = 0;
+
+text = make_buf(MAX_SIZE);
+buf = calloc(MAX_SIZE + 1, 1);
+assert(text buf);
+
+n = now(CLOCK_MONOTONIC);
+
+for (size_t i = 1; i = MAX_SIZE; i += (i 2048 ? 1 : 217)) {
+size_t j = 0, k = 0;
+int r;
+
+r = compress(text, i, buf, j);
+/* assume compresion must be succesful except for small inputs
*/
+assert(r == 0 || (i 2048 r == -ENOSPC));
+/* check for overwrites */
+assert(buf[i] == 0);
+if (r != 0) {
+skipped += i;
+continue;
+}
+
+assert(j 0);
+if (j = i)
+log_error(%s \compressed\ %zu - %zu, label, i, j);
+
+r = decompress(buf, j,
