Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package raft for openSUSE:Factory checked in at 2021-06-24 18:22:37 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/raft (Old) and /work/SRC/openSUSE:Factory/.raft.new.2625 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "raft" Thu Jun 24 18:22:37 2021 rev:13 rq:901627 version:0.11.1 Changes: -------- --- /work/SRC/openSUSE:Factory/raft/raft.changes 2021-05-05 20:40:12.378863376 +0200 +++ /work/SRC/openSUSE:Factory/.raft.new.2625/raft.changes 2021-06-24 18:22:56.960941187 +0200 @@ -1,0 +2,7 @@ +Tue Jun 22 20:58:38 UTC 2021 - Andreas Stieger <[email protected]> + +- raft 0.11.1: + * libuv implementation will use lz4 to compress snapshots + * minor bug fixes + +------------------------------------------------------------------- Old: ---- raft-0.10.1.tar.gz New: ---- raft-0.11.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ raft.spec ++++++ --- /var/tmp/diff_new_pack.miyJdr/_old 2021-06-24 18:22:57.456941737 +0200 +++ /var/tmp/diff_new_pack.miyJdr/_new 2021-06-24 18:22:57.456941737 +0200 @@ -16,21 +16,19 @@ # -%bcond_without libuv Name: raft -Version: 0.10.1 +Version: 0.11.1 Release: 0 Summary: Fully asynchronous C implementation of the Raft consensus protocol -License: LGPL-3.0-only WITH linking-exception-lgpl-3.0 +License: LGPL-3.0-only WITH LGPL-3.0-linking-exception URL: https://github.com/canonical/raft Source: https://github.com/canonical/raft/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz BuildRequires: autoconf BuildRequires: automake BuildRequires: libtool -BuildRequires: pkgconfig -%if !%{without libuv} +BuildRequires: pkgconfig >= 0.9.0 +BuildRequires: pkgconfig(liblz4) >= 1.7.1 BuildRequires: pkgconfig(libuv) >= 1.18.0 -%endif %description This library is a fully asynchronous C implementation of the Raft consensus protocol. @@ -39,9 +37,7 @@ networking and persistent storage. The algorithm supports leadership election, log replication, log compaction, and membership changes. -%if %{with libuv} A stock implementation of the I/O interface based on libuv is included. -%endif %package -n libraft0 Summary: Library implementing the Raft consensus protocol @@ -53,16 +49,12 @@ networking and persistent storage. The algorithm supports leadership election, log replication, log compaction, and membership changes. -%if %{with libuv} A stock implementation of the I/O interface based on libuv is included. -%endif %package devel Summary: Development files for the Raft library implementation of the consensus protocol Requires: libraft0 = %{version} -%if !%{without libuv} Requires: pkgconfig(libuv) >= 1.18.0 -%endif %description devel This library is a fully asynchronous C implementation of the Raft consensus protocol. @@ -80,10 +72,7 @@ %build autoreconf -iv %configure \ - --disable-static \ -%if %{without libuv} - --disable-uv \ -%endif + --disable-static %make_build @@ -100,9 +89,7 @@ %{_includedir}/raft.h %dir %{_includedir}/raft %{_includedir}/raft/fixture.h -%if !%{without libuv} %{_includedir}/raft/uv.h -%endif %{_libdir}/libraft.so %{_libdir}/pkgconfig/raft.pc ++++++ raft-0.10.1.tar.gz -> raft-0.11.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/.github/workflows/build-and-test.yml new/raft-0.11.1/.github/workflows/build-and-test.yml --- old/raft-0.10.1/.github/workflows/build-and-test.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/.github/workflows/build-and-test.yml 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,119 @@ +name: CI Tests + +on: + - push + - pull_request + +jobs: + build-and-test: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-18.04 + - ubuntu-20.04 + compiler: + - gcc + - clang + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Setup dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq lcov linux-libc-dev liblz4-dev libuv1-dev btrfs-progs xfsprogs zfsutils-linux + + - name: Build + env: + CC: ${{ matrix.compiler }} + run: | + git clone --depth 1 https://github.com/edlund/amalgamate.git + export PATH=$PATH:$PWD/amalgamate + autoreconf -i + ./configure --enable-example --enable-debug --enable-code-coverage --enable-sanitize + amalgamate.py --config=amalgamation.json --source=$(pwd) + $CC raft.c -c -D_GNU_SOURCE -DHAVE_LINUX_AIO_ABI_H -Wall -Wextra -Wpedantic -fpic + + - name: Test + env: + CC: ${{ matrix.compiler }} + run: | + ./test/lib/fs.sh setup + make check CFLAGS=-O0 $(./test/lib/fs.sh detect) || (cat ./test-suite.log && false) + ./test/lib/fs.sh teardown + + - name: Coverage + env: + CC: ${{ matrix.compiler }} + run: if [ "${CC}" = "gcc" ]; then make code-coverage-capture; fi + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + verbose: true + + build-and-test-nolz4: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-18.04 + - ubuntu-20.04 + compiler: + - gcc + - clang + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Setup dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq lcov linux-libc-dev libuv1-dev btrfs-progs xfsprogs zfsutils-linux + + - name: Build + env: + CC: ${{ matrix.compiler }} + run: | + git clone --depth 1 https://github.com/edlund/amalgamate.git + export PATH=$PATH:$PWD/amalgamate + autoreconf -i + ./configure --enable-example --enable-debug --enable-code-coverage --enable-sanitize --disable-lz4 + amalgamate.py --config=amalgamation.json --source=$(pwd) + $CC raft.c -c -D_GNU_SOURCE -DHAVE_LINUX_AIO_ABI_H -Wall -Wextra -Wpedantic -fpic + + - name: Test + env: + CC: ${{ matrix.compiler }} + run: | + ./test/lib/fs.sh setup + make check CFLAGS=-O0 $(./test/lib/fs.sh detect) || (cat ./test-suite.log && false) + ./test/lib/fs.sh teardown + + - name: Coverage + env: + CC: ${{ matrix.compiler }} + run: if [ "${CC}" = "gcc" ]; then make code-coverage-capture; fi + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + verbose: true + + build-nolz4-fail: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Setup dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq lcov linux-libc-dev libuv1-dev btrfs-progs xfsprogs zfsutils-linux + + # Expect the configure step to fail + - name: Build + env: + CC: gcc + run: | + autoreconf -i + ! ./configure --enable-example --enable-debug --enable-code-coverage --enable-sanitize diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/.github/workflows/cla-check.yml new/raft-0.11.1/.github/workflows/cla-check.yml --- old/raft-0.10.1/.github/workflows/cla-check.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/.github/workflows/cla-check.yml 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,11 @@ +name: Canonical CLA + +on: + - pull_request + +jobs: + cla-check: + runs-on: ubuntu-20.04 + steps: + - name: Check if CLA signed + uses: canonical/has-signed-canonical-cla@v1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/.github/workflows/coverity.yml new/raft-0.11.1/.github/workflows/coverity.yml --- old/raft-0.10.1/.github/workflows/coverity.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/.github/workflows/coverity.yml 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,52 @@ +name: Coverity +on: + push: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Download Coverity Build Tool + run: | + wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=$TOKEN&project=canonical/raft" -O cov-analysis-linux64.tar.gz + mkdir cov-analysis-linux64 + tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64 + env: + TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq lcov linux-libc-dev liblz4-dev libuv1-dev btrfs-progs xfsprogs zfsutils-linux + + - name: Run coverity + run: | + export PATH="$(pwd)/cov-analysis-linux64/bin:${PATH}" + + # Configure + autoreconf -i + mkdir build + cd build + ../configure + + # Build + cov-build --dir cov-int make -j4 + tar czvf raft.tgz cov-int + + # Submit the results + curl \ + --form project=canonical/raft \ + --form token=${TOKEN} \ + --form [email protected] \ + --form [email protected] \ + --form version=master \ + --form description="${GITHUB_SHA}" \ + https://scan.coverity.com/builds?project=canonical/raft + env: + TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/.travis.yml new/raft-0.11.1/.travis.yml --- old/raft-0.10.1/.travis.yml 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/.travis.yml 2021-06-03 17:22:33.000000000 +0200 @@ -1,58 +1,28 @@ language: c -env: - global: - - secure: "T0kfZEvvymfcV4LWvDxafXMYIXlvhgYBWpY8GMKFxx8NBPBZID7wLFEnhiAF/p7gSaW4491mn3aDGA6/l9mJ/2MgXJssBiGh/5Wz+9dUTKH8T176PZI0+hNSkLPI+c/L00sslhhL7ZOF41mQf4VVHUhkIlFtfI9s/5e6iClw3EefSryX4lqmmE7E/aSMuE5fAQYNV48iwwQvZvcWEZ4yg8Cie5nMv6MdFQUZhmnxXZeixQ2JJGx2/nT/AhlvxTNlPXMsS9khWYVCts4DUpdO3qv52Zj1hOB6f7QoqY7qv3RsHFTN+6YsNlgbKCmFV4JDkCZqklQSfiyFB4IqqudH2AGFNhaJw00wb6kk7Kz3l827V36ib7Jgt3jWg+iF6elTrPm/1Friu7VdW2aCxLWtQYVgkW6sl3uWi8W1Uv8nM58vgFzmtJuojts0mfy0Q2sT/2gR/OITIIcfjZZ8X25Dtm9uNh/7wCVAyU19thiTNVQWrykzEZoiJXWkp1TUZQhpT/PpU5ibwumuk2ZgqAzUSYXzxWXk1/qqOkks8bz4LuKqX9uHbePCUHhvJA3DgfR1kajGREjycteoRNvZQyt8l0hP0deCdEVNZe0GY0Ut0dN6EnjuGMszqioF4ozz9Pje4OR/4u8+H1rVFY4VVliVLIawZ+Eusu4rZgl9oSQ8CzY=" - addons: apt: packages: - lcov - linux-libc-dev - libuv1-dev + - liblz4-dev - btrfs-progs - xfsprogs - zfsutils-linux - coverity_scan: - build_script_url: https://dl.stgraber.org/coverity_travis.sh - project: - name: canonical/raft - description: "Fully asynchronous C implementation of the Raft consensus protocol" - - # Where email notification of build analysis results will be sent - notification_email: [email protected] - - build_command_prepend: "autoreconf -i && ./configure" - build_command: "make" - branch_pattern: master - jobs: include: - - compiler: gcc - dist: bionic - arch: amd64 - - - compiler: gcc - dist: focal - arch: amd64 - - - if: type != pull_request - compiler: clang - dist: bionic - arch: amd64 - - if: type != pull_request compiler: gcc dist: bionic arch: s390x - - if: type != pull_request + - if: type == pull_request compiler: gcc dist: bionic arch: arm64 - # Coverity issue is interfering with ppc64le master build - - if: type == pull_request + - if: type != pull_request compiler: clang dist: bionic arch: ppc64le @@ -63,13 +33,14 @@ script: - autoreconf -i - - ./configure --enable-example --enable-debug --enable-code-coverage --enable-sanitize + - | + if [ $TRAVIS_CPU_ARCH = "s390x" ] || [ $TRAVIS_CPU_ARCH = "arm64" ]; then + ./configure --enable-example --enable-debug + else + ./configure --enable-example --enable-debug --enable-sanitize + fi - amalgamate.py --config=amalgamation.json --source=$(pwd) - $CC raft.c -c -D_GNU_SOURCE -DHAVE_LINUX_AIO_ABI_H -Wall -Wextra -Wpedantic -fpic - ./test/lib/fs.sh setup - make check CFLAGS=-O0 $(./test/lib/fs.sh detect) || (cat ./test-suite.log && false) - - if [ $TRAVIS_COMPILER = gcc ]; then make code-coverage-capture; fi - ./test/lib/fs.sh teardown - -after_success: - - bash <(curl -s https://codecov.io/bash) -G "./src*" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/Makefile.am new/raft-0.11.1/Makefile.am --- old/raft-0.10.1/Makefile.am 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/Makefile.am 2021-06-03 17:22:33.000000000 +0200 @@ -12,6 +12,7 @@ libraft_la_SOURCES = \ src/byte.c \ src/client.c \ + src/compress.c \ src/configuration.c \ src/convert.c \ src/election.c \ @@ -56,12 +57,14 @@ test_unit_core_SOURCES = \ src/byte.c \ + src/compress.c \ src/configuration.c \ src/err.c \ src/heap.c \ src/log.c \ test/unit/main_core.c \ test/unit/test_byte.c \ + test/unit/test_compress.c \ test/unit/test_configuration.c \ test/unit/test_err.c \ test/unit/test_log.c \ @@ -69,6 +72,17 @@ test_unit_core_CFLAGS = $(AM_CFLAGS) -Wno-conversion test_unit_core_LDADD = libtest.la +if LZ4_AVAILABLE +test_unit_core_CFLAGS += -DLZ4_AVAILABLE +test_unit_core_LDFLAGS = $(LZ4_LIBS) +libraft_la_CFLAGS += -DLZ4_AVAILABLE +libraft_la_LDFLAGS += $(LZ4_LIBS) +endif # LZ4_AVAILABLE +if LZ4_ENABLED +test_unit_core_CFLAGS += -DLZ4_ENABLED +libraft_la_CFLAGS += -DLZ4_ENABLED +endif # LZ4_ENABLED + if FIXTURE_ENABLED libraft_la_SOURCES += src/fixture.c @@ -190,6 +204,14 @@ AM_CFLAGS += $(UV_CFLAGS) +if LZ4_AVAILABLE +test_integration_uv_CFLAGS += -DLZ4_AVAILABLE +test_integration_uv_LDFLAGS += $(LZ4_LIBS) +endif # LZ4_AVAILABLE +if LZ4_ENABLED +test_integration_uv_CFLAGS += -DLZ4_ENABLED +endif # LZ4_ENABLED + endif # UV_ENABLED if EXAMPLE_ENABLED @@ -227,7 +249,6 @@ endif if CODE_COVERAGE_ENABLED -AM_CFLAGS += -DCODE_COVERAGE_ENABLED include $(top_srcdir)/aminclude_static.am diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/configure.ac new/raft-0.11.1/configure.ac --- old/raft-0.10.1/configure.ac 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/configure.ac 2021-06-03 17:22:33.000000000 +0200 @@ -1,5 +1,5 @@ AC_PREREQ(2.60) -AC_INIT([raft], [0.10.1]) +AC_INIT([raft], [0.11.1]) AC_LANG([C]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([ac]) @@ -19,6 +19,52 @@ AS_IF([test "x$enable_uv" = "xyes" -a "x$have_uv" = "xno"], [AC_MSG_ERROR([libuv required but not found])], []) AM_CONDITIONAL(UV_ENABLED, test "x$have_uv" = "xyes") +# The libuv raft_io implementation is built by default to compress snapshots if liblz4 is found, unless +# explicitly disabled. +AC_ARG_ENABLE(lz4, AS_HELP_STRING([--disable-lz4], [do not use lz4 compression])) + +# Thanks to the OpenVPN configure.ac file for this part. +# If this fails, we will do another test next. +# We also add set LZ4_LIBS otherwise linker will not know about the lz4 library +PKG_CHECK_MODULES(LZ4, [liblz4 >= 1.7.1], [have_lz4="yes"], [LZ4_LIBS="-llz4"]) +if test "${have_lz4}" != "yes" ; then + AC_CHECK_HEADERS([lz4.h], + [have_lz4h="yes"], + []) + if test "${have_lz4h}" = "yes" ; then + AC_MSG_CHECKING([additionally if system LZ4 version >= 1.7.1]) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[ +#include <lz4.h> + ]], + [[ +/* Version encoding: MMNNPP (Major miNor Patch) - see lz4.h for details */ +#if LZ4_VERSION_NUMBER < 10701L +#error LZ4 is too old +#endif + ]] + )], + [ + AC_MSG_RESULT([ok]) + have_lz4="yes" + ], + [ + AC_MSG_RESULT([system LZ4 library is too old]) + have_lz4="no" + ] + ) + fi +fi + +AS_IF([test "x$enable_lz4" != "xno" -a "x$have_lz4" != "xyes"], + [AC_MSG_ERROR([liblz4 required but not found])], []) +# LZ4 Can be available without being enabled, this allows a user to activate +# it at a later stage through an API call. +AM_CONDITIONAL(LZ4_AVAILABLE, test "x$have_lz4" = "xyes") +# `LZ4_ENABLED` will cause the libuv snapshot implementation to use lz4 +# compression by default. +AM_CONDITIONAL(LZ4_ENABLED, test "x$enable_lz4" != "xno" -a "x$have_lz4" = "xyes") + # The fake I/O implementation and associated fixture is built by default, unless # explicitly disabled. AC_ARG_ENABLE(fixture, AS_HELP_STRING([--disable-fixture], [do not build the raft_fixture test helper])) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/include/raft/uv.h new/raft-0.11.1/include/raft/uv.h --- old/raft-0.10.1/include/raft/uv.h 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/include/raft/uv.h 2021-06-03 17:22:33.000000000 +0200 @@ -107,6 +107,15 @@ RAFT_API void raft_uv_set_segment_size(struct raft_io *io, size_t size); /** + * Turn snapshot compression on or off. + * Returns non-0 on failure, this can e.g. happen when compression is requested + * while no suitable compression library is found. + * + * By default snapshots are compressed if the appropriate libraries are found. + */ +RAFT_API int raft_uv_set_snapshot_compression(struct raft_io *io, bool compressed); + +/** * Set how many milliseconds to wait between subsequent retries when * establishing a connection with another server. The default is 1000 * milliseconds. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/compress.c new/raft-0.11.1/src/compress.c --- old/raft-0.10.1/src/compress.c 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/src/compress.c 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,250 @@ +#include "compress.h" + +#ifdef LZ4_AVAILABLE +#include <lz4frame.h> +#endif +#include <limits.h> +#include <string.h> + +#include "assert.h" +#include "byte.h" +#include "err.h" + +#define min(a,b) ((a) < (b) ? (a) : (b)) +#define max(a,b) ((a) > (b) ? (a) : (b)) +#define MEGABYTE 1048576 + +int Compress(struct raft_buffer bufs[], unsigned n_bufs, + struct raft_buffer *compressed, char *errmsg) +{ +#ifndef LZ4_AVAILABLE + (void) bufs; + (void) n_bufs; + (void) compressed; + ErrMsgPrintf(errmsg, "LZ4 not available"); + return RAFT_INVALID; +#else + assert(bufs != NULL); + assert(n_bufs > 0); + assert(compressed != NULL); + assert(errmsg != NULL); + + int rv = RAFT_IOERR; + size_t src_size = 0; + size_t dst_size = 0; + size_t src_offset = 0; + size_t dst_offset = 0; + size_t dst_size_needed = 0; /* Store minimal dst_size */ + size_t ret = 0; /* Return value of LZ4F_XXX functions */ + compressed->base = NULL; + compressed->len = 0; + + /* Determine total uncompressed size */ + for (unsigned i = 0; i < n_bufs; ++i) { + src_size += bufs[i].len; + } + + /* Set LZ4 preferences */ + LZ4F_preferences_t lz4_pref; + memset(&lz4_pref, 0, sizeof(lz4_pref)); + /* Detect data corruption when decompressing */ + lz4_pref.frameInfo.contentChecksumFlag = 1; + /* For allocating a suitable buffer when decompressing */ + lz4_pref.frameInfo.contentSize = src_size; + + /* Context to track compression progress */ + LZ4F_compressionContext_t ctx; + ret = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(ret)) { + ErrMsgPrintf(errmsg, "LZ4F_createDecompressionContext %s", + LZ4F_getErrorName(ret)); + rv = RAFT_NOMEM; + goto err; + } + + /* Guestimate of eventual compressed size, mainly not to allocate a huge + * buffer as `LZ4F_compressBound` calculates the worst case scenario. */ + dst_size = LZ4F_compressBound( + max(MEGABYTE, (size_t)lz4_pref.frameInfo.contentSize / 10), &lz4_pref); + dst_size += LZ4F_HEADER_SIZE_MAX_RAFT; + compressed->base = raft_malloc(dst_size); + if (compressed->base == NULL) { + rv = RAFT_NOMEM; + goto err_after_ctx_alloc; + } + + /* Returns the size of the lz4 header, data should be written after the + * header */ + dst_offset = LZ4F_compressBegin(ctx, compressed->base, dst_size, &lz4_pref); + if (LZ4F_isError(dst_offset)) { + ErrMsgPrintf(errmsg, "LZ4F_compressBegin %s", + LZ4F_getErrorName(dst_offset)); + rv = RAFT_IOERR; + goto err_after_buff_alloc; + } + + /* Compress all buffers */ + for (unsigned i = 0; i < n_bufs; ++i) { + src_offset = 0; + while (src_offset < bufs[i].len) { + /* Compress in chunks of maximum 1MB and check if there is enough + * room in the dst buffer, if not realloc */ + src_size = min(bufs[i].len - src_offset, (size_t)MEGABYTE); + dst_size_needed = LZ4F_compressBound(src_size, &lz4_pref); + if (dst_size - dst_offset < dst_size_needed) { + dst_size += max(dst_size_needed, (size_t)lz4_pref.frameInfo.contentSize / 10); + compressed->base = raft_realloc(compressed->base, dst_size); + if (compressed->base == NULL) { + rv = RAFT_NOMEM; + goto err_after_ctx_alloc; + } + } + /* There is guaranteed enough room in `dst` to perform the + * compression */ + ret = LZ4F_compressUpdate(ctx, (char*)compressed->base + dst_offset, + dst_size - dst_offset, (char*)bufs[i].base + src_offset, + src_size, NULL); + if (LZ4F_isError(ret)) { + ErrMsgPrintf(errmsg, "LZ4F_compressUpdate %s", + LZ4F_getErrorName(ret)); + rv = RAFT_IOERR; + goto err_after_buff_alloc; + } + dst_offset += ret; + src_offset += src_size; + } + } + + /* Make sure LZ4F_compressEnd has enough room to succeed */ + dst_size_needed = LZ4F_compressBound(0, &lz4_pref); + if ((dst_size - dst_offset) < dst_size_needed) { + dst_size += dst_size_needed; + compressed->base = raft_realloc(compressed->base, dst_size); + if (compressed->base == NULL) { + rv = RAFT_NOMEM; + goto err_after_ctx_alloc; + } + } + + /* Finalize compression */ + ret = LZ4F_compressEnd(ctx, (char*)compressed->base + dst_offset, + dst_size - dst_offset, NULL); + if (LZ4F_isError(ret)) { + ErrMsgPrintf(errmsg, "LZ4F_compressEnd %s", LZ4F_getErrorName(ret)); + rv = RAFT_IOERR; + goto err_after_buff_alloc; + } + + dst_offset += ret; + compressed->len = dst_offset; + + LZ4F_freeCompressionContext(ctx); + return 0; + +err_after_buff_alloc: + raft_free(compressed->base); + compressed->base = NULL; +err_after_ctx_alloc: + LZ4F_freeCompressionContext(ctx); +err: + return rv; +#endif /* LZ4_AVAILABLE */ +} + +int Decompress(struct raft_buffer buf, struct raft_buffer *decompressed, + char *errmsg) +{ +#ifndef LZ4_AVAILABLE + (void) buf; + (void) decompressed; + ErrMsgPrintf(errmsg, "LZ4 not available"); + return RAFT_INVALID; +#else + assert(decompressed != NULL); + + int rv = RAFT_IOERR; + size_t src_offset = 0; + size_t dst_offset = 0; + size_t src_size = 0; + size_t dst_size = 0; + size_t ret = 0; + + LZ4F_decompressionContext_t ctx; + if (LZ4F_isError(LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION))) { + ErrMsgPrintf(errmsg, "LZ4F_createDecompressionContext"); + rv = RAFT_NOMEM; + goto err; + } + + src_size = buf.len; + LZ4F_frameInfo_t frameInfo = {0}; + /* `src_size` will contain the size of the LZ4 Frame Header after the call, + * decompression must resume at that offset. */ + ret = LZ4F_getFrameInfo(ctx, &frameInfo, buf.base, &src_size); + if (LZ4F_isError(ret)) { + ErrMsgPrintf(errmsg, "LZ4F_getFrameInfo %s", LZ4F_getErrorName(ret)); + rv = RAFT_IOERR; + goto err_after_ctx_alloc; + } + src_offset = src_size; + + decompressed->base = raft_malloc((size_t)frameInfo.contentSize); + decompressed->len = (size_t)frameInfo.contentSize; + if (decompressed->base == NULL) { + rv = RAFT_NOMEM; + goto err_after_ctx_alloc; + } + + ret = 1; + while (ret != 0) { + src_size = buf.len - src_offset; + /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * The next line works around a bug in an older lz4 lib where the + * `size_t` dst_size parameter would overflow an `int`. + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + dst_size = min(decompressed->len - dst_offset, (size_t)INT_MAX); + /* `dst_size` will contain the number of bytes written to decompressed->base, + * while `src_size` will contain the number of bytes consumed from + * buf.base */ + ret = LZ4F_decompress(ctx, (char*)decompressed->base + dst_offset, &dst_size, + (char*)buf.base + src_offset, &src_size, NULL); + if (LZ4F_isError(ret)) { + ErrMsgPrintf(errmsg, "LZ4F_decompress %s", LZ4F_getErrorName(ret)); + rv = RAFT_IOERR; + goto err_after_buff_alloc; + } + src_offset += src_size; + dst_offset += dst_size; + } + + if (LZ4F_freeDecompressionContext(ctx) != 0) { + raft_free(decompressed->base); + decompressed->base = NULL; + return RAFT_IOERR; + } + + return 0; + +err_after_buff_alloc: + raft_free(decompressed->base); + decompressed->base = NULL; +err_after_ctx_alloc: + LZ4F_freeDecompressionContext(ctx); +err: + return rv; +#endif /* LZ4_AVAILABLE */ +} + +bool IsCompressed(const void *data, size_t sz) +{ + if (data == NULL || sz < 4) { + return false; + } + const void *cursor = data; +#ifdef LZ4F_MAGICNUMBER +#define RAFT_LZ4F_MAGICNUMBER LZ4F_MAGICNUMBER +#else +#define RAFT_LZ4F_MAGICNUMBER 0x184D2204U +#endif + return byteGet32(&cursor) == RAFT_LZ4F_MAGICNUMBER; +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/compress.h new/raft-0.11.1/src/compress.h --- old/raft-0.10.1/src/compress.h 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/src/compress.h 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,31 @@ +#ifndef COMPRESS_H_ +#define COMPRESS_H_ + +#include "../include/raft.h" + +#ifdef LZ4F_HEADER_SIZE_MAX +#define LZ4F_HEADER_SIZE_MAX_RAFT LZ4F_HEADER_SIZE_MAX +#else +#define LZ4F_HEADER_SIZE_MAX_RAFT 19UL +#endif + +/* + * Compresses the content of `bufs` into a newly allocated buffer that is + * returned to the caller through `compressed`. Returns a non-0 value upon + * failure. + */ +int Compress(struct raft_buffer bufs[], unsigned n_bufs, + struct raft_buffer *compressed, char *errmsg); + +/* + * Decompresses the content of `buf` into a newly allocated buffer that is + * returned to the caller through `decompressed`. Returns a non-0 value upon + * failure. + */ +int Decompress(struct raft_buffer buf, struct raft_buffer *decompressed, + char *errmsg); + +/* Returns `true` if `data` is compressed, `false` otherwise. */ +bool IsCompressed(const void *data, size_t sz); + +#endif /* COMPRESS_H_ */ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/election.c new/raft-0.11.1/src/election.c --- old/raft-0.10.1/src/election.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/election.c 2021-06-03 17:22:33.000000000 +0200 @@ -127,7 +127,7 @@ /* During pre-vote we don't actually increment term or persist vote, however * we reset any vote that we previously granted since we have timed out and * that vote is no longer valid. */ - if (r->candidate_state.in_pre_vote) { + if (r->candidate_state.in_pre_vote && !r->candidate_state.disrupt_leader) { /* Reset vote */ rv = r->io->set_vote(r->io, 0); if (rv != 0) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/recv_request_vote.c new/raft-0.11.1/src/recv_request_vote.c --- old/raft-0.10.1/src/recv_request_vote.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/recv_request_vote.c 2021-06-03 17:22:33.000000000 +0200 @@ -65,7 +65,7 @@ /* If this is a pre-vote request, don't actually increment our term or * persist the vote. */ - if (args->pre_vote) { + if (args->pre_vote && !args->disrupt_leader) { recvCheckMatchingTerms(r, args->term, &match); } else { rv = recvEnsureMatchingTerms(r, args->term, &match); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/uv.c new/raft-0.11.1/src/uv.c --- old/raft-0.10.1/src/uv.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/uv.c 2021-06-03 17:22:33.000000000 +0200 @@ -646,6 +646,11 @@ uv->errored = false; uv->direct_io = false; uv->async_io = false; +#ifdef LZ4_ENABLED + uv->snapshot_compression = true; +#else + uv->snapshot_compression = false; +#endif uv->segment_size = UV__MAX_SEGMENT_SIZE; uv->block_size = 0; QUEUE_INIT(&uv->clients); @@ -722,6 +727,19 @@ uv->block_size = size; } +int raft_uv_set_snapshot_compression(struct raft_io *io, bool compressed) +{ + struct uv *uv; + uv = io->impl; +#ifndef LZ4_AVAILABLE + if (compressed) { + return RAFT_INVALID; + } +#endif + uv->snapshot_compression = compressed; + return 0; +} + void raft_uv_set_connect_retry_delay(struct raft_io *io, unsigned msecs) { struct uv *uv; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/uv.h new/raft-0.11.1/src/uv.h --- old/raft-0.10.1/src/uv.h 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/uv.h 2021-06-03 17:22:33.000000000 +0200 @@ -61,6 +61,7 @@ struct raft_tracer *tracer; /* Debug tracing */ raft_id id; /* Server ID */ int state; /* Current state */ + bool snapshot_compression; /* If compression is enabled */ bool errored; /* If a disk I/O error was hit */ bool direct_io; /* Whether direct I/O is supported */ bool async_io; /* Whether async I/O is supported */ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/uv_fs.c new/raft-0.11.1/src/uv_fs.c --- old/raft-0.10.1/src/uv_fs.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/uv_fs.c 2021-06-03 17:22:33.000000000 +0200 @@ -6,6 +6,7 @@ #include <unistd.h> #include "assert.h" +#include "compress.h" #include "err.h" #include "heap.h" #include "uv_os.h" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/src/uv_snapshot.c new/raft-0.11.1/src/uv_snapshot.c --- old/raft-0.10.1/src/uv_snapshot.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/src/uv_snapshot.c 2021-06-03 17:22:33.000000000 +0200 @@ -5,6 +5,7 @@ #include "array.h" #include "assert.h" #include "byte.h" +#include "compress.h" #include "configuration.h" #include "heap.h" #include "uv.h" @@ -329,6 +330,16 @@ goto err; } + if (IsCompressed(buf.base, buf.len)) { + struct raft_buffer decompressed = {0}; + rv = Decompress(buf, &decompressed, errmsg); + if (rv != 0) { + goto err_after_read_file; + } + HeapFree(buf.base); + buf = decompressed; + } + snapshot->bufs = HeapMalloc(sizeof *snapshot->bufs); snapshot->n_bufs = 1; if (snapshot->bufs == NULL) { @@ -337,7 +348,6 @@ } snapshot->bufs[0] = buf; - return 0; err_after_read_file: @@ -464,6 +474,26 @@ return rv; } +static int makeFileCompressed(const char *dir, + const char *filename, + struct raft_buffer *bufs, + unsigned n_bufs, + char *errmsg) +{ + int rv; + + struct raft_buffer compressed = {0}; + rv = Compress(bufs, n_bufs, &compressed, errmsg); + if (rv != 0) { + ErrMsgWrapf(errmsg, "compress %s", filename); + return RAFT_IOERR; + } + + rv = UvFsMakeFile(dir, filename, &compressed, 1, errmsg); + raft_free(compressed.base); + return rv; +} + static void uvSnapshotPutWorkCb(uv_work_t *work) { struct uvSnapshotPut *put = work->data; @@ -486,8 +516,14 @@ sprintf(snapshot, UV__SNAPSHOT_TEMPLATE, put->snapshot->term, put->snapshot->index, put->meta.timestamp); - rv = UvFsMakeFile(uv->dir, snapshot, put->snapshot->bufs, - put->snapshot->n_bufs, put->errmsg); + if (uv->snapshot_compression) { + rv = makeFileCompressed(uv->dir, snapshot, put->snapshot->bufs, + put->snapshot->n_bufs, put->errmsg); + } else { + rv = UvFsMakeFile(uv->dir, snapshot, put->snapshot->bufs, + put->snapshot->n_bufs, put->errmsg); + } + if (rv != 0) { ErrMsgWrapf(put->errmsg, "write %s", snapshot); UvFsRemoveFile(uv->dir, metadata, errmsg); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/test/integration/test_transfer.c new/raft-0.11.1/test/integration/test_transfer.c --- old/raft-0.10.1/test/integration/test_transfer.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/test/integration/test_transfer.c 2021-06-03 17:22:33.000000000 +0200 @@ -190,3 +190,22 @@ munit_assert_int(CLUSTER_LEADER, ==, 1); return MUNIT_OK; } + +static char *cluster_pre_vote[] = {"0", "1", NULL}; +static char *cluster_heartbeat[] = {"1", "100", NULL}; + +static MunitParameterEnum _params[] = { + {CLUSTER_PRE_VOTE_PARAM, cluster_pre_vote}, + {CLUSTER_HEARTBEAT_PARAM, cluster_heartbeat}, + {NULL, NULL}, +}; + +/* It's possible to transfer leadership also when pre-vote is active */ +TEST(raft_transfer, preVote, setUp, tearDown, 0, _params) +{ + struct fixture *f = data; + TRANSFER(0, 2); + CLUSTER_STEP_UNTIL_HAS_LEADER(1000); + munit_assert_int(CLUSTER_LEADER, ==, 1); + return MUNIT_OK; +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/test/integration/test_uv_snapshot_put.c new/raft-0.11.1/test/integration/test_uv_snapshot_put.c --- old/raft-0.10.1/test/integration/test_uv_snapshot_put.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/test/integration/test_uv_snapshot_put.c 2021-06-03 17:22:33.000000000 +0200 @@ -238,6 +238,18 @@ return MUNIT_OK; } +/* Request to install a snapshot without compression. */ +TEST(snapshot_put, installNoCompression, setUp, tearDown, 0, NULL) +{ + struct fixture *f = data; + raft_uv_set_snapshot_compression(&f->io, false); + APPEND(4, 8); + SNAPSHOT_PUT(0, /* trailing */ + 1 /* index */ + ); + return MUNIT_OK; +} + /* Request to install a snapshot, no previous entry is present. */ TEST(snapshot_put, installWithoutPreviousEntries, setUp, tearDown, 0, NULL) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/test/lib/cluster.h new/raft-0.11.1/test/lib/cluster.h --- old/raft-0.10.1/test/lib/cluster.h 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/test/lib/cluster.h 2021-06-03 17:22:33.000000000 +0200 @@ -24,6 +24,7 @@ do { \ unsigned _n = DEFAULT_N; \ bool _pre_vote = false; \ + unsigned _hb = 0; \ unsigned _i; \ int _rv; \ if (munit_parameters_get(params, CLUSTER_N_PARAM) != NULL) { \ @@ -33,6 +34,10 @@ _pre_vote = \ atoi(munit_parameters_get(params, CLUSTER_PRE_VOTE_PARAM)); \ } \ + if (munit_parameters_get(params, CLUSTER_HEARTBEAT_PARAM) != NULL) { \ + _hb = \ + atoi(munit_parameters_get(params, CLUSTER_HEARTBEAT_PARAM)); \ + } \ munit_assert_int(_n, >, 0); \ for (_i = 0; _i < _n; _i++) { \ FsmInit(&f->fsms[_i]); \ @@ -41,6 +46,10 @@ munit_assert_int(_rv, ==, 0); \ for (_i = 0; _i < _n; _i++) { \ raft_set_pre_vote(raft_fixture_get(&f->cluster, _i), _pre_vote); \ + if (_hb) { \ + raft_set_heartbeat_timeout(raft_fixture_get(&f->cluster, _i),\ + _hb); \ + } \ } \ } while (0) @@ -63,6 +72,9 @@ /* Munit parameter for enabling pre-vote */ #define CLUSTER_PRE_VOTE_PARAM "cluster-pre-vote" +/* Munit parameter for setting HeartBeat timeout */ +#define CLUSTER_HEARTBEAT_PARAM "cluster-heartbeat" + /* Get the number of servers in the cluster. */ #define CLUSTER_N raft_fixture_n(&f->cluster) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/test/lib/munit.c new/raft-0.11.1/test/lib/munit.c --- old/raft-0.10.1/test/lib/munit.c 2021-04-29 13:11:07.000000000 +0200 +++ new/raft-0.11.1/test/lib/munit.c 2021-06-03 17:22:33.000000000 +0200 @@ -143,10 +143,6 @@ # define MUNIT_NO_BUFFER #endif -#if defined(CODE_COVERAGE_ENABLED) -extern void __gcov_flush(void); -#endif - /*** Logging ***/ static MunitLogLevel munit_log_level_visible = MUNIT_LOG_INFO; @@ -1392,10 +1388,7 @@ if (stderr_buf != NULL) { munit_log_errno(MUNIT_LOG_ERROR, stderr, "unable to write to pipe"); } -#if defined(CODE_COVERAGE_ENABLED) - __gcov_flush(); -#endif - _exit(EXIT_FAILURE); + exit(EXIT_FAILURE); } bytes_written += write_res; } while ((size_t) bytes_written < sizeof(report)); @@ -1404,10 +1397,7 @@ fclose(stderr_buf); close(pipefd[1]); -#if defined(CODE_COVERAGE_ENABLED) - __gcov_flush(); -#endif - _exit(EXIT_SUCCESS); + exit(EXIT_SUCCESS); } else if (fork_pid == -1) { close(pipefd[0]); close(pipefd[1]); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/raft-0.10.1/test/unit/test_compress.c new/raft-0.11.1/test/unit/test_compress.c --- old/raft-0.10.1/test/unit/test_compress.c 1970-01-01 01:00:00.000000000 +0100 +++ new/raft-0.11.1/test/unit/test_compress.c 2021-06-03 17:22:33.000000000 +0200 @@ -0,0 +1,269 @@ +#include "../../src/byte.h" +#include "../../src/compress.h" +#include "../lib/munit.h" +#include "../lib/runner.h" + +#include <sys/random.h> +#ifdef LZ4_AVAILABLE +#include <lz4frame.h> +#endif + +SUITE(Compress) + +struct raft_buffer getBufWithRandom(size_t len) +{ + struct raft_buffer buf = {0}; + buf.len = len; + buf.base = munit_malloc(buf.len); + munit_assert_ptr_not_null(buf.base); + + size_t offset = 0; + /* Write as many random ints in buf as possible */ + for(size_t n = buf.len / sizeof(int); n > 0; n--) { + *((int*)(buf.base) + offset) = rand(); + offset += 1; + } + + /* Fill the remaining bytes */ + size_t rem = buf.len % sizeof(int); + /* Offset will now be used in char* arithmetic */ + offset *= sizeof(int); + if (rem) { + int r_int = rand(); + for (unsigned i = 0; i < rem; i++) { + *((char*)buf.base + offset) = *((char*)&r_int + i); + offset++; + } + } + + munit_assert_ulong(offset, ==, buf.len); + return buf; +} + +struct raft_buffer getBufWithNonRandom(size_t len) +{ + struct raft_buffer buf = {0}; + buf.len = len; + buf.base = munit_malloc(buf.len); + munit_assert_ptr_not_null(buf.base); + + memset(buf.base, 0xAC, buf.len); + return buf; +} + +#ifdef LZ4_AVAILABLE + +static void sha1(struct raft_buffer bufs[], unsigned n_bufs, uint8_t value[20]) +{ + struct byteSha1 sha; + byteSha1Init(&sha); + for (unsigned i = 0; i < n_bufs; i++) { + byteSha1Update(&sha, (const uint8_t *)bufs[i].base, (uint32_t)bufs[i].len); + } + byteSha1Digest(&sha, value); +} + +static char* len_one_params[] = { +/* 16B 1KB 64KB 4MB 128MB */ + "16", "1024", "65536", "4194304", "134217728", +/* Around Blocksize*/ + "65516", "65517", "65518", "65521", "65535", + "65537", "65551", "65555", "65556", +/* Ugly lengths */ + "1", "9", "123450", "1337", "6655111", + NULL +}; + +static MunitParameterEnum random_one_params[] = { + { "len_one", len_one_params }, + { NULL, NULL }, +}; + +TEST(Compress, compressDecompressRandomOne, NULL, NULL, 0, + random_one_params) +{ + char errmsg[RAFT_ERRMSG_BUF_SIZE] = {0}; + struct raft_buffer compressed = {0}; + struct raft_buffer decompressed = {0}; + uint8_t sha1_virgin[20] = {0}; + uint8_t sha1_decompressed[20] = {1}; + + /* Fill a buffer with random data */ + size_t len = strtoul(munit_parameters_get(params, "len_one"), NULL, 0); + struct raft_buffer buf = getBufWithRandom(len); + + /* Assert that after compression and decompression the data is unchanged */ + sha1(&buf, 1, sha1_virgin); + munit_assert_int(Compress(&buf, 1, &compressed, errmsg), ==, 0); + free(buf.base); + munit_assert_true(IsCompressed(compressed.base, compressed.len)); + munit_assert_int(Decompress(compressed, &decompressed, errmsg), ==, 0); + munit_assert_ulong(decompressed.len, ==, len); + sha1(&decompressed, 1, sha1_decompressed); + munit_assert_int(memcmp(sha1_virgin, sha1_decompressed, 20), ==, 0); + + raft_free(compressed.base); + raft_free(decompressed.base); + return MUNIT_OK; +} + +static char* len_nonrandom_one_params[] = { +/* 4KB 64KB 4MB 1GB 3GB */ + "4096", "65536", "4194304", "1073741824", "3221225472", +/* Around Blocksize*/ + "65516", "65517", "65518", "65521", "65535", + "65537", "65551", "65555", "65556", +/* Ugly lengths */ + "993450", "31337", "83883825", + NULL +}; + +static MunitParameterEnum nonrandom_one_params[] = { + { "len_one", len_nonrandom_one_params }, + { NULL, NULL }, +}; + +TEST(Compress, compressDecompressNonRandomOne, NULL, NULL, 0, + nonrandom_one_params) +{ + char errmsg[RAFT_ERRMSG_BUF_SIZE] = {0}; + struct raft_buffer compressed = {0}; + struct raft_buffer decompressed = {0}; + uint8_t sha1_virgin[20] = {0}; + uint8_t sha1_decompressed[20] = {1}; + + /* Fill a buffer with non-random data */ + size_t len = strtoul(munit_parameters_get(params, "len_one"), NULL, 0); + struct raft_buffer buf = getBufWithNonRandom(len); + + /* Assert that after compression and decompression the data is unchanged and + * that the compressed data is actually smaller */ + sha1(&buf, 1, sha1_virgin); + munit_assert_int(Compress(&buf, 1, &compressed, errmsg), ==, 0); + free(buf.base); + munit_assert_true(IsCompressed(compressed.base, compressed.len)); + munit_assert_ulong(compressed.len, <, buf.len); + munit_assert_int(Decompress(compressed, &decompressed, errmsg), ==, 0); + munit_assert_ulong(decompressed.len, ==, len); + sha1(&decompressed, 1, sha1_decompressed); + munit_assert_int(memcmp(sha1_virgin, sha1_decompressed, 20), ==, 0); + + raft_free(compressed.base); + raft_free(decompressed.base); + return MUNIT_OK; +} + +static char* len_two_params[] = { + "4194304", "13373", "66", + NULL +}; + +static MunitParameterEnum random_two_params[] = { + { "len_one", len_one_params }, + { "len_two", len_two_params }, + { NULL, NULL }, +}; + +TEST(Compress, compressDecompressRandomTwo, NULL, NULL, 0, + random_two_params) +{ + char errmsg[RAFT_ERRMSG_BUF_SIZE] = {0}; + struct raft_buffer compressed = {0}; + struct raft_buffer decompressed = {0}; + uint8_t sha1_virgin[20] = {0}; + uint8_t sha1_decompressed[20] = {1}; + + /* Fill two buffers with random data */ + size_t len1 = strtoul(munit_parameters_get(params, "len_one"), NULL, 0); + struct raft_buffer buf1 = getBufWithRandom(len1); + size_t len2 = strtoul(munit_parameters_get(params, "len_two"), NULL, 0); + struct raft_buffer buf2 = getBufWithRandom(len2); + struct raft_buffer bufs[2] = { buf1, buf2 }; + + /* Assert that after compression and decompression the data is unchanged */ + sha1(bufs, 2, sha1_virgin); + munit_assert_int(Compress(bufs, 2, &compressed, errmsg), ==, 0); + free(buf1.base); + free(buf2.base); + munit_assert_true(IsCompressed(compressed.base, compressed.len)); + munit_assert_int(Decompress(compressed, &decompressed, errmsg), ==, 0); + munit_assert_ulong(decompressed.len, ==, buf1.len + buf2.len); + sha1(&decompressed, 1, sha1_decompressed); + munit_assert_int(memcmp(sha1_virgin, sha1_decompressed, 20), ==, 0); + + raft_free(compressed.base); + raft_free(decompressed.base); + return MUNIT_OK; +} + +TEST(Compress, compressDecompressCorruption, NULL, NULL, 0, NULL) +{ + char errmsg[RAFT_ERRMSG_BUF_SIZE] = {0}; + struct raft_buffer compressed = {0}; + struct raft_buffer decompressed = {0}; + + /* Fill a buffer with random data */ + size_t len = 2048; + struct raft_buffer buf = getBufWithRandom(len); + + munit_assert_int(Compress(&buf, 1, &compressed, errmsg), ==, 0); + munit_assert_true(IsCompressed(compressed.base, compressed.len)); + + /* Corrupt the a data byte after the header */ + munit_assert_ulong(LZ4F_HEADER_SIZE_MAX_RAFT, <, compressed.len); + ((char*)compressed.base)[LZ4F_HEADER_SIZE_MAX_RAFT] += 1; + + munit_assert_int(Decompress(compressed, &decompressed, errmsg), !=, 0); + munit_assert_string_equal(errmsg, "LZ4F_decompress ERROR_contentChecksum_invalid"); + munit_assert_ptr_null(decompressed.base); + + raft_free(compressed.base); + free(buf.base); + return MUNIT_OK; +} + +#else + +TEST(Compress, lz4Disabled, NULL, NULL, 0, NULL) +{ + char errmsg[RAFT_ERRMSG_BUF_SIZE] = {0}; + struct raft_buffer compressed = {0}; + + /* Fill a buffer with random data */ + size_t len = 2048; + struct raft_buffer buf = getBufWithRandom(len); + + munit_assert_int(Compress(&buf, 1, &compressed, errmsg), ==, RAFT_INVALID); + munit_assert_ptr_null(compressed.base); + + free(buf.base); + return MUNIT_OK; +} + +#endif /* LZ4_AVAILABLE */ + +static const char LZ4_MAGIC[4] = {0x04, 0x22, 0x4d, 0x18}; +TEST(Compress, isCompressedTooSmall, NULL, NULL, 0, NULL) +{ + munit_assert_false(IsCompressed(&LZ4_MAGIC[1], sizeof(LZ4_MAGIC)-1)); + return MUNIT_OK; +} + +TEST(Compress, isCompressedNull, NULL, NULL, 0, NULL) +{ + munit_assert_false(IsCompressed(NULL, sizeof(LZ4_MAGIC))); + return MUNIT_OK; +} + +TEST(Compress, isCompressed, NULL, NULL, 0, NULL) +{ + munit_assert_true(IsCompressed(LZ4_MAGIC, sizeof(LZ4_MAGIC))); + return MUNIT_OK; +} + +TEST(Compress, notCompressed, NULL, NULL, 0, NULL) +{ + char not_compressed[4] = {0x18, 0x4d, 0x22, 0x04}; + munit_assert_false(IsCompressed(not_compressed, sizeof(not_compressed))); + return MUNIT_OK; +}
