from:"Sven Schmidt"

[PATCH v8 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-02-15 Thread Sven Schmidt

Update the unlz4 wrapper to work with the updated LZ4 kernel module
version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

[PATCH v8 5/5] lib/lz4: Remove back-compat wrappers

2017-02-15 Thread Sven Schmidt

Remove the functions introduced as wrappers for providing backwards
compatibility to the prior LZ4 version.  They're not needed anymore since
there's no callers left.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  | 69 
 lib/lz4/lz4_compress.c   | 22 ---
 lib/lz4/lz4_decompress.c | 42 -
 lib/lz4/lz4hc_compress.c | 23 
 4 files changed, 156 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 1b0f8ca..394e3d9 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -173,18 +173,6 @@ static inline int LZ4_compressBound(size_t isize)
 }
 
 /**
- * lz4_compressbound() - For backwards compatibility; see LZ4_compressBound
- * @isize: Size of the input data
- *
- * Return: Max. size LZ4 may output in a "worst case" szenario
- * (data not compressible)
- */
-static inline int lz4_compressbound(size_t isize)
-{
-   return LZ4_COMPRESSBOUND(isize);
-}
-
-/**
  * LZ4_compress_default() - Compress data from source to dest
  * @source: source address of the original data
  * @dest: output buffer address of the compressed data
@@ -257,20 +245,6 @@ int LZ4_compress_fast(const char *source, char *dest, int 
inputSize,
 int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
int targetDestSize, void *wrkmem);
 
-/*
- * lz4_compress() - For backward compatibility, see LZ4_compress_default
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND
- * @dst_len: is the output size, which is returned after compress done
- * @workmem: address of the working memory.
- *
- * Return: Success if return 0, Error if return < 0
- */
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
-   size_t *dst_len, void *wrkmem);
-
 /*-
  * Decompression Functions
  **/
@@ -346,34 +320,6 @@ int LZ4_decompress_safe(const char *source, char *dest, 
int compressedSize,
 int LZ4_decompress_safe_partial(const char *source, char *dest,
int compressedSize, int targetOutputSize, int maxDecompressedSize);
 
-/*
- * lz4_decompress_unknownoutputsize() - For backwards compatibility,
- * see LZ4_decompress_safe
- * @src: source address of the compressed data
- * @src_len: is the input size, therefore the compressed size
- * @dest: output buffer address of the decompressed data
- * which must be already allocated
- * @dest_len: is the max size of the destination buffer, which is
- * returned with actual size of decompressed data after decompress done
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
-   unsigned char *dest, size_t *dest_len);
-
-/**
- * lz4_decompress() - For backwards cocmpatibility, see LZ4_decompress_fast
- * @src: source address of the compressed data
- * @src_len: is the input size, which is returned after decompress done
- * @dest: output buffer address of the decompressed data,
- * which must be already allocated
- * @actual_dest_len: is the size of uncompressed data, supposing it's known
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress(const unsigned char *src, size_t *src_len,
-   unsigned char *dest, size_t actual_dest_len);
-
 /*-
  * LZ4 HC Compression
  **/
@@ -401,21 +347,6 @@ int LZ4_compress_HC(const char *src, char *dst, int 
srcSize, int dstCapacity,
int compressionLevel, void *wrkmem);
 
 /**
- * lz4hc_compress() - For backwards compatibility, see LZ4_compress_HC
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND.
- * @dst_len: is the output size, which is returned after compress done
- * @wrkmem: address of the working memory.
- * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- *
- * Return  : Success if return 0, Error if return (< 0)
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char 
*dst,
-   size_t *dst_len, void *wrkmem);
-
-/**
  * LZ4_resetStreamHC() - Init an allocated 'LZ4_streamHC_t' structure
  * @streamHCPtr: pointer to the 'LZ4_streamHC_t' structure
  * @compressionLevel: Recommended values are between 4 and 9, although any
diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c
index 53f313f..cc7b6d4 100644
--- a/lib/lz4/lz4_compress.c
+++ b/lib/l

[PATCH v8 1/5] lib: Update LZ4 compressor module

2017-02-15 Thread Sven Schmidt

Update the LZ4 kernel module to LZ4 v1.7.3 by Yann Collet.  The kernel
module is inspired by the previous work by Chanho Min.  The updated LZ4
module will not break existing code since the patchset contains
appropriate changes.

API changes:

New method LZ4_compress_fast which differs from the variant available in
kernel by the new acceleration parameter, allowing to trade compression
ratio for more compression speed and vice versa.

LZ4_decompress_fast is the respective decompression method, featuring a
very fast decoder (multiple GB/s per core), able to reach RAM speed in
multi-core systems.  The decompressor allows to decompress data compressed
with LZ4 fast as well as the LZ4 HC (high compression) algorithm.

Also the useful functions LZ4_decompress_safe_partial and
LZ4_compress_destsize were added.  The latter reverses the logic by trying
to compress as much data as possible from source to dest while the former
aims to decompress partial blocks of data.

A bunch of streaming functions were also added which allow
compressig/decompressing data in multiple steps (so called "streaming
mode").

The methods lz4_compress and lz4_decompress_unknownoutputsize are now
known as LZ4_compress_default respectivley LZ4_decompress_safe. The old
methods will be removed since there's no callers left in the code.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  |  762 +++---
 lib/lz4/Makefile |2 +
 lib/lz4/lz4_compress.c   | 1161 +-
 lib/lz4/lz4_decompress.c |  705 ++--
 lib/lz4/lz4defs.h|  338 --
 lib/lz4/lz4hc_compress.c |  867 ++
 6 files changed, 2758 insertions(+), 1077 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 6b784c5..1b0f8ca 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -1,87 +1,717 @@
-#ifndef __LZ4_H__
-#define __LZ4_H__
-/*
- * LZ4 Kernel Interface
+/* LZ4 Kernel Interface
  *
  * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik@lge.com>
+ * Copyright (C) 2016, Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
+ *
+ * This file is based on the original header file
+ * for LZ4 - Fast LZ compression algorithm.
+ *
+ * LZ4 - Fast LZ compression algorithm
+ * Copyright (C) 2011-2016, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
  */
-#define LZ4_MEM_COMPRESS   (16384)
-#define LZ4HC_MEM_COMPRESS (262144 + (2 * sizeof(unsigned char *)))
 
+#ifndef __LZ4_H__
+#define __LZ4_H__
+
+#include 
+#include/* memset, memcpy */
+
+/*-
+ * CONSTANTS
+ **/
 /*
- * lz4_compressbound()
- * Provides the maximum size that LZ4 may output in a "worst case" scenario
- * (input data not compressible)
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes
+ * (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, f

[PATCH v8 3/5] crypto: Change LZ4 modules to work with new LZ4 module version

2017-02-15 Thread Sven Schmidt

Update the crypto modules using LZ4 compression as well as the test cases
in testmgr.h to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c |  23 -
 crypto/lz4hc.c   |  23 -
 crypto/testmgr.h | 142 +++
 3 files changed, 120 insertions(+), 68 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..71eff9b 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, *dlen, ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
-   return -EINVAL;
+   if (out_len < 0)
+   return out_len;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return 0;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..03a34a8 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   *dlen, LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
-   return -EINVAL;
+   if (out_len < 0)
+   return out_len;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return 0;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9b656be..98d4be0 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -34498,31 +34498,62 @@ static struct hash_testvec bfin_crc_tv_template[] = {
 
 static struct comp_testvec lz4_comp_tv_template[] = {
{
-   .inlen  = 70,
-   .outlen = 45,
-   .input  = "Join us now and share the software "
- "Join us now and share the software ",
-   .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
+   .inlen  = 255,
+   .outlen = 218,
+   .input  = "LZ4 is lossless compression algorithm, providing"
+" compression speed at 400 MB/s per core, scalable "
+"with multi-cores CPU. It features an extremely fast "
+"decoder, with speed in multiple GB/s per core, "
+"typically reaching RAM speed limits on multi-core "
+"systems.",
+   .output = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+

[PATCH v8 0/5] Update LZ4 compressor module

2017-02-15 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html

[PATCH 1/5] lib: Update LZ4 compressor module
[PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module 
version
[PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 
version
[PATCH 5/5] lib/lz4: Remove back-compat wrappers

Changes:
v8:
- Rewrote the architecture-dependent definitions in lz4defs.h, such as 
LZ4_read*,
  LZ4_write* and LZ4_NbCommonBytes as proposed by Eric Biggers
- Added -O3 compiler flag to Makefile, also suggested by Eric
- Defined FORCE_INLINE, as used in upstream LZ4, as __always_inline and
  force-inlined most of the small functions in lz4defs.h
- lz4_decompress: Wrapped the EXPORT_SYMBOL and MODULE_* macros in a
  #ifdef STATIC the way suggested by Andrew Morton, fixing the breakage
  of CONFIG_KERNEL_LZ4 in x86 as reported by Arnd Bergman

v7:
- Fixed errors reported by the Smatch tool
- Changed function documentation comments in lz4.h to match kernel-doc style
- Fixed a misbehaviour of LZ4HC caused by the wrong level of indentation
  concerning two for loops introduced after I refactored the code style using
  checkpatch.pl (upstream LZ4 put dozens of stuff in just one line, gnah)
- Updated the crypto tests for LZ4 since they did fail for the new code
  and hence zram did fail to allocate memory for LZ4

v6:
- Fixed LZ4_NBCOMMONBYTES() for 64-bit little endian
- Reset LZ4_MEMORY_USAGE to 14 (which is the value used in
  upstream LZ4 as well as the previous kernel module)
- Fixed that weird double-indentation in lz4defs.h and lz4.h
- Adjusted general styling issues in lz4defs.h
  (e.g. lines consisting of more than one instruction)
- Removed the architecture-dependent typedef to reg_t
  since upstream LZ4 is just using size_t and that works fine
- Changed error messages in pstore/platform.c:
  * LZ4_compress_default always returns 0 in case of an error
(no need to print the return value)
  * LZ4_decompress_safe returns a negative error message
(return value _does_ matter)

v5:
- Added a fifth patch to remove the back-compat wrappers introduced
  to ensure bisectibility between the patches (the functions are no longer
  needed since there's no callers left)

v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
  to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
  and triggers an unused function warning when false

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
  are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
  different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

[PATCH v8 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-02-15 Thread Sven Schmidt

Update fs/pstore and fs/squashfs to use the updated functions from the new
LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 22 +-
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..efab7b6 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,35 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
-   pr_err("lz4_compress error, ret = %d!\n", ret);
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
+   pr_err("LZ4_compress_default error; compression failed!\n");
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
-   pr_err("lz4_decompress error, ret = %d!\n", ret);
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
+   /*
+* LZ4_decompress_safe will return an error code
+* (< 0) if decompression failed
+*/
+   pr_err("LZ4_decompress_safe error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

Re: [PATCH] lz4: fix performance regressions

2017-02-14 Thread Sven Schmidt

Hey Eric,

On Sun, Feb 12, 2017 at 03:38:02PM -0800, Eric Biggers wrote:
> Hi Sven,
> 
> On Sun, Feb 12, 2017 at 12:16:18PM +0100, Sven Schmidt wrote:
> >  /*-
> >   * Reading and writing into memory
> >   **/
> > +typedef union {
> > +   U16 u16;
> > +   U32 u32;
> > +   size_t uArch;
> > +} __packed unalign;
> > 
> > -static inline U16 LZ4_read16(const void *memPtr)
> > +static FORCE_INLINE __maybe_unused U16 LZ4_read16(const void *ptr)
> >  {
> > -   U16 val;
> > -
> > -   memcpy(, memPtr, sizeof(val));
> > -
> > -   return val;
> > +   return ((const unalign *)ptr)->u16;
> >  }
> > 
> > -static inline U32 LZ4_read32(const void *memPtr)
> > +static FORCE_INLINE __maybe_unused U32 LZ4_read32(const void *ptr)
> >  {
> > -   U32 val;
> > -
> > -   memcpy(, memPtr, sizeof(val));
> > -
> > -   return val;
> > +   return ((const unalign *)ptr)->u32;
> >  }
> > 
> > -static inline size_t LZ4_read_ARCH(const void *memPtr)
> > +static FORCE_INLINE __maybe_unused size_t LZ4_read_ARCH(const void *ptr)
> >  {
> > -   size_t val;
> > -
> > -   memcpy(, memPtr, sizeof(val));
> > -
> > -   return val;
> > +   return ((const unalign *)ptr)->uArch;
> >  }
> > 
> > -static inline void LZ4_write16(void *memPtr, U16 value)
> > +static FORCE_INLINE __maybe_unused void LZ4_write16(void *memPtr, U16 
> > value)
> >  {
> > -   memcpy(memPtr, , sizeof(value));
> > +   ((unalign *)memPtr)->u16 = value;
> >  }
> > 
> > -static inline void LZ4_write32(void *memPtr, U32 value)
> > -{
> > -   memcpy(memPtr, , sizeof(value));
> > +static FORCE_INLINE __maybe_unused void LZ4_write32(void *memPtr, U32 
> > value) {
> > +   ((unalign *)memPtr)->u32 = value;
> >  }
> > 
> > -static inline U16 LZ4_readLE16(const void *memPtr)
> > +static FORCE_INLINE __maybe_unused U16 LZ4_readLE16(const void *memPtr)
> >  {
> > -#ifdef __LITTLE_ENDIAN__
> > +#if LZ4_LITTLE_ENDIAN
> > return LZ4_read16(memPtr);
> >  #else
> > const BYTE *p = (const BYTE *)memPtr;
> > @@ -137,19 +143,19 @@ static inline U16 LZ4_readLE16(const void *memPtr)
> >  #endif
> >  }
> 
> Since upstream LZ4 is intended to be compiled at -O3, this may allow it to get
> away with using memcpy() for unaligned memory accesses.  The reason it uses
> memcpy() is that, other than a byte-by-byte copy, it is the only portable way 
> to
> express unaligned memory accesses.  But the Linux kernel is sometimes compiled
> optimized for size (-Os), and I wouldn't be *too* surprised if some of the
> memcpy()'s don't always get inlined then, which could be causing the 
> performance
> regression being observed.  (Of course, this could be verified by checking
> whether CONFIG_CC_OPTIMIZE_FOR_SIZE=y is set, then reading the assembly.)
> 
> But I don't think accessing a __packed structure directly is the right
> alternative.  Instead, Linux already includes macros for unaligned memory
> accesses which have been optimized for every supported architecture.  Those
> should just be used instead, e.g. like this:
> 
> static FORCE_INLINE U16 LZ4_read16(const void *ptr)
> {
>   return get_unaligned((const u16 *)ptr);
> }
> 
> static FORCE_INLINE U32 LZ4_read32(const void *ptr)
> {
>   return get_unaligned((const u32 *)ptr);
> }
> 
> static FORCE_INLINE size_t LZ4_read_ARCH(const void *ptr)
> {
>   return get_unaligned((const size_t *)ptr);
> }
> 
> static FORCE_INLINE void LZ4_write16(void *memPtr, U16 value)
> {
>   put_unaligned(value, (u16 *)memPtr);
> }
> 
> static FORCE_INLINE void LZ4_write32(void *memPtr, U32 value)
> {
>   put_unaligned(value, (u32 *)memPtr);
> }
> 
> static FORCE_INLINE U16 LZ4_readLE16(const void *memPtr)
> {
>   return get_unaligned_le16(memPtr);
> }
> 
> static FORCE_INLINE void LZ4_writeLE16(void *memPtr, U16 value)
> {
>   return put_unaligned_le16(value, memPtr);
> }
> 
> static FORCE_INLINE void LZ4_copy8(void *dst, const void *src)
> {
>   if (LZ4_64bits()) {
>   u64 a = get_unaligned((const u64 *)src);
>   put_unaligned(a, (u64 *)dst);
>   } else {
>   u32 a = get_unaligned((const u32 *)src);
>   u32 b = get_unaligned((const u32 *)src + 1);
>   put_unaligned(a, (u32 *)dst);
>   put_unaligned(b, (u32 *)dst + 1);
>   }
> }
> 
> 
> Note that I dropped __maybe

Re: [PATCH v7 0/5] Update LZ4 compressor module

2017-02-13 Thread Sven Schmidt

On Mon, Feb 13, 2017 at 09:03:24AM +0900, Minchan Kim wrote:
> Hi Sven,
> 
> On Sun, Feb 12, 2017 at 12:16:17PM +0100, Sven Schmidt wrote:
> > 
> > 
> > 
> > On 02/10/2017 01:13 AM, Minchan Kim wrote:
> > > Hello Sven,
> > >
> > > On Thu, Feb 09, 2017 at 11:56:17AM +0100, Sven Schmidt wrote:
> > >> Hey Minchan,
> > >>
> > >> On Thu, Feb 09, 2017 at 08:31:21AM +0900, Minchan Kim wrote:
> > >>> Hello Sven,
> > >>>
> > >>> On Sun, Feb 05, 2017 at 08:09:03PM +0100, Sven Schmidt wrote:
> > >>>>
> > >>>> This patchset is for updating the LZ4 compression module to a version 
> > >>>> based
> > >>>> on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 
> > >>>> fast
> > >>>> which provides an "acceleration" parameter as a tradeoff between
> > >>>> high compression ratio and high compression speed.
> > >>>>
> > >>>> We want to use LZ4 fast in order to support compression in lustre
> > >>>> and (mostly, based on that) investigate data reduction techniques in 
> > >>>> behalf of
> > >>>> storage systems.
> > >>>>
> > >>>> Also, it will be useful for other users of LZ4 compression, as with 
> > >>>> LZ4 fast
> > >>>> it is possible to enable applications to use fast and/or high 
> > >>>> compression
> > >>>> depending on the usecase.
> > >>>> For instance, ZRAM is offering a LZ4 backend and could benefit from an 
> > >>>> updated
> > >>>> LZ4 in the kernel.
> > >>>>
> > >>>> LZ4 homepage: http://www.lz4.org/
> > >>>> LZ4 source repository: https://github.com/lz4/lz4
> > >>>> Source version: 1.7.3
> > >>>>
> > >>>> Benchmark (taken from [1], Core i5-4300U @1.9GHz):
> > >>>> |--||--
> > >>>> Compressor  | Compression  | Decompression  | Ratio
> > >>>> |--||--
> > >>>> memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
> > >>>> LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
> > >>>> LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
> > >>>> LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
> > >>>> LZ4 default |   385 MB/s   |  1850 MB/s | 2.101
> > >>>>
> > >>>> [1] 
> > >>>> http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
> > >>>>
> > >>>> [PATCH 1/5] lib: Update LZ4 compressor module
> > >>>> [PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 
> > >>>> module version
> > >>>> [PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module 
> > >>>> version
> > >>>> [PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with 
> > >>>> new LZ4 version
> > >>>> [PATCH 5/5] lib/lz4: Remove back-compat wrappers
> > >>>
> > >>> Today, I did zram-lz4 performance test with fio in current mmotm and
> > >>> found it makes regression about 20%.
> > >>>
> > >>> "lz4-update" means current mmots(git://git.cmpxchg.org/linux-mmots.git) 
> > >>> so
> > >>> applied your 5 patches. (But now sure current mmots has recent uptodate
> > >>> patches)
> > >>> "revert" means I reverted your 5 patches in current mmots.
> > >>>
> > >>>  revertlz4-update
> > >>>
> > >>>   seq-write   1547   1339  86.55%
> > >>>  rand-write  22775  19381  85.10%
> > >>>seq-read   7035   5589  79.45%
> > >>>   rand-read  78556  68479  87.17%
> > >>>mixed-seq(R)   1305   1066  81.69%
> > >>>mixed-seq(W)   1205984  81.66%
> > >>>   mixed-rand(R)  17421  14993  86.06%
> > >>>   mixed-rand(W)  17391  14968  86.07%
> > >>
> > >> which parts of the output (as well as units) are these values

Re: [PATCH] lz4: fix performance regressions

2017-02-13 Thread Sven Schmidt

On Sun, Feb 12, 2017 at 10:41:17PM +0100, Willy Tarreau wrote:
> On Sun, Feb 12, 2017 at 04:20:00PM +0100, Sven Schmidt wrote:
> > On Sun, Feb 12, 2017 at 02:05:08PM +0100, Willy Tarreau wrote:
> > > Hi Sven,
> > > 
> > > On Sun, Feb 12, 2017 at 12:16:18PM +0100, Sven Schmidt wrote:
> > > > Fix performance regressions compared to current kernel LZ4
> > > 
> > > Your patch contains mostly style cleanups which certainly are welcome
> > > but make the whole patch hard to review. These cleanups would have been
> > > better into a separate, preliminary patch IMHO.
> > > 
> > > Regards,
> > > Willy
> > 
> > Hi Willy,
> > 
> > the problem was, I wanted to compare my version to the upstream LZ4 to find 
> > bugs (as with my last patch version: wrong indentation in LZ4HC 
> > in two for loops). But since the LZ4 code is a pain to read, I made 
> > additional style cleanups "on the way".
> 
> Oh I can easily understand!
> 
> > Hope you can manage to review the patch though, because it is difficult to 
> > separate the cleanups now.
> 
> When I need to split a patch into pieces, usually what I do is that I
> revert it, re-apply it without committing, then "git add -p", validate
> all the hunks to be taken as the first patch (ie here the cleanups),
> commit, then commit the rest as a separate one. It seems to me that the
> fix is in the last few hunks though I'm not sure yet.
> 
> Thanks,
> Willy

Hi Willy,

I didn't know about this 'trick' until now. Thanks for sharing it! I gave it a 
short try recently, that's really cool!

Since the problem discussed in this branch of this thread seems to be solved 
(see Minchans E-Mail), I won't split the patches, though.
Or is there an actual need for doing so? I will send an updated patchset 
(containing these patches + the other ones suggested by Eric) later.

Regards,

Sven

Re: [PATCH] lz4: fix performance regressions

2017-02-12 Thread Sven Schmidt

On Sun, Feb 12, 2017 at 02:05:08PM +0100, Willy Tarreau wrote:
> Hi Sven,
> 
> On Sun, Feb 12, 2017 at 12:16:18PM +0100, Sven Schmidt wrote:
> > Fix performance regressions compared to current kernel LZ4
> 
> Your patch contains mostly style cleanups which certainly are welcome
> but make the whole patch hard to review. These cleanups would have been
> better into a separate, preliminary patch IMHO.
> 
> Regards,
> Willy

Hi Willy,

the problem was, I wanted to compare my version to the upstream LZ4 to find 
bugs (as with my last patch version: wrong indentation in LZ4HC 
in two for loops). But since the LZ4 code is a pain to read, I made additional 
style cleanups "on the way".
Hope you can manage to review the patch though, because it is difficult to 
separate the cleanups now.
Please feel free to ask if you stumble upon something.

Greetings,

Sven

[PATCH] lz4: fix performance regressions

2017-02-12 Thread Sven Schmidt

Fix performance regressions compared to current kernel LZ4

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  |   2 +-
 lib/lz4/lz4_compress.c   | 157 +++-
 lib/lz4/lz4_decompress.c |  50 
 lib/lz4/lz4defs.h| 203 ---
 lib/lz4/lz4hc_compress.c |   8 +-
 5 files changed, 281 insertions(+), 139 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index a3912d7..394e3d9 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -82,7 +82,7 @@
 /*-
  * STREAMING CONSTANTS AND STRUCTURES
  **/
-#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE - 3)) + 4)
 #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))

 #define LZ4_STREAMHCSIZE262192
diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c
index 697dbda..2cbbf99 100644
--- a/lib/lz4/lz4_compress.c
+++ b/lib/lz4/lz4_compress.c
@@ -39,27 +39,33 @@
 #include 
 #include 

+static const int LZ4_minLength = (MFLIMIT + 1);
+static const int LZ4_64Klimit = ((64 * KB) + (MFLIMIT - 1));
+
 /*-**
  * Compression functions
  /
-static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
+static FORCE_INLINE U32 LZ4_hash4(
+   U32 sequence,
+   tableType_t const tableType)
 {
if (tableType == byU16)
return ((sequence * 2654435761U)
-   >> ((MINMATCH*8) - (LZ4_HASHLOG + 1)));
+   >> ((MINMATCH * 8) - (LZ4_HASHLOG + 1)));
else
return ((sequence * 2654435761U)
-   >> ((MINMATCH*8) - LZ4_HASHLOG));
+   >> ((MINMATCH * 8) - LZ4_HASHLOG));
 }

-#if LZ4_ARCH64
-static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
+static FORCE_INLINE __maybe_unused U32 LZ4_hash5(
+   U64 sequence,
+   tableType_t const tableType)
 {
const U32 hashLog = (tableType == byU16)
? LZ4_HASHLOG + 1
: LZ4_HASHLOG;

-#ifdef __LITTLE_ENDIAN__
+#if LZ4_LITTLE_ENDIAN
static const U64 prime5bytes = 889523592379ULL;

return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
@@ -69,9 +75,10 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const 
tableType)
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 #endif
 }
-#endif

-static U32 LZ4_hashPosition(const void *p, tableType_t tableType)
+static FORCE_INLINE U32 LZ4_hashPosition(
+   const void *p,
+   tableType_t const tableType)
 {
 #if LZ4_ARCH64
if (tableType == byU32)
@@ -81,8 +88,12 @@ static U32 LZ4_hashPosition(const void *p, tableType_t 
tableType)
return LZ4_hash4(LZ4_read32(p), tableType);
 }

-static void LZ4_putPositionOnHash(const BYTE *p, U32 h, void *tableBase,
-   tableType_t const tableType, const BYTE *srcBase)
+static void LZ4_putPositionOnHash(
+   const BYTE *p,
+   U32 h,
+   void *tableBase,
+   tableType_t const tableType,
+   const BYTE *srcBase)
 {
switch (tableType) {
case byPtr:
@@ -109,16 +120,22 @@ static void LZ4_putPositionOnHash(const BYTE *p, U32 h, 
void *tableBase,
}
 }

-static inline void LZ4_putPosition(const BYTE *p, void *tableBase,
-   tableType_t tableType, const BYTE *srcBase)
+static FORCE_INLINE void LZ4_putPosition(
+   const BYTE *p,
+   void *tableBase,
+   tableType_t tableType,
+   const BYTE *srcBase)
 {
U32 const h = LZ4_hashPosition(p, tableType);

LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }

-static const BYTE *LZ4_getPositionOnHash(U32 h, void *tableBase,
-   tableType_t tableType, const BYTE *srcBase)
+static const BYTE *LZ4_getPositionOnHash(
+   U32 h,
+   void *tableBase,
+   tableType_t tableType,
+   const BYTE *srcBase)
 {
if (tableType == byPtr) {
const BYTE **hashTable = (const BYTE **) tableBase;
@@ -135,12 +152,16 @@ static const BYTE *LZ4_getPositionOnHash(U32 h, void 
*tableBase,
{
/* default, to ensure a return */
const U16 * const hashTable = (U16 *) tableBase;
+
return hashTable[h] + srcBase;
}
 }

-static inline const BYTE *LZ4_getPosition(const BYTE *p, void *tableBase,
-   tableType_t tableType, const BYTE *srcBase)
+static FORCE_INLINE const BYTE *LZ4_getPosition(
+   const BYTE *p,
+   void *tableBase,
+   tableType_t tableType,
+   const BYTE *srcBase)
 {
U32 const h = LZ4_hashPosition(p, tableType);

@@ -152,7 +173,7 @@ static

Re: [PATCH v7 0/5] Update LZ4 compressor module

2017-02-12 Thread Sven Schmidt




On 02/10/2017 01:13 AM, Minchan Kim wrote:
> Hello Sven,
>
> On Thu, Feb 09, 2017 at 11:56:17AM +0100, Sven Schmidt wrote:
>> Hey Minchan,
>>
>> On Thu, Feb 09, 2017 at 08:31:21AM +0900, Minchan Kim wrote:
>>> Hello Sven,
>>>
>>> On Sun, Feb 05, 2017 at 08:09:03PM +0100, Sven Schmidt wrote:
>>>>
>>>> This patchset is for updating the LZ4 compression module to a version based
>>>> on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
>>>> which provides an "acceleration" parameter as a tradeoff between
>>>> high compression ratio and high compression speed.
>>>>
>>>> We want to use LZ4 fast in order to support compression in lustre
>>>> and (mostly, based on that) investigate data reduction techniques in 
>>>> behalf of
>>>> storage systems.
>>>>
>>>> Also, it will be useful for other users of LZ4 compression, as with LZ4 
>>>> fast
>>>> it is possible to enable applications to use fast and/or high compression
>>>> depending on the usecase.
>>>> For instance, ZRAM is offering a LZ4 backend and could benefit from an 
>>>> updated
>>>> LZ4 in the kernel.
>>>>
>>>> LZ4 homepage: http://www.lz4.org/
>>>> LZ4 source repository: https://github.com/lz4/lz4
>>>> Source version: 1.7.3
>>>>
>>>> Benchmark (taken from [1], Core i5-4300U @1.9GHz):
>>>> |--||--
>>>> Compressor  | Compression  | Decompression  | Ratio
>>>> |--||--
>>>> memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
>>>> LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
>>>> LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
>>>> LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
>>>> LZ4 default |   385 MB/s   |  1850 MB/s | 2.101
>>>>
>>>> [1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
>>>>
>>>> [PATCH 1/5] lib: Update LZ4 compressor module
>>>> [PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 
>>>> module version
>>>> [PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
>>>> [PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new 
>>>> LZ4 version
>>>> [PATCH 5/5] lib/lz4: Remove back-compat wrappers
>>>
>>> Today, I did zram-lz4 performance test with fio in current mmotm and
>>> found it makes regression about 20%.
>>>
>>> "lz4-update" means current mmots(git://git.cmpxchg.org/linux-mmots.git) so
>>> applied your 5 patches. (But now sure current mmots has recent uptodate
>>> patches)
>>> "revert" means I reverted your 5 patches in current mmots.
>>>
>>>  revertlz4-update
>>>
>>>   seq-write   1547   1339  86.55%
>>>  rand-write  22775  19381  85.10%
>>>seq-read   7035   5589  79.45%
>>>   rand-read  78556  68479  87.17%
>>>mixed-seq(R)   1305   1066  81.69%
>>>mixed-seq(W)   1205984  81.66%
>>>   mixed-rand(R)  17421  14993  86.06%
>>>   mixed-rand(W)  17391  14968  86.07%
>>
>> which parts of the output (as well as units) are these values exactly?
>> I did not work with fio until now, so I think I might ask before 
>> misinterpreting my results.
>
> It is IOPS.
>
>>  
>>> My fio description file
>>>
>>> [global]
>>> bs=4k
>>> ioengine=sync
>>> size=100m
>>> numjobs=1
>>> group_reporting
>>> buffer_compress_percentage=30
>>> scramble_buffers=0
>>> filename=/dev/zram0
>>> loops=10
>>> fsync_on_close=1
>>>
>>> [seq-write]
>>> bs=64k
>>> rw=write
>>> stonewall
>>>
>>> [rand-write]
>>> rw=randwrite
>>> stonewall
>>>
>>> [seq-read]
>>> bs=64k
>>> rw=read
>>> stonewall
>>>
>>> [rand-read]
>>> rw=randread
>>> stonewall
>>>
>>> [mixed-seq]
>>> bs=64k
>>> rw=rw
>>> stonewall
>>>
>>> [mixed-rand]
>>> rw=randrw
>>> stonewal

Re: [PATCH v7 0/5] Update LZ4 compressor module

2017-02-09 Thread Sven Schmidt

Hey Eric,

On Wed, Feb 08, 2017 at 09:24:25PM -0800, Eric Biggers wrote:
> Also I noticed another bug, this time in LZ4_count():
> 
> > #if defined(CONFIG_64BIT)
> > #define LZ4_ARCH64 1
> > #else
> > #define LZ4_ARCH64 0
> > #endif
> ...
> > #ifdef LZ4_ARCH64
> >if ((pIn < (pInLimit-3))
> >&& (LZ4_read32(pMatch) == LZ4_read32(pIn))) {
> >pIn += 4; pMatch += 4;
> >}
> > #endif
> 
> Because of how LZ4_ARCH64 is defined, it needs to be '#if LZ4_ARCH64'.
> 
> But I also think the way upstream LZ4 does 64-bit detection could have just 
> been
> left as-is; it has a function which gets inlined:
> 
>   static unsigned LZ4_64bits(void) { return sizeof(void*)==8; }
> 
> Eric

does this apply for LZ4_isLittleEndian() as well? As a reminder:

static unsigned LZ4_isLittleEndian(void)
{
/* don't use static : performance detrimental */
const union { U32 u; BYTE c[4]; } one = { 1 };

return one.c[0];
}

It is surely easier to read and understand using these functions in favor of 
the macros.

Thanks,

Sven

Re: [PATCH v7 0/5] Update LZ4 compressor module

2017-02-09 Thread Sven Schmidt

Hey Eric,

On Wed, Feb 08, 2017 at 04:24:36PM -0800, Eric Biggers wrote:
> On Thu, Feb 09, 2017 at 08:31:21AM +0900, Minchan Kim wrote:
> > 
> > Today, I did zram-lz4 performance test with fio in current mmotm and
> > found it makes regression about 20%.
> > 
> 
> This may or may not be the cause of the specific regression you're observing,
> but I just noticed that the proposed patch drops a lot of FORCEINLINE
> annotations from upstream LZ4.  The FORCEINLINE's are there for a reason,
> especially for the main decompression and compression functions which are
> basically "templates" that take in different sets of constant parameters, and
> should be left in.  We should #define FORCEINLINE to __always_inline 
> somewhere,
> or just do a s/FORCEINLINE/__always_inline/g.
>

I generally just replaced "FORCE_INLINE" by "static inline". At least I thought 
so.
I rechecked and realised, I missed at least two of them (why did I not just use 
"search+replace"?).
So I think it's maybe safer and easier to eventually just use "FORCE_INLINE"
with the definition you suggested. Will try that.

> Note that the upstream LZ4 code is very carefully optimized, so we should not,
> in general, be changing things like when functions are force-inlined, what the
> hash table size is, etc.
> 
> [Also, for some reason linux-crypto is apparently still not receiving patch 
> 1/5
> in the series.  It's missing from the linux-crypto archive at
> http://www.spinics.net/lists/linux-crypto/, so it's not just me.]
> 

I don't really know what to do about this. I think the matter is the size of 
the E-Mail.
Are there filters or something like that? Since in linux-kernel the patch seems 
to get delivered.
I could otherwise CC you if you wish.

Thanks,

Sven

Re: [PATCH v7 0/5] Update LZ4 compressor module

2017-02-09 Thread Sven Schmidt

Hey Minchan,

On Thu, Feb 09, 2017 at 08:31:21AM +0900, Minchan Kim wrote:
> Hello Sven,
> 
> On Sun, Feb 05, 2017 at 08:09:03PM +0100, Sven Schmidt wrote:
> > 
> > This patchset is for updating the LZ4 compression module to a version based
> > on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
> > which provides an "acceleration" parameter as a tradeoff between
> > high compression ratio and high compression speed.
> > 
> > We want to use LZ4 fast in order to support compression in lustre
> > and (mostly, based on that) investigate data reduction techniques in behalf 
> > of
> > storage systems.
> > 
> > Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
> > it is possible to enable applications to use fast and/or high compression
> > depending on the usecase.
> > For instance, ZRAM is offering a LZ4 backend and could benefit from an 
> > updated
> > LZ4 in the kernel.
> > 
> > LZ4 homepage: http://www.lz4.org/
> > LZ4 source repository: https://github.com/lz4/lz4
> > Source version: 1.7.3
> > 
> > Benchmark (taken from [1], Core i5-4300U @1.9GHz):
> > |--||--
> > Compressor  | Compression  | Decompression  | Ratio
> > |--||--
> > memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
> > LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
> > LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
> > LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
> > LZ4 default |   385 MB/s   |  1850 MB/s | 2.101
> > 
> > [1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
> > 
> > [PATCH 1/5] lib: Update LZ4 compressor module
> > [PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module 
> > version
> > [PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
> > [PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new 
> > LZ4 version
> > [PATCH 5/5] lib/lz4: Remove back-compat wrappers
> 
> Today, I did zram-lz4 performance test with fio in current mmotm and
> found it makes regression about 20%.
> 
> "lz4-update" means current mmots(git://git.cmpxchg.org/linux-mmots.git) so
> applied your 5 patches. (But now sure current mmots has recent uptodate
> patches)
> "revert" means I reverted your 5 patches in current mmots.
> 
>  revertlz4-update
> 
>   seq-write   1547   1339  86.55%
>  rand-write  22775  19381  85.10%
>seq-read   7035   5589  79.45%
>   rand-read  78556  68479  87.17%
>mixed-seq(R)   1305   1066  81.69%
>mixed-seq(W)   1205984  81.66%
>   mixed-rand(R)  17421  14993  86.06%
>   mixed-rand(W)  17391  14968  86.07%

which parts of the output (as well as units) are these values exactly?
I did not work with fio until now, so I think I might ask before 
misinterpreting my results.
 
> My fio description file
> 
> [global]
> bs=4k
> ioengine=sync
> size=100m
> numjobs=1
> group_reporting
> buffer_compress_percentage=30
> scramble_buffers=0
> filename=/dev/zram0
> loops=10
> fsync_on_close=1
> 
> [seq-write]
> bs=64k
> rw=write
> stonewall
> 
> [rand-write]
> rw=randwrite
> stonewall
> 
> [seq-read]
> bs=64k
> rw=read
> stonewall
> 
> [rand-read]
> rw=randread
> stonewall
> 
> [mixed-seq]
> bs=64k
> rw=rw
> stonewall
> 
> [mixed-rand]
> rw=randrw
> stonewall
> 

Great, this makes it easy for me to reproduce your test.

Thanks,

Sven

[PATCH v7 5/5] lib/lz4: Remove back-compat wrappers

2017-02-05 Thread Sven Schmidt

This patch removes the functions introduced as wrappers for providing
backwards compatibility to the prior LZ4 version.
They're not needed anymore since there's no callers left.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  | 69 
 lib/lz4/lz4_compress.c   | 22 ---
 lib/lz4/lz4_decompress.c | 42 -
 lib/lz4/lz4hc_compress.c | 23 
 4 files changed, 156 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 1b7ab2a..a3912d7 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -173,18 +173,6 @@ static inline int LZ4_compressBound(size_t isize)
 }
 
 /**
- * lz4_compressbound() - For backwards compatibility; see LZ4_compressBound
- * @isize: Size of the input data
- *
- * Return: Max. size LZ4 may output in a "worst case" szenario
- * (data not compressible)
- */
-static inline int lz4_compressbound(size_t isize)
-{
-   return LZ4_COMPRESSBOUND(isize);
-}
-
-/**
  * LZ4_compress_default() - Compress data from source to dest
  * @source: source address of the original data
  * @dest: output buffer address of the compressed data
@@ -257,20 +245,6 @@ int LZ4_compress_fast(const char *source, char *dest, int 
inputSize,
 int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
int targetDestSize, void *wrkmem);
 
-/*
- * lz4_compress() - For backward compatibility, see LZ4_compress_default
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND
- * @dst_len: is the output size, which is returned after compress done
- * @workmem: address of the working memory.
- *
- * Return: Success if return 0, Error if return < 0
- */
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
-   size_t *dst_len, void *wrkmem);
-
 /*-
  * Decompression Functions
  **/
@@ -346,34 +320,6 @@ int LZ4_decompress_safe(const char *source, char *dest, 
int compressedSize,
 int LZ4_decompress_safe_partial(const char *source, char *dest,
int compressedSize, int targetOutputSize, int maxDecompressedSize);
 
-/*
- * lz4_decompress_unknownoutputsize() - For backwards compatibility,
- * see LZ4_decompress_safe
- * @src: source address of the compressed data
- * @src_len: is the input size, therefore the compressed size
- * @dest: output buffer address of the decompressed data
- * which must be already allocated
- * @dest_len: is the max size of the destination buffer, which is
- * returned with actual size of decompressed data after decompress done
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
-   unsigned char *dest, size_t *dest_len);
-
-/**
- * lz4_decompress() - For backwards cocmpatibility, see LZ4_decompress_fast
- * @src: source address of the compressed data
- * @src_len: is the input size, which is returned after decompress done
- * @dest: output buffer address of the decompressed data,
- * which must be already allocated
- * @actual_dest_len: is the size of uncompressed data, supposing it's known
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress(const unsigned char *src, size_t *src_len,
-   unsigned char *dest, size_t actual_dest_len);
-
 /*-
  * LZ4 HC Compression
  **/
@@ -401,21 +347,6 @@ int LZ4_compress_HC(const char *src, char *dst, int 
srcSize, int dstCapacity,
int compressionLevel, void *wrkmem);
 
 /**
- * lz4hc_compress() - For backwards compatibility, see LZ4_compress_HC
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND.
- * @dst_len: is the output size, which is returned after compress done
- * @wrkmem: address of the working memory.
- * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- *
- * Return  : Success if return 0, Error if return (< 0)
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char 
*dst,
-   size_t *dst_len, void *wrkmem);
-
-/**
  * LZ4_resetStreamHC() - Init an allocated 'LZ4_streamHC_t' structure
  * @streamHCPtr: pointer to the 'LZ4_streamHC_t' structure
  * @compressionLevel: Recommended values are between 4 and 9, although any
diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c
index 6aa7ac3..697dbda 100644
--- a/lib/lz4/lz4_compress.c

[PATCH v7 3/5] crypto: Change LZ4 modules to work with new LZ4 module version

2017-02-05 Thread Sven Schmidt

This patch updates the crypto modules using LZ4 compression as well as the
test cases in testmgr.h to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c |  23 -
 crypto/lz4hc.c   |  23 -
 crypto/testmgr.h | 142 +++
 3 files changed, 120 insertions(+), 68 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..71eff9b 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, *dlen, ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
-   return -EINVAL;
+   if (out_len < 0)
+   return out_len;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return 0;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..03a34a8 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   *dlen, LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
-   return -EINVAL;
+   if (out_len < 0)
+   return out_len;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return 0;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9b656be..98d4be0 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -34498,31 +34498,62 @@ static struct hash_testvec bfin_crc_tv_template[] = {
 
 static struct comp_testvec lz4_comp_tv_template[] = {
{
-   .inlen  = 70,
-   .outlen = 45,
-   .input  = "Join us now and share the software "
- "Join us now and share the software ",
-   .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
+   .inlen  = 255,
+   .outlen = 218,
+   .input  = "LZ4 is lossless compression algorithm, providing"
+" compression speed at 400 MB/s per core, scalable "
+"with multi-cores CPU. It features an extremely fast "
+"decoder, with speed in multiple GB/s per core, "
+"typically reaching RAM speed limits on multi-core "
+"systems.",
+   .output = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+

[PATCH v7 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-02-05 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

[PATCH v7 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-02-05 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 22 +-
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..efab7b6 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,35 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
-   pr_err("lz4_compress error, ret = %d!\n", ret);
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
+   pr_err("LZ4_compress_default error; compression failed!\n");
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
-   pr_err("lz4_decompress error, ret = %d!\n", ret);
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
+   /*
+* LZ4_decompress_safe will return an error code
+* (< 0) if decompression failed
+*/
+   pr_err("LZ4_decompress_safe error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

[PATCH v7 0/5] Update LZ4 compressor module

2017-02-05 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html

[PATCH 1/5] lib: Update LZ4 compressor module
[PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module 
version
[PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 
version
[PATCH 5/5] lib/lz4: Remove back-compat wrappers

Changes:
v7:
- Fixed errors reported by the Smatch tool
- Changed function documentation comments in lz4.h to match kernel-doc style
- Fixed a misbehaviour of LZ4HC caused by the wrong level of indentation
  concerning two for loops introduced after I refactored the code style using
  checkpatch.pl (upstream LZ4 put dozens of stuff in just one line, gnah)
- Updated the crypto tests for LZ4 since they did fail for the new code
  and hence zram did fail to allocate memory for LZ4

v6:
- Fixed LZ4_NBCOMMONBYTES() for 64-bit little endian
- Reset LZ4_MEMORY_USAGE to 14 (which is the value used in
  upstream LZ4 as well as the previous kernel module)
- Fixed that weird double-indentation in lz4defs.h and lz4.h
- Adjusted general styling issues in lz4defs.h
  (e.g. lines consisting of more than one instruction)
- Removed the architecture-dependent typedef to reg_t
  since upstream LZ4 is just using size_t and that works fine
- Changed error messages in pstore/platform.c:
  * LZ4_compress_default always returns 0 in case of an error
(no need to print the return value)
  * LZ4_decompress_safe returns a negative error message
(return value _does_ matter)

v5:
- Added a fifth patch to remove the back-compat wrappers introduced
  to ensure bisectibility between the patches (the functions are no longer
  needed since there's no callers left)

v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
  to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
  and triggers an unused function warning when false

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
  are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
  different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

Re: [PATCH v6 1/5] lib: Update LZ4 compressor module

2017-02-01 Thread Sven Schmidt

On Tue, Jan 31, 2017 at 03:27:44PM -0700, Jonathan Corbet wrote:
> On Fri, 27 Jan 2017 23:02:00 +0100
> Sven Schmidt <4ssch...@informatik.uni-hamburg.de> wrote:
> 
> I have one quick question...
> 
> >  /*
> > + * LZ4_compress_default()
> > + * Compresses 'sourceSize' bytes from buffer 'source'
> > + * into already allocated 'dest' buffer of size 'maxOutputSize'.
> > + * Compression is guaranteed to succeed if
> > + * 'maxOutputSize' >= LZ4_compressBound(inputSize).
> > + * It also runs faster, so it's a recommended setting.
> > + * If the function cannot compress 'source'
> > + * into a more limited 'dest' budget,
> > + * compression stops *immediately*,
> > + * and the function result is zero.
> > + * As a consequence, 'dest' content is not valid.
> > + *
> > + * source   : source address of the original data
> > + * dest : output buffer address
> > + * of the compressed data
> > + * inputSize: Max supported value is
> > + * LZ4_MAX_INPUT_SIZE
> > + * maxOutputSize: full or partial size of buffer 'dest'
> > + * (which must be already allocated)
> > + * workmem  : address of the working memory.
> > + * This requires 'workmem' of size LZ4_MEM_COMPRESS.
> > + * return   : the number of bytes written into buffer 'dest'
> > + *   (necessarily <= maxOutputSize) or 0 if compression fails
> > + */
> > +int LZ4_compress_default(const char *source, char *dest, int inputSize,
> > +   int maxOutputSize, void *wrkmem);
> 
> Is there any chance you could format these as kerneldoc comments?  You're
> not too far from it now, and that would allow the LZ4 interface to be
> pulled into the documentation.
> 
> Thanks,
> 
> jon

Hi Jon,

of course, that makes sense. I already checked the documentation and you're 
right, I'm not that far from it.
Will do the necessary changes.

Thanks,

Sven

[PATCH v6 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-01-27 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 22 +-
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..efab7b6 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,35 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
-   pr_err("lz4_compress error, ret = %d!\n", ret);
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
+   pr_err("LZ4_compress_default error; compression failed!\n");
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
-   pr_err("lz4_decompress error, ret = %d!\n", ret);
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
+   /*
+* LZ4_decompress_safe will return an error code
+* (< 0) if decompression failed
+*/
+   pr_err("LZ4_decompress_safe error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v6 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-01-27 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v6 3/5] crypto: Change LZ4 modules to work with new LZ4 module version

2017-01-27 Thread Sven Schmidt

This patch updates the crypto modules using LZ4 compression
to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c   | 21 -
 crypto/lz4hc.c | 21 -
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..40fd2c2 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, (int)((size_t)dlen), ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..6f16f96 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   (int)((size_t)dlen), LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (out_len == 0)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v6 5/5] lib/lz4: Remove back-compat wrappers

2017-01-27 Thread Sven Schmidt

This patch removes the functions introduced as wrappers for providing
backwards compatibility to the prior LZ4 version.
They're not needed anymore since there's no callers left.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  | 73 
 lib/lz4/lz4_compress.c   | 22 ---
 lib/lz4/lz4_decompress.c | 42 
 lib/lz4/lz4hc_compress.c | 23 ---
 4 files changed, 160 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index ed59cb9..2844e6f 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -173,14 +173,6 @@ static inline int LZ4_compressBound(size_t isize)
 }
 
 /*
- * For backward compatibility
- */
-static inline int lz4_compressbound(size_t isize)
-{
-   return LZ4_COMPRESSBOUND(isize);
-}
-
-/*
  * LZ4_compress_default()
  * Compresses 'sourceSize' bytes from buffer 'source'
  * into already allocated 'dest' buffer of size 'maxOutputSize'.
@@ -249,23 +241,6 @@ int LZ4_compress_fast(const char *source, char *dest, int 
inputSize,
 int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
int targetDestSize, void *wrkmem);
 
-/*
- * lz4_compress()
- * src: source address of the original data
- * src_len: size of the original data
- * dst: output buffer address of the compressed data
- *   This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len: is the output size, which is returned after compress done
- * workmem: address of the working memory.
- *   This requires 'workmem' of size LZ4_MEM_COMPRESS.
- * return  : Success if return 0
- *Error if return (< 0)
- * note :  Destination buffer and workmem must be already allocated with
- * the defined size.
- */
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
-   size_t *dst_len, void *wrkmem);
-
 /*-
  * Decompression Functions
  **/
@@ -340,37 +315,6 @@ int LZ4_decompress_safe(const char *source, char *dest, 
int compressedSize,
 int LZ4_decompress_safe_partial(const char *source, char *dest,
int compressedSize, int targetOutputSize, int maxDecompressedSize);
 
-
-/*
- * lz4_decompress_unknownoutputsize() :
- * src : source address of the compressed data
- * src_len : is the input size, therefore the compressed size
- * dest: output buffer address of the decompressed data
- * dest_len: is the max size of the destination buffer, which is
- *returned with actual size of decompressed data after
- *decompress done
- * return: Success if return 0
- * Error if return (< 0)
- * note:   Destination buffer must be already allocated.
- */
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
-   unsigned char *dest, size_t *dest_len);
-
-/*
- * lz4_decompress() :
- * src: source address of the compressed data
- * src_len: is the input size,
- *   which is returned after decompress done
- * dest   : output buffer address of the decompressed data
- * actual_dest_len: is the size of uncompressed data, supposing it's known
- * return: Success if return 0
- *  Error if return (< 0)
- * note  : Destination buffer must be already allocated.
- *  slightly faster than lz4_decompress_unknownoutputsize()
- */
-int lz4_decompress(const unsigned char *src, size_t *src_len,
-   unsigned char *dest, size_t actual_dest_len);
-
 /*-
  * LZ4 HC Compression
  **/
@@ -399,23 +343,6 @@ int LZ4_compress_HC(const char *src, char *dst, int 
srcSize, int dstCapacity,
int compressionLevel, void *wrkmem);
 
 /*
- * lz4hc_compress()
- * src: source address of the original data
- * src_len: size of the original data
- * dst: output buffer address of the compressed data
- *   This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len: is the output size, which is returned after compress done
- * workmem: address of the working memory.
- *   This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- * return  : Success if return 0
- *  Error if return (< 0)
- * note   : Destination buffer and workmem must be already allocated with
- *  the defined size.
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char 
*dst,
-   size_t *dst_len, void *wrkmem);
-
-/*
  * These functions compress data in successive blocks of any size,
  * using previous blocks as dictionary. One key assumption is that previ

[PATCH v6 0/5] Update LZ4 compressor module

2017-01-27 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

[PATCH 1/5] lib: Update LZ4 compressor module
[PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module 
version
[PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 
version
[PATCH 5/5] lib/lz4: Remove back-compat wrappers

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
  are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
  different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions

v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
  to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
  and triggers an unused function warning when false

v5:
- Added a fifth patch to remove the back-compat wrappers introduced
  to ensure bisectibility between the patches (the functions are no longer
  needed since there's no callers left)

v6:
- Fixed LZ4_NBCOMMONBYTES() for 64-bit little endian
- Reset LZ4_MEMORY_USAGE to 14 (which is the value used in
  upstream LZ4 as well as the previous kernel module)
- Fixed that weird double-indentation in lz4defs.h and lz4.h
- Adjusted general styling issues in lz4defs.h
  (e.g. lines consisting of more than one instruction)
- Removed the architecture-dependent typedef to reg_t
  since upstream LZ4 is just using size_t and that works fine
- Changed error messages in pstore/platform.c:
  * LZ4_compress_default always returns 0 in case of an error
(no need to print the return value)
  * LZ4_decompress_safe returns a negative error message
(return value _does_ matter)

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v5 0/5] Update LZ4 compressor module

2017-01-26 Thread Sven Schmidt

On Thu, Jan 26, 2017 at 01:19:53AM -0800, Eric Biggers wrote:
> On Thu, Jan 26, 2017 at 08:57:30AM +0100, Sven Schmidt wrote:
> > 
> > This patchset is for updating the LZ4 compression module to a version based
> > on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
> > which provides an "acceleration" parameter as a tradeoff between
> > high compression ratio and high compression speed.
> > 
> > We want to use LZ4 fast in order to support compression in lustre
> > and (mostly, based on that) investigate data reduction techniques in behalf 
> > of
> > storage systems.
> > 
> > Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
> > it is possible to enable applications to use fast and/or high compression
> > depending on the usecase.
> > For instance, ZRAM is offering a LZ4 backend and could benefit from an 
> > updated
> > LZ4 in the kernel.
> > 
>

Hey Eric,
 
> Hi Sven,
> 
> [For some reason I didn't receive patch 1/5 and had to get it from 
> patchwork...
> I'm not sure why.  I'm subscribed to linux-crypto but not linux-kernel.]

that's weird. I just experienced the first patch takes a little longer to get 
delivered because of its size.
Please let me know if the problem occurs again.
 
> The proposed patch defines LZ4_MEMORY_USAGE to 10 which means that LZ4
> compression will use a hash table of only 1024 bytes, containing only 256
> entries, to find matches.  This differs from upstream LZ4 1.7.3, which uses
> LZ4_MEMORY_USAGE of 14, as well as the previous LZ4 included in the Linux
> kernel, both of which specify the hash table size to be 16384 bytes, 
> containing
> 4096 entries.
> 
> Given that varying the hash table size is a trade-off between memory usage,
> speed, and compression ratio, is this an intentional difference and has it 
> been
> benchmarked?
> 

I believe I had some troubles with LZ4_MEMORY_USAGE of 14. But I may be wrong.
I will test that again and eventually adapt that value.

> Also, in lz4defs.h:
> 
> > #if defined(__x86_64__)
> >typedef U64 reg_t;   /* 64-bits in x32 mode */
> > #else
> >typedef size_t reg_t;/* 32-bits in x32 mode */
> > #endif
> 
> Are you sure this really needed over just always using size_t?
>

No, actually there's just one use of that value and the upstream version uses 
size_t instead of reg_t 
in that particular place. So I will replace it with size_t.
 
> > #if LZ4_ARCH64
> > #ifdef __BIG_ENDIAN__
> > #define LZ4_NBCOMMONBYTES(val) (__builtin_clzll(val) >> 3)
> > #else
> > #define LZ4_NBCOMMONBYTES(val) (__builtin_clzll(val) >> 3)
> > #endif
> > #else
> > #ifdef __BIG_ENDIAN__
> > #define LZ4_NBCOMMONBYTES(val) (__builtin_clz(val) >> 3)
> > #else
> > #define LZ4_NBCOMMONBYTES(val) (__builtin_ctz(val) >> 3)
> > #endif
> > #endif
> 
> LZ4_NBCOMMONBYTES() is defined incorrectly for 64-bit little endian; it should
> be using __builtin_ctzll().
> 

Indeed! Using the same values in if and else does not make sense at all.
Thank you for pointing that one out. I will fix it.

> Nit: can you also clean up the weird indentation (e.g. double tabs) in
> lz4defs.h?
> 
> Thanks,
> 
> Eric
> 

I'm wondering why checkpatch does not point out this kind of styling problem?
I did fix that in the other files but I think I missed lz4defs.h. Will fix the 
indentation.

Thanks,

Sven
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v5 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-01-25 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v5 5/5] lib/lz4: Remove back-compat wrappers

2017-01-25 Thread Sven Schmidt

This patch removes the functions introduced as wrappers for providing
backwards compatibility to the prior LZ4 version.
They're not needed anymore since there's no callers left.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 include/linux/lz4.h  | 73 
 lib/lz4/lz4_compress.c   | 22 ---
 lib/lz4/lz4_decompress.c | 42 
 lib/lz4/lz4hc_compress.c | 23 ---
 4 files changed, 160 deletions(-)

diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 2072255..5958f7d 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -173,14 +173,6 @@ static inline int LZ4_compressBound(size_t isize)
 }
 
 /*
- * For backward compatibility
- */
-static inline int lz4_compressbound(size_t isize)
-{
-   return LZ4_COMPRESSBOUND(isize);
-}
-
-/*
  * LZ4_compress_default()
  * Compresses 'sourceSize' bytes from buffer 'source'
  * into already allocated 'dest' buffer of size 'maxOutputSize'.
@@ -249,23 +241,6 @@ int LZ4_compress_fast(const char *source, char *dest, int 
inputSize,
 int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
int targetDestSize, void *wrkmem);
 
-/*
- * lz4_compress()
- * src: source address of the original data
- * src_len: size of the original data
- * dst: output buffer address of the compressed data
- *   This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len: is the output size, which is returned after compress done
- * workmem: address of the working memory.
- *   This requires 'workmem' of size LZ4_MEM_COMPRESS.
- * return  : Success if return 0
- *Error if return (< 0)
- * note :  Destination buffer and workmem must be already allocated with
- * the defined size.
- */
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
-   size_t *dst_len, void *wrkmem);
-
 /*-
  * Decompression Functions
  **/
@@ -340,37 +315,6 @@ int LZ4_decompress_safe(const char *source, char *dest, 
int compressedSize,
 int LZ4_decompress_safe_partial(const char *source, char *dest,
int compressedSize, int targetOutputSize, int maxDecompressedSize);
 
-
-/*
- * lz4_decompress_unknownoutputsize() :
- * src : source address of the compressed data
- * src_len : is the input size, therefore the compressed size
- * dest: output buffer address of the decompressed data
- * dest_len: is the max size of the destination buffer, which is
- *returned with actual size of decompressed data after
- *decompress done
- * return: Success if return 0
- * Error if return (< 0)
- * note:   Destination buffer must be already allocated.
- */
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
-   unsigned char *dest, size_t *dest_len);
-
-/*
- * lz4_decompress() :
- * src: source address of the compressed data
- * src_len: is the input size,
- *   which is returned after decompress done
- * dest   : output buffer address of the decompressed data
- * actual_dest_len: is the size of uncompressed data, supposing it's known
- * return: Success if return 0
- *  Error if return (< 0)
- * note  : Destination buffer must be already allocated.
- *  slightly faster than lz4_decompress_unknownoutputsize()
- */
-int lz4_decompress(const unsigned char *src, size_t *src_len,
-   unsigned char *dest, size_t actual_dest_len);
-
 /*-
  * LZ4 HC Compression
  **/
@@ -399,23 +343,6 @@ int LZ4_compress_HC(const char *src, char *dst, int 
srcSize, int dstCapacity,
int compressionLevel, void *wrkmem);
 
 /*
- * lz4hc_compress()
- * src: source address of the original data
- * src_len: size of the original data
- * dst: output buffer address of the compressed data
- *   This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len: is the output size, which is returned after compress done
- * workmem: address of the working memory.
- *   This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- * return  : Success if return 0
- *  Error if return (< 0)
- * note   : Destination buffer and workmem must be already allocated with
- *  the defined size.
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char 
*dst,
-   size_t *dst_len, void *wrkmem);
-
-/*
  * These functions compress data in successive blocks of any size,
  * using previous blocks as dictionary. One key assumption is that previ

[PATCH v5 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-01-25 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 14 +++---
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..85c4c58 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,31 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
pr_err("lz4_compress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
pr_err("lz4_decompress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v5 3/5] crypto: Change LZ4 modules to work with new LZ4 module version

2017-01-25 Thread Sven Schmidt

This patch updates the crypto modules using LZ4 compression
to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c   | 21 -
 crypto/lz4hc.c | 21 -
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..40fd2c2 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, (int)((size_t)dlen), ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..6f16f96 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   (int)((size_t)dlen), LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (out_len == 0)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v5 0/5] Update LZ4 compressor module

2017-01-25 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

[PATCH 1/5] lib: Update LZ4 compressor module
[PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module 
version
[PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 
version
[PATCH 5/5] lib/lz4: Remove back-compat wrappers

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
  are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
  different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions

v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
  to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
  and triggers an unused function warning when false

v5:
- Added a fifth patch to remove the back-compat wrappers introduced
  to ensure bisectibility between the patches (the functions are no longer
  needed since there's no callers left)

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 1/4] lib: Update LZ4 compressor module

2017-01-24 Thread Sven Schmidt

On Mon, Jan 23, 2017 at 04:23:57PM -0800, Andrew Morton wrote:
> On Sun, 22 Jan 2017 20:35:14 +0100 Sven Schmidt 
> <4ssch...@informatik.uni-hamburg.de> wrote:
> 
> > This patch updates LZ4 kernel module to LZ4 v1.7.3 by Yann Collet.
> > The kernel module is inspired by the previous work by Chanho Min.
> > The updated LZ4 module will not break existing code since there were alias
> > methods added to ensure backwards compatibility.
> > 
> > API changes:
> > 
> > New method LZ4_compress_fast which differs from the variant available
> > in kernel by the new acceleration parameter,
> > allowing to trade compression ratio for more compression speed
> > and vice versa.
> > 
> > LZ4_decompress_fast is the respective decompression method, featuring a very
> > fast decoder (multiple GB/s per core), able to reach RAM speed in multi-core
> > systems. The decompressor allows to decompress data compressed with
> > LZ4 fast as well as the LZ4 HC (high compression) algorithm.
> > 
> > Also the useful functions LZ4_decompress_safe_partial
> > LZ4_compress_destsize were added. The latter reverses the logic by trying to
> > compress as much data as possible from source to dest while the former aims
> > to decompress partial blocks of data.
> > 
> > A bunch of streaming functions were also added
> > which allow compressig/decompressing data in multiple steps
> > (so called "streaming mode").
> > 
> > The methods lz4_compress and lz4_decompress_unknownoutputsize
> > are now known as LZ4_compress_default respectivley LZ4_decompress_safe.
> > The old methods are still available for providing backwards compatibility.
> > 
> > ...
> >
> > +/*
> > + * For backward compatibility
> > + */
> > +static inline int lz4_compressbound(size_t isize)
> >  {
> 
> Do we actually need the back-compat wrappers?  After your other three
> patches we have no callers, correct?  If so, we should go ahead and
> eliminate such back-compatibility interfaces.
> 
> 
> > -   return isize + (isize / 255) + 16;
> > +   return LZ4_COMPRESSBOUND(isize);
> >  }
> 

Hey Andrew,

you're absolutely right. Since I changed the callers to use the new functions, 
we do not need these wrappers anymore.

I will add a patch removing them and send an update.

Thanks,

Sven
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 4/4] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-01-22 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 14 +++---
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..85c4c58 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,31 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
pr_err("lz4_compress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
pr_err("lz4_decompress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 3/4] crypto: Change LZ4 modules to work with new LZ4 module version

2017-01-22 Thread Sven Schmidt

This patch updates the crypto modules using LZ4 compression
to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c   | 21 -
 crypto/lz4hc.c | 21 -
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..40fd2c2 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, (int)((size_t)dlen), ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..6f16f96 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   (int)((size_t)dlen), LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (out_len == 0)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 0/4] Update LZ4 compressor module

2017-01-22 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

[PATCH 1/4] lib: Update LZ4 compressor module
[PATCH 2/4] lib/decompress_unlz4: Change module to work with new LZ4 module 
version
[PATCH 3/4] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/4] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 
version

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
  are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
  different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions

v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
  to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
  and triggers an unused function warning when false

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 2/4] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-01-22 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 3/4] crypto: Change LZ4 modules to work with new LZ4 module version

2017-01-21 Thread Sven Schmidt

This patch updates the crypto modules using LZ4 compression
to work with the new LZ4 module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 crypto/lz4.c   | 21 -
 crypto/lz4hc.c | 21 -
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..40fd2c2 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
 static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_default(src, dst,
+   slen, (int)((size_t)dlen), ctx);
 
-   err = lz4_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (!out_len)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..6f16f96 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
 static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
   u8 *dst, unsigned int *dlen, void *ctx)
 {
-   size_t tmp_len = *dlen;
-   int err;
+   int out_len = LZ4_compress_HC(src, dst, slen,
+   (int)((size_t)dlen), LZ4HC_DEFAULT_CLEVEL, ctx);
 
-   err = lz4hc_compress(src, slen, dst, _len, ctx);
-
-   if (err < 0)
+   if (out_len == 0)
return -EINVAL;
 
-   *dlen = tmp_len;
+   *dlen = out_len;
return 0;
 }
 
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, 
const u8 *src,
 static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
 u8 *dst, unsigned int *dlen, void *ctx)
 {
-   int err;
-   size_t tmp_len = *dlen;
-   size_t __slen = slen;
+   int out_len = LZ4_decompress_safe(src, dst, slen, (int)((size_t)dlen));
 
-   err = lz4_decompress_unknownoutputsize(src, __slen, dst, _len);
-   if (err < 0)
+   if (out_len < 0)
return -EINVAL;
 
-   *dlen = tmp_len;
-   return err;
+   *dlen = out_len;
+   return out_len;
 }
 
 static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 4/4] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version

2017-01-21 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 14 +++---
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..85c4c58 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,31 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
-   if (ret) {
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+   if (!ret) {
pr_err("lz4_compress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
pr_err("lz4_decompress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output,
+   length, output->length);
+
+   if (res < 0)
return -EIO;
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/4] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-01-21 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v3 0/4] Update LZ4 compressor module

2017-01-21 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html

[PATCHv2 1/4] lib: Update LZ4  compressor module based on LZ4 v1.7.2
[PATCHv2 2/4] lib: Update decompress_unlz4 wrapper to work with new LZ4 module
[PATCHv2 3/4] crypto: Change lz4 modules to work with new LZ4 module
[PATCHv2 4/4] fs/pstore: fs/squashfs: Change LZ4 compressor functions to work 
with new LZ4 module

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h

v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
different return type
- Corrected version number (was LZ4 1.7.3)
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 4/4] fs/pstore: fs/squashfs: Change usage of LZ4 to comply with new LZ4 module version

2017-01-10 Thread Sven Schmidt

Hi Kees,

On 01/07/2017 10:33 PM, Kees Cook wrote:
 >On Sat, Jan 7, 2017 at 8:55 AM, Sven Schmidt
 ><4ssch...@informatik.uni-hamburg.de> wrote:
 >> This patch updates fs/pstore and fs/squashfs to use the updated functions 
 >> from
 >> the new LZ4 module.
 >>
 >> Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
 >> ---
 >>  fs/pstore/platform.c  | 14 --
 >>  fs/squashfs/lz4_wrapper.c | 12 ++--
 >>  2 files changed, 14 insertions(+), 12 deletions(-)
 >>
 >> diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
 >> index 729677e..a0d8ca8 100644
 >> --- a/fs/pstore/platform.c
 >> +++ b/fs/pstore/platform.c
 >> @@ -342,31 +342,33 @@ static int compress_lz4(const void *in, void *out, 
 >> size_t inlen, size_t outlen)
 >>  {
 >> int ret;
 >>
 >> -   ret = lz4_compress(in, inlen, out, , workspace);
 >> +   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
 >> if (ret) {
 >> +   // ret is 0 means an error occured
 >
 >If that's true, then shouldn't the "if" logic be changed? Also, here
 >and in all following comments are C++ style instead of kernel C-style.
 >This should be "/* ret == 0 means an error occured */", though really,
 >that should be obvious from the code and the comment isn't really
 >needed.

indeed, it should. I fixed that one.

 >> pr_err("lz4_compress error, ret = %d!\n", ret);
 >If it's always going to be zero here, is there a better place to get
 >details on why it failed?

 It is always going to be zero. Honestly, after looking at the current LZ4 in 
kernel again
 I don't get why there actually was a need to print the return value since 
lz4_compress
 will (as far as I see) always return -1 in case of an error while the new 
lz4_compress_fast/default
 will return 0 in such case. Maybe I should just stick with the error?

 Thanks,

 Sven
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 1/4] lib: Update LZ4 compressor module based on LZ4 v1.7.2.

2017-01-10 Thread Sven Schmidt

On 01/08/2017 12:23 PM, Greg KH wrote:
> On Sat, Jan 07, 2017 at 05:55:43PM +0100, Sven Schmidt wrote:
>> This patch updates the unlz4 wrapper to work with the new LZ4 kernel module 
>> version.
>>
>> Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
>> ---
>>  lib/decompress_unlz4.c | 13 -
>>  1 file changed, 8 insertions(+), 5 deletions(-)
>>
>> diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
>> index 036fc88..1b0baf3 100644
>> --- a/lib/decompress_unlz4.c
>> +++ b/lib/decompress_unlz4.c
>> @@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
>>  error("NULL input pointer and missing fill function");
>>  goto exit_1;
>>  } else {
>> -inp = large_malloc(lz4_compressbound(uncomp_chunksize));
>> +inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
>
> Having functions differ by different cases of the characters is ripe for
> abuse and confusion.  Please never do that, especially as these "new"
> functions you created don't follow the correct kernel coding style rules
> :(
>
> thanks,
>
> greg k-h

Hi Greg,

you're right about that. I think I put a little too much effort in keeping old 
function names here.
I will get rid of that.

I also want to quote your previous E-Mail here:

> And follow the proper kernel coding style rules, putting your patches
> through scripts/checkpatch.pl should help you out here.

Sorry, I didn't know about that particular script. I already put Patches 2, 3 
and 4 through checkpatch.pl;
patch 1 is a little more to rework but I will, of course, do that.

Thanks,

Sven
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 1/4] lib: Update LZ4 compressor module based on LZ4 v1.7.2.

2017-01-10 Thread Sven Schmidt

On 01/08/2017 12:25 PM, Greg KH wrote:
>On Sat, Jan 07, 2017 at 05:55:42PM +0100, Sven Schmidt wrote:
>> This patch updates LZ4 kernel module to LZ4 v1.7.2 by Yann Collet.
>> The kernel module is inspired by the previous work by Chanho Min.
>> The updated LZ4 module will not break existing code since there were alias
>> methods added to ensure backwards compatibility.
>
> Meta-comment.  Does this update include all of the security fixes that
> we have made over the past few years to the lz4 code?  I don't want to
> be adding back insecure functions that will cause us problems.
>
> Specifically look at the changes I made in 2014 in this directory for an
> example of what I am talking about here.
>

Hi Greg,

it doesn't. I didn't have that in mind until now.
But I had a look at the related commits after I received your E-Mail and will 
review what I have to change
(and, obviously, do the changes :)).

Thanks,

Sven
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 4/4] fs/pstore: fs/squashfs: Change usage of LZ4 to comply with new LZ4 module version

2017-01-07 Thread Sven Schmidt

This patch updates fs/pstore and fs/squashfs to use the updated functions from
the new LZ4 module.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 fs/pstore/platform.c  | 14 --
 fs/squashfs/lz4_wrapper.c | 12 ++--
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..a0d8ca8 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,33 @@ static int compress_lz4(const void *in, void *out, size_t 
inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_compress(in, inlen, out, , workspace);
+   ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
if (ret) {
+   // ret is 0 means an error occured
pr_err("lz4_compress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
 {
int ret;
 
-   ret = lz4_decompress_unknownoutputsize(in, inlen, out, );
-   if (ret) {
+   ret = LZ4_decompress_safe(in, out, inlen, outlen);
+   if (ret < 0) {
+   // return value is < 0 in case of error
pr_err("lz4_decompress error, ret = %d!\n", ret);
return -EIO;
}
 
-   return outlen;
+   return ret;
 }
 
 static void allocate_lz4(void)
 {
-   big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+   big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..a512399 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
-   size_t dest_len = output->length;
 
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
put_bh(bh[i]);
}
 
-   res = lz4_decompress_unknownoutputsize(stream->input, length,
-   stream->output, _len);
-   if (res)
+   res = LZ4_decompress_safe(stream->input, stream->output, length, 
output->length);
+   if (res < 0) {
+   // res of less than 0 means an error occured
return -EIO;
+   }
 
-   bytes = dest_len;
+   bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, 
void *strm,
}
squashfs_finish_page(output);
 
-   return dest_len;
+   return res;
 }
 
 const struct squashfs_decompressor squashfs_lz4_comp_ops = {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 0/4] Update LZ4 compressor module

2017-01-07 Thread Sven Schmidt


This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.2 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.

We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.

Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.

LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.2

Benchmark (taken from [1], Core i5-4300U @1.9GHz):
|--||--
Compressor  | Compression  | Decompression  | Ratio
|--||--
memcpy  |  4200 MB/s   |  4200 MB/s | 1.000
LZ4 fast 50 |  1080 MB/s   |  2650 MB/s | 1.375
LZ4 fast 17 |   680 MB/s   |  2220 MB/s | 1.607
LZ4 fast 5  |   475 MB/s   |  1920 MB/s | 1.886
LZ4 default |   385 MB/s   |  1850 MB/s | 2.101

[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html

[PATCHv2 1/4] lib: Update LZ4  compressor module based on LZ4 v1.7.2
[PATCHv2 2/4] lib: Update decompress_unlz4 wrapper to work with new LZ4 module
[PATCHv2 3/4] crypto: Change lz4 modules to work with new LZ4 module
[PATCHv2 4/4] fs/pstore: fs/squashfs: Change LZ4 compressor functions to work 
with new LZ4 module

v2:
- Changed order of the patches since in the initial patchset the lz4.h was in 
the
  last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 2/4] lib/decompress_unlz4: Change module to work with new LZ4 module version

2017-01-07 Thread Sven Schmidt

This patch updates the unlz4 wrapper to work with the new LZ4 kernel module 
version.

Signed-off-by: Sven Schmidt <4ssch...@informatik.uni-hamburg.de>
---
 lib/decompress_unlz4.c | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
-   inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+   inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
-   if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+   if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
-   ret = lz4_decompress(inp, , outp, dest_len);
+
+   ret = LZ4_decompress_fast(inp, outp, dest_len);
+   chunksize = ret;
 #else
dest_len = uncomp_chunksize;
-   ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
-   _len);
+
+   ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+   dest_len = ret;
 #endif
if (ret < 0) {
error("Decoding failed");
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

47 matches

Mail list logo