LGTM, so ack. Thanks, Tao Liu
On Fri, Mar 1, 2024 at 7:19 PM Tang Yulong <[email protected]> wrote: > > In Linux 5.1, the ZRAM block driver has changed its default compressor from > "lzo" to "lzo-rle" to enhance LZO compression support. However, crash does > not support the improved LZO algorithm, resulting in failure when reading > memory. > > change default compressor : ce82f19fd5809f0cf87ea9f753c5cc65ca0673d6 > > > The issue was discovered when using the extension 'gcore' to generate a > process coredump, which was found to be incomplete and unable to be opened > properly with gdb. > > > This patch is for Crash-utility tool, it enables the Crash-utility to support > decompression of the "lzo-rle" compression algorithm used in zram. The patch > has been tested with vmcore files from kernel version 5.4, and successfully > allows reading of memory compressed with the zram compression algorithm. > > > > Testing: > ======== > before apply this patch : > crash> gcore -v 0 1 > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > after: > crash> gcore -v 0 1 > Saved core.1.init > > Changelog: > ========== > v2: keep the "if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)" related > code of the copied kernel code, but change the "if defined" macro into a > runtime check . > > > Thanks and regards, > Yulong > > Patch: > ========== > > From 0e91de69e08c3d4087b84018bcbfe10be07b7f7b Mon Sep 17 00:00:00 2001 > From: "yulong.tang" <[email protected]> > Date: Tue, 20 Feb 2024 15:09:49 +0800 > Subject: [PATCH] Adding the zram decompression algorithm "lzo-rle" to support > kernel versions >= 5.1. > > Port the improved decompression method for "lzo" in the kernel to support > decompression of "lzorle". After the 5.1 version of the kernel, the default > compression algorithm for zram was changed from "lzo" to "lzo-rle". the crash > utility only supports decompression for "lzo", when parsing vmcore files that > utilize zram compression, such as when using the gcore command to detach > process core dump files, parsing cannot be completed successfully. > > before: > crash> gcore -v 0 1 > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > gcore: WARNING: only the lzo compressor is supported > after: > crash> gcore -v 0 1 > Saved core.1.init > > Signed-off-by: yulong.tang <[email protected]> > Reviewed-by: Tao Liu <[email protected]> > --- > Makefile | 9 +- > defs.h | 1 + > diskdump.c | 15 +++ > lzorle_decompress.c | 314 ++++++++++++++++++++++++++++++++++++++++++++ > lzorle_decompress.h | 81 ++++++++++++ > 5 files changed, 418 insertions(+), 2 deletions(-) > create mode 100644 lzorle_decompress.c > create mode 100644 lzorle_decompress.h > > diff --git a/Makefile b/Makefile > index 9e97313..ce0c070 100644 > --- a/Makefile > +++ b/Makefile > @@ -74,7 +74,8 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c > help.c task.c \ > xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \ > xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \ > ramdump.c vmware_vmss.c vmware_guestdump.c \ > - xen_dom0.c kaslr_helper.c sbitmap.c maple_tree.c > + xen_dom0.c kaslr_helper.c sbitmap.c maple_tree.c \ > + lzorle_decompress.c > > SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \ > ${REDHAT_CFILES} ${REDHAT_HFILES} ${UNWIND_HFILES} \ > @@ -94,7 +95,8 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o > filesys.o help.o task.o \ > xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \ > xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \ > ramdump.o vmware_vmss.o vmware_guestdump.o \ > - xen_dom0.o kaslr_helper.o sbitmap.o maple_tree.o > + xen_dom0.o kaslr_helper.o sbitmap.o maple_tree.o \ > + lzorle_decompress.o > > MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c > memory_driver/README > > @@ -546,6 +548,9 @@ bpf.o: ${GENERIC_HFILES} bpf.c > maple_tree.o: ${GENERIC_HFILES} ${MAPLE_TREE_HFILES} maple_tree.c > ${CC} -c ${CRASH_CFLAGS} maple_tree.c ${WARNING_OPTIONS} > ${WARNING_ERROR} > > +lzorle_decompress.o: lzorle_decompress.c > + ${CC} -c ${CRASH_CFLAGS} lzorle_decompress.c ${WARNING_OPTIONS} > ${WARNING_ERROR} > + > ${PROGRAM}: force > @$(MAKE) all > > diff --git a/defs.h b/defs.h > index 98650e8..c2c3fba 100644 > --- a/defs.h > +++ b/defs.h > @@ -51,6 +51,7 @@ > #include <regex.h> > #ifdef LZO > #include <lzo/lzo1x.h> > +#include "lzorle_decompress.h" > #endif > #ifdef SNAPPY > #include <snappy-c.h> > diff --git a/diskdump.c b/diskdump.c > index 3ae7bf2..e324a2e 100644 > --- a/diskdump.c > +++ b/diskdump.c > @@ -3068,6 +3068,21 @@ try_zram_decompress(ulonglong pte_val, unsigned char > *buf, ulong len, ulonglong > "zram decompress error: this executable needs to be > built" > " with lzo library\n"); > return 0; > +#endif > + } else if (STREQ(name, "lzo-rle")) { > +#ifdef LZO > + if (!(dd->flags & LZO_SUPPORTED)) { > + if (lzo_init() == LZO_E_OK) > + dd->flags |= LZO_SUPPORTED; > + else > + return 0; > + } > + decompressor = (void *)&lzorle_decompress_safe; > +#else > + error(WARNING, > + "zram decompress error: this executable needs to be > built" > + " with lzo-rle library\n"); > + return 0; > #endif > } else { /* todo: support more compressor */ > error(WARNING, "only the lzo compressor is supported\n"); > diff --git a/lzorle_decompress.c b/lzorle_decompress.c > new file mode 100644 > index 0000000..55900c3 > --- /dev/null > +++ b/lzorle_decompress.c > @@ -0,0 +1,314 @@ > +/* lzorle_decompress.h > + * > + * from kernel lib/lzo/lzo1x_decompress_safe.c > + * > + * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer <[email protected]> > + * Copyright (C) 2024 NIO > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; either version 2 of the License, or > + * (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + */ > + > +#include "lzorle_decompress.h" > +#include <stdint.h> > +#include <string.h> > + > +#include "defs.h" > + > +/* This MAX_255_COUNT is the maximum number of times we can add 255 to a base > + * count without overflowing an integer. The multiply will overflow when > + * multiplying 255 by more than MAXINT/255. The sum will overflow earlier > + * depending on the base count. Since the base count is taken from a u8 > + * and a few bits, it is safe to assume that it will always be lower than > + * or equal to 2*255, thus we can always prevent any overflow by accepting > + * two less 255 steps. See Documentation/lzo.txt for more information. > + */ > +#define MAX_255_COUNT ((((unsigned long)~0) / 255) - 2) > + > +static uint16_t get_unaligned_le16(const void *p) { > + uint16_t value; > + memcpy(&value, p, sizeof(uint16_t)); > + return value; > +} > + > +int lzorle_decompress_safe(const unsigned char *in, unsigned long in_len, > + unsigned char *out, unsigned long *out_len, void > *other/* NOT USED */) { > + unsigned char *op; > + const unsigned char *ip; > + unsigned long t, next; > + unsigned long state = 0; > + const unsigned char *m_pos; > + const unsigned char * const ip_end = in + in_len; > + unsigned char * const op_end = out + *out_len; > + > + unsigned char bitstream_version; > + > + bool efficient_unaligned_access = > (get_kernel_config("CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS", NULL) == > IKCONFIG_Y); > + > + op = out; > + ip = in; > + > + if (in_len < 3) > + goto input_overrun; > + > + if (in_len >= 5 && *ip == 17) { > + bitstream_version = ip[1]; > + ip += 2; > + } else { > + bitstream_version = 0; > + } > + > + if (*ip > 17) { > + t = *ip++ - 17; > + if (t < 4) { > + next = t; > + goto match_next; > + } > + goto copy_literal_run; > + } > + > + for (;;) { > + t = *ip++; > + if (t < 16) { > + if (state == 0) { > + if (t == 0) { > + unsigned long offset; > + const unsigned char *ip_last = ip; > + > + while (*ip == 0) { > + ip++; > + NEED_IP(1); > + } > + offset = ip - ip_last; > + if (offset > MAX_255_COUNT) > + return LZO_E_ERROR; > + > + offset = (offset << 8) - offset; > + t += offset + 15 + *ip++; > + } > + t += 3; > +copy_literal_run: > + if (efficient_unaligned_access) { > + if (HAVE_IP(t + 15) && HAVE_OP(t + > 15)) { > + const unsigned char *ie = ip > + t; > + unsigned char *oe = op + t; > + do { > + COPY8(op, ip); > + op += 8; > + ip += 8; > + COPY8(op, ip); > + op += 8; > + ip += 8; > + } while (ip < ie); > + ip = ie; > + op = oe; > + } else { > + NEED_OP(t); > + NEED_IP(t + 3); > + do { > + *op++ = *ip++; > + } while (--t > 0); > + } > + } else { > + NEED_OP(t); > + NEED_IP(t + 3); > + do { > + *op++ = *ip++; > + } while (--t > 0); > + } > + state = 4; > + continue; > + } else if (state != 4) { > + next = t & 3; > + m_pos = op - 1; > + m_pos -= t >> 2; > + m_pos -= *ip++ << 2; > + TEST_LB(m_pos); > + NEED_OP(2); > + op[0] = m_pos[0]; > + op[1] = m_pos[1]; > + op += 2; > + goto match_next; > + } else { > + next = t & 3; > + m_pos = op - (1 + M2_MAX_OFFSET); > + m_pos -= t >> 2; > + m_pos -= *ip++ << 2; > + t = 3; > + } > + } else if (t >= 64) { > + next = t & 3; > + m_pos = op - 1; > + m_pos -= (t >> 2) & 7; > + m_pos -= *ip++ << 3; > + t = (t >> 5) - 1 + (3 - 1); > + } else if (t >= 32) { > + t = (t & 31) + (3 - 1); > + if (t == 2) { > + unsigned long offset; > + const unsigned char *ip_last = ip; > + > + while (*ip == 0) { > + ip++; > + NEED_IP(1); > + } > + offset = ip - ip_last; > + if (offset > MAX_255_COUNT) > + return LZO_E_ERROR; > + > + offset = (offset << 8) - offset; > + t += offset + 31 + *ip++; > + NEED_IP(2); > + } > + m_pos = op - 1; > + > + next = get_unaligned_le16(ip); > + ip += 2; > + m_pos -= next >> 2; > + next &= 3; > + } else { > + NEED_IP(2); > + next = get_unaligned_le16(ip); > + if (((next & 0xfffc) == 0xfffc) && > + ((t & 0xf8) == 0x18) && > + bitstream_version) { > + NEED_IP(3); > + t &= 7; > + t |= ip[2] << 3; > + t += MIN_ZERO_RUN_LENGTH; > + NEED_OP(t); > + memset(op, 0, t); > + op += t; > + next &= 3; > + ip += 3; > + goto match_next; > + } else { > + m_pos = op; > + m_pos -= (t & 8) << 11; > + t = (t & 7) + (3 - 1); > + if (t == 2) { > + unsigned long offset; > + const unsigned char *ip_last = ip; > + > + while (*ip == 0) { > + ip++; > + NEED_IP(1); > + } > + offset = ip - ip_last; > + if (offset > MAX_255_COUNT) > + return LZO_E_ERROR; > + > + offset = (offset << 8) - offset; > + t += offset + 7 + *ip++; > + NEED_IP(2); > + next = get_unaligned_le16(ip); > + } > + ip += 2; > + m_pos -= next >> 2; > + next &= 3; > + if (m_pos == op) > + goto eof_found; > + m_pos -= 0x4000; > + } > + } > + TEST_LB(m_pos); > + > + if (efficient_unaligned_access) { > + if (op - m_pos >= 8) { > + unsigned char *oe = op + t; > + if (HAVE_OP(t + 15)) { > + do { > + COPY8(op, m_pos); > + op += 8; > + m_pos += 8; > + COPY8(op, m_pos); > + op += 8; > + m_pos += 8; > + } while (op < oe); > + op = oe; > + if (HAVE_IP(6)) { > + state = next; > + COPY4(op, ip); > + op += next; > + ip += next; > + continue; > + } > + } else { > + NEED_OP(t); > + do { > + *op++ = *m_pos++; > + } while (op < oe); > + } > + } else { > + unsigned char *oe = op + t; > + NEED_OP(t); > + op[0] = m_pos[0]; > + op[1] = m_pos[1]; > + op += 2; > + m_pos += 2; > + do { > + *op++ = *m_pos++; > + } while (op < oe); > + } > + } else { > + unsigned char *oe = op + t; > + NEED_OP(t); > + op[0] = m_pos[0]; > + op[1] = m_pos[1]; > + op += 2; > + m_pos += 2; > + do { > + *op++ = *m_pos++; > + } while (op < oe); > + } > +match_next: > + state = next; > + t = next; > + if (efficient_unaligned_access) { > + if (HAVE_IP(6) && HAVE_OP(4)) { > + COPY4(op, ip); > + op += t; > + ip += t; > + } else { > + NEED_IP(t + 3); > + NEED_OP(t); > + while (t > 0) { > + *op++ = *ip++; > + t--; > + } > + } > + } else { > + NEED_IP(t + 3); > + NEED_OP(t); > + while (t > 0) { > + *op++ = *ip++; > + t--; > + } > + } > + } > + > +eof_found: > + *out_len = op - out; > + return (t != 3 ? LZO_E_ERROR : > + ip == ip_end ? LZO_E_OK : > + ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : > LZO_E_INPUT_OVERRUN); > + > +input_overrun: > + *out_len = op - out; > + return LZO_E_INPUT_OVERRUN; > + > +output_overrun: > + *out_len = op - out; > + return LZO_E_OUTPUT_OVERRUN; > + > +lookbehind_overrun: > + *out_len = op - out; > + return LZO_E_LOOKBEHIND_OVERRUN; > +} > \ No newline at end of file > diff --git a/lzorle_decompress.h b/lzorle_decompress.h > new file mode 100644 > index 0000000..c7dfd70 > --- /dev/null > +++ b/lzorle_decompress.h > @@ -0,0 +1,81 @@ > +/* lzorle_decompress.h > + * > + * from kernel lib/lzo/lzodefs.h > + * > + * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer <[email protected]> > + * Copyright (C) 2024 NIO > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; either version 2 of the License, or > + * (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + */ > + > +#ifndef LZODEFS_H > +#define LZODEFS_H > + > +#define COPY4(dst, src) memcpy((dst), (src), sizeof(uint32_t)) > +#define COPY8(dst, src) memcpy((dst), (src), sizeof(uint64_t)) > + > +#define M1_MAX_OFFSET 0x0400 > +#define M2_MAX_OFFSET 0x0800 > +#define M3_MAX_OFFSET 0x4000 > +#define M4_MAX_OFFSET_V0 0xbfff > +#define M4_MAX_OFFSET_V1 0xbffe > + > +#define M1_MIN_LEN 2 > +#define M1_MAX_LEN 2 > +#define M2_MIN_LEN 3 > +#define M2_MAX_LEN 8 > +#define M3_MIN_LEN 3 > +#define M3_MAX_LEN 33 > +#define M4_MIN_LEN 3 > +#define M4_MAX_LEN 9 > + > +#define M1_MARKER 0 > +#define M2_MARKER 64 > +#define M3_MARKER 32 > +#define M4_MARKER 16 > + > +#define MIN_ZERO_RUN_LENGTH 4 > +#define MAX_ZERO_RUN_LENGTH (2047 + MIN_ZERO_RUN_LENGTH) > + > +#define lzo_dict_t unsigned short > +#define D_BITS 13 > +#define D_SIZE (1u << D_BITS) > +#define D_MASK (D_SIZE - 1) > +#define D_HIGH ((D_MASK >> 1) + 1) > + > +#define LZO_E_OK 0 > +#define LZO_E_ERROR (-1) > +#define LZO_E_OUT_OF_MEMORY (-2) > +#define LZO_E_NOT_COMPRESSIBLE (-3) > +#define LZO_E_INPUT_OVERRUN (-4) > +#define LZO_E_OUTPUT_OVERRUN (-5) > +#define LZO_E_LOOKBEHIND_OVERRUN (-6) > +#define LZO_E_EOF_NOT_FOUND (-7) > +#define LZO_E_INPUT_NOT_CONSUMED (-8) > +#define LZO_E_NOT_YET_IMPLEMELZO_HFILESNTED (-9) > +#define LZO_E_INVALID_ARGUMENT (-10) > + > +#define HAVE_IP(x) ((unsigned long)(ip_end - ip) >= (unsigned long)(x)) > +#define HAVE_OP(x) ((unsigned long)(op_end - op) >= (unsigned long)(x)) > +#define NEED_IP(x) > \ > + if (!HAVE_IP(x)) > \ > + goto input_overrun > +#define NEED_OP(x) > \ > + if (!HAVE_OP(x)) > \ > + goto output_overrun > +#define TEST_LB(m_pos) > \ > + if ((m_pos) < out) > \ > + goto lookbehind_overrun > + > +int lzorle_decompress_safe(const unsigned char *in, unsigned long in_len, > + unsigned char *out, unsigned long *out_len, void > *other/* NOT USED */); > + > +#endif > \ No newline at end of file > -- > 2.25.1 > -- > Crash-utility mailing list -- [email protected] > To unsubscribe send an email to [email protected] > https://${domain_name}/admin/lists/devel.lists.crash-utility.osci.io/ > Contribution Guidelines: https://github.com/crash-utility/crash/wiki -- Crash-utility mailing list -- [email protected] To unsubscribe send an email to [email protected] https://${domain_name}/admin/lists/devel.lists.crash-utility.osci.io/ Contribution Guidelines: https://github.com/crash-utility/crash/wiki
