Unlike gzip and bzip2, LZ77-based compressors can continue to improve their compression ratio quite a bit in some cases by using more memory. For this reason, the .lzma format allows a dictionary size (and thus memory usage) up to 4 GiB, which can present problems for the decompressor, including making a system unresponsive or summoning the dreaded Linux OOM killer. Make sure dpkg does not use more than 100 MiB, nor 40% of available RAM, when decompressing an lzma-compressed package to unpack it or examine its contents.
If the lzma command is provided by XZ Utils, also make sure _not_ to set a memory usage limit below 10 MiB. Without this change, dpkg would refuse to install packages compressed with the default lzma settings on memory-starved systems (with less than 20 MiB of physical memory). Add a --memlimit command-line option to allow overriding the memory usage limit in case it is too low. Signed-off-by: Jonathan Nieder <[email protected]> --- configure.ac | 2 + dpkg-deb/dpkg-deb.h | 3 + dpkg-deb/extract.c | 2 +- dpkg-deb/main.c | 24 ++++++++++ lib/dpkg/Makefile.am | 5 ++ lib/dpkg/compression-backend.c | 100 +++++++++++++++++++++++++++++++++++++++- lib/dpkg/compression-backend.h | 6 ++- lib/dpkg/compression.c | 8 ++- lib/dpkg/dpkg.h | 4 +- man/dpkg-deb.1 | 8 +++ 10 files changed, 153 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 52f019c..7a7458c 100644 --- a/configure.ac +++ b/configure.ac @@ -75,6 +75,7 @@ fi # Checks for header files. AC_HEADER_STDC +AC_HEADER_STDBOOL AC_CHECK_HEADERS([stddef.h error.h locale.h libintl.h kvm.h \ sys/cdefs.h sys/syscall.h]) DPKG_CHECK_DEFINE(TIOCNOTTY, [sys/ioctl.h]) @@ -102,6 +103,7 @@ DPKG_CHECK_DECL([WCOREDUMP], [sys/wait.h]) DPKG_CHECK_COMPAT_FUNCS([getopt getopt_long obstack_free \ strnlen strerror strsignal \ scandir alphasort unsetenv]) +TUKLIB_PHYSMEM AC_CHECK_FUNCS([strtoul isascii bcopy memcpy lchown setsid getdtablesize]) DPKG_COMPILER_WARNINGS diff --git a/dpkg-deb/dpkg-deb.h b/dpkg-deb/dpkg-deb.h index 2bd6d88..edb8a8f 100644 --- a/dpkg-deb/dpkg-deb.h +++ b/dpkg-deb/dpkg-deb.h @@ -22,6 +22,8 @@ #ifndef DPKG_DEB_H #define DPKG_DEB_H +#include <stdint.h> + typedef void dofunction(const char *const *argv); dofunction do_build DPKG_ATTR_NORET; dofunction do_contents, do_control, do_showinfo; @@ -37,6 +39,7 @@ void extracthalf(const char *debar, const char *directory, extern const char *compression; extern const char* showformat; extern enum compress_type compress_type; +extern uint64_t compress_memlimit; #define ARCHIVEVERSION "2.0" diff --git a/dpkg-deb/extract.c b/dpkg-deb/extract.c index 4c429d7..f0abce8 100644 --- a/dpkg-deb/extract.c +++ b/dpkg-deb/extract.c @@ -292,7 +292,7 @@ void extracthalf(const char *debar, const char *directory, m_dup2(readfromfd,0); if (admininfo) close(p1[0]); if (taroption) { m_dup2(p2[1],1); close(p2[0]); close(p2[1]); } - decompress_cat(compress_type, 0, 1, _("data")); + decompress_cat(compress_type, 0, 1, compress_memlimit, _("data")); } if (readfromfd != fileno(ar)) close(readfromfd); if (taroption) close(p2[1]); diff --git a/dpkg-deb/main.c b/dpkg-deb/main.c index b478ba9..0f2ef19 100644 --- a/dpkg-deb/main.c +++ b/dpkg-deb/main.c @@ -25,6 +25,7 @@ #include <stdio.h> #include <string.h> +#include <stdint.h> #include <stdlib.h> #include <signal.h> #include <sys/stat.h> @@ -107,6 +108,8 @@ usage(const struct cmdinfo *cip, const char *value) " -z# Set the compression level when building.\n" " -Z<type> Set the compression type used when building.\n" " Allowed values: gzip, bzip2, lzma, none.\n" +" -M, --memlimit=<bytes> Set the memory usage limit used when\n" +" examining lzma compressed packages.\n" "\n")); printf(_( @@ -137,11 +140,13 @@ const char printforhelp[]= int debugflag=0, nocheckflag=0, oldformatflag=BUILDOLDPKGFORMAT; const char* compression=NULL; enum compress_type compress_type = compress_type_gzip; +uint64_t compress_memlimit = 0; const struct cmdinfo *cipaction = NULL; dofunction *action = NULL; static void setaction(const struct cmdinfo *cip, const char *value); static void setcompresstype(const struct cmdinfo *cip, const char *value); +static void setmemlimit(const struct cmdinfo *cip, const char *value); static dofunction *const dofunctions[]= { do_build, @@ -174,6 +179,7 @@ static const struct cmdinfo cmdinfos[]= { { "nocheck", 0, 0, &nocheckflag, NULL, NULL, 1 }, { "compression", 'z', 1, NULL, &compression, NULL, 1 }, { "compress_type", 'Z', 1, NULL, NULL, setcompresstype }, + { "memlimit", 'M', 1, NULL, NULL, setmemlimit }, { "showformat", 0, 1, NULL, &showformat, NULL }, { "help", 'h', 0, NULL, NULL, usage }, { "version", 0, 0, NULL, NULL, printversion }, @@ -206,6 +212,24 @@ static void setcompresstype(const struct cmdinfo *cip, const char *value) { ohshit(_("unknown compression type `%s'!"), value); } +static void setmemlimit(const struct cmdinfo *cip, const char *value) { + const char *endp; + unsigned long long limit; + + if (strchr(value, '-') != NULL) + ohshit(_("invalid integer for -M: '%s'"), value); + + errno = 0; + limit = strtoull(value, (char **)&endp, 10); + + if (value == endp || *endp != '\0') + ohshit(_("invalid integer for -M: '%s'"), value); + if (errno == ERANGE || limit > UINT64_MAX) + ohshit(_("argument to -M out of range: '%s'"), value); + + compress_memlimit = (uint64_t)limit; +} + int main(int argc, const char *const *argv) { jmp_buf ejbuf; diff --git a/lib/dpkg/Makefile.am b/lib/dpkg/Makefile.am index 7428f7c..22cbeff 100644 --- a/lib/dpkg/Makefile.am +++ b/lib/dpkg/Makefile.am @@ -23,6 +23,11 @@ libdpkg_a_SOURCES = \ cleanup.c \ compression.c \ compression-backend.c compression-backend.h \ + $(top_srcdir)/lib/tuklib/tuklib_physmem.c \ + $(top_srcdir)/lib/tuklib/tuklib_physmem.h \ + $(top_srcdir)/lib/tuklib/tuklib_common.h \ + $(top_srcdir)/lib/tuklib/tuklib_config.h \ + $(top_srcdir)/lib/tuklib/sysdefs.h \ database.c \ dbmodify.c \ dump.c \ diff --git a/lib/dpkg/compression-backend.c b/lib/dpkg/compression-backend.c index 9bfbcee..3cf561b 100644 --- a/lib/dpkg/compression-backend.c +++ b/lib/dpkg/compression-backend.c @@ -4,9 +4,12 @@ #include <dpkg/i18n.h> #include <stdarg.h> +#include <stdint.h> +#include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> +#include <sys/resource.h> #include <errno.h> #ifdef WITH_ZLIB @@ -15,10 +18,12 @@ #ifdef WITH_BZ2 #include <bzlib.h> #endif +#include <tuklib/tuklib_physmem.h> #include <dpkg/dpkg.h> #include <dpkg/varbuf.h> #include <dpkg/buffer.h> +#include <dpkg/subproc.h> #include <dpkg/macros.h> #include "compression-backend.h" @@ -58,6 +63,29 @@ fd_fd_filter(int fd_in, int fd_out, const char *desc, varbuffree(&argbuf); } +/* Default memory usage limit for LZ77-based decompressors. */ +static uint64_t +default_memlimit() +{ + /* + * The command-line decoder from XZ Utils limits itself to 40% of + * available RAM, so take that as a reasonable default memory + * limit. If tuklib_physmem() fails, this is zero. + */ + uint64_t limit = tuklib_physmem() * 2 / 5; + + /* + * Assume one can manage to find 10 MiB even on memory-starved + * systems, so dpkg won’t refuse to unpack packages shipped by + * Debian. Do not use more than 100 MiB, as a safety measure + * for servers processing untrusted packages. + */ + limit = max(limit, 10 << 20); + limit = min(limit, 100 << 20); + + return limit; +} + #define DECOMPRESS(format, zFile, zdopen, zread, zerror, ERR_ERRNO, \ fd_in, fd_out, desc) do \ { \ @@ -180,10 +208,78 @@ compress_bzip2(int fd_in, int fd_out, char compression, const char *desc) } #endif +static bool +input_matches(FILE *in, const char *str) +{ + char ch; + + while (ch = *str++) + if (fgetc(in) != ch) + return false; + + return true; +} + +static bool +lzma_is_xz(const char *desc) +{ + int pipefd[2]; + pid_t cpid; + + m_pipe(pipefd); + cpid = m_fork(); + + if (cpid == 0) { + m_dup2(pipefd[1], 1); + close(pipefd[0]); + close(pipefd[1]); + execlp(LZMA, "lzma", "--version", NULL); + ohshite(_("%s: failed to exec '%s %s'"), + desc, "lzma", "--version"); + } else { + FILE *pipef; + bool ret; + + close(pipefd[1]); + pipef = fdopen(pipefd[0], "r"); + + ret = input_matches(pipef, "xz "); + + if (ferror(pipef)) + ohshite(_("%s: error reading lzma's pipe"), desc); + if (fclose(pipef)) + ohshite(_("%s: error closing lzma's pipe"), desc); + waitsubproc(cpid, "lzma --version", PROCPIPE); + + return ret; + } +} + void -decompress_lzma(int fd_in, int fd_out, const char *desc) +decompress_lzma(int fd_in, int fd_out, uint64_t memlimit, const char *desc) { - fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma", "-dc"); + if (memlimit == 0) + memlimit = default_memlimit(); + + if (lzma_is_xz(desc)) { + fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma", + "-dcM%" PRIu64, memlimit); + } else { + struct rlimit lim; + + if (getrlimit(RLIMIT_AS, &lim)) + ohshite(_("%s: failed to get address space limit"), + desc); + if (memlimit > lim.rlim_max) + lim.rlim_cur = lim.rlim_max; + else + lim.rlim_cur = (rlim_t)memlimit; + + if (setrlimit(RLIMIT_AS, &lim)) + ohshite(_("%s: failed to set address space limit"), + desc); + fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma", "-dc"); + } } void diff --git a/lib/dpkg/compression-backend.h b/lib/dpkg/compression-backend.h index 7f3c5d6..86c55f2 100644 --- a/lib/dpkg/compression-backend.h +++ b/lib/dpkg/compression-backend.h @@ -12,14 +12,16 @@ #include <config.h> #include <compat.h> +#include <stdint.h> + #include <dpkg/macros.h> void decompress_gzip(int fd_in, int fd_out, const char *desc) DPKG_ATTR_NORET; void decompress_bzip2(int fd_in, int fd_out, const char *desc) DPKG_ATTR_NORET; -void decompress_lzma(int fd_in, int fd_out, const char *desc) - DPKG_ATTR_NORET; +void decompress_lzma(int fd_in, int fd_out, uint64_t memlimit, + const char *desc) DPKG_ATTR_NORET; void decompress_noop(int fd_in, int fd_out, const char *desc) DPKG_ATTR_NORET; diff --git a/lib/dpkg/compression.c b/lib/dpkg/compression.c index 813526f..61068be 100644 --- a/lib/dpkg/compression.c +++ b/lib/dpkg/compression.c @@ -1,14 +1,16 @@ #include <config.h> #include <compat.h> -#include <stdlib.h> #include <stdarg.h> +#include <stdint.h> +#include <stdlib.h> #include <dpkg/dpkg.h> #include <dpkg/varbuf.h> #include <dpkg/compression-backend.h> -void decompress_cat(enum compress_type type, int fd_in, int fd_out, char *desc, ...) { +void decompress_cat(enum compress_type type, int fd_in, int fd_out, + uint64_t memlimit, char *desc, ...) { va_list al; struct varbuf v = VARBUF_INIT; @@ -22,7 +24,7 @@ void decompress_cat(enum compress_type type, int fd_in, int fd_out, char *desc, case compress_type_bzip2: decompress_bzip2(fd_in, fd_out, v.buf); case compress_type_lzma: - decompress_lzma(fd_in, fd_out, v.buf); + decompress_lzma(fd_in, fd_out, memlimit, v.buf); case compress_type_cat: decompress_noop(fd_in, fd_out, v.buf); default: diff --git a/lib/dpkg/dpkg.h b/lib/dpkg/dpkg.h index afe650f..94a5214 100644 --- a/lib/dpkg/dpkg.h +++ b/lib/dpkg/dpkg.h @@ -30,6 +30,7 @@ DPKG_BEGIN_DECLS #include <setjmp.h> #include <stdarg.h> #include <stdio.h> +#include <stdint.h> #include <sys/types.h> #ifdef HAVE_SYS_CDEFS_H @@ -224,7 +225,8 @@ enum compress_type { }; void decompress_cat(enum compress_type type, int fd_in, int fd_out, - char *desc, ...) DPKG_ATTR_NORET DPKG_ATTR_PRINTF(4); + uint64_t memlimit, char *desc, ...) + DPKG_ATTR_NORET DPKG_ATTR_PRINTF(5); void compress_cat(enum compress_type type, int fd_in, int fd_out, const char *compression, char *desc, ...) DPKG_ATTR_NORET DPKG_ATTR_PRINTF(5); diff --git a/man/dpkg-deb.1 b/man/dpkg-deb.1 index bb08dc9..37b43b3 100644 --- a/man/dpkg-deb.1 +++ b/man/dpkg-deb.1 @@ -197,6 +197,14 @@ Specify which compression type to use when building a package. Allowed values are \fIgzip\fP, \fIbzip2\fP, \fIlzma\fP, and \fInone\fP (default is \fIgzip\fP). .TP +.BR \-M ", " \-\-memlimit= \fImemory_limit\fP +Specify a maximum in bytes for memory usage when decompressing an lzma +or xz compressed package. The default is 40% of the installed RAM, +clamped to at most 100 MiB and at least 10 MiB, which allows +decompression of any package built at the default compression level. +This option allows one to increase the limit to allow decompression of +packages built with a higher compression level than the default. +.TP .BR \-\-new Ensures that .B dpkg\-deb -- 1.6.5.rc1.199.g596ec -- To UNSUBSCRIBE, email to [email protected] with a subject of "unsubscribe". Trouble? Contact [email protected]
