This is an automated email from the git hooks/post-receive script. guillem pushed a commit to branch main in repository dpkg.
View the commit online: https://git.dpkg.org/cgit/dpkg/dpkg.git/commit/?id=cad30c7db931f7aaf598a9c24cf5b38fda386424 commit cad30c7db931f7aaf598a9c24cf5b38fda386424 Author: Guillem Jover <[email protected]> AuthorDate: Sat Jul 17 22:58:01 2021 +0200 dpkg-deb: Add support for --threads-max and DPKG_DEB_THREADS_MAX These will make it possible to limit the maximum number of threads used by compressors that support multi-threading. Prompted-by: vv221 on IRC --- lib/dpkg/compress.c | 35 ++++++++++++++++++++++++++++++++--- lib/dpkg/compress.h | 1 + man/dpkg-deb.pod | 12 ++++++++++++ src/deb/extract.c | 1 + src/deb/main.c | 9 +++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/lib/dpkg/compress.c b/lib/dpkg/compress.c index 264eeb018..dfb6a6dea 100644 --- a/lib/dpkg/compress.c +++ b/lib/dpkg/compress.c @@ -697,15 +697,37 @@ filter_xz_get_memlimit(void) return mt_memlimit; } +static long +parse_threads_max(const char *str) +{ + long value; + char *end; + + errno = 0; + value = strtol(str, &end, 10); + if (str == end || *end != '\0' || errno != 0) + return 0; + + return value; +} + static uint32_t -filter_xz_get_cputhreads(void) +filter_xz_get_cputhreads(struct compress_params *params) { - uint32_t threads_max; + long threads_max; + const char *env; threads_max = lzma_cputhreads(); if (threads_max == 0) threads_max = 1; + if (params->threads_max >= 0) + return clamp(params->threads_max, 1, threads_max); + + env = getenv("DPKG_DEB_THREADS_MAX"); + if (str_is_set(env)) + return clamp(parse_threads_max(env), 1, threads_max); + return threads_max; } #endif @@ -749,7 +771,7 @@ filter_xz_init(struct io_lzma *io, lzma_stream *s) #ifdef HAVE_LZMA_MT_ENCODER mt_options.preset = preset; mt_memlimit = filter_xz_get_memlimit(); - mt_options.threads = filter_xz_get_cputhreads(); + mt_options.threads = filter_xz_get_cputhreads(io->params); /* Guess whether we have enough RAM to use the multi-threaded encoder, * and decrease them up to single-threaded to reduce memory usage. */ @@ -838,15 +860,22 @@ compress_xz(struct compress_params *params, int fd_in, int fd_out, const char *desc) { struct command cmd; + char *threads_opt = NULL; command_compress_init(&cmd, XZ, desc, params->level); if (params->strategy == COMPRESSOR_STRATEGY_EXTREME) command_add_arg(&cmd, "-e"); + if (params->threads_max > 0) { + threads_opt = str_fmt("-T%d", params->threads_max); + command_add_arg(&cmd, threads_opt); + } + fd_fd_filter(&cmd, fd_in, fd_out, env_xz); command_destroy(&cmd); + free(threads_opt); } #endif diff --git a/lib/dpkg/compress.h b/lib/dpkg/compress.h index e075e525a..f9b66ba0e 100644 --- a/lib/dpkg/compress.h +++ b/lib/dpkg/compress.h @@ -58,6 +58,7 @@ struct compress_params { enum compressor_type type; enum compressor_strategy strategy; int level; + int threads_max; }; enum compressor_type compressor_find_by_name(const char *name); diff --git a/man/dpkg-deb.pod b/man/dpkg-deb.pod index fb9c950c7..9848d6646 100644 --- a/man/dpkg-deb.pod +++ b/man/dpkg-deb.pod @@ -284,6 +284,11 @@ The B<--no-uniform-compression> option disables uniform compression (since dpkg 1.19.0). Uniform compression is the default (since dpkg 1.19.0). +=item B<--threads-max=>I<threads> + +Sets the maximum number of threads allowed for compressors that support +multi-threaded operations (since dpkg 1.21.9). + =item B<--root-owner-group> Set the owner and group for each entry in the filesystem tree data to @@ -344,6 +349,13 @@ memory allocations, etc. =over +=item B<DPKG_DEB_THREADS_MAX> + +Sets the maximum number of threads allowed for compressors that support +multi-threaded operations (since dpkg 1.21.9). + +The B<--threads-max> option overrides this value. + =item B<DPKG_COLORS> Sets the color mode (since dpkg 1.18.5). diff --git a/src/deb/extract.c b/src/deb/extract.c index 89819eaf3..a09853962 100644 --- a/src/deb/extract.c +++ b/src/deb/extract.c @@ -122,6 +122,7 @@ extracthalf(const char *debar, const char *dir, bool header_done; struct compress_params decompress_params = { .type = COMPRESSOR_TYPE_GZIP, + .threads_max = compress_params.threads_max, }; ar = dpkg_ar_open(debar); diff --git a/src/deb/main.c b/src/deb/main.c index 17ed92b18..d1e6fbb08 100644 --- a/src/deb/main.c +++ b/src/deb/main.c @@ -105,6 +105,7 @@ usage(const struct cmdinfo *cip, const char *value) " --nocheck Suppress control file check (build bad\n" " packages).\n" " --root-owner-group Forces the owner and groups to root.\n" +" --threads-max=<threads> Use at most <threads> with compressor.\n" " --[no-]uniform-compression Use the compression params on all members.\n" " -z# Set the compression level when building.\n" " -Z<type> Set the compression type used when building.\n" @@ -166,6 +167,7 @@ struct compress_params compress_params = { .type = DPKG_DEB_DEFAULT_COMPRESSOR, .strategy = COMPRESSOR_STRATEGY_NONE, .level = -1, + .threads_max = -1, }; static void @@ -200,6 +202,12 @@ set_compress_type(const struct cmdinfo *cip, const char *value) badusage(_("obsolete compression type '%s'; use xz or gzip instead"), value); } +static void +set_threads_max(const struct cmdinfo *cip, const char *value) +{ + compress_params.threads_max = dpkg_options_parse_arg_int(cip, value); +} + static const struct cmdinfo cmdinfos[]= { ACTION("build", 'b', 0, do_build), ACTION("contents", 'c', 0, do_contents), @@ -218,6 +226,7 @@ static const struct cmdinfo cmdinfos[]= { { "verbose", 'v', 0, &opt_verbose, NULL, NULL, 1 }, { "nocheck", 0, 0, &nocheckflag, NULL, NULL, 1 }, { "root-owner-group", 0, 0, &opt_root_owner_group, NULL, NULL, 1 }, + { "threads-max", 0, 1, NULL, NULL, set_threads_max }, { "uniform-compression", 0, 0, &opt_uniform_compression, NULL, NULL, 1 }, { "no-uniform-compression", 0, 0, &opt_uniform_compression, NULL, NULL, 0 }, { NULL, 'z', 1, NULL, NULL, set_compress_level }, -- Dpkg.Org's dpkg
