On Linux there is the field `MemAvailable' in `/proc/meminfo' which holds the amount of memory which is available as of now. It considers the fact that the page cache can purge (if not all) some memory can be reclaimed (for instance by writting back dirty inodes) and some memory should remain free just in case. This amount of memory can be used without the need to swap-out some memory. The complete definition can be located at [0].
The advantage over PHYS_MEM/2 is that it considers the current status/usage of the system with assumung that half of what is physically avilable is usable. [0] https://www.kernel.org/doc/html/latest/filesystems/proc.html Signed-off-by: Sebastian Andrzej Siewior <sebast...@breakpoint.cc> --- This is a bit of my multithreaded XZ decompressor for dpkg. For compression the PhysMem/2 estimation is probably good enough since mostly used the buildd and owns the system.. For decompression the amount of memory should be close to reality so it does not start threads and allocate a lot of memory if the system is quite busy at the amout if package upgrade/installation. lib/dpkg/compress.c | 95 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/lib/dpkg/compress.c b/lib/dpkg/compress.c index 41991317afe53..7ec9144a56290 100644 --- a/lib/dpkg/compress.c +++ b/lib/dpkg/compress.c @@ -523,6 +523,88 @@ filter_lzma_error(struct io_lzma *io, lzma_ret ret) dpkg_lzma_strerror(ret, io->status)); } +#ifdef HAVE_LZMA_MT +# ifdef __linux__ + +#include <sys/stat.h> +#include <fcntl.h> + +/* + * An estimate of how much memory is available. Swap will not be used, the page + * cache may be purged, not everything will be reclaimed what might be + * reclaimed, watermarks are considers. + */ +static char str_MemAvailable[] = "MemAvailable"; + +static int +get_avail_mem(uint64_t *val) +{ + char buf[4096]; + char *str; + ssize_t bytes; + int fd; + + *val = 0; + + fd = open("/proc/meminfo", O_RDONLY); + if (fd < 0) + return -1; + + bytes = read(fd, buf, sizeof(buf)); + close(fd); + + if (bytes <= 0) + return -1; + + buf[bytes] = '\0'; + + str = buf; + while (1) { + char *end; + + end = strchr(str, ':'); + if (!end) + break; + if ((end - str) == sizeof(str_MemAvailable) - 1) { + if (!strncmp(str, str_MemAvailable, + sizeof(str_MemAvailable) - 1)) { + uint64_t num; + + str = end + 1; + num = strtoull(str, &end, 10); + if (!num) + return -1; + if (num == ULLONG_MAX) + return -1; + /* it should end with ' kb\n' */ + if (*end != ' ') + return -1; + + num *= 1024; + *val = num; + return 0; + } + } + + end = strchr(end + 1, '\n'); + if (!end) + break; + str = end + 1; + } + return -1; +} + +# else + +static int +get_avail_mem(uint64_t *val) +{ + return -1; +} + +# endif +#endif + static void filter_unxz_init(struct io_lzma *io, lzma_stream *s) { @@ -562,11 +644,14 @@ filter_xz_init(struct io_lzma *io, lzma_stream *s) #ifdef HAVE_LZMA_MT mt_options.preset = preset; - /* Initialize the multi-threaded memory limit to half the physical - * RAM, or to 128 MiB if we cannot infer the number. */ - mt_memlimit = lzma_physmem() / 2; - if (mt_memlimit == 0) - mt_memlimit = 128 * 1024 * 1024; + /* Ask the kernel what is currently available for us. If this fails + * initialize the memory limit to half the physical RAM, or to 128 MiB + * if we cannot infer the number. */ + if (get_avail_mem(&mt_memlimit) < 0) { + mt_memlimit = lzma_physmem() / 2; + if (mt_memlimit == 0) + mt_memlimit = 128 * 1024 * 1024; + } /* Clamp the multi-threaded memory limit to half the addressable * memory on this architecture. */ if (mt_memlimit > INTPTR_MAX) -- 2.30.0