Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package oomstaller for openSUSE:Factory checked in at 2025-03-27 22:34:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/oomstaller (Old) and /work/SRC/openSUSE:Factory/.oomstaller.new.2696 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "oomstaller" Thu Mar 27 22:34:27 2025 rev:3 rq:1264627 version:0.4.0 Changes: -------- --- /work/SRC/openSUSE:Factory/oomstaller/oomstaller.changes 2024-12-13 22:41:47.426701895 +0100 +++ /work/SRC/openSUSE:Factory/.oomstaller.new.2696/oomstaller.changes 2025-03-27 22:35:08.447658312 +0100 @@ -1,0 +2,9 @@ +Thu Mar 27 17:23:55 UTC 2025 - Andreas Stieger <andreas.stie...@gmx.de> + +- update to 0.4.0: + * adopt a new algorithm that eliminates necessity of specifying + values with --thres and --thrash options + * fix a bug where oomstaller crashes when files under /proc + cannot be read + +------------------------------------------------------------------- Old: ---- oomstaller-0.3.0.tar.gz New: ---- oomstaller-0.4.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ oomstaller.spec ++++++ --- /var/tmp/diff_new_pack.jSBwMZ/_old 2025-03-27 22:35:09.039682818 +0100 +++ /var/tmp/diff_new_pack.jSBwMZ/_new 2025-03-27 22:35:09.043682983 +0100 @@ -1,7 +1,7 @@ # # spec file for package oomstaller # -# Copyright (c) 2024 Andreas Stieger <andreas.stie...@gmx.de> +# Copyright (c) 2025 Andreas Stieger <andreas.stie...@gmx.de> # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: oomstaller -Version: 0.3.0 +Version: 0.4.0 Release: 0 Summary: A tool for suppressing swap thrashing at build time License: BSL-1.0 ++++++ oomstaller-0.3.0.tar.gz -> oomstaller-0.4.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/oomstaller-0.3.0/CODE_OF_CONDUCT.md new/oomstaller-0.4.0/CODE_OF_CONDUCT.md --- old/oomstaller-0.3.0/CODE_OF_CONDUCT.md 2024-12-09 14:15:44.000000000 +0100 +++ new/oomstaller-0.4.0/CODE_OF_CONDUCT.md 2025-03-26 17:12:24.000000000 +0100 @@ -1,7 +1,7 @@ # Community Guidelines for Sustainability of Open-Source Ecosystem -Version 1.0, 2024-12-01 +Version 1.0.1, 2024-12-29 Copyright Naoki Shibata 2024. https://github.com/shibatch/nofreelunch @@ -392,7 +392,7 @@ * Project members strive to promote awareness of the existence, philosophy, purpose, and content of these guidelines to those who - use open source and free software, either directly or indirectly. + directly or indirectly use open source or free software. * Project members strive to promote the awareness that it is natural for organizations that make a profit from the commercial use of open @@ -400,9 +400,9 @@ increased profits from the use of the software to relevant projects, where the contribution is made in order for the project to serve society as a whole. This contribution must not be biased to benefit - any specific organizations deviating from the purpose or original - nature of the project. On top of that, the contribution herein - refers to providing the following items. + any specific organizations beyond the purpose or original nature of + the project. On top of that, the contribution herein refers to + providing the following items. * Financial support * Contributing code or documentation diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/oomstaller-0.3.0/README.md new/oomstaller-0.4.0/README.md --- old/oomstaller-0.3.0/README.md 2024-12-09 14:15:44.000000000 +0100 +++ new/oomstaller-0.4.0/README.md 2025-03-26 17:12:24.000000000 +0100 @@ -100,12 +100,6 @@ ### Options -`--thres <percentage>` default: 75.0 - -This tool suspends processes so that memory usage by running build -processes does not exceed the specified percentage of available -memory. - `--max-parallel <number of processes>` default: 0 Suspends processes so that the number of running build processes does @@ -122,11 +116,6 @@ Specifies the interval at which memory usage of each process is checked and processes are controlled. -`--thrash <minimum available memory (MB)>` default: 256.0 - -If the amount of available memory falls below the specified value, it is -assumed that swap thrashing is occurring. - `--show-stat` Displays statistics when finished. @@ -134,16 +123,6 @@ ### Tips -Lowering the value of --thres results in increased time where some -cores are not used in order to reduce memory usage. Increasing this -value too much would increase the time where only a few processes can -run due to swap thrashing. - -When the available memory is less than the value set by --thrash -option, oomstaller assumes that swap thrashing is occurring and -suspends all processes except the process occupying the largest amount -of memory. This has the effect of minimizing swapping. - Although oomstaller is designed primarily to suppress swapping during builds by suspending the build process, it can also be configured to allow swapping while attempting to reduce the build time. The number @@ -163,6 +142,9 @@ only consume extra memory in situations where there is not enough memory in which you might want to use this tool. +Before benchmarking, run "sudo swapoff -a; sudo swapon -a" to reset +the condition of the swap space. + If you kill this tool with SIGKILL, a large number of build processes will remain suspended with SIGSTOP. To prevent this from happening, use SIGTERM or SIGINT to kill this tool. You can send SIGCONT to all diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/oomstaller-0.3.0/oomstaller.1 new/oomstaller-0.4.0/oomstaller.1 --- old/oomstaller-0.3.0/oomstaller.1 2024-12-09 14:15:44.000000000 +0100 +++ new/oomstaller-0.4.0/oomstaller.1 2025-03-26 17:12:24.000000000 +0100 @@ -1,6 +1,6 @@ .\" Manpage for oomstaller .\" Contact shiba...@users.sourceforge.net to correct errors. -.TH oomstaller 1 "01 Dec 2024" "0.3.0" "oomstaller man page" +.TH oomstaller 1 "01 Jan 2025" "0.4.0" "oomstaller man page" .SH NAME oomstaller \- suppress swap thrashing at build time .SH SYNOPSIS @@ -18,9 +18,6 @@ .UNINDENT .SH OPTIONS .TP -.BR \-\-thres " " <percentage, " " default=75> -This tool suspends processes so that memory usage by running build processes does not exceed the specified percentage of available memory. -.TP .BR \-\-max\-parallel " " <number " " of " " processes, " " default=0> Suspends processes so that the number of running build processes does not exceed the specified number. 0 means no limit. A process is counted as one process even if it has multiple threads. .TP @@ -30,9 +27,6 @@ .BR \-\-period " " <seconds, " " default=1> Specifies the interval at which memory usage of each process is checked and processes are controlled. .TP -.BR \-\-thrash " " <minimum " "available " "memory " "(MB), " " default=256> -If the amount of available memory falls below the specified value, it is assumed that swap thrashing is occurring. -.TP .BR \-\-show\-stat Displays statistics when finished. .SH TIPS diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/oomstaller-0.3.0/oomstaller.cpp new/oomstaller-0.4.0/oomstaller.cpp --- old/oomstaller-0.3.0/oomstaller.cpp 2024-12-09 14:15:44.000000000 +0100 +++ new/oomstaller-0.4.0/oomstaller.cpp 2025-03-26 17:12:24.000000000 +0100 @@ -6,14 +6,20 @@ #include <unordered_set> #include <set> #include <vector> -#include <cstdio> -#include <cstdint> -#include <cstdlib> -#include <cstring> #include <thread> #include <mutex> #include <condition_variable> #include <memory> +#include <chrono> + +#include <cstdio> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <climits> +#include <cmath> +#include <ctime> + #include <signal.h> #include <unistd.h> #include <dirent.h> @@ -23,13 +29,13 @@ using namespace std; -double memThres = 0.75, period = 1.0, minFreeMem = 256 * 1024; -int maxParallel = 0, maxParallelThrash = 1; - const long pageSize = sysconf(_SC_PAGESIZE); uid_t uid = getuid(); const pid_t pid = getpid(); +double period = 1.0; +int maxParallel = 0, maxParallelThrash = 1; + bool showStat = false; unordered_map<int, long> statInfo; @@ -52,16 +58,17 @@ } struct ProcInfo { - string comm; + string comm, d_name; int pid, ppid, pgrp, session; long long unsigned starttime; - long unsigned vsize; + long unsigned vsize, vmswap = 0; long int rss, num_threads; char state; ProcInfo() {} ProcInfo(FILE *fpstat, const char *dn) { + d_name = dn; vector<char> line(1024); if (fgets(line.data(), line.size(), fpstat) == NULL) throw(runtime_error("Could not read /proc/<pid>/stat")); @@ -83,6 +90,23 @@ // + { + FILE *fp = fopen((string("/proc/") + d_name + "/status").c_str(), "r"); + + if (fp) { + while(!feof(fp)) { + if (fgets(line.data(), line.size(), fp) == NULL) break; + if (strncmp(line.data(), "VmSwap:", 7) == 0) { + if (sscanf(line.data(), "%*s %lu", &vmswap) == 1) vmswap = vmswap * 1024 / pageSize; + break; + } + } + fclose(fp); + } + } + + // + if (num_threads > 1) { DIR *taskdir = opendir((string("/proc/") + dn + "/task").c_str()); if (!taskdir) return; @@ -128,8 +152,10 @@ if (!fp) continue; if (fstat(fileno(fp), &statbuf) == 0 && statbuf.st_uid == uid) { - ProcInfo pi(fp, entry->d_name); - ret[pi.pid] = pi; + try { + ProcInfo pi(fp, entry->d_name); + ret[pi.pid] = pi; + } catch(exception &ex) {} } fclose(fp); @@ -157,7 +183,28 @@ condition_variable condvar; void loop(shared_ptr<thread> childTh) { + // Clear the thrashing detection if swap space is used but total + // swap has not increased for 10 seconds + const int THRASHCOUNT1 = 10.0 / period; + + // Clear the thrashing detection 3 seconds after total size of swap + // space is reduced + const int THRASHCOUNT2 = 3.0 / period; + + // The estimated amount of memory that each process could + // potentially occupy is attenuated by this amount each time one + // process finishes + const double MEMMAXDECAY = pow(0.5, 1.0 / 10.0); + + // + unique_lock<mutex> lock(mtx); + unordered_set<int> lastActivePids; + double memMax = 0; + + const long totalMem = readMemInfo("MemTotal:") * 1024 / pageSize; + long lastSwapFree = readMemInfo("SwapFree:"); + int thrashTimer = 0; while(!exiting) { set<ProcInfo, bool(*)(const ProcInfo &lhs, const ProcInfo &rhs)> @@ -166,8 +213,10 @@ if (lhs.starttime < rhs.starttime) return false; return lhs.pid > rhs.pid; } }; - long usedMem = 0; - int pidLargestRSS = 0; + long usedMem = 0, memMax2 = 0; + int pidLargest = 0; + unordered_set<int> activePids; + bool swapUsed = false; { auto m = getProcesses(); @@ -177,30 +226,58 @@ for(auto e : m) { if ((e.second.state != 'R' && e.second.state != 'T' && e.second.state != 'D') || !isTarget(m, &e.second)) continue; + if (e.second.rss + e.second.vmswap > (unsigned long)memMax2) memMax2 = e.second.rss + e.second.vmswap; usedMem += e.second.rss; + if (e.second.vmswap > 0) swapUsed = true; proc.insert(e.second); - if (pidLargestRSS == 0 || e.second.rss > m.at(pidLargestRSS).rss) { - pidLargestRSS = e.second.pid; - } + activePids.insert(e.second.pid); + if (pidLargest == 0 || (e.second.rss + e.second.vmswap > m.at(pidLargest).rss + m.at(pidLargest).vmswap)) + pidLargest = e.second.pid; } } - long freeMem = readMemInfo("MemAvailable:"), usableMem = freeMem * 1024 / pageSize + usedMem; - bool thrashDetected = freeMem < minFreeMem; + long freeMem = readMemInfo("MemAvailable:") * 1024 / pageSize, usableMem = freeMem + usedMem; + if (usableMem > totalMem) usableMem = totalMem; + + // Detection of swap thrashing + + long swapFree = readMemInfo("SwapFree:"); + if (swapFree < lastSwapFree) { + thrashTimer = THRASHCOUNT1; + } else if (swapFree > lastSwapFree) { + if (thrashTimer > THRASHCOUNT2) thrashTimer = THRASHCOUNT2; + } else if (!swapUsed) { + thrashTimer = 0; + } else { + if (thrashTimer > 0) thrashTimer--; + } + lastSwapFree = swapFree; + + // Calculate the number of parallel executions + + for(auto a : lastActivePids) if (activePids.count(a) == 0) memMax *= MEMMAXDECAY; + lastActivePids = activePids; + + // memMax2 is the maximum occupied memory by the currently running processes + // memMax is the estimated amount of memory that each process could potentially occupy + if (memMax2 > memMax) memMax = memMax2; + + int maxParallelOP = INT_MAX; + if (memMax != 0) maxParallelOP = usableMem / memMax + 1; - unordered_set<int> activePids, removedPids; - long m = usedMem, n = proc.size(); + // Determine the next state of each process + + unordered_set<int> removedPids; + long n = proc.size(); int nRunningProcs = 0; for(auto e : proc) { - activePids.insert(e.pid); char nextState = '\0'; - if (e.pid == pidLargestRSS) { + if (e.pid == pidLargest) { nextState = 'R'; } else { - if (m >= usableMem * memThres || (maxParallel > 0 && n > maxParallel) || - (thrashDetected && maxParallelThrash > 0 && n > maxParallelThrash)) { + if (n > maxParallelOP || (maxParallel > 0 && n > maxParallel) || + ((thrashTimer > 0 || freeMem == 0) && maxParallelThrash > 0 && n > maxParallelThrash)) { nextState = 'T'; - m -= e.rss; n--; } else { nextState = 'R'; @@ -228,7 +305,7 @@ // Update statistics info - if (thrashDetected) statInfo[-1]++; + if (thrashTimer > 0 || freeMem == 0) statInfo[-1]++; statInfo[nRunningProcs]++; // @@ -288,11 +365,6 @@ cerr << " build, and suspends processes as necessary to prevent swap thrashing" << endl; cerr << " from occurring." << endl; cerr << endl; - cerr << " --thres <percentage> default: 75.0" << endl; - cerr << endl; - cerr << " This tool suspends processes so that memory usage by running build" << endl; - cerr << " processes does not exceed the specified percentage of available memory." << endl; - cerr << endl; cerr << " --max-parallel <number of processes> default: 0" << endl; cerr << endl; cerr << " Suspends processes so that the number of running build processes" << endl; @@ -309,11 +381,6 @@ cerr << " Specifies the interval at which memory usage of each process is checked" << endl; cerr << " and processes are controlled." << endl; cerr << endl; - cerr << " --thrash <minimum available memory (MB)> default: 256.0" << endl; - cerr << endl; - cerr << " If the amount of available memory falls below the specified value, it is" << endl; - cerr << " assumed that swap thrashing is occurring." << endl; - cerr << endl; cerr << " --show-stat" << endl; cerr << endl; cerr << " Displays statistics when finished." << endl; @@ -331,7 +398,7 @@ cerr << endl; cerr << " See https://github.com/shibatch/oomstaller"; << endl; cerr << endl; - cerr << "oomstaller 0.3.0" << endl; + cerr << "oomstaller 0.4.0" << endl; cerr << endl; exit(-1); @@ -342,14 +409,7 @@ int nextArg; for(nextArg = 1;nextArg < argc;nextArg++) { - if (string(argv[nextArg]) == "--thres") { - if (nextArg+1 >= argc) showUsage(argv[0]); - char *p; - memThres = strtod(argv[nextArg+1], &p) * 0.01; - if (p == argv[nextArg+1] || *p || memThres < 0) - showUsage(argv[0], "A non-negative real value is expected after --thres."); - nextArg++; - } else if (string(argv[nextArg]) == "--max-parallel") { + if (string(argv[nextArg]) == "--max-parallel") { if (nextArg+1 >= argc) showUsage(argv[0]); char *p; maxParallel = strtol(argv[nextArg+1], &p, 0); @@ -370,13 +430,6 @@ if (p == argv[nextArg+1] || *p || period <= 0) showUsage(argv[0], "A positive value is expected after --period."); nextArg++; - } else if (string(argv[nextArg]) == "--thrash") { - if (nextArg+1 >= argc) showUsage(argv[0]); - char *p; - minFreeMem = strtod(argv[nextArg+1], &p) * 1024; - if (p == argv[nextArg+1] || *p || minFreeMem < 0) - showUsage(argv[0], "A non-negative real value is expected after --thrash."); - nextArg++; } else if (string(argv[nextArg]) == "--uid") { if (nextArg+1 >= argc) showUsage(argv[0]); char *p;
