https://github.com/python/cpython/commit/3cfc87e8bdb4711a0757d59000459202fb1a82f6 commit: 3cfc87e8bdb4711a0757d59000459202fb1a82f6 branch: main author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-08-19T11:42:08+03:00 summary:
gh-75876: Report the peak memory use of a bigmem test (GH-156024) In verbose mode, report how much memory the test really used, next to how much it declared. The subprocess measures its own peak; if it died before reporting it, the parent reports what it sampled, as a lower bound. Report also the number of major page faults, if any: the test did not fit in memory, so its timing means little. Co-authored-by: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst M Lib/test/support/__init__.py M Lib/test/support/subprocess_runner.py diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 31e5508dd9b790..be71575a6ea06a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None: def _memory_watchdog(pid): - """Return a function printing the memory usage of process *pid*.""" + """Return a function printing the memory usage of process *pid*. + + The largest value it saw is kept in its ``peak`` attribute. + """ # Imported here: test.support does not depend on test.libregrtest. from test.libregrtest.utils import get_process_memory_usage def watch(): mem = get_process_memory_usage(pid) if mem is not None: + watch.peak = max(watch.peak, mem) print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB", flush=True) + watch.peak = 0 return watch @@ -1333,7 +1338,23 @@ def wrapper(self): qualname = f'{cls.__qualname__}.{f.__name__}' proc = isolation._start_test(cls.__module__, qualname) watchdog = _memory_watchdog(proc.pid) if verbose else None - isolation._replay_test(self, *proc.wait(tick=watchdog)) + payload, output, returncode = proc.wait(tick=watchdog) + if watchdog: + # The subprocess measures its own peak exactly. What the + # parent sampled is only a lower bound. + maxrss = payload and payload.get('maxrss') + peak = maxrss or watchdog.peak + if peak: + print(f" ... peak memory use: " + f"{peak / (1024 ** 3):.1f} GiB" + f"{'' if maxrss else ' or more'}", flush=True) + majflt = payload and payload.get('majflt') + if majflt: + # The test did not fit in memory, so its timing means + # little. + print(f" ... {majflt} major page faults: the test " + f"waited for the disk", flush=True) + isolation._replay_test(self, payload, output, returncode) return return f(self, maxsize) diff --git a/Lib/test/support/subprocess_runner.py b/Lib/test/support/subprocess_runner.py index 90d74cc757d878..bb4ab059e2b8b6 100644 --- a/Lib/test/support/subprocess_runner.py +++ b/Lib/test/support/subprocess_runner.py @@ -72,7 +72,45 @@ def _outcome(kind, test, detail): for t, tb in result.expectedFailures] outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped] -payload = {'outcomes': outcomes, 'durations': result.id_durations} +def _usage(): + """What this process used: peak resident set size in bytes, and the + number of major page faults it took, either of which can be None. + + A major page fault is served from disk, so a non-zero count means swapping. + + The modules are imported here, after the test has run, so that the test + does not see them. + """ + try: + import resource + except ImportError: + pass + else: + usage = resource.getrusage(resource.RUSAGE_SELF) + # Solaris and illumos leave these fields at 0, which no live process + # has, so treat it as "not supported". + if not usage.ru_maxrss: + return {'maxrss': None, 'majflt': None} + # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs. + maxrss = usage.ru_maxrss + return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024, + 'majflt': usage.ru_majflt} + try: + import os + import _winapi + handle = _winapi.OpenProcess( + _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid()) + except (ImportError, OSError): + return {'maxrss': None, 'majflt': None} + try: + info = _winapi.GetProcessMemoryInfo(handle) + finally: + _winapi.CloseHandle(handle) + # PageFaultCount counts all faults, not only the ones served from disk. + return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None} + + +payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()} with open(outfile, 'wb') as f: marshal.dump(payload, f) diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst new file mode 100644 index 00000000000000..863b49d2f91d8c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst @@ -0,0 +1,3 @@ +In verbose mode, a test decorated with :func:`~test.support.bigmemtest` now +reports how much memory it really used, and the number of major page faults +it took, if any. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
