https://github.com/python/cpython/commit/1d28f9aaded28d6bc1c6a38634e1f94f4982fd0e
commit: 1d28f9aaded28d6bc1c6a38634e1f94f4982fd0e
branch: main
author: Victor Stinner <[email protected]>
committer: vstinner <[email protected]>
date: 2026-05-22T19:05:45+02:00
summary:
gh-150114: Log the memory usage in regrtest (#150255)
On Linux, log the total memory usage of all Python test processes.
Read the private memory in /proc/pid/smaps.
files:
A Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst
M Lib/test/libregrtest/logger.py
M Lib/test/libregrtest/run_workers.py
M Lib/test/libregrtest/utils.py
M Lib/test/test_regrtest.py
diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py
index fa1d4d575c8fd4..4e011ef06f8a91 100644
--- a/Lib/test/libregrtest/logger.py
+++ b/Lib/test/libregrtest/logger.py
@@ -1,5 +1,6 @@
import os
import time
+from typing import Callable
from test.support import MS_WINDOWS
from .results import TestResults
@@ -19,16 +20,27 @@ def __init__(self, results: TestResults, quiet: bool, pgo:
bool):
self._results: TestResults = results
self._quiet: bool = quiet
self._pgo: bool = pgo
+ self.get_mem_usage: Callable[[], int | None] | None = None
def log(self, line: str = '') -> None:
empty = not line
- # add the system load prefix: "load avg: 1.80 "
+ # Add the memory usage: "mem: 1 GiB "
+ if self.get_mem_usage is not None:
+ mem = self.get_mem_usage()
+ if mem:
+ mib = mem / (1024*1024)
+ if mib >= 1024:
+ line = f"mem: {mib / 1024:.1f} GiB {line}"
+ else:
+ line = f"mem: {mib:.1f} MiB {line}"
+
+ # Add the system load prefix: "load avg: 1.80 "
load_avg = self.get_load_avg()
if load_avg is not None:
line = f"load avg: {load_avg:.2f} {line}"
- # add the timestamp prefix: "0:01:05 "
+ # Add the timestamp prefix: "0:01:05 "
log_time = time.perf_counter() - self.start_time
mins, secs = divmod(int(log_time), 60)
diff --git a/Lib/test/libregrtest/run_workers.py
b/Lib/test/libregrtest/run_workers.py
index 424085a0050eb5..befdac7ee77f10 100644
--- a/Lib/test/libregrtest/run_workers.py
+++ b/Lib/test/libregrtest/run_workers.py
@@ -22,7 +22,7 @@
from .single import PROGRESS_MIN_TIME
from .utils import (
StrPath, TestName,
- format_duration, print_warning, count, plural)
+ format_duration, print_warning, count, plural, get_process_memory_usage)
from .worker import create_worker_process, USE_PROCESS_GROUP
if MS_WINDOWS:
@@ -452,6 +452,12 @@ def wait_stopped(self, start_time: float) -> None:
print_warning(f"Failed to join {self} in
{format_duration(dt)}")
break
+ def get_mem_usage(self):
+ popen = self._popen
+ if popen is None:
+ return
+ return get_process_memory_usage(popen.pid)
+
def get_running(workers: list[WorkerThread]) -> str | None:
running: list[str] = []
@@ -473,6 +479,7 @@ def __init__(self, num_workers: int, runtests: RunTests,
logger: Logger, results: TestResults) -> None:
self.num_workers = num_workers
self.runtests = runtests
+ self.logger = logger
self.log = logger.log
self.display_progress = logger.display_progress
self.results: TestResults = results
@@ -598,9 +605,21 @@ def _process_result(self, item: QueueOutput) -> TestResult:
return result
+ def get_mem_usage(self):
+ usage = 0
+ main_mem = get_process_memory_usage(os.getpid())
+ if main_mem:
+ usage += main_mem
+ for worker in self.workers:
+ worker_mem = worker.get_mem_usage()
+ if worker_mem:
+ usage += worker_mem
+ return usage
+
def run(self) -> None:
fail_fast = self.runtests.fail_fast
fail_env_changed = self.runtests.fail_env_changed
+ self.logger.get_mem_usage = self.get_mem_usage
self.start_workers()
@@ -625,3 +644,4 @@ def run(self) -> None:
# worker when we exit this function
self.pending.stop()
self.stop_workers()
+ self.logger.get_mem_usage = None
diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
index 00703d6c074855..1b4cb96406d6f6 100644
--- a/Lib/test/libregrtest/utils.py
+++ b/Lib/test/libregrtest/utils.py
@@ -752,3 +752,26 @@ def display_title(title):
print(title)
print("#" * len(title))
print(flush=True)
+
+
+def get_process_memory_usage(pid: int) -> int | None:
+ """
+ Read the private memory in bytes from /proc/pid/smaps.
+ """
+ try:
+ fp = open(f"/proc/{pid}/smaps", "rb")
+ except OSError:
+ return None
+
+ try:
+ total = 0
+ with fp:
+ for line in fp:
+ # Include both Private_Clean and Private_Dirty sections.
+ line = line.rstrip()
+ if line.startswith(b"Private_") and line.endswith(b'kB'):
+ parts = line.split()
+ total += int(parts[1]) * 1024
+ return total
+ except ProcessLookupError:
+ return None
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 02f6e0c74b5ce8..207b144d01d925 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -41,7 +41,11 @@
ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
-LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
+LOG_PREFIX = (
+ r'[0-9]+:[0-9]+:[0-9]+ '
+ r'(?:load avg: [0-9]+\.[0-9]{2} )?'
+ r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?'
+)
RESULT_REGEX = (
'passed',
'failed',
diff --git
a/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst
b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst
new file mode 100644
index 00000000000000..a140bf921972ed
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst
@@ -0,0 +1,2 @@
+On Linux, regrtest now logs the total memory usage of all Python processes.
+Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner.
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
