https://github.com/python/cpython/commit/566f815d746ecb1b0e02101e3041d52c4149a4c8 commit: 566f815d746ecb1b0e02101e3041d52c4149a4c8 branch: 3.14 author: Victor Stinner <[email protected]> committer: vstinner <[email protected]> date: 2026-10-16T23:44:19Z summary:
[3.14] gh-156701: Backport test_dtrace from main (#159596) * gh-98894: Fix DTrace test_check_probes for shared builds (#151122) When building with --enable-shared, the SystemTap/DTrace notes live in libpython. Add detection logic to be used by readelf. Force the C locale on readelf output. (cherry picked from commit 3a08e9373977788e7691ec0a973fbe1abe170c91) * gh-151496: Use process groups in test_dtrace (#151512) Create a new process group to run bpftrace commands, so it's possible to kill also child processes on timeout. (cherry picked from commit a064b323f4350305e7486c7b1090cf12b19e7738) * gh-151496: Use process groups in TraceBackend in test_dtrace (#152039) Run the generic DTrace/SystemTap commands in a new process group and terminate the whole group on timeout. This prevents a forked tracer child from keeping stdout/stderr pipes open after the direct tracer process is killed. (cherry picked from commit 1785f4b35f899704df0be54cba3776906186b2b1) * gh-98894: Check readelf failures in test_dtrace (#152345) Report readelf command failures directly instead of later failing with missing-probe assertions. (cherry picked from commit d60a6779e8dd5fbf1f2f84b34bb5a20f3e8cd7fd) * gh-98894: Check tracer exit status in test_dtrace (#152893) Fail functional dtrace and SystemTap cases directly when the tracer exits non-zero. (cherry picked from commit 311e7e79f4e305f1217ef548a62ea219a47c0d7b) * gh-98894: Quote test_dtrace tracer subcommands (#152901) Use shlex.join() when building tracer -c command strings for avoiding issues with spaces. (cherry picked from commit a159c6851f47549d2bc0cd37a0f555951bb11d7b) * gh-98894: Fix dtrace tests in shared builds (#153372) Generate SystemTap probe definitions targeting libpython for shared builds and use centralized USDT probe object discovery for readelf and BPFTrace. (cherry picked from commit 298bef3a0043dfea9cf7ffe319183c4b8f0a24f5) * gh-156703: Set LD_LIBRARY_PATH in SystemTap trace commands (GH-156424) Prefix commands passed to stap -c with LD_LIBRARY_PATH via env, when probes reside in libpython. This allows the interpreter to find libpython when staprun does not preserve the loader path. (cherry picked from commit faf4e524356634ac95d9de3f7f115603f7440a9b) * gh-156722, test_dtrace: Terminate tracing processes gracefully on timeout (#156753) Send SIGTERM to the tracing process group before falling back to SIGKILL. (cherry picked from commit 55ecc0a40918ba0346bc83ef639b655b8a883645) * gh-156701: Increase the SystemTap usability timeout (#156720) SystemTap may compile a kernel module during its usability check (if no cached module is available). We need to allow a larger timeout for that case. (cherry picked from commit e1be6132b20fc7a65edac8c671420fa61eabe46c) --------- Co-authored-by: stratakis <[email protected]> Co-authored-by: stratakis <[email protected]> files: M Lib/test/dtracedata/call_stack.stp M Lib/test/dtracedata/gc.stp M Lib/test/test_dtrace.py diff --git a/Lib/test/dtracedata/call_stack.stp b/Lib/test/dtracedata/call_stack.stp index 54082c202f66aa4..d4455fc8489af20 100644 --- a/Lib/test/dtracedata/call_stack.stp +++ b/Lib/test/dtracedata/call_stack.stp @@ -10,7 +10,7 @@ function basename:string(path:string) return last_token; } -probe process.mark("function__entry") +probe @PYTHON_SYSTEMTAP_PROBE@("function__entry") { funcname = user_string($arg2); @@ -19,7 +19,8 @@ probe process.mark("function__entry") } } -probe process.mark("function__entry"), process.mark("function__return") +probe @PYTHON_SYSTEMTAP_PROBE@("function__entry"), + @PYTHON_SYSTEMTAP_PROBE@("function__return") { filename = user_string($arg1); funcname = user_string($arg2); @@ -31,7 +32,7 @@ probe process.mark("function__entry"), process.mark("function__return") } } -probe process.mark("function__return") +probe @PYTHON_SYSTEMTAP_PROBE@("function__return") { funcname = user_string($arg2); diff --git a/Lib/test/dtracedata/gc.stp b/Lib/test/dtracedata/gc.stp index 162c6d3a2209b98..11d2715e6c721a8 100644 --- a/Lib/test/dtracedata/gc.stp +++ b/Lib/test/dtracedata/gc.stp @@ -1,6 +1,6 @@ global tracing -probe process.mark("function__entry") +probe @PYTHON_SYSTEMTAP_PROBE@("function__entry") { funcname = user_string($arg2); @@ -9,14 +9,15 @@ probe process.mark("function__entry") } } -probe process.mark("gc__start"), process.mark("gc__done") +probe @PYTHON_SYSTEMTAP_PROBE@("gc__start"), + @PYTHON_SYSTEMTAP_PROBE@("gc__done") { if (tracing) { printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1); } } -probe process.mark("function__return") +probe @PYTHON_SYSTEMTAP_PROBE@("function__return") { funcname = user_string($arg2); diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index d2ff10c57a0982c..7813ca3d3f652e1 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -1,14 +1,18 @@ import dis import os.path import re +import shlex +import signal import subprocess import sys import sysconfig +import tempfile import types import unittest from test import support from test.support import findfile, MS_WINDOWS +from test.support import os_helper if not support.has_subprocess_support: @@ -23,6 +27,31 @@ def abspath(filename): return os.path.abspath(findfile(filename, subdir="dtracedata")) +def get_probe_binary(): + binary = sys.executable + if sysconfig.get_config_var("Py_ENABLE_SHARED"): + lib_dir = sysconfig.get_config_var("LIBDIR") + if not lib_dir or sysconfig.is_python_build(): + lib_dir = os.path.abspath(os.path.dirname(sys.executable)) + + lib_names = [] + for name in ( + sysconfig.get_config_var("INSTSONAME"), + sysconfig.get_config_var("LDLIBRARY"), + ): + if name and name not in lib_names: + lib_names.append(name) + + if lib_dir: + for name in lib_names: + libpython_path = os.path.join(lib_dir, name) + if os.path.exists(libpython_path): + binary = libpython_path + break + + return binary + + def normalize_trace_output(output): """Normalize DTrace output for comparison. @@ -48,16 +77,77 @@ def normalize_trace_output(output): ) +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +TERMINATE_TIMEOUT = 10 + +def create_process_group(*args, **kwargs): + if USE_PROCESS_GROUP: + kwargs['start_new_session'] = True + return subprocess.Popen(*args, **kwargs) + +def terminate_process_group(proc): + if USE_PROCESS_GROUP: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + else: + proc.terminate() + + try: + proc.communicate(timeout=TERMINATE_TIMEOUT) + except subprocess.TimeoutExpired: + if USE_PROCESS_GROUP: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + proc.kill() + proc.communicate(timeout=TERMINATE_TIMEOUT) # Clean up + + +def run_readelf(cmd): + # Force the C locale to disable localization. + env = dict(os.environ, LC_ALL="C") + try: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + except OSError: + raise unittest.SkipTest("Couldn't find readelf on the path") + + with proc: + stdout, stderr = proc.communicate() + + if proc.returncode: + raise AssertionError( + f"Command {shlex.join(cmd)!r} failed " + f"with exit code {proc.returncode}: " + f"stdout={stdout!r} stderr={stderr!r}" + ) + + return stdout + + class TraceBackend: EXTENSION = None COMMAND = None COMMAND_ARGS = [] + USABILITY_TIMEOUT = 10 def run_case(self, name, optimize_python=None): - actual_output = normalize_trace_output(self.trace_python( - script_file=abspath(name + self.EXTENSION), - python_file=abspath(name + ".py"), - optimize_python=optimize_python)) + try: + actual_output = normalize_trace_output(self.trace_python( + script_file=abspath(name + self.EXTENSION), + python_file=abspath(name + ".py"), + optimize_python=optimize_python)) + except subprocess.TimeoutExpired: + raise AssertionError(f"{self.COMMAND[0]} timed out") with open(abspath(name + self.EXTENSION + ".expected")) as f: expected_output = f.read().rstrip() @@ -70,25 +160,42 @@ def generate_trace_command(self, script_file, subcommand=None): command += ["-c", subcommand] return command - def trace(self, script_file, subcommand=None): + def trace(self, script_file, subcommand=None, *, timeout=None, + check_returncode=False): command = self.generate_trace_command(script_file, subcommand) - stdout, _ = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True).communicate() + proc = create_process_group(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + try: + stdout, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + terminate_process_group(proc) + raise + if check_returncode and proc.returncode: + raise AssertionError( + f"Command {shlex.join(command)!r} failed " + f"with exit code {proc.returncode}: output={stdout!r}" + ) return stdout def trace_python(self, script_file, python_file, optimize_python=None): python_flags = [] if optimize_python: python_flags.extend(["-O"] * optimize_python) - subcommand = " ".join([sys.executable] + python_flags + [python_file]) - return self.trace(script_file, subcommand) + subcommand = shlex.join([sys.executable] + python_flags + [python_file]) + return self.trace(script_file, subcommand, timeout=60, + check_returncode=True) def assert_usable(self): try: - output = self.trace(abspath("assert_usable" + self.EXTENSION)) + output = self.trace(abspath("assert_usable" + self.EXTENSION), + timeout=self.USABILITY_TIMEOUT) output = output.strip() + except subprocess.TimeoutExpired: + raise unittest.SkipTest( + f"{self.COMMAND[0]} timed out during usability check" + ) except (FileNotFoundError, NotADirectoryError, PermissionError) as fnfe: output = str(fnfe) if output != "probe: success": @@ -105,6 +212,57 @@ class DTraceBackend(TraceBackend): class SystemTapBackend(TraceBackend): EXTENSION = ".stp" COMMAND = ["stap", "-g"] + PROBE_PLACEHOLDER = "@PYTHON_SYSTEMTAP_PROBE@" + USABILITY_TIMEOUT = 60 + + @staticmethod + def quote_systemtap_string(value): + return value.replace("\\", "\\\\").replace('"', '\\"') + + def python_probe(self): + executable = self.quote_systemtap_string(sys.executable) + probe_binary = get_probe_binary() + if probe_binary == sys.executable: + return f'process("{executable}").mark' + + # Python built with --enable-shared + probe_binary = self.quote_systemtap_string(probe_binary) + return f'process("{executable}").library("{probe_binary}").mark' + + def render_script(self, filename): + with open(filename) as fp: + script = fp.read() + + return script.replace(self.PROBE_PLACEHOLDER, self.python_probe()) + + def generate_trace_command(self, script_file, subcommand=None): + probe_binary = get_probe_binary() + if subcommand and probe_binary != sys.executable: + library_path = os.path.dirname(probe_binary) + if existing_path := os.environ.get("LD_LIBRARY_PATH"): + library_path = os.pathsep.join((library_path, existing_path)) + env = shlex.join(["env", f"LD_LIBRARY_PATH={library_path}"]) + subcommand = f"{env} {subcommand}" + + return super().generate_trace_command(script_file, subcommand) + + def trace(self, script_file, subcommand=None, *, timeout=None, + check_returncode=False): + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", suffix=self.EXTENSION, delete=False + ) as script: + script.write(self.render_script(script_file)) + generated_script_file = script.name + + try: + return super().trace( + generated_script_file, + subcommand, + timeout=timeout, + check_returncode=check_returncode, + ) + finally: + os_helper.unlink(generated_script_file) @unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.") @@ -189,25 +347,7 @@ def setUpClass(cls): @staticmethod def get_readelf_version(): - try: - cmd = ["readelf", "--version"] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - with proc: - version, stderr = proc.communicate() - - if proc.returncode: - raise Exception( - f"Command {' '.join(cmd)!r} failed " - f"with exit code {proc.returncode}: " - f"stdout={version!r} stderr={stderr!r}" - ) - except OSError: - raise unittest.SkipTest("Couldn't find readelf on the path") + version = run_readelf(["readelf", "--version"]) # Regex to parse: # 'GNU readelf (GNU Binutils) 2.40.0\n' -> 2.40 @@ -218,14 +358,7 @@ def get_readelf_version(): return int(match.group(1)), int(match.group(2)) def get_readelf_output(self): - command = ["readelf", "-n", sys.executable] - stdout, _ = subprocess.Popen( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ).communicate() - return stdout + return run_readelf(["readelf", "-n", get_probe_binary()]) def test_check_probes(self): readelf_output = self.get_readelf_output() _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
