https://github.com/python/cpython/commit/13b1659d6e8cb39436bf6b772725f61f33e6ec47
commit: 13b1659d6e8cb39436bf6b772725f61f33e6ec47
branch: 3.15
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-09-24T23:52:22Z
summary:
[3.15] gh-154090: Store profiling mode and capture settings in binary profiles
(GH-154105) (#158144)
(cherry picked from commit d3663ef943d984f4c8e349d705b61f393385d817)
files:
A Misc/NEWS.d/next/Library/2026-07-19-14-00-00.gh-issue-154090.q2kH1b.rst
M Include/internal/pycore_global_objects_fini_generated.h
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_runtime_init_generated.h
M Include/internal/pycore_unicodeobject_generated.h
M InternalDocs/profiling_binary_format.md
M Lib/profiling/sampling/binary_collector.py
M Lib/profiling/sampling/binary_reader.py
M Lib/profiling/sampling/cli.py
M Lib/profiling/sampling/stack_collector.py
M Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
M Lib/test/test_profiling/test_sampling_profiler/test_cli.py
M Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
M Modules/_remote_debugging/binary_io.h
M Modules/_remote_debugging/binary_io_reader.c
M Modules/_remote_debugging/binary_io_writer.c
M Modules/_remote_debugging/clinic/module.c.h
M Modules/_remote_debugging/module.c
diff --git a/Include/internal/pycore_global_objects_fini_generated.h
b/Include/internal/pycore_global_objects_fini_generated.h
index bb7524bfc7cff4..c536079498e9cb 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -1641,6 +1641,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capture_features));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cb_type));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(certfile));
diff --git a/Include/internal/pycore_global_strings.h
b/Include/internal/pycore_global_strings.h
index 7f5e4c423b60ea..daa1f2ad2f109d 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -364,6 +364,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(canonical)
STRUCT_FOR_ID(capath)
STRUCT_FOR_ID(capitals)
+ STRUCT_FOR_ID(capture_features)
STRUCT_FOR_ID(category)
STRUCT_FOR_ID(cb_type)
STRUCT_FOR_ID(certfile)
diff --git a/Include/internal/pycore_runtime_init_generated.h
b/Include/internal/pycore_runtime_init_generated.h
index 05e22b21560575..5c6635e6c3c232 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -1639,6 +1639,7 @@ extern "C" {
INIT_ID(canonical), \
INIT_ID(capath), \
INIT_ID(capitals), \
+ INIT_ID(capture_features), \
INIT_ID(category), \
INIT_ID(cb_type), \
INIT_ID(certfile), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h
b/Include/internal/pycore_unicodeobject_generated.h
index da8ced205f3438..ddda6b467ce1c1 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -1236,6 +1236,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp)
{
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(capture_features);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(category);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
diff --git a/InternalDocs/profiling_binary_format.md
b/InternalDocs/profiling_binary_format.md
index 51b1944aa5c8a0..0947e946f41bd0 100644
--- a/InternalDocs/profiling_binary_format.md
+++ b/InternalDocs/profiling_binary_format.md
@@ -84,15 +84,23 @@ with a single seek to `file_size - 32`, without first
reading the header.
| | | | reserved) |
| 12 | 8 | uint64 | Start timestamp (microseconds) |
| 20 | 8 | uint64 | Sample interval (microseconds) |
-| 28 | 4 | uint32 | Total sample count |
-| 32 | 4 | uint32 | Thread count |
-| 36 | 8 | uint64 | String table offset |
-| 44 | 8 | uint64 | Frame table offset |
-| 52 | 4 | uint32 | Compression type (0=none, 1=zstd) |
-| 56 | 8 | bytes | Reserved (zero-filled) |
+| 28 | 8 | uint64 | Total sample count |
+| 36 | 4 | uint32 | Thread count |
+| 40 | 8 | uint64 | String table offset |
+| 48 | 8 | uint64 | Frame table offset |
+| 56 | 4 | uint32 | Compression type (0=none, 1=zstd) |
+| 60 | 4 | uint32 | Profiling configuration bit field |
+--------+------+---------+----------------------------------------+
```
+The low three configuration bits contain the
+`_remote_debugging.PROFILING_MODE_*` value plus one. Zero means that the mode
+was not recorded, which is also the value in binaries written before this
+field was defined. Bit 3 indicates that capture features are known; when set,
+bits 4 through 8 respectively record `--all-threads`, `--native`, GC frames,
+`--opcodes`, and `--blocking`. Remaining bits are reserved for future capture
+features.
+
The magic number `0x54414348` ("TACH" for Tachyon) identifies the file format
and also serves as an **endianness marker**. When read on a system with
different byte order than the writer, it appears as `0x48434154`. The reader
@@ -567,9 +575,10 @@ one write() call (or feeds through the compression stream).
## Future Considerations
The optional profile-statistics block provides an extensible metadata area.
-The 16-byte checksum field in the footer is currently unused. The version
-field allows incompatible changes with graceful rejection. New compression
-types could be added (compression_type > 1).
+The Python-version field retains one reserved byte. The 16-byte checksum
+field in the footer is currently unused. The version field allows
+incompatible changes with graceful rejection. New compression types could
+be added (compression_type > 1).
Any changes that alter the meaning of existing fields or the parsing logic
should increment the version number to prevent older readers from
diff --git a/Lib/profiling/sampling/binary_collector.py
b/Lib/profiling/sampling/binary_collector.py
index ec6020078985ca..7a35044b1ee1d7 100644
--- a/Lib/profiling/sampling/binary_collector.py
+++ b/Lib/profiling/sampling/binary_collector.py
@@ -10,6 +10,23 @@
COMPRESSION_NONE = 0
COMPRESSION_ZSTD = 1
+CAPTURE_FEATURES = {
+ "all_threads": 1 << 0,
+ "native": 1 << 1,
+ "gc": 1 << 2,
+ "opcodes": 1 << 3,
+ "blocking": 1 << 4,
+}
+
+
+def encode_capture_config(capture_config):
+ if capture_config is None:
+ return -1
+ return sum(
+ bit for name, bit in CAPTURE_FEATURES.items()
+ if capture_config.get(name, False)
+ )
+
def _resolve_compression(compression):
"""Resolve compression type from string or int.
@@ -49,7 +66,7 @@ class BinaryCollector(Collector):
"""
def __init__(self, filename, sample_interval_usec, *, skip_idle=False,
- compression='auto'):
+ compression='auto', mode=None, capture_config=None):
"""Create a new binary collector.
Args:
@@ -57,6 +74,9 @@ def __init__(self, filename, sample_interval_usec, *,
skip_idle=False,
sample_interval_usec: Sampling interval in microseconds
skip_idle: If True, skip idle threads (not used in binary format)
compression: 'auto', 'zstd', 'none', or int (0=none, 1=zstd)
+ mode: Profiling mode, or None if unknown
+ capture_config: Mapping of capture feature names to booleans, or
+ None if the capture configuration is unknown
"""
self.filename = filename
self.sample_interval_usec = sample_interval_usec
@@ -65,7 +85,10 @@ def __init__(self, filename, sample_interval_usec, *,
skip_idle=False,
compression_type = _resolve_compression(compression)
start_time_us = int(time.monotonic() * 1_000_000)
self._writer = _remote_debugging.BinaryWriter(
- filename, sample_interval_usec, start_time_us,
compression=compression_type
+ filename, sample_interval_usec, start_time_us,
+ compression=compression_type,
+ mode=-1 if mode is None else mode,
+ capture_features=encode_capture_config(capture_config),
)
def collect(self, stack_frames, timestamp_us=None):
diff --git a/Lib/profiling/sampling/binary_reader.py
b/Lib/profiling/sampling/binary_reader.py
index e611dfd45523f1..2087c0da7c43b6 100644
--- a/Lib/profiling/sampling/binary_reader.py
+++ b/Lib/profiling/sampling/binary_reader.py
@@ -6,6 +6,7 @@
from .stack_collector import FlamegraphCollector, CollapsedStackCollector
from .jsonl_collector import JsonlCollector
from .pstats_collector import PstatsCollector
+from .binary_collector import CAPTURE_FEATURES
class BinaryReader:
@@ -50,10 +51,21 @@ def get_info(self):
- string_count: Number of unique strings
- frame_count: Number of unique frames
- compression: Compression type used
+ - mode: Profiling mode, or None if not recorded
+ - capture_config: Capture feature mapping, or None if not
+ recorded
"""
if self._reader is None:
raise RuntimeError("Reader not open. Use as context manager.")
- return self._reader.get_info()
+ info = self._reader.get_info()
+ capture_features = info.pop("capture_features")
+ info["capture_config"] = (
+ None if capture_features is None else {
+ name: bool(capture_features & bit)
+ for name, bit in CAPTURE_FEATURES.items()
+ }
+ )
+ return info
def replay_samples(self, collector, progress_callback=None):
"""Replay samples from binary file through a collector.
@@ -119,7 +131,7 @@ def convert_binary_to_format(input_file, output_file,
output_format,
elif output_format == 'gecko':
collector = GeckoCollector(interval)
elif output_format == "jsonl":
- collector = JsonlCollector(interval)
+ collector = JsonlCollector(interval, mode=info.get("mode"))
else:
raise ValueError(f"Unknown output format: {output_format}")
@@ -127,6 +139,8 @@ def convert_binary_to_format(input_file, output_file,
output_format,
count = reader.replay_samples(collector, progress_callback)
if hasattr(collector, "set_replay_stats"):
collector.set_replay_stats(info)
+ if hasattr(collector, "set_mode"):
+ collector.set_mode(info.get("mode"))
# Export to target format
collector.export(output_file)
diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py
index db50aefbf7bf52..67adf4c935038b 100644
--- a/Lib/profiling/sampling/cli.py
+++ b/Lib/profiling/sampling/cli.py
@@ -629,9 +629,20 @@ def _sort_to_mode(sort_choice):
}
return sort_map.get(sort_choice, SORT_MODE_NSAMPLES)
+
+def _capture_config_from_args(args):
+ return {
+ "all_threads": args.all_threads,
+ "native": args.native,
+ "gc": args.gc,
+ "opcodes": args.opcodes,
+ "blocking": args.blocking,
+ }
+
+
def _create_collector(format_type, sample_interval_usec, skip_idle,
opcodes=False,
mode=None, output_file=None, compression='auto',
- diff_baseline=None):
+ diff_baseline=None, capture_config=None):
"""Create the appropriate collector based on format type.
Args:
@@ -645,6 +656,7 @@ def _create_collector(format_type, sample_interval_usec,
skip_idle, opcodes=Fals
output_file: Output file path (required for binary format)
compression: Compression type for binary format ('auto', 'zstd',
'none')
diff_baseline: Path to baseline binary file for differential flamegraph
+ capture_config: Capture feature mapping for binary profiles and diffs
Returns:
A collector instance of the appropriate type
@@ -661,7 +673,9 @@ def _create_collector(format_type, sample_interval_usec,
skip_idle, opcodes=Fals
return collector_class(
sample_interval_usec,
baseline_binary_path=diff_baseline,
- skip_idle=skip_idle
+ skip_idle=skip_idle,
+ mode=mode,
+ capture_config=capture_config,
)
# Binary format requires output file and compression
@@ -669,7 +683,8 @@ def _create_collector(format_type, sample_interval_usec,
skip_idle, opcodes=Fals
if output_file is None:
raise ValueError("Binary format requires an output file")
return collector_class(output_file, sample_interval_usec,
skip_idle=skip_idle,
- compression=compression)
+ compression=compression, mode=mode,
+ capture_config=capture_config)
# Gecko format never skips idle (it needs both GIL and CPU data)
# and is the only format that uses opcodes for interval markers
@@ -760,7 +775,9 @@ def _replay_with_reader(args, reader):
collector = _create_collector(
args.format, interval, skip_idle=False,
- diff_baseline=args.diff_baseline
+ mode=info.get("mode"),
+ diff_baseline=args.diff_baseline,
+ capture_config=info.get("capture_config"),
)
def progress_callback(current, total):
@@ -778,6 +795,8 @@ def progress_callback(current, total):
count = reader.replay_samples(collector, progress_callback)
if hasattr(collector, "set_replay_stats"):
collector.set_replay_stats(info)
+ if hasattr(collector, "set_mode"):
+ collector.set_mode(info.get("mode"))
print()
if args.format == "pstats":
@@ -791,7 +810,8 @@ def progress_callback(current, total):
sort_mode = _sort_to_mode(sort_choice)
collector.print_stats(
sort_mode, limit, not args.no_summary,
- PROFILING_MODE_WALL
+ info.get("mode") if info.get("mode") is not None
+ else PROFILING_MODE_WALL
)
else:
filename = (
@@ -1179,7 +1199,8 @@ def _handle_attach(args):
args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode,
output_file=output_file,
compression=getattr(args, 'compression', 'auto'),
- diff_baseline=args.diff_baseline
+ diff_baseline=args.diff_baseline,
+ capture_config=_capture_config_from_args(args),
)
with _get_child_monitor_context(args, args.pid):
@@ -1286,7 +1307,8 @@ def _handle_run(args):
args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode,
output_file=output_file,
compression=getattr(args, 'compression', 'auto'),
- diff_baseline=args.diff_baseline
+ diff_baseline=args.diff_baseline,
+ capture_config=_capture_config_from_args(args),
)
with _get_child_monitor_context(args, process.pid):
diff --git a/Lib/profiling/sampling/stack_collector.py
b/Lib/profiling/sampling/stack_collector.py
index 8de460856666d7..1610f4a3655882 100644
--- a/Lib/profiling/sampling/stack_collector.py
+++ b/Lib/profiling/sampling/stack_collector.py
@@ -161,6 +161,8 @@ def set_replay_stats(self, info):
missed_samples=info.get("missed_samples"),
mode=self.stats.get("mode"),
)
+ def set_mode(self, mode):
+ self.stats["mode"] = mode
def export(self, filename):
flamegraph_data = self._convert_to_flamegraph_format()
@@ -565,13 +567,16 @@ def _create_flamegraph_html(self, data):
class DiffFlamegraphCollector(FlamegraphCollector):
"""Differential flamegraph collector that compares against a baseline
binary profile."""
- def __init__(self, sample_interval_usec, *, baseline_binary_path,
skip_idle=False):
+ def __init__(self, sample_interval_usec, *, baseline_binary_path,
+ skip_idle=False, mode=None, capture_config=None):
super().__init__(sample_interval_usec, skip_idle=skip_idle)
if not os.path.exists(baseline_binary_path):
raise ValueError(f"Baseline file not found:
{baseline_binary_path}")
self.baseline_binary_path = baseline_binary_path
self._baseline_collector = None
self._elided_paths = set()
+ self.mode = mode
+ self.capture_config = capture_config
def _load_baseline(self):
"""Load baseline profile from binary file."""
@@ -580,6 +585,32 @@ def _load_baseline(self):
with BinaryReader(self.baseline_binary_path) as reader:
info = reader.get_info()
+ baseline_mode = info.get("mode")
+ if (
+ baseline_mode is not None
+ and self.mode is not None
+ and baseline_mode != self.mode
+ ):
+ raise ValueError(
+ "Baseline profiling mode does not match current mode"
+ )
+
+ baseline_config = info.get("capture_config")
+ if baseline_config is not None and self.capture_config is not None:
+ names = baseline_config.keys() | self.capture_config.keys()
+ mismatches = [
+ name for name in names
+ if baseline_config.get(name, False)
+ != self.capture_config.get(name, False)
+ ]
+ else:
+ mismatches = []
+ if mismatches:
+ raise ValueError(
+ "Baseline capture configuration does not match current "
+ f"configuration: {', '.join(sorted(mismatches))}"
+ )
+
baseline_collector = FlamegraphCollector(
sample_interval_usec=info['sample_interval_us'],
skip_idle=self.skip_idle
diff --git
a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
index 253f419dae3c2e..ff944b3163ec3e 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
@@ -25,6 +25,7 @@
)
from profiling.sampling.binary_collector import BinaryCollector
from profiling.sampling.binary_reader import BinaryReader,
convert_binary_to_format
+ from profiling.sampling.constants import PROFILING_MODE_CPU
from profiling.sampling.gecko_collector import GeckoCollector
ZSTD_AVAILABLE = _remote_debugging.zstd_available()
@@ -151,19 +152,24 @@ def tearDown(self):
if os.path.exists(f):
os.unlink(f)
- def create_binary_file(self, samples, interval=1000, compression="none"):
+ def create_binary_file(self, samples, interval=1000, compression="none",
+ mode=None, capture_config=None):
"""Create a test binary file and track it for cleanup."""
- filename, _ = self.write_binary_file(samples, interval, compression)
+ filename, _ = self.write_binary_file(
+ samples, interval, compression, mode, capture_config
+ )
return filename
- def write_binary_file(self, samples, interval=1000, compression="none"):
+ def write_binary_file(self, samples, interval=1000, compression="none",
+ mode=None, capture_config=None):
"""Like create_binary_file but also returns the writer collector."""
with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
filename = f.name
self.temp_files.append(filename)
collector = BinaryCollector(
- filename, interval, compression=compression
+ filename, interval, compression=compression, mode=mode,
+ capture_config=capture_config,
)
for sample in samples:
collector.collect(sample)
@@ -516,6 +522,37 @@ def test_sample_interval_preserved(self):
info = reader.get_info()
self.assertEqual(info["sample_interval_us"], interval)
+ def test_profiling_mode_preserved(self):
+ filename = self.create_binary_file([], mode=PROFILING_MODE_CPU)
+ with BinaryReader(filename) as reader:
+ self.assertEqual(reader.get_info()["mode"], PROFILING_MODE_CPU)
+
+ def test_missing_profiling_mode_is_unknown(self):
+ filename = self.create_binary_file([])
+ with BinaryReader(filename) as reader:
+ self.assertIsNone(reader.get_info()["mode"])
+
+ def test_capture_config_preserved(self):
+ capture_config = {
+ "all_threads": True,
+ "native": True,
+ "gc": False,
+ "opcodes": True,
+ "blocking": False,
+ }
+ filename = self.create_binary_file(
+ [], capture_config=capture_config
+ )
+ with BinaryReader(filename) as reader:
+ self.assertEqual(
+ reader.get_info()["capture_config"], capture_config
+ )
+
+ def test_missing_capture_config_is_unknown(self):
+ filename = self.create_binary_file([])
+ with BinaryReader(filename) as reader:
+ self.assertIsNone(reader.get_info()["capture_config"])
+
def test_threads_interleaved_samples(self):
"""Multiple threads with interleaved varying samples."""
samples = []
@@ -1673,8 +1710,10 @@ def test_original_stats_extension_remains_readable(self):
class TestBinaryReplayToJsonl(BinaryFormatTestBase):
"""Tests for binary -> JSONL replay via convert_binary_to_format."""
- def _replay_to_jsonl(self, samples, interval=1000):
- bin_path = self.create_binary_file(samples, interval=interval)
+ def _replay_to_jsonl(self, samples, interval=1000, mode=None):
+ bin_path = self.create_binary_file(
+ samples, interval=interval, mode=mode
+ )
with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f:
jsonl_path = f.name
self.temp_files.append(jsonl_path)
@@ -1704,6 +1743,16 @@ def test_binary_replay_to_jsonl_basic(self):
self.assertEqual(len(frame_defs), 1)
self.assertEqual(frame_defs[0]["line"], 99)
+ def test_binary_replay_to_jsonl_preserves_mode(self):
+ frame = make_frame("hot.py", 99, "hot_func")
+ records = self._replay_to_jsonl(
+ [[make_interpreter(0, [make_thread(1, [frame])])]],
+ mode=PROFILING_MODE_CPU,
+ )
+
+ meta = next(record for record in records if record["type"] == "meta")
+ self.assertEqual(meta["mode"], "cpu")
+
def test_binary_replay_to_jsonl_rle_weight_propagation(self):
"""RLE-batched identical samples land as a single agg entry with the
right total."""
frame = make_frame("rle.py", 42, "rle_func")
diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py
b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py
index 3448258eca5d6c..e5dcd72c27ef5c 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py
@@ -23,11 +23,13 @@
requires_remote_subprocess_debugging,
)
+from profiling.sampling.binary_reader import BinaryReader
from profiling.sampling.cli import (
FORMAT_EXTENSIONS,
_create_collector,
_generate_output_filename,
_handle_output,
+ _replay_with_reader,
main,
)
from profiling.sampling.constants import (
@@ -963,6 +965,41 @@ def test_cli_replay_reader_errors_exit_cleanly(self):
"Error: Unsupported format version 2",
)
+ def test_cli_replay_propagates_recorded_mode(self):
+ reader = mock.MagicMock()
+ reader.get_info.return_value = {
+ "sample_interval_us": 1000,
+ "sample_count": 0,
+ "compression_type": 0,
+ "mode": PROFILING_MODE_CPU,
+ "capture_config": {"all_threads": True},
+ }
+ reader.replay_samples.return_value = 0
+ collector = mock.MagicMock()
+ collector.export.return_value = True
+ args = SimpleNamespace(
+ format="diff_flamegraph",
+ input_file="current.bin",
+ diff_baseline="baseline.bin",
+ outfile="diff.html",
+ browser=False,
+ )
+
+ with mock.patch(
+ "profiling.sampling.cli._create_collector",
+ return_value=collector,
+ ) as create_collector:
+ _replay_with_reader(args, reader)
+
+ create_collector.assert_called_once_with(
+ "diff_flamegraph",
+ 1000,
+ skip_idle=False,
+ mode=PROFILING_MODE_CPU,
+ diff_baseline="baseline.bin",
+ capture_config={"all_threads": True},
+ )
+
def test_cli_jsonl_format_mutually_exclusive_with_pstats(self):
"""--jsonl and --pstats cannot be combined (mutually exclusive
group)."""
with (
@@ -1005,6 +1042,26 @@ def
test_cli_jsonl_create_collector_propagates_mode(self):
meta = next(r for r in records if r["type"] == "meta")
self.assertEqual(meta["mode"], "cpu")
+ def test_cli_binary_create_collector_propagates_mode(self):
+ with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
+ binary_path = f.name
+ self.addCleanup(os.unlink, binary_path)
+ collector = _create_collector(
+ "binary",
+ sample_interval_usec=1000,
+ skip_idle=True,
+ mode=PROFILING_MODE_CPU,
+ output_file=binary_path,
+ compression="none",
+ capture_config={"native": True},
+ )
+ collector.export(None)
+
+ with BinaryReader(binary_path) as reader:
+ info = reader.get_info()
+ self.assertEqual(info["mode"], PROFILING_MODE_CPU)
+ self.assertTrue(info["capture_config"]["native"])
+
def test_cli_jsonl_rejects_opcodes_combination(self):
"""--opcodes is incompatible with --jsonl per
opcodes_compatible_formats."""
test_args = [
diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
index 069a72eb1c88a6..533ca36376e569 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
@@ -1638,6 +1638,50 @@ def test_diff_flamegraph_changed_functions(self):
self.assertAlmostEqual(cold_node["diff"], -1.0)
self.assertAlmostEqual(cold_node["diff_pct"], -50.0)
+ def test_diff_flamegraph_rejects_mismatched_profiling_modes(self):
+ from profiling.sampling.binary_collector import BinaryCollector
+ from profiling.sampling.stack_collector import DiffFlamegraphCollector
+
+ bin_file = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
+ self.addCleanup(close_and_unlink, bin_file)
+ writer = BinaryCollector(
+ bin_file.name,
+ sample_interval_usec=1000,
+ compression="none",
+ mode=PROFILING_MODE_CPU,
+ )
+ writer.export(None)
+
+ diff = DiffFlamegraphCollector(
+ 1000,
+ baseline_binary_path=bin_file.name,
+ mode=PROFILING_MODE_WALL,
+ )
+ with self.assertRaisesRegex(ValueError, "profiling mode"):
+ diff._convert_to_flamegraph_format()
+
+ def test_diff_flamegraph_rejects_mismatched_capture_config(self):
+ from profiling.sampling.binary_collector import BinaryCollector
+ from profiling.sampling.stack_collector import DiffFlamegraphCollector
+
+ bin_file = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
+ self.addCleanup(close_and_unlink, bin_file)
+ writer = BinaryCollector(
+ bin_file.name,
+ sample_interval_usec=1000,
+ compression="none",
+ capture_config={"all_threads": True},
+ )
+ writer.export(None)
+
+ diff = DiffFlamegraphCollector(
+ 1000,
+ baseline_binary_path=bin_file.name,
+ capture_config={"all_threads": False},
+ )
+ with self.assertRaisesRegex(ValueError, "all_threads"):
+ diff._convert_to_flamegraph_format()
+
def test_diff_flamegraph_scale_factor(self):
"""Scale factor adjusts when sample counts differ."""
baseline_frames = [
diff --git
a/Misc/NEWS.d/next/Library/2026-07-19-14-00-00.gh-issue-154090.q2kH1b.rst
b/Misc/NEWS.d/next/Library/2026-07-19-14-00-00.gh-issue-154090.q2kH1b.rst
new file mode 100644
index 00000000000000..f2d2b370241db0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-19-14-00-00.gh-issue-154090.q2kH1b.rst
@@ -0,0 +1,2 @@
+Record the profiling mode in Tachyon binary profiles and reject differential
+comparisons whose known modes do not match.
diff --git a/Modules/_remote_debugging/binary_io.h
b/Modules/_remote_debugging/binary_io.h
index 615e3626e1cbc6..c936d3372e5acd 100644
--- a/Modules/_remote_debugging/binary_io.h
+++ b/Modules/_remote_debugging/binary_io.h
@@ -59,7 +59,9 @@ extern "C" {
#define HDR_SIZE_FRAME_TABLE 8
#define HDR_OFF_COMPRESSION (HDR_OFF_FRAME_TABLE + HDR_SIZE_FRAME_TABLE)
#define HDR_SIZE_COMPRESSION 4
-#define FILE_HEADER_SIZE (HDR_OFF_COMPRESSION + HDR_SIZE_COMPRESSION)
+#define HDR_OFF_CONFIG (HDR_OFF_COMPRESSION + HDR_SIZE_COMPRESSION)
+#define HDR_SIZE_CONFIG 4
+#define FILE_HEADER_SIZE (HDR_OFF_CONFIG + HDR_SIZE_CONFIG)
#define FILE_HEADER_PLACEHOLDER_SIZE 64
static_assert(FILE_HEADER_SIZE <= FILE_HEADER_PLACEHOLDER_SIZE,
@@ -135,6 +137,19 @@ static_assert(PROFILE_STATS_SIZE == 56,
#define COMPRESSION_NONE 0
#define COMPRESSION_ZSTD 1
+/* Profiling configuration. The mode occupies the low three bits and is
+ * stored plus one so zero remains compatible with files written before this
+ * field was defined. Capture features are valid when bit 3 is set. */
+#define PROFILING_CONFIG_MODE_MASK 0x7U
+#define PROFILING_CONFIG_FEATURES_KNOWN (1U << 3)
+#define PROFILING_CONFIG_FEATURES_SHIFT 4
+#define PROFILING_FEATURE_ALL_THREADS (1U << 0)
+#define PROFILING_FEATURE_NATIVE (1U << 1)
+#define PROFILING_FEATURE_GC (1U << 2)
+#define PROFILING_FEATURE_OPCODES (1U << 3)
+#define PROFILING_FEATURE_BLOCKING (1U << 4)
+#define PROFILING_FEATURE_MASK 0x1FU
+
/* Stack encoding types for delta compression */
#define STACK_REPEAT 0x00 /* RLE: identical to previous, with
count */
#define STACK_FULL 0x01 /* Full stack (first sample or no match)
*/
@@ -298,6 +313,8 @@ typedef struct {
double missed_samples;
uint32_t profile_stats_present;
int has_profile_stats;
+ int profiling_mode;
+ int capture_features;
/* String hash table: PyObject* -> uint32_t index */
_Py_hashtable_t *string_hash;
@@ -373,6 +390,8 @@ typedef struct {
int has_profile_stats;
uint64_t string_table_offset;
uint64_t frame_table_offset;
+ int profiling_mode;
+ int capture_features;
/* Parsed string table: array of Python string objects */
PyObject **strings;
@@ -559,6 +578,8 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t
*capacity, size_t elem_
* sample_interval_us: Sampling interval in microseconds
* compression_type: COMPRESSION_NONE or COMPRESSION_ZSTD
* start_time_us: Start timestamp in microseconds (from time.monotonic() *
1e6)
+ * profiling_mode: PROFILING_MODE_* value, or -1 if unknown
+ * capture_features: PROFILING_FEATURE_* bit mask, or -1 if unknown
*
* Returns:
* New BinaryWriter* on success, NULL on failure (PyErr set)
@@ -567,7 +588,9 @@ BinaryWriter *binary_writer_create(
PyObject *path,
uint64_t sample_interval_us,
int compression_type,
- uint64_t start_time_us
+ uint64_t start_time_us,
+ int profiling_mode,
+ int capture_features
);
/*
diff --git a/Modules/_remote_debugging/binary_io_reader.c
b/Modules/_remote_debugging/binary_io_reader.c
index 4e5ccaaabc7270..9625ee6f301f05 100644
--- a/Modules/_remote_debugging/binary_io_reader.c
+++ b/Modules/_remote_debugging/binary_io_reader.c
@@ -85,7 +85,7 @@ reader_parse_header(BinaryReader *reader, const uint8_t
*data, size_t file_size)
/* Read header fields with byte-swapping if needed */
uint64_t start_time_us, sample_interval_us, string_table_offset,
frame_table_offset;
uint64_t sample_count;
- uint32_t thread_count, compression_type;
+ uint32_t thread_count, compression_type, profiling_config;
memcpy(&start_time_us, &data[HDR_OFF_START_TIME], HDR_SIZE_START_TIME);
memcpy(&sample_interval_us, &data[HDR_OFF_INTERVAL], HDR_SIZE_INTERVAL);
@@ -94,6 +94,7 @@ reader_parse_header(BinaryReader *reader, const uint8_t
*data, size_t file_size)
memcpy(&string_table_offset, &data[HDR_OFF_STR_TABLE], HDR_SIZE_STR_TABLE);
memcpy(&frame_table_offset, &data[HDR_OFF_FRAME_TABLE],
HDR_SIZE_FRAME_TABLE);
memcpy(&compression_type, &data[HDR_OFF_COMPRESSION],
HDR_SIZE_COMPRESSION);
+ memcpy(&profiling_config, &data[HDR_OFF_CONFIG], HDR_SIZE_CONFIG);
reader->start_time_us = SWAP64_IF(reader->needs_swap, start_time_us);
reader->sample_interval_us = SWAP64_IF(reader->needs_swap,
sample_interval_us);
@@ -102,6 +103,20 @@ reader_parse_header(BinaryReader *reader, const uint8_t
*data, size_t file_size)
reader->string_table_offset = SWAP64_IF(reader->needs_swap,
string_table_offset);
reader->frame_table_offset = SWAP64_IF(reader->needs_swap,
frame_table_offset);
reader->compression_type = (int)SWAP32_IF(reader->needs_swap,
compression_type);
+ profiling_config = SWAP32_IF(reader->needs_swap, profiling_config);
+ uint32_t profiling_mode =
+ profiling_config & PROFILING_CONFIG_MODE_MASK;
+ if (profiling_mode > PROFILING_MODE_EXCEPTION + 1) {
+ PyErr_Format(PyExc_ValueError,
+ "Invalid profiling mode in header: %u", profiling_mode);
+ return -1;
+ }
+ reader->profiling_mode = (int)profiling_mode - 1;
+ reader->capture_features =
+ profiling_config & PROFILING_CONFIG_FEATURES_KNOWN
+ ? (int)((profiling_config >> PROFILING_CONFIG_FEATURES_SHIFT) &
+ PROFILING_FEATURE_MASK)
+ : -1;
return 0;
}
@@ -1388,8 +1403,31 @@ binary_reader_get_info(BinaryReader *reader)
Py_DECREF(error_rate);
return NULL;
}
+ PyObject *profiling_mode = reader->profiling_mode < 0
+ ? Py_NewRef(Py_None)
+ : PyLong_FromLong(reader->profiling_mode);
+ if (profiling_mode == NULL) {
+ Py_DECREF(py_version);
+ Py_DECREF(duration);
+ Py_DECREF(sample_rate);
+ Py_DECREF(error_rate);
+ Py_DECREF(missed_samples);
+ return NULL;
+ }
+ PyObject *capture_features = reader->capture_features < 0
+ ? Py_NewRef(Py_None)
+ : PyLong_FromLong(reader->capture_features);
+ if (capture_features == NULL) {
+ Py_DECREF(py_version);
+ Py_DECREF(profiling_mode);
+ Py_DECREF(duration);
+ Py_DECREF(sample_rate);
+ Py_DECREF(error_rate);
+ Py_DECREF(missed_samples);
+ return NULL;
+ }
return Py_BuildValue(
- "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i, s:N, s:N, s:N, s:N}",
+ "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i, s:N, s:N, s:N, s:N,
s:N, s:N}",
"version", BINARY_FORMAT_VERSION,
"python_version", py_version,
"start_time_us", reader->start_time_us,
@@ -1402,7 +1440,9 @@ binary_reader_get_info(BinaryReader *reader)
"duration_sec", duration,
"sample_rate", sample_rate,
"error_rate", error_rate,
- "missed_samples", missed_samples
+ "missed_samples", missed_samples,
+ "mode", profiling_mode,
+ "capture_features", capture_features
);
}
diff --git a/Modules/_remote_debugging/binary_io_writer.c
b/Modules/_remote_debugging/binary_io_writer.c
index 753d0b0cc966d0..6af81515e7131d 100644
--- a/Modules/_remote_debugging/binary_io_writer.c
+++ b/Modules/_remote_debugging/binary_io_writer.c
@@ -728,7 +728,8 @@ write_sample_with_encoding(BinaryWriter *writer,
ThreadEntry *entry,
BinaryWriter *
binary_writer_create(PyObject *path, uint64_t sample_interval_us, int
compression_type,
- uint64_t start_time_us)
+ uint64_t start_time_us, int profiling_mode,
+ int capture_features)
{
BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter));
if (!writer) {
@@ -739,6 +740,8 @@ binary_writer_create(PyObject *path, uint64_t
sample_interval_us, int compressio
writer->start_time_us = start_time_us;
writer->sample_interval_us = sample_interval_us;
writer->compression_type = compression_type;
+ writer->profiling_mode = profiling_mode;
+ writer->capture_features = capture_features;
writer->write_buffer = PyMem_Malloc(WRITE_BUFFER_SIZE);
if (!writer->write_buffer) {
@@ -1202,6 +1205,12 @@ binary_writer_finalize(BinaryWriter *writer)
uint64_t frame_table_offset_u64 = (uint64_t)frame_table_offset;
uint32_t thread_count_u32 = (uint32_t)writer->thread_count;
uint32_t compression_type_u32 = (uint32_t)writer->compression_type;
+ uint32_t profiling_config_u32 = (uint32_t)(writer->profiling_mode + 1);
+ if (writer->capture_features >= 0) {
+ profiling_config_u32 |= PROFILING_CONFIG_FEATURES_KNOWN;
+ profiling_config_u32 |= (uint32_t)writer->capture_features
+ << PROFILING_CONFIG_FEATURES_SHIFT;
+ }
uint8_t header[FILE_HEADER_SIZE] = {0};
uint32_t magic = BINARY_FORMAT_MAGIC;
@@ -1218,6 +1227,7 @@ binary_writer_finalize(BinaryWriter *writer)
memcpy(header + HDR_OFF_STR_TABLE, &string_table_offset_u64,
HDR_SIZE_STR_TABLE);
memcpy(header + HDR_OFF_FRAME_TABLE, &frame_table_offset_u64,
HDR_SIZE_FRAME_TABLE);
memcpy(header + HDR_OFF_COMPRESSION, &compression_type_u32,
HDR_SIZE_COMPRESSION);
+ memcpy(header + HDR_OFF_CONFIG, &profiling_config_u32, HDR_SIZE_CONFIG);
if (fwrite_checked_allow_threads(header, FILE_HEADER_SIZE, writer->fp) <
0) {
return -1;
}
diff --git a/Modules/_remote_debugging/clinic/module.c.h
b/Modules/_remote_debugging/clinic/module.c.h
index 341ba000a37a75..30fd0bbea020e5 100644
--- a/Modules/_remote_debugging/clinic/module.c.h
+++ b/Modules/_remote_debugging/clinic/module.c.h
@@ -717,7 +717,7 @@ _remote_debugging_GCMonitor_get_gc_stats(PyObject *self,
PyObject *const *args,
PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__,
"BinaryWriter(filename, sample_interval_us, start_time_us, *,\n"
-" compression=0)\n"
+" compression=0, mode=-1, capture_features=-1)\n"
"--\n"
"\n"
"High-performance binary writer for profiling data.\n"
@@ -728,6 +728,9 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__,
" start_time_us: Start timestamp in microseconds (from\n"
" time.monotonic() * 1e6)\n"
" compression: 0=none, 1=zstd (default: 0)\n"
+" mode: Profiling mode, or -1 if unknown (default: -1)\n"
+" capture_features: Capture feature bit mask, or -1 if unknown\n"
+" (default: -1)\n"
"\n"
"Use as a context manager or call finalize() when done.");
@@ -736,7 +739,8 @@
_remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self,
PyObject *filename,
unsigned long long
sample_interval_us,
unsigned long long start_time_us,
- int compression);
+ int compression, int mode,
+ int capture_features);
static int
_remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args,
PyObject *kwargs)
@@ -744,7 +748,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self,
PyObject *args, PyObject
int return_value = -1;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
- #define NUM_KEYWORDS 4
+ #define NUM_KEYWORDS 6
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
@@ -753,7 +757,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self,
PyObject *args, PyObject
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
- .ob_item = { &_Py_ID(filename), &_Py_ID(sample_interval_us),
&_Py_ID(start_time_us), &_Py_ID(compression), },
+ .ob_item = { &_Py_ID(filename), &_Py_ID(sample_interval_us),
&_Py_ID(start_time_us), &_Py_ID(compression), &_Py_ID(mode),
&_Py_ID(capture_features), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@@ -762,14 +766,14 @@ _remote_debugging_BinaryWriter___init__(PyObject *self,
PyObject *args, PyObject
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
- static const char * const _keywords[] = {"filename", "sample_interval_us",
"start_time_us", "compression", NULL};
+ static const char * const _keywords[] = {"filename", "sample_interval_us",
"start_time_us", "compression", "mode", "capture_features", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "BinaryWriter",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
- PyObject *argsbuf[4];
+ PyObject *argsbuf[6];
PyObject * const *fastargs;
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3;
@@ -777,6 +781,8 @@ _remote_debugging_BinaryWriter___init__(PyObject *self,
PyObject *args, PyObject
unsigned long long sample_interval_us;
unsigned long long start_time_us;
int compression = 0;
+ int mode = -1;
+ int capture_features = -1;
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs,
kwargs, NULL, &_parser,
/*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
@@ -793,12 +799,30 @@ _remote_debugging_BinaryWriter___init__(PyObject *self,
PyObject *args, PyObject
if (!noptargs) {
goto skip_optional_kwonly;
}
- compression = PyLong_AsInt(fastargs[3]);
- if (compression == -1 && PyErr_Occurred()) {
+ if (fastargs[3]) {
+ compression = PyLong_AsInt(fastargs[3]);
+ if (compression == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_kwonly;
+ }
+ }
+ if (fastargs[4]) {
+ mode = PyLong_AsInt(fastargs[4]);
+ if (mode == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_kwonly;
+ }
+ }
+ capture_features = PyLong_AsInt(fastargs[5]);
+ if (capture_features == -1 && PyErr_Occurred()) {
goto exit;
}
skip_optional_kwonly:
- return_value =
_remote_debugging_BinaryWriter___init___impl((BinaryWriterObject *)self,
filename, sample_interval_us, start_time_us, compression);
+ return_value =
_remote_debugging_BinaryWriter___init___impl((BinaryWriterObject *)self,
filename, sample_interval_us, start_time_us, compression, mode,
capture_features);
exit:
return return_value;
@@ -1685,4 +1709,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject
*const *args, Py_ssize
exit:
return return_value;
}
-/*[clinic end generated code: output=287128fc776bf710 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=88bcb21da5a25526 input=a9049054013a1b77]*/
diff --git a/Modules/_remote_debugging/module.c
b/Modules/_remote_debugging/module.c
index 6d23f89c961fd8..abe996b46c1a4b 100644
--- a/Modules/_remote_debugging/module.c
+++ b/Modules/_remote_debugging/module.c
@@ -1717,6 +1717,8 @@ _remote_debugging.BinaryWriter.__init__
start_time_us: unsigned_long_long
*
compression: int = 0
+ mode: int = -1
+ capture_features: int = -1
High-performance binary writer for profiling data.
@@ -1726,6 +1728,9 @@ High-performance binary writer for profiling data.
start_time_us: Start timestamp in microseconds (from
time.monotonic() * 1e6)
compression: 0=none, 1=zstd (default: 0)
+ mode: Profiling mode, or -1 if unknown (default: -1)
+ capture_features: Capture feature bit mask, or -1 if unknown
+ (default: -1)
Use as a context manager or call finalize() when done.
[clinic start generated code]*/
@@ -1735,14 +1740,26 @@
_remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self,
PyObject *filename,
unsigned long long
sample_interval_us,
unsigned long long start_time_us,
- int compression)
-/*[clinic end generated code: output=00446656ea2e5986 input=2e3f298c69fc7666]*/
+ int compression, int mode,
+ int capture_features)
+/*[clinic end generated code: output=3c1c9576795658ce input=98add735b20403ad]*/
{
+ if (mode < -1 || mode > PROFILING_MODE_EXCEPTION) {
+ PyErr_SetString(PyExc_ValueError, "invalid profiling mode");
+ return -1;
+ }
+ if (capture_features < -1 ||
+ capture_features > (int)PROFILING_FEATURE_MASK) {
+ PyErr_SetString(PyExc_ValueError, "invalid capture features");
+ return -1;
+ }
if (self->writer) {
binary_writer_destroy(self->writer);
}
- self->writer = binary_writer_create(filename, sample_interval_us,
compression, start_time_us);
+ self->writer = binary_writer_create(
+ filename, sample_interval_us, compression, start_time_us, mode,
+ capture_features);
if (!self->writer) {
return -1;
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
