https://github.com/python/cpython/commit/7f0ccd6c0e3f85fbaeceb2f67b06ab3631db0480
commit: 7f0ccd6c0e3f85fbaeceb2f67b06ab3631db0480
branch: main
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-08-26T10:24:00+01:00
summary:
gh-154060: Preserve replay duration and rate (#154101)
files:
A Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst
M Include/internal/pycore_global_objects_fini_generated.h
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_runtime_init_generated.h
M Include/internal/pycore_unicodeobject_generated.h
M InternalDocs/profiling_binary_format.md
M Lib/profiling/sampling/binary_collector.py
M Lib/profiling/sampling/binary_reader.py
M Lib/profiling/sampling/cli.py
M Lib/profiling/sampling/stack_collector.py
M Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
M Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
M Modules/_remote_debugging/binary_io.h
M Modules/_remote_debugging/binary_io_reader.c
M Modules/_remote_debugging/binary_io_writer.c
M Modules/_remote_debugging/clinic/module.c.h
M Modules/_remote_debugging/module.c
diff --git a/Include/internal/pycore_global_objects_fini_generated.h
b/Include/internal/pycore_global_objects_fini_generated.h
index c982f76431c52d1..9ab20be70614def 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -1724,6 +1724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration_sec));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory));
@@ -1737,6 +1738,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(entrypoint));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(env));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(error_rate));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask));
@@ -1921,6 +1923,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(missed_samples));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module));
@@ -2053,6 +2056,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_rate));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script));
diff --git a/Include/internal/pycore_global_strings.h
b/Include/internal/pycore_global_strings.h
index c430d0fb99bc41d..51d9fbe89b34235 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -447,6 +447,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(dont_inherit)
STRUCT_FOR_ID(dst)
STRUCT_FOR_ID(dst_dir_fd)
+ STRUCT_FOR_ID(duration_sec)
STRUCT_FOR_ID(eager_start)
STRUCT_FOR_ID(effective_ids)
STRUCT_FOR_ID(element_factory)
@@ -460,6 +461,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(endpos)
STRUCT_FOR_ID(entrypoint)
STRUCT_FOR_ID(env)
+ STRUCT_FOR_ID(error_rate)
STRUCT_FOR_ID(errors)
STRUCT_FOR_ID(event)
STRUCT_FOR_ID(eventmask)
@@ -644,6 +646,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(milliseconds)
STRUCT_FOR_ID(minute)
STRUCT_FOR_ID(minutes)
+ STRUCT_FOR_ID(missed_samples)
STRUCT_FOR_ID(mod)
STRUCT_FOR_ID(mode)
STRUCT_FOR_ID(module)
@@ -776,6 +779,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(rounding)
STRUCT_FOR_ID(salt)
STRUCT_FOR_ID(sample_interval_us)
+ STRUCT_FOR_ID(sample_rate)
STRUCT_FOR_ID(sched_priority)
STRUCT_FOR_ID(scheduler)
STRUCT_FOR_ID(script)
diff --git a/Include/internal/pycore_runtime_init_generated.h
b/Include/internal/pycore_runtime_init_generated.h
index 5a194478f562314..88ca09e6ba245f0 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -1722,6 +1722,7 @@ extern "C" {
INIT_ID(dont_inherit), \
INIT_ID(dst), \
INIT_ID(dst_dir_fd), \
+ INIT_ID(duration_sec), \
INIT_ID(eager_start), \
INIT_ID(effective_ids), \
INIT_ID(element_factory), \
@@ -1735,6 +1736,7 @@ extern "C" {
INIT_ID(endpos), \
INIT_ID(entrypoint), \
INIT_ID(env), \
+ INIT_ID(error_rate), \
INIT_ID(errors), \
INIT_ID(event), \
INIT_ID(eventmask), \
@@ -1919,6 +1921,7 @@ extern "C" {
INIT_ID(milliseconds), \
INIT_ID(minute), \
INIT_ID(minutes), \
+ INIT_ID(missed_samples), \
INIT_ID(mod), \
INIT_ID(mode), \
INIT_ID(module), \
@@ -2051,6 +2054,7 @@ extern "C" {
INIT_ID(rounding), \
INIT_ID(salt), \
INIT_ID(sample_interval_us), \
+ INIT_ID(sample_rate), \
INIT_ID(sched_priority), \
INIT_ID(scheduler), \
INIT_ID(script), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h
b/Include/internal/pycore_unicodeobject_generated.h
index 25529c5ca713a95..3c4d7d664537a8a 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -1568,6 +1568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp)
{
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(duration_sec);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(eager_start);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
@@ -1620,6 +1624,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp)
{
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(error_rate);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(errors);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
@@ -2356,6 +2364,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp)
{
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(missed_samples);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(mod);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
@@ -2884,6 +2896,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp)
{
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(sample_rate);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(sched_priority);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
diff --git a/InternalDocs/profiling_binary_format.md
b/InternalDocs/profiling_binary_format.md
index 7e4592a0d897055..51b1944aa5c8a00 100644
--- a/InternalDocs/profiling_binary_format.md
+++ b/InternalDocs/profiling_binary_format.md
@@ -34,7 +34,7 @@ by 10-50x compared to text formats while also enabling faster
I/O.
## File Layout
-The file consists of five sections:
+The file consists of five required sections and one optional extension:
```
+------------------+ Offset 0
@@ -47,6 +47,8 @@ The file consists of five sections:
| String Table | Variable size
+------------------+ frame_table_offset
| Frame Table | Variable size
++------------------+ file_size - 64 (when stats are present)
+| Profile Stats | 32 bytes (optional)
+------------------+ file_size - 32
| Footer | 32 bytes (fixed)
+------------------+ file_size
@@ -354,6 +356,38 @@ location. Zigzag encoding ensures these small negative
values encode
efficiently (−1 becomes 1, which is one byte) rather than requiring the
maximum varint length.
+## Profile Statistics
+
+New files can store measured duration, sampling rate, error rate, and missed
+sample percentage in an optional 56-byte extension immediately before the
+footer. Older readers ignore these
+bytes after parsing the declared number of frame-table entries, and newer
+readers treat a missing extension as unavailable statistics.
+
+```
+ Offset Size Type Description
++--------+------+---------+----------------------------------------+
+| 0 | 8 | double | Measured duration (seconds) |
+| 8 | 8 | double | Measured sample rate (samples/second) |
+| 16 | 8 | double | Failed sample percentage |
+| 24 | 8 | double | Missed sample percentage |
+| 32 | 4 | uint32 | Optional field presence flags |
+| 36 | 4 | uint32 | Reserved |
+| 40 | 8 | bytes | Signature ("TACHSTAT") |
+| 48 | 4 | uint32 | Extension version (1) |
+| 52 | 4 | uint32 | Extension size (56) |
++--------+------+---------+----------------------------------------+
+```
+
+Putting the signature, version, and size at the end lets readers discover
+the extension from its fixed position relative to the footer while allowing
+future versions to add fields before that trailer. Multi-byte values use the
+same native byte order as the rest of the file and are byte-swapped by
+cross-endian readers.
+
+Readers also accept the original 32-byte extension, which only contains the
+duration and sampling rate.
+
## Footer
```
@@ -448,8 +482,9 @@ compress less; higher levels (6+) compress more but slow
down writing. Level
4. Flush remaining buffered data and finalize compression
5. Write the string table (length-prefixed strings in index order)
6. Write the frame table (varint-encoded entries in index order)
-7. Write the footer with final counts
-8. Seek to offset 0 and write the header with actual values
+7. Write measured profile statistics, when available
+8. Write the footer with final counts
+9. Seek to offset 0 and write the header with actual values
The writer maintains two dictionaries: one mapping strings to indices, one
mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These
@@ -461,12 +496,13 @@ enable O(1) lookup during interning.
if the magic appears byte-swapped)
2. Validate version and read remaining header fields (byte-swapping if needed)
3. Seek to end − 32 and read the footer (byte-swapping counts if needed)
-4. Allocate string array of `string_count` elements
-5. Parse the string table, populating the array
-6. Allocate frame array of `frame_count * 3` uint32 elements
-7. Parse the frame table, populating the array
-8. If compressed, decompress the sample data region
-9. Iterate through samples, resolving indices to strings/frames
+4. Read measured profile statistics when the optional extension is present
+5. Allocate string array of `string_count` elements
+6. Parse the string table, populating the array
+7. Allocate the frame array
+8. Parse the frame table, populating the array
+9. If compressed, decompress the sample data region
+10. Iterate through samples, resolving indices to strings/frames
(byte-swapping thread_id and interpreter_id if needed)
The reader builds lookup arrays rather than dictionaries since it only needs
@@ -530,11 +566,10 @@ one write() call (or feeds through the compression
stream).
## Future Considerations
-The format reserves space for future extensions. The 12 reserved bytes in
-the header could hold additional metadata. The 16-byte checksum field in
-the footer is currently unused. The version field allows incompatible
-changes with graceful rejection. New compression types could be added
-(compression_type > 1).
+The optional profile-statistics block provides an extensible metadata area.
+The 16-byte checksum field in the footer is currently unused. The version
+field allows incompatible changes with graceful rejection. New compression
+types could be added (compression_type > 1).
Any changes that alter the meaning of existing fields or the parsing logic
should increment the version number to prevent older readers from
diff --git a/Lib/profiling/sampling/binary_collector.py
b/Lib/profiling/sampling/binary_collector.py
index afbbc8292690678..ec6020078985caf 100644
--- a/Lib/profiling/sampling/binary_collector.py
+++ b/Lib/profiling/sampling/binary_collector.py
@@ -87,6 +87,13 @@ def collect_failed_sample(self):
"""Record a failed sample attempt (no-op for binary format)."""
pass
+ def set_stats(self, sample_interval_usec, duration_sec, sample_rate,
+ error_rate=None, missed_samples=None, **kwargs):
+ """Persist measured statistics for later replay."""
+ self._writer.set_stats(
+ duration_sec, sample_rate, error_rate, missed_samples
+ )
+
def export(self, filename=None):
"""Finalize and close the binary file.
diff --git a/Lib/profiling/sampling/binary_reader.py
b/Lib/profiling/sampling/binary_reader.py
index a29dad91ae339d1..e611dfd45523f1d 100644
--- a/Lib/profiling/sampling/binary_reader.py
+++ b/Lib/profiling/sampling/binary_reader.py
@@ -125,6 +125,8 @@ def convert_binary_to_format(input_file, output_file,
output_format,
# Replay samples through collector
count = reader.replay_samples(collector, progress_callback)
+ if hasattr(collector, "set_replay_stats"):
+ collector.set_replay_stats(info)
# Export to target format
collector.export(output_file)
diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py
index b47b166c6cd9408..83af313e1efcbdb 100644
--- a/Lib/profiling/sampling/cli.py
+++ b/Lib/profiling/sampling/cli.py
@@ -776,6 +776,8 @@ def progress_callback(current, total):
)
count = reader.replay_samples(collector, progress_callback)
+ if hasattr(collector, "set_replay_stats"):
+ collector.set_replay_stats(info)
print()
if args.format == "pstats":
diff --git a/Lib/profiling/sampling/stack_collector.py
b/Lib/profiling/sampling/stack_collector.py
index dd86d5d54a9aaa3..8de460856666d7f 100644
--- a/Lib/profiling/sampling/stack_collector.py
+++ b/Lib/profiling/sampling/stack_collector.py
@@ -103,7 +103,6 @@ def collect(self, stack_frames, timestamps_us=None):
"""Override to track thread status statistics before processing
frames."""
# Weight is number of timestamps (samples with identical stack)
weight = len(timestamps_us) if timestamps_us else 1
-
# Increment sample count by weight
self._sample_count += weight
@@ -148,6 +147,21 @@ def set_stats(self, sample_interval_usec, duration_sec,
sample_rate,
"mode": mode
}
+ def set_replay_stats(self, info):
+ """Restore measured statistics stored in a binary profile."""
+ duration_sec = info.get("duration_sec")
+ sample_rate = info.get("sample_rate")
+ if duration_sec is None or sample_rate is None:
+ return
+ self.set_stats(
+ self.sample_interval_usec,
+ duration_sec,
+ sample_rate,
+ error_rate=info.get("error_rate"),
+ missed_samples=info.get("missed_samples"),
+ mode=self.stats.get("mode"),
+ )
+
def export(self, filename):
flamegraph_data = self._convert_to_flamegraph_format()
diff --git
a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
index e4963dca9c96636..253f419dae3c2e7 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
@@ -1580,6 +1580,96 @@ def test_timestamp_preservation_with_rle(self):
self.assertEqual(ts_collector.all_timestamps, expected_timestamps)
+class TestBinaryReplayToFlamegraph(BinaryFormatTestBase):
+ def test_replay_includes_persisted_stats(self):
+ frames = [
+ make_frame("hot.py", 99, "hot_func"),
+ make_frame("main.py", 1, "main"),
+ ]
+ samples = [
+ [
+ make_interpreter(
+ 0,
+ [make_thread(1, frames, THREAD_STATUS_HAS_GIL)],
+ )
+ ]
+ for _ in range(5)
+ ]
+ with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as file:
+ bin_path = file.name
+ self.temp_files.append(bin_path)
+ collector = BinaryCollector(bin_path, 2000, compression="none")
+ for sample in samples:
+ collector.collect(sample)
+ collector.set_stats(
+ 2000, 1.25, 4.0, error_rate=2.5, missed_samples=1.5
+ )
+ collector.export(None)
+
+ with BinaryReader(bin_path) as reader:
+ info = reader.get_info()
+ self.assertEqual(info["duration_sec"], 1.25)
+ self.assertEqual(info["sample_rate"], 4.0)
+ self.assertEqual(info["error_rate"], 2.5)
+ self.assertEqual(info["missed_samples"], 1.5)
+
+ with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as file:
+ html_path = file.name
+ self.temp_files.append(html_path)
+
+ convert_binary_to_format(bin_path, html_path, "flamegraph")
+
+ with open(html_path, encoding="utf-8") as file:
+ content = file.read()
+ self.assertIn('"duration_sec": 1.25', content)
+ self.assertIn('"sample_rate": 4.0', content)
+ self.assertIn('"error_rate": 2.5', content)
+ self.assertIn('"missed_samples": 1.5', content)
+
+ def test_legacy_binary_has_no_measured_stats(self):
+ frame = make_frame("hot.py", 99, "hot_func")
+ bin_path = self.create_binary_file([
+ [make_interpreter(0, [make_thread(1, [frame])])]
+ ])
+
+ with BinaryReader(bin_path) as reader:
+ info = reader.get_info()
+
+ self.assertIsNone(info["duration_sec"])
+ self.assertIsNone(info["sample_rate"])
+ self.assertIsNone(info["error_rate"])
+ self.assertIsNone(info["missed_samples"])
+
+ def test_original_stats_extension_remains_readable(self):
+ frame = make_frame("hot.py", 99, "hot_func")
+ with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as file:
+ bin_path = file.name
+ self.temp_files.append(bin_path)
+ collector = BinaryCollector(bin_path, 2000, compression="none")
+ collector.collect([
+ make_interpreter(0, [make_thread(1, [frame])])
+ ])
+ collector.set_stats(2000, 1.25, 4.0)
+ collector.export(None)
+
+ with open(bin_path, "rb") as file:
+ data = file.read()
+ stats = data[-88:-32]
+ footer = bytearray(data[-32:])
+ old_stats = stats[:16] + b"TACHSTAT" + struct.pack("=II", 1, 32)
+ old_data = bytearray(data[:-88] + old_stats + footer)
+ struct.pack_into("=Q", old_data, -24, len(old_data))
+ with open(bin_path, "wb") as file:
+ file.write(old_data)
+
+ with BinaryReader(bin_path) as reader:
+ info = reader.get_info()
+ self.assertEqual(info["duration_sec"], 1.25)
+ self.assertEqual(info["sample_rate"], 4.0)
+ self.assertIsNone(info["error_rate"])
+ self.assertIsNone(info["missed_samples"])
+
+
class TestBinaryReplayToJsonl(BinaryFormatTestBase):
"""Tests for binary -> JSONL replay via convert_binary_to_format."""
diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
index 13e112f78014b9f..069a72eb1c88a6a 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py
@@ -1159,6 +1159,30 @@ def test_flamegraph_collector_stats_accumulation(self):
collector.collect(stack_frames_gc)
self.assertEqual(collector.samples_with_gc_frames, 2)
+ def test_flamegraph_collector_restores_replay_stats(self):
+ """Replay restores measured values from binary metadata."""
+ collector = FlamegraphCollector(1000)
+ collector.set_replay_stats({
+ "duration_sec": 1.25,
+ "sample_rate": 4.0,
+ "error_rate": 2.5,
+ "missed_samples": 1.5,
+ })
+
+ self.assertEqual(collector.stats["duration_sec"], 1.25)
+ self.assertEqual(collector.stats["sample_rate"], 4.0)
+ self.assertEqual(collector.stats["error_rate"], 2.5)
+ self.assertEqual(collector.stats["missed_samples"], 1.5)
+
+ def test_flamegraph_collector_leaves_legacy_replay_stats_unavailable(self):
+ collector = FlamegraphCollector(1000)
+ original_stats = collector.stats.copy()
+ collector.set_replay_stats({
+ "duration_sec": None,
+ "sample_rate": None,
+ })
+ self.assertEqual(collector.stats, original_stats)
+
def test_flamegraph_collector_per_thread_stats(self):
"""Test per-thread statistics tracking in FlamegraphCollector."""
collector = FlamegraphCollector(sample_interval_usec=1000)
diff --git
a/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst
b/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst
new file mode 100644
index 000000000000000..a9df0e6ad399631
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-19-12-10-00.gh-issue-154060.cR6tYn.rst
@@ -0,0 +1,2 @@
+Store measured duration and sampling rate in Tachyon binary profiles so they
+remain available in replayed flamegraphs.
diff --git a/Modules/_remote_debugging/binary_io.h
b/Modules/_remote_debugging/binary_io.h
index 8c34668ab15ae7e..615e3626e1cbc6f 100644
--- a/Modules/_remote_debugging/binary_io.h
+++ b/Modules/_remote_debugging/binary_io.h
@@ -91,6 +91,37 @@ static_assert(SAMPLE_HEADER_FIXED_SIZE == 13,
static_assert(FILE_FOOTER_SIZE == 32,
"FILE_FOOTER_SIZE must remain 32");
+/* Optional profiling statistics immediately precede the footer. The
+ * signature and size live at the end so readers can discover extensions
+ * without changing the fixed header or moving streamed sample data. */
+#define PROFILE_STATS_MAGIC "TACHSTAT"
+#define PROFILE_STATS_MAGIC_SIZE 8
+#define PROFILE_STATS_VERSION 1
+#define PST_OFF_DURATION 0
+#define PST_SIZE_DURATION 8
+#define PST_OFF_SAMPLE_RATE (PST_OFF_DURATION + PST_SIZE_DURATION)
+#define PST_SIZE_SAMPLE_RATE 8
+#define PST_OFF_ERROR_RATE (PST_OFF_SAMPLE_RATE + PST_SIZE_SAMPLE_RATE)
+#define PST_SIZE_ERROR_RATE 8
+#define PST_OFF_MISSED_SAMPLES (PST_OFF_ERROR_RATE + PST_SIZE_ERROR_RATE)
+#define PST_SIZE_MISSED_SAMPLES 8
+#define PST_OFF_PRESENT (PST_OFF_MISSED_SAMPLES +
PST_SIZE_MISSED_SAMPLES)
+#define PST_SIZE_PRESENT 4
+#define PST_OFF_RESERVED (PST_OFF_PRESENT + PST_SIZE_PRESENT)
+#define PST_SIZE_RESERVED 4
+#define PST_OFF_MAGIC (PST_OFF_RESERVED + PST_SIZE_RESERVED)
+#define PST_OFF_VERSION (PST_OFF_MAGIC + PROFILE_STATS_MAGIC_SIZE)
+#define PST_SIZE_VERSION 4
+#define PST_OFF_SIZE (PST_OFF_VERSION + PST_SIZE_VERSION)
+#define PST_SIZE_SIZE 4
+#define PROFILE_STATS_SIZE (PST_OFF_SIZE + PST_SIZE_SIZE)
+#define PROFILE_STATS_V1_SIZE 32
+#define PROFILE_STATS_ERROR_RATE 0x01
+#define PROFILE_STATS_MISSED 0x02
+
+static_assert(PROFILE_STATS_SIZE == 56,
+ "PROFILE_STATS_SIZE must remain 56");
+
/* Minimum on-disk bytes of a string (1) and frame (7) table entry. */
#define MIN_STRING_ENTRY_SIZE 1
#define MIN_FRAME_ENTRY_SIZE 7
@@ -261,6 +292,12 @@ typedef struct {
uint64_t start_time_us;
uint64_t sample_interval_us;
uint64_t total_samples;
+ double duration_sec;
+ double sample_rate;
+ double error_rate;
+ double missed_samples;
+ uint32_t profile_stats_present;
+ int has_profile_stats;
/* String hash table: PyObject* -> uint32_t index */
_Py_hashtable_t *string_hash;
@@ -328,6 +365,12 @@ typedef struct {
uint64_t sample_interval_us;
uint64_t sample_count;
uint32_t thread_count;
+ double duration_sec;
+ double sample_rate;
+ double error_rate;
+ double missed_samples;
+ uint32_t profile_stats_present;
+ int has_profile_stats;
uint64_t string_table_offset;
uint64_t frame_table_offset;
@@ -556,6 +599,16 @@ int binary_writer_write_sample(
*/
int binary_writer_finalize(BinaryWriter *writer);
+/* Store measured statistics to write during finalization. */
+int binary_writer_set_stats(
+ BinaryWriter *writer,
+ double duration_sec,
+ double sample_rate,
+ double error_rate,
+ double missed_samples,
+ uint32_t present
+);
+
/*
* Destroy a binary writer and free all resources.
* Safe to call even if writer is partially initialized.
diff --git a/Modules/_remote_debugging/binary_io_reader.c
b/Modules/_remote_debugging/binary_io_reader.c
index 07352f94cc805d7..4e5ccaaabc72700 100644
--- a/Modules/_remote_debugging/binary_io_reader.c
+++ b/Modules/_remote_debugging/binary_io_reader.c
@@ -12,6 +12,7 @@
#include "binary_io.h"
#include "_remote_debugging.h"
#include "pycore_bitutils.h" /* _Py_bswap32, _Py_bswap64 for cross-endian
reading */
+#include <math.h>
#include <string.h>
#ifdef HAVE_ZSTD
@@ -125,6 +126,88 @@ reader_parse_footer(BinaryReader *reader, const uint8_t
*data, size_t file_size)
return 0;
}
+static inline int
+reader_parse_profile_stats(BinaryReader *reader, const uint8_t *data,
+ size_t file_size)
+{
+ size_t minimum_size = FILE_FOOTER_SIZE + PROFILE_STATS_V1_SIZE;
+ if (file_size < minimum_size) {
+ return 0;
+ }
+
+ const uint8_t *stats_end = data + file_size - FILE_FOOTER_SIZE;
+ const uint8_t *stats_tail = stats_end - PROFILE_STATS_MAGIC_SIZE -
+ PST_SIZE_VERSION - PST_SIZE_SIZE;
+ if (memcmp(stats_tail,
+ PROFILE_STATS_MAGIC, PROFILE_STATS_MAGIC_SIZE) != 0) {
+ return 0;
+ }
+
+ uint32_t version, size;
+ memcpy(&version, stats_tail + PROFILE_STATS_MAGIC_SIZE,
+ PST_SIZE_VERSION);
+ memcpy(&size, stats_tail + PROFILE_STATS_MAGIC_SIZE + PST_SIZE_VERSION,
+ PST_SIZE_SIZE);
+ version = SWAP32_IF(reader->needs_swap, version);
+ size = SWAP32_IF(reader->needs_swap, size);
+
+ if (size < PROFILE_STATS_V1_SIZE ||
+ size > file_size - FILE_FOOTER_SIZE) {
+ PyErr_SetString(PyExc_ValueError,
+ "Invalid profiling statistics size");
+ return -1;
+ }
+ if (version != PROFILE_STATS_VERSION) {
+ return 0;
+ }
+
+ const uint8_t *trailer = stats_end - size;
+ uint64_t duration_bits, sample_rate_bits;
+ memcpy(&duration_bits, trailer + PST_OFF_DURATION, PST_SIZE_DURATION);
+ memcpy(&sample_rate_bits, trailer + PST_OFF_SAMPLE_RATE,
+ PST_SIZE_SAMPLE_RATE);
+ duration_bits = SWAP64_IF(reader->needs_swap, duration_bits);
+ sample_rate_bits = SWAP64_IF(reader->needs_swap, sample_rate_bits);
+ memcpy(&reader->duration_sec, &duration_bits, sizeof(duration_bits));
+ memcpy(&reader->sample_rate, &sample_rate_bits, sizeof(sample_rate_bits));
+
+ if (!isfinite(reader->duration_sec) || reader->duration_sec < 0.0 ||
+ !isfinite(reader->sample_rate) || reader->sample_rate < 0.0) {
+ PyErr_SetString(PyExc_ValueError,
+ "Invalid profiling statistics values");
+ return -1;
+ }
+ reader->has_profile_stats = 1;
+
+ if (size >= PROFILE_STATS_SIZE) {
+ uint64_t error_rate_bits, missed_samples_bits;
+ uint32_t present;
+ memcpy(&error_rate_bits, trailer + PST_OFF_ERROR_RATE,
+ PST_SIZE_ERROR_RATE);
+ memcpy(&missed_samples_bits, trailer + PST_OFF_MISSED_SAMPLES,
+ PST_SIZE_MISSED_SAMPLES);
+ memcpy(&present, trailer + PST_OFF_PRESENT, PST_SIZE_PRESENT);
+ error_rate_bits = SWAP64_IF(reader->needs_swap, error_rate_bits);
+ missed_samples_bits = SWAP64_IF(reader->needs_swap,
+ missed_samples_bits);
+ present = SWAP32_IF(reader->needs_swap, present);
+ memcpy(&reader->error_rate, &error_rate_bits,
+ sizeof(error_rate_bits));
+ memcpy(&reader->missed_samples, &missed_samples_bits,
+ sizeof(missed_samples_bits));
+ if (((present & PROFILE_STATS_ERROR_RATE) &&
+ (!isfinite(reader->error_rate) || reader->error_rate < 0.0)) ||
+ ((present & PROFILE_STATS_MISSED) &&
+ !isfinite(reader->missed_samples))) {
+ PyErr_SetString(PyExc_ValueError,
+ "Invalid profiling statistics values");
+ return -1;
+ }
+ reader->profile_stats_present = present;
+ }
+ return 0;
+}
+
#ifdef HAVE_ZSTD
/* Maximum decompression buffer size to prevent memory exhaustion (1GB) */
#define MAX_DECOMPRESS_SIZE (1ULL << 30)
@@ -538,6 +621,9 @@ binary_reader_open(PyObject *path)
if (reader_parse_footer(reader, data, file_size) < 0) {
goto error;
}
+ if (reader_parse_profile_stats(reader, data, file_size) < 0) {
+ goto error;
+ }
/* Validate table offsets are within file bounds */
if (reader->string_table_offset > file_size) {
@@ -1270,8 +1356,40 @@ binary_reader_get_info(BinaryReader *reader)
if (py_version == NULL) {
return NULL;
}
+ PyObject *duration = reader->has_profile_stats
+ ? PyFloat_FromDouble(reader->duration_sec) : Py_NewRef(Py_None);
+ if (duration == NULL) {
+ Py_DECREF(py_version);
+ return NULL;
+ }
+ PyObject *sample_rate = reader->has_profile_stats
+ ? PyFloat_FromDouble(reader->sample_rate) : Py_NewRef(Py_None);
+ if (sample_rate == NULL) {
+ Py_DECREF(py_version);
+ Py_DECREF(duration);
+ return NULL;
+ }
+ PyObject *error_rate =
+ reader->profile_stats_present & PROFILE_STATS_ERROR_RATE
+ ? PyFloat_FromDouble(reader->error_rate) : Py_NewRef(Py_None);
+ if (error_rate == NULL) {
+ Py_DECREF(py_version);
+ Py_DECREF(duration);
+ Py_DECREF(sample_rate);
+ return NULL;
+ }
+ PyObject *missed_samples =
+ reader->profile_stats_present & PROFILE_STATS_MISSED
+ ? PyFloat_FromDouble(reader->missed_samples) : Py_NewRef(Py_None);
+ if (missed_samples == NULL) {
+ Py_DECREF(py_version);
+ Py_DECREF(duration);
+ Py_DECREF(sample_rate);
+ Py_DECREF(error_rate);
+ return NULL;
+ }
return Py_BuildValue(
- "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i}",
+ "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i, s:N, s:N, s:N, s:N}",
"version", BINARY_FORMAT_VERSION,
"python_version", py_version,
"start_time_us", reader->start_time_us,
@@ -1280,7 +1398,11 @@ binary_reader_get_info(BinaryReader *reader)
"thread_count", reader->thread_count,
"string_count", reader->strings_count,
"frame_count", reader->frames_count,
- "compression_type", reader->compression_type
+ "compression_type", reader->compression_type,
+ "duration_sec", duration,
+ "sample_rate", sample_rate,
+ "error_rate", error_rate,
+ "missed_samples", missed_samples
);
}
diff --git a/Modules/_remote_debugging/binary_io_writer.c
b/Modules/_remote_debugging/binary_io_writer.c
index e2ac40d96b39bee..753d0b0cc966d08 100644
--- a/Modules/_remote_debugging/binary_io_writer.c
+++ b/Modules/_remote_debugging/binary_io_writer.c
@@ -12,6 +12,7 @@
#include "binary_io.h"
#include "_remote_debugging.h"
#include "pycore_opcode_utils.h" // MAX_REAL_OPCODE
+#include <math.h>
#include <string.h>
#ifdef HAVE_ZSTD
@@ -1146,6 +1147,30 @@ binary_writer_finalize(BinaryWriter *writer)
}
}
+ if (writer->has_profile_stats) {
+ uint8_t profile_stats[PROFILE_STATS_SIZE] = {0};
+ uint32_t version = PROFILE_STATS_VERSION;
+ uint32_t size = PROFILE_STATS_SIZE;
+ memcpy(profile_stats + PST_OFF_DURATION,
+ &writer->duration_sec, PST_SIZE_DURATION);
+ memcpy(profile_stats + PST_OFF_SAMPLE_RATE,
+ &writer->sample_rate, PST_SIZE_SAMPLE_RATE);
+ memcpy(profile_stats + PST_OFF_ERROR_RATE,
+ &writer->error_rate, PST_SIZE_ERROR_RATE);
+ memcpy(profile_stats + PST_OFF_MISSED_SAMPLES,
+ &writer->missed_samples, PST_SIZE_MISSED_SAMPLES);
+ memcpy(profile_stats + PST_OFF_PRESENT,
+ &writer->profile_stats_present, PST_SIZE_PRESENT);
+ memcpy(profile_stats + PST_OFF_MAGIC,
+ PROFILE_STATS_MAGIC, PROFILE_STATS_MAGIC_SIZE);
+ memcpy(profile_stats + PST_OFF_VERSION, &version, PST_SIZE_VERSION);
+ memcpy(profile_stats + PST_OFF_SIZE, &size, PST_SIZE_SIZE);
+ if (fwrite_checked_allow_threads(
+ profile_stats, PROFILE_STATS_SIZE, writer->fp) < 0) {
+ return -1;
+ }
+ }
+
/* Footer: string_count(4) + frame_count(4) + file_size(8) + checksum(16)
*/
file_offset_t footer_offset = FTELL64(writer->fp);
if (footer_offset < 0) {
@@ -1207,6 +1232,41 @@ binary_writer_finalize(BinaryWriter *writer)
return 0;
}
+int
+binary_writer_set_stats(BinaryWriter *writer, double duration_sec,
+ double sample_rate, double error_rate,
+ double missed_samples, uint32_t present)
+{
+ if (!isfinite(duration_sec) || duration_sec < 0.0) {
+ PyErr_SetString(PyExc_ValueError,
+ "duration must be a finite non-negative value");
+ return -1;
+ }
+ if (!isfinite(sample_rate) || sample_rate < 0.0) {
+ PyErr_SetString(PyExc_ValueError,
+ "sample rate must be a finite non-negative value");
+ return -1;
+ }
+ if ((present & PROFILE_STATS_ERROR_RATE) &&
+ (!isfinite(error_rate) || error_rate < 0.0)) {
+ PyErr_SetString(PyExc_ValueError,
+ "error rate must be a finite non-negative value");
+ return -1;
+ }
+ if ((present & PROFILE_STATS_MISSED) && !isfinite(missed_samples)) {
+ PyErr_SetString(PyExc_ValueError,
+ "missed samples must be a finite value");
+ return -1;
+ }
+ writer->duration_sec = duration_sec;
+ writer->sample_rate = sample_rate;
+ writer->error_rate = error_rate;
+ writer->missed_samples = missed_samples;
+ writer->profile_stats_present = present;
+ writer->has_profile_stats = 1;
+ return 0;
+}
+
void
binary_writer_destroy(BinaryWriter *writer)
{
diff --git a/Modules/_remote_debugging/clinic/module.c.h
b/Modules/_remote_debugging/clinic/module.c.h
index ad66d947d19c52f..efbf529922b33df 100644
--- a/Modules/_remote_debugging/clinic/module.c.h
+++ b/Modules/_remote_debugging/clinic/module.c.h
@@ -873,6 +873,103 @@ _remote_debugging_BinaryWriter_write_sample(PyObject
*self, PyObject *const *arg
return return_value;
}
+PyDoc_STRVAR(_remote_debugging_BinaryWriter_set_stats__doc__,
+"set_stats($self, /, duration_sec, sample_rate, error_rate=None,\n"
+" missed_samples=None)\n"
+"--\n"
+"\n"
+"Store measured profile statistics in the binary file.");
+
+#define _REMOTE_DEBUGGING_BINARYWRITER_SET_STATS_METHODDEF \
+ {"set_stats", _PyCFunction_CAST(_remote_debugging_BinaryWriter_set_stats),
METH_FASTCALL|METH_KEYWORDS, _remote_debugging_BinaryWriter_set_stats__doc__},
+
+static PyObject *
+_remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self,
+ double duration_sec,
+ double sample_rate,
+ PyObject *error_rate,
+ PyObject *missed_samples);
+
+static PyObject *
+_remote_debugging_BinaryWriter_set_stats(PyObject *self, PyObject *const
*args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 4
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ Py_hash_t ob_hash;
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_hash = -1,
+ .ob_item = { &_Py_ID(duration_sec), &_Py_ID(sample_rate),
&_Py_ID(error_rate), &_Py_ID(missed_samples), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"duration_sec", "sample_rate",
"error_rate", "missed_samples", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "set_stats",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[4];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) -
2;
+ double duration_sec;
+ double sample_rate;
+ PyObject *error_rate = Py_None;
+ PyObject *missed_samples = Py_None;
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser,
+ /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (PyFloat_CheckExact(args[0])) {
+ duration_sec = PyFloat_AS_DOUBLE(args[0]);
+ }
+ else
+ {
+ duration_sec = PyFloat_AsDouble(args[0]);
+ if (duration_sec == -1.0 && PyErr_Occurred()) {
+ goto exit;
+ }
+ }
+ if (PyFloat_CheckExact(args[1])) {
+ sample_rate = PyFloat_AS_DOUBLE(args[1]);
+ }
+ else
+ {
+ sample_rate = PyFloat_AsDouble(args[1]);
+ if (sample_rate == -1.0 && PyErr_Occurred()) {
+ goto exit;
+ }
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[2]) {
+ error_rate = args[2];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ missed_samples = args[3];
+skip_optional_pos:
+ return_value =
_remote_debugging_BinaryWriter_set_stats_impl((BinaryWriterObject *)self,
duration_sec, sample_rate, error_rate, missed_samples);
+
+exit:
+ return return_value;
+}
+
PyDoc_STRVAR(_remote_debugging_BinaryWriter_finalize__doc__,
"finalize($self, /)\n"
"--\n"
@@ -1588,4 +1685,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject
*const *args, Py_ssize
exit:
return return_value;
}
-/*[clinic end generated code: output=22511c10d9224b28 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=5f20a08be0d0ed5a input=a9049054013a1b77]*/
diff --git a/Modules/_remote_debugging/module.c
b/Modules/_remote_debugging/module.c
index 4f365f303f14c77..708632bf603ddca 100644
--- a/Modules/_remote_debugging/module.c
+++ b/Modules/_remote_debugging/module.c
@@ -1781,6 +1781,53 @@
_remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self,
Py_RETURN_NONE;
}
+/*[clinic input]
+_remote_debugging.BinaryWriter.set_stats
+ duration_sec: double
+ sample_rate: double
+ error_rate: object = None
+ missed_samples: object = None
+
+Store measured profile statistics in the binary file.
+[clinic start generated code]*/
+
+static PyObject *
+_remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self,
+ double duration_sec,
+ double sample_rate,
+ PyObject *error_rate,
+ PyObject *missed_samples)
+/*[clinic end generated code: output=28ab1bdd7c631a97 input=1646e7182f4c2259]*/
+{
+ if (!self->writer) {
+ PyErr_SetString(PyExc_ValueError, "Writer is closed");
+ return NULL;
+ }
+ uint32_t present = 0;
+ double error_rate_value = 0.0;
+ double missed_samples_value = 0.0;
+ if (error_rate != Py_None) {
+ error_rate_value = PyFloat_AsDouble(error_rate);
+ if (error_rate_value == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ present |= PROFILE_STATS_ERROR_RATE;
+ }
+ if (missed_samples != Py_None) {
+ missed_samples_value = PyFloat_AsDouble(missed_samples);
+ if (missed_samples_value == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ present |= PROFILE_STATS_MISSED;
+ }
+ if (binary_writer_set_stats(self->writer, duration_sec, sample_rate,
+ error_rate_value, missed_samples_value,
+ present) < 0) {
+ return NULL;
+ }
+ Py_RETURN_NONE;
+}
+
/* Finalize the writer, cache total_samples, and destroy it.
*
* The cache assignment must happen AFTER binary_writer_finalize(): finalize
@@ -1928,6 +1975,7 @@ static PyGetSetDef BinaryWriter_getset[] = {
static PyMethodDef BinaryWriter_methods[] = {
_REMOTE_DEBUGGING_BINARYWRITER_WRITE_SAMPLE_METHODDEF
+ _REMOTE_DEBUGGING_BINARYWRITER_SET_STATS_METHODDEF
_REMOTE_DEBUGGING_BINARYWRITER_FINALIZE_METHODDEF
_REMOTE_DEBUGGING_BINARYWRITER_CLOSE_METHODDEF
_REMOTE_DEBUGGING_BINARYWRITER___ENTER___METHODDEF
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
