https://github.com/python/cpython/commit/639a5525fb1b54973dcf55a636efc10be4042239
commit: 639a5525fb1b54973dcf55a636efc10be4042239
branch: main
author: Maurycy Pawłowski-Wieroński <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-07-04T20:03:01+01:00
summary:
gh-151378: Fix unbounded memory growth when recording binary and the thread is
sleeping (#152089)
files:
A Misc/NEWS.d/next/Library/2026-06-24-16-40-15.gh-issue-151378.iYWa_6.rst
M Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
M Modules/_remote_debugging/binary_io.h
M Modules/_remote_debugging/binary_io_writer.c
M Modules/_remote_debugging/module.c
diff --git
a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
index 5efc60a92111754..56dac2cf86a7647 100644
--- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
+++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py
@@ -932,6 +932,29 @@ def
test_writer_total_samples_after_finalize_matches_reader(self):
self.assertEqual(writer_collector.total_samples, len(samples))
self.assertEqual(writer_collector.total_samples, replayed)
+ def test_rle_buffer_flushes_before_finalize(self):
+ """RLE buffer flushes before the writer is destroyed during
finalize()."""
+ with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f:
+ filename = f.name
+ self.temp_files.append(filename)
+
+ frame = make_frame("rle_boundary.py", 1, "hot")
+ sample = [make_interpreter(0, [make_thread(1, [frame])])]
+
+ writer = _remote_debugging.BinaryWriter(filename, 1000, 0,
compression=0)
+ for i in range(10_000):
+ writer.write_sample(sample, i * 1000)
+ self.assertGreater(writer.get_stats()["repeat_records"], 0)
+
+ writer.finalize()
+
+ reader_collector = RawCollector()
+ with BinaryReader(filename) as reader:
+ replayed = reader.replay_samples(reader_collector)
+
+ self.assertEqual(replayed, 10_000)
+ self.assertEqual(writer.total_samples, 10_000)
+
def test_writer_total_samples_after_context_manager_matches_reader(self):
"""total_samples after `with BinaryWriter(...)` matches the reader's
count.
diff --git
a/Misc/NEWS.d/next/Library/2026-06-24-16-40-15.gh-issue-151378.iYWa_6.rst
b/Misc/NEWS.d/next/Library/2026-06-24-16-40-15.gh-issue-151378.iYWa_6.rst
new file mode 100644
index 000000000000000..0b3eb860ba8d329
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-06-24-16-40-15.gh-issue-151378.iYWa_6.rst
@@ -0,0 +1,3 @@
+Fix a bug in the binary collector in Tachyon that caused unbounded memory
+growth while profiling a thread that stayed asleep. Patch by Maurycy
+Pawłowski-Wieroński.
diff --git a/Modules/_remote_debugging/binary_io.h
b/Modules/_remote_debugging/binary_io.h
index d4188335c0b6d0a..b8ea9a0e2cc41c4 100644
--- a/Modules/_remote_debugging/binary_io.h
+++ b/Modules/_remote_debugging/binary_io.h
@@ -109,9 +109,6 @@ static_assert(FILE_FOOTER_SIZE == 32,
/* Maximum stack depth we'll buffer for delta encoding */
#define MAX_STACK_DEPTH 256
-/* Initial capacity for RLE pending buffer */
-#define INITIAL_RLE_CAPACITY 64
-
/* Initial capacities for dynamic arrays - sized to reduce reallocations */
#define INITIAL_STRING_CAPACITY 4096
#define INITIAL_FRAME_CAPACITY 4096
@@ -226,12 +223,6 @@ typedef struct {
uint8_t opcode;
} FrameKey;
-/* Pending RLE sample - buffered for run-length encoding */
-typedef struct {
- uint64_t timestamp_delta;
- uint8_t status;
-} PendingRLESample;
-
/* Thread entry - tracks per-thread state for delta encoding */
typedef struct {
uint64_t thread_id;
@@ -244,10 +235,9 @@ typedef struct {
size_t prev_stack_capacity;
/* RLE pending buffer - samples waiting to be written as a repeat group */
- PendingRLESample *pending_rle;
- size_t pending_rle_count;
- size_t pending_rle_capacity;
- int has_pending_rle; /* Flag: do we have buffered repeats? */
+ uint8_t *pending_rle;
+ size_t pending_rle_bytes;
+ size_t pending_rle_samples;
} ThreadEntry;
/* Main binary writer structure */
diff --git a/Modules/_remote_debugging/binary_io_writer.c
b/Modules/_remote_debugging/binary_io_writer.c
index 341f9f7dc8ac457..5d300103ff16f8b 100644
--- a/Modules/_remote_debugging/binary_io_writer.c
+++ b/Modules/_remote_debugging/binary_io_writer.c
@@ -29,6 +29,10 @@
/* Frame buffer: depth varint (max 2 bytes for 256) + 256 frames * 5
bytes/varint + margin */
#define MAX_FRAME_BUFFER_SIZE ((MAX_STACK_DEPTH * MAX_VARINT_SIZE_U32) +
MAX_VARINT_SIZE_U32 + 16)
+/* RLE pending buffer (per thread): one entry is a u64 delta varint + status
byte */
+#define MAX_RLE_BUF_SIZE (16 * 1024)
+#define MAX_RLE_ENTRY_SIZE (MAX_VARINT_SIZE + sizeof(uint8_t))
+
/* Helper macro: convert PyLong to int32, using default_val if conversion
fails */
#define PYLONG_TO_INT32_OR_DEFAULT(obj, var, default_val) \
do { \
@@ -134,15 +138,6 @@ writer_write_varint_u32(BinaryWriter *writer, uint32_t
value)
return writer_write_bytes(writer, buf, len);
}
-/* Encode and write a varint u64 - returns 0 on success, -1 on error */
-static inline int
-writer_write_varint_u64(BinaryWriter *writer, uint64_t value)
-{
- uint8_t buf[MAX_VARINT_SIZE];
- size_t len = encode_varint_u64(buf, value);
- return writer_write_bytes(writer, buf, len);
-}
-
/* ============================================================================
* UTILITY FUNCTIONS
@@ -511,7 +506,6 @@ writer_get_or_create_thread_entry(BinaryWriter *writer,
uint64_t thread_id,
entry->interpreter_id = interpreter_id;
entry->prev_timestamp = writer->start_time_us;
entry->prev_stack_capacity = MAX_STACK_DEPTH;
- entry->pending_rle_capacity = INITIAL_RLE_CAPACITY;
entry->prev_stack = PyMem_Calloc(entry->prev_stack_capacity,
sizeof(uint32_t));
if (!entry->prev_stack) {
@@ -519,13 +513,6 @@ writer_get_or_create_thread_entry(BinaryWriter *writer,
uint64_t thread_id,
return NULL;
}
- entry->pending_rle = PyMem_Malloc(entry->pending_rle_capacity *
sizeof(PendingRLESample));
- if (!entry->pending_rle) {
- PyMem_Free(entry->prev_stack);
- PyErr_NoMemory();
- return NULL;
- }
-
writer->thread_count++;
if (is_new) {
*is_new = 1;
@@ -626,14 +613,9 @@ write_sample_header(BinaryWriter *writer, ThreadEntry
*entry, uint8_t encoding)
static int
flush_pending_rle(BinaryWriter *writer, ThreadEntry *entry)
{
- if (!entry->has_pending_rle || entry->pending_rle_count == 0) {
+ if (entry->pending_rle_samples == 0) {
return 0;
}
- if (entry->pending_rle_count > UINT32_MAX - writer->total_samples) {
- PyErr_SetString(PyExc_OverflowError,
- "too many samples for binary format");
- return -1;
- }
/* Write RLE record:
* [thread_id: 8] [interpreter_id: 4] [STACK_REPEAT: 1] [count: varint]
@@ -644,27 +626,21 @@ flush_pending_rle(BinaryWriter *writer, ThreadEntry
*entry)
return -1;
}
- if (writer_write_varint_u32(writer, (uint32_t)entry->pending_rle_count) <
0) {
+ if (writer_write_varint_u32(writer, (uint32_t)entry->pending_rle_samples)
< 0) {
return -1;
}
- for (size_t i = 0; i < entry->pending_rle_count; i++) {
- if (writer_write_varint_u64(writer,
entry->pending_rle[i].timestamp_delta) < 0) {
- return -1;
- }
- if (writer_write_bytes(writer, &entry->pending_rle[i].status, 1) < 0) {
- return -1;
- }
- writer->total_samples++;
+ if (writer_write_bytes(writer, entry->pending_rle,
entry->pending_rle_bytes) < 0) {
+ return -1;
}
writer->stats.repeat_records++;
- writer->stats.repeat_samples += entry->pending_rle_count;
+ writer->stats.repeat_samples += entry->pending_rle_samples;
/* Each RLE sample saves writing the entire stack */
- writer->stats.frames_saved += entry->pending_rle_count *
entry->prev_stack_depth;
+ writer->stats.frames_saved += entry->pending_rle_samples *
entry->prev_stack_depth;
- entry->pending_rle_count = 0;
- entry->has_pending_rle = 0;
+ entry->pending_rle_bytes = 0;
+ entry->pending_rle_samples = 0;
return 0;
}
@@ -678,12 +654,6 @@ write_sample_with_encoding(BinaryWriter *writer,
ThreadEntry *entry,
const uint32_t *frame_indices, size_t stack_depth,
size_t shared_count, size_t pop_count, size_t
push_count)
{
- if (writer->total_samples == UINT32_MAX) {
- PyErr_SetString(PyExc_OverflowError,
- "too many samples for binary format");
- return -1;
- }
-
/* Header: thread_id(8) + interpreter_id(4) + encoding(1) + delta(varint)
+ status(1) */
uint8_t header_buf[SAMPLE_HEADER_MAX_SIZE];
memcpy(header_buf + SMP_OFF_THREAD_ID, &entry->thread_id,
SMP_SIZE_THREAD_ID);
@@ -752,7 +722,6 @@ write_sample_with_encoding(BinaryWriter *writer,
ThreadEntry *entry,
}
writer->stats.total_frames_written += frames_written;
- writer->total_samples++;
return 0;
}
@@ -955,6 +924,12 @@ static int
process_thread_sample(BinaryWriter *writer, PyObject *thread_info,
uint32_t interpreter_id, uint64_t timestamp_us)
{
+ if (writer->total_samples >= UINT32_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "too many samples for binary format");
+ return -1;
+ }
+
PyObject *thread_id_obj = PyStructSequence_GET_ITEM(thread_info, 0);
PyObject *status_obj = PyStructSequence_GET_ITEM(thread_info, 1);
PyObject *frame_list = PyStructSequence_GET_ITEM(thread_info, 2);
@@ -1002,20 +977,25 @@ process_thread_sample(BinaryWriter *writer, PyObject
*thread_info,
* STACK_REPEAT against an empty curr_stack (depth 0). Buffering
* it here is correct; the RLE flush path emits it as a normal
* STACK_REPEAT record. */
- if (GROW_ARRAY(entry->pending_rle, entry->pending_rle_count,
- entry->pending_rle_capacity, PendingRLESample) < 0) {
+ if (entry->pending_rle == NULL) {
+ entry->pending_rle = PyMem_Malloc(MAX_RLE_BUF_SIZE);
+ if (!entry->pending_rle) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ }
+ if (entry->pending_rle_bytes + MAX_RLE_ENTRY_SIZE > MAX_RLE_BUF_SIZE
+ && flush_pending_rle(writer, entry) < 0) {
return -1;
}
- entry->pending_rle[entry->pending_rle_count].timestamp_delta = delta;
- entry->pending_rle[entry->pending_rle_count].status = status;
- entry->pending_rle_count++;
- entry->has_pending_rle = 1;
+ entry->pending_rle_bytes += encode_varint_u64(
+ entry->pending_rle + entry->pending_rle_bytes, delta);
+ entry->pending_rle[entry->pending_rle_bytes++] = status;
+ entry->pending_rle_samples++;
} else {
/* Stack changed - flush any pending RLE first */
- if (entry->has_pending_rle) {
- if (flush_pending_rle(writer, entry) < 0) {
- return -1;
- }
+ if (flush_pending_rle(writer, entry) < 0) {
+ return -1;
}
if (write_sample_with_encoding(writer, entry, delta, status, encoding,
@@ -1028,6 +1008,7 @@ process_thread_sample(BinaryWriter *writer, PyObject
*thread_info,
entry->prev_stack_depth = curr_depth;
}
+ writer->total_samples++;
return 0;
}
@@ -1075,10 +1056,8 @@ int
binary_writer_finalize(BinaryWriter *writer)
{
for (size_t i = 0; i < writer->thread_count; i++) {
- if (writer->thread_entries[i].has_pending_rle) {
- if (flush_pending_rle(writer, &writer->thread_entries[i]) < 0) {
- return -1;
- }
+ if (flush_pending_rle(writer, &writer->thread_entries[i]) < 0) {
+ return -1;
}
}
diff --git a/Modules/_remote_debugging/module.c
b/Modules/_remote_debugging/module.c
index 7979cd43c6a1274..62d5315f361fa8a 100644
--- a/Modules/_remote_debugging/module.c
+++ b/Modules/_remote_debugging/module.c
@@ -1784,9 +1784,9 @@
_remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self,
/* Finalize the writer, cache total_samples, and destroy it.
*
* The cache assignment must happen AFTER binary_writer_finalize(): finalize
- * flushes pending RLE samples via flush_pending_rle(), which increments
- * writer->total_samples for each one. Caching before finalize would lose
- * those trailing samples. */
+ * flushes pending RLE records and updates the file header before the writer is
+ * destroyed. Caching before finalize would preserve a count for a file that
+ * might still fail to finish. */
static int
binary_writer_finalize_and_cache(BinaryWriterObject *self)
{
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
