https://github.com/python/cpython/commit/262625fa30e5a1b5cf33c9dbce5d2b713093c7be
commit: 262625fa30e5a1b5cf33c9dbce5d2b713093c7be
branch: main
author: Grant Herman <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-06-06T01:58:35Z
summary:
gh-150207: Raise MemoryError on tokenizer allocation failure instead of
crashing (#150275)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
M Parser/lexer/state.c
M Parser/tokenizer/file_tokenizer.c
M Parser/tokenizer/helpers.c
M Parser/tokenizer/readline_tokenizer.c
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
new file mode 100644
index 000000000000000..12fbffcd170684c
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
@@ -0,0 +1 @@
+Fix a crash when a memory allocation fails during tokenizer initialization. A
proper :exc:`MemoryError` is now raised instead.
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 3663dc3eb7f9f69..5cf9b4d768c3ebb 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void)
struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
1,
sizeof(struct tok_state));
- if (tok == NULL)
+ if (tok == NULL) {
+ PyErr_NoMemory();
return NULL;
+ }
+
tok->buf = tok->cur = tok->inp = NULL;
tok->fp_interactive = 0;
tok->interactive_src_start = NULL;
diff --git a/Parser/tokenizer/file_tokenizer.c
b/Parser/tokenizer/file_tokenizer.c
index 8c836a3f7258296..a11702557a07af3 100644
--- a/Parser/tokenizer/file_tokenizer.c
+++ b/Parser/tokenizer/file_tokenizer.c
@@ -378,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc,
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
_PyTokenizer_Free(tok);
+ PyErr_NoMemory();
return NULL;
}
tok->cur = tok->inp = tok->buf;
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index 9542969ad3127b9..c69e66d0ab9b7a8 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -193,6 +193,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len,
struct tok_state *tok)
char* result = (char *)PyMem_Malloc(len + 1);
if (!result) {
tok->done = E_NOMEM;
+ PyErr_NoMemory();
return NULL;
}
memcpy(result, s, len);
@@ -221,6 +222,7 @@ _PyTokenizer_translate_newlines(const char *s, int
exec_input, int preserve_crlf
buf = PyMem_Malloc(needed_length);
if (buf == NULL) {
tok->done = E_NOMEM;
+ PyErr_NoMemory();
return NULL;
}
for (current = buf; *s; s++, current++) {
diff --git a/Parser/tokenizer/readline_tokenizer.c
b/Parser/tokenizer/readline_tokenizer.c
index 0f7769aeb8fd570..917f7b40cfbbfed 100644
--- a/Parser/tokenizer/readline_tokenizer.c
+++ b/Parser/tokenizer/readline_tokenizer.c
@@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char*
enc,
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
_PyTokenizer_Free(tok);
+ PyErr_NoMemory();
return NULL;
}
tok->cur = tok->inp = tok->buf;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
