https://github.com/python/cpython/commit/6a40e1c8d00882724f91a03348951d6b19e3ac37 commit: 6a40e1c8d00882724f91a03348951d6b19e3ac37 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: pablogsal <[email protected]> date: 2026-06-06T03:47:00+01:00 summary:
[3.13] gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275) (#150998) gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275) (cherry picked from commit 262625fa30e5a1b5cf33c9dbce5d2b713093c7be) Co-authored-by: Grant Herman <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst M Parser/lexer/state.c M Parser/tokenizer/file_tokenizer.c M Parser/tokenizer/helpers.c M Parser/tokenizer/readline_tokenizer.c diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst new file mode 100644 index 000000000000000..12fbffcd170684c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst @@ -0,0 +1 @@ +Fix a crash when a memory allocation fails during tokenizer initialization. A proper :exc:`MemoryError` is now raised instead. diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 1665debea30b301..7d3b9e9ae59cbc4 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void) struct tok_state *tok = (struct tok_state *)PyMem_Calloc( 1, sizeof(struct tok_state)); - if (tok == NULL) + if (tok == NULL) { + PyErr_NoMemory(); return NULL; + } + tok->buf = tok->cur = tok->inp = NULL; tok->fp_interactive = 0; tok->interactive_src_start = NULL; diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 2750527da484aa5..3e711eb8792c1ed 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -355,6 +355,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index aa6da82319e1c23..976d039f571554d 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -183,6 +183,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) char* result = (char *)PyMem_Malloc(len + 1); if (!result) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } memcpy(result, s, len); @@ -211,6 +212,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf buf = PyMem_Malloc(needed_length); if (buf == NULL) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } for (current = buf; *s; s++, current++) { diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index a2637af9902dd3b..b325cbd6d1cd51d 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
