https://github.com/python/cpython/commit/e5fbabbb47f45f738d42d0a558f37d221937adf0 commit: e5fbabbb47f45f738d42d0a558f37d221937adf0 branch: main author: Pablo Galindo Salgado <[email protected]> committer: pablogsal <[email protected]> date: 2026-09-14T20:31:59Z summary:
gh-153568: Don't materialize parser token text that is never read (#153576) * gh-153568: Don't materialize parser token text that is never read Only tokens whose text is actually consumed get a bytes object; operators and structural tokens no longer allocate one. * Update Parser/pegen.c Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> * gh-153568: Preserve keyword token text --------- Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst M Parser/pegen.c diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst new file mode 100644 index 00000000000000..36504dceb86bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst @@ -0,0 +1,2 @@ +Speed up the parser by not materializing the text of tokens whose text is +never read. diff --git a/Parser/pegen.c b/Parser/pegen.c index c70244b1b36a1f..b13cb8d400f9c3 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -197,6 +197,32 @@ _get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) return NAME; } +// Token types whose text is consumed by grammar actions or helpers, other +// than NAME-derived tokens (identifiers and keywords), which always keep +// their text: error actions may print keyword text (e.g. invalid_kwarg's +// "cannot assign to True"). For every other type the token text is never +// read again, so materializing a PyBytes for it is wasted work. +static inline int +token_needs_text(int type) +{ + switch (type) { + case NAME: + case NUMBER: + case STRING: + case FSTRING_START: + case FSTRING_MIDDLE: + case FSTRING_END: + case TSTRING_START: + case TSTRING_MIDDLE: + case TSTRING_END: + case TYPE_COMMENT: + case NOTEQUAL: // _PyPegen_check_barry_as_flufl() reads its text + return 1; + default: + return 0; + } +} + static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); @@ -205,13 +231,18 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to const char *text = _PyToken_TextView(p->tok, new_token, &length); parser_token->type = token_type == NAME ? _get_keyword_or_name_type(p, text, length) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(text, length); - if (parser_token->bytes == NULL) { - return -1; + if (token_type == NAME || token_needs_text(parser_token->type)) { + parser_token->bytes = PyBytes_FromStringAndSize(text, length); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + else { + parser_token->bytes = NULL; } parser_token->metadata = NULL; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
