https://github.com/python/cpython/commit/0d6671d7448fa412e290617022dab2de61ebfab9
commit: 0d6671d7448fa412e290617022dab2de61ebfab9
branch: main
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-09-14T16:24:02Z
summary:
gh-153568: Skip newline translation for source without carriage returns
(#153584)
The tokenizer copied every input byte by byte to normalize newlines,
but source without a carriage return needs no translation and can be
copied verbatim.
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst
M Parser/tokenizer/decoder.c
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst
new file mode 100644
index 00000000000000..deaa82c96fd3ac
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst
@@ -0,0 +1,2 @@
+Speed up the tokenizer by copying source text verbatim when it contains no
+carriage returns.
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index 5b588572f1fb7e..ab8ae393d03032 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -87,15 +87,22 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
int preserve_crlf,
return NULL;
}
Py_ssize_t write = 0;
- for (Py_ssize_t read = 0; read < len; read++) {
- char c = data[read];
- if (!preserve_crlf && c == '\r') {
- if (read + 1 < len && data[read + 1] == '\n') {
- read++;
+ if (memchr(data, '\r', len) == NULL) {
+ // No carriage returns: nothing to translate, copy verbatim.
+ memcpy(result, data, len);
+ write = len;
+ }
+ else {
+ for (Py_ssize_t read = 0; read < len; read++) {
+ char c = data[read];
+ if (!preserve_crlf && c == '\r') {
+ if (read + 1 < len && data[read + 1] == '\n') {
+ read++;
+ }
+ c = '\n';
}
- c = '\n';
+ result[write++] = c;
}
- result[write++] = c;
}
int implicit = add_final_newline && write > 0 && result[write - 1] != '\n';
if (implicit) {
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
