https://github.com/python/cpython/commit/f892b11f22f513c7297e403535fa8511fd0bb0fe
commit: f892b11f22f513c7297e403535fa8511fd0bb0fe
branch: main
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-10-13T13:01:17Z
summary:
gh-153569: Simplify tokenizer source views and reader ownership (#158849)
Move source access and logical spans into the tokenizer source API, and keep
input and encoding ownership in the reader. Export the incremental decoder
helper for parser generator extensions.
files:
M Include/internal/pycore_codecs.h
M Parser/lexer/lexer.c
M Parser/lexer/lexer_internal.h
M Parser/lexer/state.c
M Parser/lexer/state.h
M Parser/lexer/string.c
M Parser/tokenizer/api.c
M Parser/tokenizer/decoder.c
M Parser/tokenizer/helpers.c
M Parser/tokenizer/reader.c
M Parser/tokenizer/reader_internal.h
M Parser/tokenizer/source.h
M Parser/tokenizer/tokenizer.h
M Python/Python-tokenize.c
diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h
index bfa10eadf735731..4f8ee383efb22ad 100644
--- a/Include/internal/pycore_codecs.h
+++ b/Include/internal/pycore_codecs.h
@@ -59,10 +59,9 @@ extern PyObject* _PyCodec_DecodeText(
const char *encoding,
const char *errors);
-/* These two aren't actually text encoding specific, but _io.TextIOWrapper
- * is the only current API consumer.
- */
-extern PyObject* _PyCodecInfo_GetIncrementalDecoder(
+/* Helpers to create incremental codecs from a codec info object. */
+// Export for test_peg_generator.
+PyAPI_FUNC(PyObject*) _PyCodecInfo_GetIncrementalDecoder(
PyObject *codec_info,
const char *errors);
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a4fdb9484abd8de..83071e9e0b8ddf0 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -13,12 +13,6 @@
tokenizing. */
static const char* type_comment_prefix = "# type: ";
-static inline int
-contains_null_bytes(const char* str, size_t size)
-{
- return memchr(str, 0, size) != NULL;
-}
-
int
_PyLexer_refill(struct tok_state *tok)
{
@@ -29,8 +23,9 @@ _PyLexer_refill(struct tok_state *tok)
#if defined(Py_DEBUG)
if (tok->debug) {
fprintf(stderr, "line[%d] = ", tok->lineno);
- _PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok,
tok->cur),
- tok->inp - tok->cur);
+ _PyTokenizer_print_escape(
+ stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
+ tok->inp - tok->cur);
fprintf(stderr, " tok->done = %d\n", tok->done);
}
#endif
@@ -39,8 +34,8 @@ _PyLexer_refill(struct tok_state *tok)
return 0;
}
tok->line_start = tok->cur;
- if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
- tok->inp - tok->line_start)) {
+ _PyTok_Span line = {tok->line_start, tok->inp};
+ if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
tok->cur = tok->inp;
return 0;
@@ -56,7 +51,7 @@ _PyLexer_backup(struct tok_state *tok, int c)
if (--tok->cur < tok->buf_offset) {
Py_FatalError("tokenizer beginning of buffer");
}
- if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) !=
Py_CHARMASK(c)) {
+ if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
}
@@ -74,7 +69,8 @@ verify_identifier(struct tok_state *tok)
PyObject *s;
if (tok_failed(tok))
return 0;
- s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur
- tok->start, NULL);
+ s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
+ tok->cur - tok->start, NULL);
if (s == NULL) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_DECODE;
@@ -105,13 +101,13 @@ verify_identifier(struct tok_state *tok)
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
_PyTokenizer_syntaxerror_at(
- tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid character '%c' (U+%04X)", ch, ch);
}
else {
_PyTokenizer_syntaxerror_at(
- tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid non-printable character U+%04X", ch);
}
@@ -172,10 +168,6 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
/* Skip comment, unless it's a type comment */
if (c == '#') {
- const char* p = NULL;
- const char *prefix, *type_start;
- int current_starting_col_offset;
-
while (c != EOF && c != '\n' && c != '\r') {
c = tok_nextc(tok);
}
@@ -192,23 +184,20 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
}
- if (tok->tok_extra_tokens) {
- p = _PyLexer_BufferPointer(tok, tok->start);
- }
-
+ _PyTok_Off comment = tok->start;
if (tok->type_comments) {
- p = _PyLexer_BufferPointer(tok, tok->start);
- current_starting_col_offset = tok->start_loc.byte_col;
- prefix = type_comment_prefix;
- while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
+ const char *prefix = type_comment_prefix;
+ while (*prefix && comment < tok->cur) {
if (*prefix == ' ') {
- while (*p == ' ' || *p == '\t') {
- p++;
- current_starting_col_offset++;
+ while (comment < tok->cur) {
+ int ch = _PyTok_SourceByte(&tok->source, comment);
+ if (ch != ' ' && ch != '\t') {
+ break;
+ }
+ comment++;
}
- } else if (*prefix == *p) {
- p++;
- current_starting_col_offset++;
+ } else if (*prefix == _PyTok_SourceByte(&tok->source,
comment)) {
+ comment++;
} else {
break;
}
@@ -218,35 +207,26 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
/* This is a type comment if we matched all of
type_comment_prefix. */
if (!*prefix) {
- int is_type_ignore = 1;
- // +6 in order to skip the word 'ignore'
- const char *ignore_end = p + 6;
- const int ignore_end_col_offset = current_starting_col_offset
+ 6;
tok_backup(tok, c); /* don't eat the newline or EOF */
-
- type_start = p;
-
/* A TYPE_IGNORE is "type: ignore" followed by the end of the
token
* or anything ASCII and non-alphanumeric. */
- is_type_ignore = (
- _PyLexer_BufferPointer(tok, tok->cur) >= ignore_end &&
memcmp(p, "ignore", 6) == 0
- && !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
- && ((unsigned char)ignore_end[0] >= 128 ||
Py_ISALNUM(ignore_end[0]))));
+ int is_type_ignore = tok->cur - comment >= 6 &&
+ memcmp(_PyTok_SourcePointer(&tok->source, comment),
+ "ignore", 6) == 0;
+ if (is_type_ignore && comment + 6 < tok->cur) {
+ int ch = _PyTok_SourceByte(&tok->source, comment + 6);
+ is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
+ }
int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
- int start_col_offset = is_type_ignore
- ? ignore_end_col_offset : current_starting_col_offset;
+ p_start = comment + (is_type_ignore ? 6 : 0);
p_end = tok->cur;
- if (is_type_ignore) {
- p_start = _PyLexer_BufferOffset(tok, ignore_end);
-
- /* If this type ignore is the only thing on the line,
consume the newline also. */
- if (blankline) {
- tok_nextc(tok);
- tok->layout.at_bol = 1;
- }
- } else {
- p_start = _PyLexer_BufferOffset(tok, type_start);
+ int start_col_offset = tok->start_loc.byte_col +
+ (int)(p_start - tok->start);
+ /* Consume the newline after a standalone type ignore. */
+ if (is_type_ignore && blankline) {
+ tok_nextc(tok);
+ tok->layout.at_bol = 1;
}
_PyLexer_token_setup(tok, token, type, p_start, p_end);
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
@@ -257,7 +237,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
- p_start = _PyLexer_BufferOffset(tok, p);
+ p_start = comment;
p_end = tok->cur;
tok->layout.comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 652512a3e88c22d..922a23bd97ef620 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -40,14 +40,11 @@ tok_nextc(struct tok_state *tok)
return EOF;
}
}
- assert(tok->cur >= tok->source.base_offset);
- assert(tok->cur - tok->source.base_offset < tok->source.len);
if (tok->cur - tok->line_start >= INT_MAX) {
tok->done = E_COLUMNOVERFLOW;
return EOF;
}
- return Py_CHARMASK(
- tok->source.bytes[tok->cur++ - tok->source.base_offset]);
+ return _PyTok_SourceByte(&tok->source, tok->cur++);
}
/* Return -1 on error, otherwise whether the line is blank. */
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 75ff26f16d47ba7..71f1c86461c4bfb 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -58,9 +58,6 @@ _PyLexer_PopFTString(struct tok_state *tok)
void
_PyTokenizer_Free(struct tok_state *tok)
{
- if (tok->encoding != NULL) {
- PyMem_Free(tok->encoding);
- }
Py_XDECREF(tok->filename);
Py_XDECREF(tok->module);
_PyTok_ReaderFree(tok);
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 3b5d47e12a6c38f..51ad278b13c40db 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -80,7 +80,6 @@ struct tok_state {
_PyTok_SourceText source;
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
- FILE *fp; /* Rest of input; NULL if tokenizing a string */
lexer_layout_state layout;
int lineno; /* Current line number */
_PyTok_Loc start_loc;
@@ -92,9 +91,6 @@ struct tok_state {
int parencolstack[MAXLEVEL];
PyObject *filename;
PyObject *module;
- /* Stuff for PEP 0263 */
- char *encoding; /* Source encoding. */
-
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
@@ -135,33 +131,6 @@ _PyLexer_FTStringBracketDepth(const struct tok_state *tok,
return tok->level - state->paren_level;
}
-static inline _PyTok_Off
-_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
-{
- const char *base = _PyTok_SourceData(&tok->source);
- assert(position >= base && position <= base + tok->source.len);
- return tok->source.base_offset + (position - base);
-}
-
-static inline const char *
-_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
-{
- assert(offset >= tok->source.base_offset);
- assert(offset - tok->source.base_offset <= tok->source.len);
- return _PyTok_SourceData(&tok->source) + (offset -
tok->source.base_offset);
-}
-
-static inline const char *
-_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
- Py_ssize_t *length)
-{
- assert(length != NULL);
- assert(_PyTok_SpanIsValid(span));
- *length = span.end - span.start;
- (void)_PyLexer_BufferPointer(tok, span.end);
- return _PyLexer_BufferPointer(tok, span.start);
-}
-
static inline int
_PyLexer_ByteColumn(const struct tok_state *tok)
{
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index 99e45578b6ca16e..d0f568dfa857580 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -76,13 +76,11 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state
*state,
if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
return 0;
}
- Py_ssize_t expr_len;
- const char *expr = _PyLexer_BufferSpanView(
- tok, state->expr_span, &expr_len);
tokenizer_comments *comments = state->comments;
PyObject *res;
if (comments != NULL && comments->count > 0) {
- Py_ssize_t stripped_size = expr_len;
+ Py_ssize_t stripped_size =
+ state->expr_span.end - state->expr_span.start;
Py_ssize_t comment_count = 0;
for (Py_ssize_t i = 0; i < comments->count; i++) {
_PyTok_Span comment = comments->spans[i];
@@ -101,26 +99,28 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state
*state,
PyErr_NoMemory();
return -1;
}
- _PyTok_Off copied_to = state->expr_span.start;
+ _PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
Py_ssize_t stripped_len = 0;
for (Py_ssize_t i = 0; i < comment_count; i++) {
- _PyTok_Span comment = comments->spans[i];
- Py_ssize_t length = comment.start - copied_to;
- memcpy(stripped + stripped_len,
- expr + copied_to - state->expr_span.start,
- (size_t)length);
+ kept.end = comments->spans[i].start;
+ Py_ssize_t length;
+ const char *text = _PyTok_SourceSpanView(
+ &tok->source, kept, &length);
+ memcpy(stripped + stripped_len, text, (size_t)length);
stripped_len += length;
- copied_to = comment.end;
+ kept.start = comments->spans[i].end;
}
- Py_ssize_t length = state->expr_span.end - copied_to;
- memcpy(stripped + stripped_len,
- expr + copied_to - state->expr_span.start,
- (size_t)length);
- stripped_len += length;
- res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
+ kept.end = state->expr_span.end;
+ Py_ssize_t length;
+ const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
+ memcpy(stripped + stripped_len, text, (size_t)length);
+ res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
PyMem_Free(stripped);
}
else {
+ Py_ssize_t expr_len;
+ const char *expr = _PyTok_SourceSpanView(
+ &tok->source, state->expr_span, &expr_len);
res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
}
@@ -339,7 +339,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token
*token, int c)
}
int end_lineno = tok->lineno;
_PyTok_Loc location = tok->start_loc;
- const char *line = _PyLexer_BufferPointer(tok, tok->start) -
location.byte_col;
+ const char *line = _PyTok_SourcePointer(
+ &tok->source, tok->start - location.byte_col);
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
const ftstring_state *state = _PyLexer_CurrentFTString(tok);
@@ -460,7 +461,8 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state
*current, struct tok
int end_lineno = tok->lineno;
_PyTok_Loc location = current->start_loc;
- const char *line = _PyLexer_BufferPointer(tok, current->start) -
location.byte_col;
+ const char *line = _PyTok_SourcePointer(
+ &tok->source, current->start - location.byte_col);
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
if (quote_size == 3) {
diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c
index 3f5efa8dc8f26e6..d955c6b335e1db6 100644
--- a/Parser/tokenizer/api.c
+++ b/Parser/tokenizer/api.c
@@ -4,6 +4,7 @@
#include "tokenizer.h"
#include "reader.h"
+#include "reader_internal.h"
#include "../lexer/state.h"
_PyTokenizer_Info
@@ -21,10 +22,10 @@ _PyTokenizer_GetInfo(const struct tok_state *tok)
.delimiter_loc = {-1, -1},
.in_formatted_string = tok->ftstring_depth != 0,
.is_interactive = _PyTok_ReaderIsInteractive(tok),
- .is_file = tok->fp != NULL && tok->fp != stdin,
+ .is_file = tok->reader->fp != NULL && tok->reader->fp != stdin,
.filename = tok->filename,
.module = tok->module,
- .encoding = tok->encoding,
+ .encoding = tok->reader->encoding,
};
if (tok->level > 0) {
int level = tok->level - 1;
@@ -45,14 +46,14 @@ _PyToken_TextView(const struct tok_state *tok, const struct
token *token,
*length = 0;
return "";
}
- return _PyLexer_BufferSpanView(tok, token->span, length);
+ return _PyTok_SourceSpanView(&tok->source, token->span, length);
}
const char *
_PyTokenizer_SpanView(const struct tok_state *tok, _PyTok_Span span,
Py_ssize_t *length)
{
- return _PyLexer_BufferSpanView(tok, span, length);
+ return _PyTok_SourceSpanView(&tok->source, span, length);
}
void
@@ -60,19 +61,13 @@ _PyToken_GetView(const struct tok_state *tok, const struct
token *token,
_PyToken_View *view)
{
assert(view != NULL);
- assert((token->span.start == -1 && token->span.end == -1) ||
- _PyTok_SpanIsValid(token->span));
- if (token->span.start >= 0) {
- (void)_PyLexer_BufferPointer(tok, token->span.end);
- }
- view->text = token->span.start < 0
- ? NULL : _PyLexer_BufferPointer(tok, token->span.start);
- view->length = token->span.end - token->span.start;
- view->end_line = _PyLexer_BufferPointer(tok, tok->line_start);
- view->line = ISSTRINGLIT(token->type)
- ? view->text - token->start_loc.byte_col : view->end_line;
- view->line_length = tok->inp - tok->line_start +
- (view->end_line - view->line);
+ view->text = _PyToken_TextView(tok, token, &view->length);
+ view->end_line_start = tok->line_start;
+ view->line_span = (_PyTok_Span){
+ ISSTRINGLIT(token->type)
+ ? token->span.start - token->start_loc.byte_col : tok->line_start,
+ tok->inp,
+ };
view->implicit_newline = tok->implicit_newline;
view->at_eof = tok->done == E_EOF;
}
@@ -111,22 +106,28 @@ _PyTokenizer_ImplyDedents(struct tok_state *tok)
int
_PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
{
- const char *cur = _PyLexer_BufferPointer(tok, tok->cur);
- char c = *cur;
- for (;;) {
- while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
- c = *++cur;
- }
- if (!c) {
+ _PyTok_Off cur = tok->cur;
+ _PyTok_Off end = tok->source.base_offset + tok->source.len;
+ while (cur < end) {
+ int c = _PyTok_SourceByte(&tok->source, cur++);
+ if (c == '\0') {
return 0;
}
+ if (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
+ continue;
+ }
if (c != '#') {
return 1;
}
- while (c && c != '\n') {
- c = *++cur;
+ while (cur < end) {
+ c = _PyTok_SourceByte(&tok->source, cur);
+ if (c == '\0' || c == '\n') {
+ break;
+ }
+ cur++;
}
}
+ return 0;
}
int
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index 220fed65ad4e450..48f906845651aab 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -117,8 +117,8 @@ _PyTok_SetEncoding(struct tok_state *tok, const char
*encoding)
tok->done = E_NOMEM;
return -1;
}
- PyMem_Free(tok->encoding);
- tok->encoding = copy;
+ PyMem_Free(tok->reader->encoding);
+ tok->reader->encoding = copy;
return 0;
}
@@ -246,8 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const
_PyTok_Chunk *first,
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
- PyMem_Free(tok->encoding);
- tok->encoding = cookie;
+ PyMem_Free(tok->reader->encoding);
+ tok->reader->encoding = cookie;
return _PYTOK_ENCODING_DONE;
}
@@ -393,9 +393,10 @@ _PyTok_PrepareString(struct tok_state *tok, const char
*input, int utf8_only,
.len = raw_len,
.ownership = _PYTOK_CHUNK_BORROWED,
};
- if (tok->encoding != NULL && strcmp(tok->encoding, "utf-8") != 0) {
+ const char *encoding = tok->reader->encoding;
+ if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
if (_PyTok_DecodeOnce(
- tok, &decoded, tok->encoding, NULL) < 0) {
+ tok, &decoded, encoding, NULL) < 0) {
return -1;
}
}
@@ -407,7 +408,7 @@ _PyTok_PrepareString(struct tok_state *tok, const char
*input, int utf8_only,
return -1;
}
if (!utf8_only &&
- (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
+ (encoding == NULL || strcmp(encoding, "utf-8") == 0) &&
!_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok,
1)) {
return -1;
}
@@ -418,22 +419,18 @@ int
_PyTok_StartDecoder(struct tok_state *tok, const char *errors)
{
_PyTok_Reader *reader = tok->reader;
- if (tok->encoding == NULL || reader->decoder != NULL) {
+ if (reader->encoding == NULL || reader->decoder != NULL) {
return 0;
}
if (reader->kind == _PYTOK_READER_FILE &&
- strcmp(tok->encoding, "utf-8") == 0) {
+ strcmp(reader->encoding, "utf-8") == 0) {
return 0;
}
- PyObject *codec = _PyCodec_LookupTextEncoding(tok->encoding, NULL);
+ PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
if (codec != NULL) {
- PyObject *factory = PyObject_GetAttrString(codec,
"incrementaldecoder");
+ reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
Py_DECREF(codec);
- if (factory != NULL) {
- reader->decoder = PyObject_CallFunction(factory, "s", errors);
- Py_DECREF(factory);
- }
}
if (reader->decoder == NULL) {
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index 0d3ea85109ec49e..c39e6756476b97c 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -91,9 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char
*format, ...)
// These errors are cleaned on startup. Todo: Fix it.
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok,
tok->line_start),
- tok->cur - tok->line_start, tok->lineno,
- format, -1, -1, vargs);
+ int ret = _syntaxerror_range(
+ tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
+ tok->cur - tok->line_start, tok->lineno, format, -1, -1, vargs);
va_end(vargs);
return ret;
}
@@ -105,9 +105,10 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
{
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok,
tok->line_start),
- tok->cur - tok->line_start, tok->lineno,
- format, col_offset, end_col_offset, vargs);
+ int ret = _syntaxerror_range(
+ tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
+ tok->cur - tok->line_start, tok->lineno,
+ format, col_offset, end_col_offset, vargs);
va_end(vargs);
return ret;
}
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 36f910e3199756d..ce2bb1cf5c4bc30 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -34,6 +34,7 @@ _PyTok_ReaderFree(struct tok_state *tok)
i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) {
_PyTok_ChunkClear(&reader->prefetched_lines[i]);
}
+ PyMem_Free(reader->encoding);
PyMem_Free(reader->file_buffer);
PyMem_Free(reader->decoded);
PyMem_Free(reader);
@@ -173,15 +174,13 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
if (tok->lineno >= tok->source.nlines) {
return _PYTOK_READ_EOF;
}
- const char *start = _PyLexer_BufferPointer(tok, tok->inp);
- const char *newline = memchr(
- start, '\n', tok->source.bytes + tok->source.len - start);
- _PyTok_Off end = newline != NULL
- ? newline - tok->source.bytes + 1 : tok->source.len;
- chunk->data = (char *)start;
- chunk->len = tok->source.bytes + end - start;
+ _PyTok_Span tail = {tok->inp, tok->source.base_offset + tok->source.len};
+ _PyTok_Off newline = _PyTok_SourceFindByte(&tok->source, tail, '\n');
+ _PyTok_Span line = {tail.start, newline >= 0 ? newline + 1 : tail.end};
+ chunk->data = (char *)_PyTok_SourceSpanView(
+ &tok->source, line, &chunk->len);
chunk->ownership = _PYTOK_CHUNK_BORROWED;
- chunk->implicit_newline = end == tok->source.len &&
+ chunk->implicit_newline = line.end == tail.end &&
tok->reader->prepared_final_newline_is_implicit;
return _PYTOK_READ_LINE;
}
@@ -203,7 +202,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk)
int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX);
size_t read = 0;
char *result = _Py_UniversalNewlineFgetsWithSize(
- reader->file_buffer + len, available, tok->fp, NULL, &read);
+ reader->file_buffer + len, available, reader->fp, NULL, &read);
if (result == NULL) {
if (len == 0) {
return _PYTOK_READ_EOF;
@@ -228,7 +227,7 @@ initialize_file(struct tok_state *tok)
{
_PyTok_Reader *reader = tok->reader;
reader->file_initialized = 1;
- if (tok->encoding != NULL) {
+ if (reader->encoding != NULL) {
return _PyTok_StartDecoder(tok, "strict");
}
@@ -411,7 +410,7 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
}
_PyTok_Chunk input = {0};
- if (tok->encoding != NULL) {
+ if (reader->encoding != NULL) {
if (!PyBytes_Check(raw)) {
PyErr_SetString(PyExc_TypeError,
"readline() returned a non-bytes object");
@@ -434,7 +433,7 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
input.ownership = _PYTOK_CHUNK_PYOBJECT;
int decoded;
if (reader->decoder == NULL &&
- strcmp(tok->encoding, "utf-8") == 0 &&
+ strcmp(reader->encoding, "utf-8") == 0 &&
chunk_is_line(&input)) {
decoded = _PyTok_DecodeOnce(
tok, &input, "utf-8", "replace");
@@ -510,7 +509,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_STOPPED;
}
char *input = PyOS_Readline(
- tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
+ reader->fp != NULL ? reader->fp : stdin, stdout, reader->prompt);
if (reader->nextprompt != NULL) {
reader->prompt = reader->nextprompt;
}
@@ -527,9 +526,9 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
.len = len,
.ownership = _PYTOK_CHUNK_PYMEM,
};
- if (tok->encoding != NULL &&
+ if (reader->encoding != NULL &&
_PyTok_DecodeOnce(
- tok, &decoded, tok->encoding, NULL) < 0) {
+ tok, &decoded, reader->encoding, NULL) < 0) {
_PyTok_ChunkClear(&decoded);
return _PYTOK_READ_ERROR;
}
@@ -605,10 +604,11 @@ int
_PyTok_ReaderUnderflow(struct tok_state *tok)
{
assert(tok->cur >= tok->buf_offset && tok->cur <= tok->inp);
- _PyTok_ReaderKind kind = tok->reader->kind;
+ _PyTok_Reader *reader = tok->reader;
+ _PyTok_ReaderKind kind = reader->kind;
int prepared = kind == _PYTOK_READER_PREPARED;
int streaming = reader_is_streaming(kind);
- int reset_buffer = !prepared && tok->start < 0 &&
+ int reset_buffer = tok->start < 0 &&
_PyLexer_CurrentFTString(tok) == NULL;
_PyTok_Chunk chunk;
@@ -661,23 +661,24 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
tok->cur = source_start;
tok->buf_offset = source_start;
tok->line_start = tok->buf_offset;
- tok->start = -1;
}
tok->inp = source_start + chunk.len;
}
- if (prepared) {
- if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
- tok->buf_offset = tok->source.base_offset +
- (chunk.data - tok->source.bytes);
+ else {
+ if (reset_buffer) {
+ tok->buf_offset = tok->inp;
}
- tok->inp = _PyLexer_BufferOffset(tok, chunk.data) + chunk.len;
+ tok->inp += chunk.len;
}
tok->implicit_newline = chunk.implicit_newline;
tok->lineno++;
if (kind == _PYTOK_READER_FILE &&
- (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(_PyLexer_BufferPointer(tok, tok->cur),
tok, tok->lineno)) {
+ (reader->encoding == NULL ||
+ strcmp(reader->encoding, "utf-8") == 0) &&
+ !_PyTokenizer_ensure_utf8(
+ _PyTok_SourcePointer(&tok->source, tok->cur),
+ tok, tok->lineno)) {
_PyTok_ChunkClear(&chunk);
return 0;
}
@@ -779,7 +780,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
_PyTokenizer_Free(tok);
return NULL;
}
- tok->fp = fp;
+ tok->reader->fp = fp;
tok->reader->prompt = ps1;
tok->reader->nextprompt = ps2;
return tok;
@@ -840,8 +841,8 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject
*filename)
tok->filename = Py_NewRef(filename != NULL ? filename :
&_Py_STR(anon_string));
char *encoding = NULL;
if (initialize_file(tok) == 0) {
- encoding = tok->encoding;
- tok->encoding = NULL;
+ encoding = tok->reader->encoding;
+ tok->reader->encoding = NULL;
}
fclose(fp);
_PyTokenizer_Free(tok);
diff --git a/Parser/tokenizer/reader_internal.h
b/Parser/tokenizer/reader_internal.h
index f74063de8d653a4..a7a68269fffaa4a 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -41,6 +41,8 @@ typedef struct {
} _PyTok_Chunk;
typedef struct _PyTok_Reader {
+ FILE *fp; // Borrowed input stream; NULL for string and readline input.
+ char *encoding; // Owned source encoding.
PyObject *readline;
PyObject *decoder;
const char *prompt;
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index 66980a41fb7c481..0fd9fe1877bed2b 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -19,6 +19,46 @@ _PyTok_SourceData(const _PyTok_SourceText *source)
return source->bytes != NULL ? source->bytes : "";
}
+/* Convert positions within the retained source window. Pointers and views
+ are borrowed; append, discard, and clear invalidate them. */
+static inline const char *
+_PyTok_SourcePointer(const _PyTok_SourceText *source, _PyTok_Off offset)
+{
+ assert(offset >= source->base_offset);
+ assert(offset - source->base_offset <= source->len);
+ return _PyTok_SourceData(source) + (offset - source->base_offset);
+}
+
+static inline unsigned char
+_PyTok_SourceByte(const _PyTok_SourceText *source, _PyTok_Off offset)
+{
+ assert(offset >= source->base_offset);
+ assert(offset - source->base_offset < source->len);
+ return (unsigned char)source->bytes[offset - source->base_offset];
+}
+
+static inline const char *
+_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ assert(_PyTok_SpanIsValid(span));
+ assert(span.end - source->base_offset <= source->len);
+ *length = span.end - span.start;
+ return _PyTok_SourcePointer(source, span.start);
+}
+
+/* Return the first matching offset within span, or -1 if absent. */
+static inline _PyTok_Off
+_PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span,
+ int byte)
+{
+ Py_ssize_t length;
+ const char *data = _PyTok_SourceSpanView(source, span, &length);
+ const char *found = memchr(data, byte, length);
+ return found != NULL ? span.start + (found - data) : -1;
+}
+
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
/* Clear invalidates all spans and views for the source. */
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h
index 2ef87add3646c3c..bc4009700fdc967 100644
--- a/Parser/tokenizer/tokenizer.h
+++ b/Parser/tokenizer/tokenizer.h
@@ -21,9 +21,8 @@ struct token {
typedef struct {
const char *text;
Py_ssize_t length;
- const char *line;
- Py_ssize_t line_length;
- const char *end_line;
+ _PyTok_Span line_span;
+ _PyTok_Off end_line_start;
int implicit_newline;
int at_eof;
} _PyToken_View;
@@ -73,8 +72,9 @@ _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state
*);
/* An absent token span has a nonnull empty text view. */
const char *_PyToken_TextView(
const struct tok_state *, const struct token *, Py_ssize_t *);
-/* Use the token from the most recent Get. text is NULL for an absent span;
- line includes the token's complete physical line range. */
+/* Use the token from the most recent Get. An absent span has nonnull empty
text.
+ line_span covers the token's complete physical line range. end_line_start
+ is the logical offset of its final physical line. */
void _PyToken_GetView(
const struct tok_state *tok, const struct token *token,
_PyToken_View *view);
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index e91141bd1b9e610..731444e13a02750 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -158,12 +158,16 @@ _tokenizer_error(tokenizeriterobject *it)
static PyObject *
_get_current_line(tokenizeriterobject *it, int current_lineno,
- const char *line_start, Py_ssize_t size, int *line_changed)
+ const _PyToken_View *view, int *line_changed)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
if (current_lineno != it->last_lineno) {
- // Line has changed since last token, so we fetch the new line and
cache it
- // in the iter object.
+ Py_ssize_t size;
+ const char *line_start = _PyTokenizer_SpanView(
+ it->tok, view->line_span, &size);
+ if (size > 0 && view->implicit_newline) {
+ size--;
+ }
Py_XDECREF(it->last_line);
it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace");
it->byte_col_offset_diff = 0;
@@ -180,14 +184,13 @@ _get_col_offsets(tokenizeriterobject *it, const struct
token *token,
Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
- const char *token_start = view->text;
- const char *token_end = token_start == NULL
- ? NULL : token_start + view->length;
+ _PyTok_Off token_start = token->span.start;
+ _PyTok_Off token_end = token->span.end;
Py_ssize_t lineno = token->start_loc.lineno;
Py_ssize_t end_lineno = token->end_loc.lineno;
Py_ssize_t byte_offset = -1;
- if (token_start != NULL && token_start >= view->line) {
- byte_offset = token_start - view->line;
+ if (token_start >= view->line_span.start) {
+ byte_offset = token_start - view->line_span.start;
if (line_changed) {
*col_offset = _PyPegen_byte_offset_to_character_offset_line(line,
0, byte_offset);
if (*col_offset < 0) {
@@ -200,8 +203,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct
token *token,
}
}
- if (token_end != NULL && token_end >= view->end_line) {
- Py_ssize_t end_byte_offset = token_end - view->end_line;
+ if (token_end >= view->end_line_start) {
+ Py_ssize_t end_byte_offset = token_end - view->end_line_start;
if (lineno == end_lineno) {
// Avoid rescanning the prefix of a very long line.
Py_ssize_t token_col_offset =
_PyPegen_byte_offset_to_character_offset_line(line, byte_offset,
end_byte_offset);
@@ -213,7 +216,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct
token *token,
}
else {
*end_col_offset = _PyPegen_byte_offset_to_character_offset_line(
- line, view->end_line - view->line, token_end - view->line);
+ line, view->end_line_start - view->line_span.start,
+ token_end - view->line_span.start);
if (*end_col_offset < 0) {
return -1;
}
@@ -251,15 +255,7 @@ tokenizeriter_next(PyObject *op)
}
_PyToken_View view;
_PyToken_GetView(it->tok, &token, &view);
- const char *token_start = view.text;
- PyObject *str;
- if (token.span.start < 0) {
- assert(token.span.start == -1 && token.span.end == -1);
- str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
- }
- else {
- str = PyUnicode_FromStringAndSize(token_start, view.length);
- }
+ PyObject *str = PyUnicode_FromStringAndSize(view.text, view.length);
if (str == NULL) {
goto exit;
}
@@ -271,13 +267,8 @@ tokenizeriter_next(PyObject *op)
if (it->extra_tokens && is_trailing_token) {
line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
} else {
- Py_ssize_t size = view.line_length;
- if (size >= 1 && view.implicit_newline) {
- size -= 1;
- }
-
line = _get_current_line(
- it, token.end_loc.lineno, view.line, size, &line_changed);
+ it, token.end_loc.lineno, &view, &line_changed);
}
if (line == NULL) {
Py_DECREF(str);
@@ -307,8 +298,7 @@ tokenizeriter_next(PyObject *op)
else if (type == NEWLINE) {
if (!view.implicit_newline) {
Py_DECREF(str);
- assert(token_start != NULL);
- if (token_start[0] == '\r') {
+ if (view.text[0] == '\r') {
str = PyUnicode_FromString("\r\n");
} else {
str = PyUnicode_FromString("\n");
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
